63 lines
3.4 KiB
Docker
63 lines
3.4 KiB
Docker
# syntax=docker/dockerfile:1
|
|
# quant_unified: 单镜像双环境量化镜像
|
|
# /opt/conda/envs/data py3.12 训练镜像同版本集 (lerobot 直读数据集)
|
|
# /opt/conda/envs/quant py3.10 pi_quant 全套 (horizon 工具链 + vendor 量化链)
|
|
# 层序: 大依赖层在前 (freeze/wheels), 源码层最后 (改代码只重建末层)
|
|
|
|
FROM continuumio/miniconda3:24.7.1-0
|
|
|
|
ARG http_proxy=
|
|
ARG https_proxy=
|
|
ENV http_proxy=${http_proxy} https_proxy=${https_proxy} no_proxy=localhost,127.0.0.1
|
|
|
|
# 双 conda env (python 版本分别由两侧工具链锁定)
|
|
RUN conda create -y -n data python=3.12 && conda create -y -n quant python=3.10
|
|
|
|
# venv_data: 训练镜像同版本集 (大层, 少动); lerobot 本体 --no-deps (绕开实体机器人依赖链)
|
|
# pip cache mount: 层失效时 wheel 缓存跨构建复用, 免重下大包
|
|
COPY docker/freeze_data.txt /tmp/freeze_data.txt
|
|
RUN --mount=type=cache,target=/root/.cache/pip \
|
|
/opt/conda/envs/data/bin/pip install -r /tmp/freeze_data.txt \
|
|
&& /opt/conda/envs/data/bin/pip install --no-deps lerobot==0.4.4
|
|
|
|
# venv_quant: pi_quant pip 集 (大层, 少动)
|
|
COPY docker/freeze_quant.txt /tmp/freeze_quant.txt
|
|
RUN --mount=type=cache,target=/root/.cache/pip \
|
|
/opt/conda/envs/quant/bin/pip install -r /tmp/freeze_quant.txt
|
|
|
|
# horizon wheels (本地 vendored, cp310/torch280 锁定)
|
|
COPY deps/wheels /tmp/wheels
|
|
RUN --mount=type=cache,target=/root/.cache/pip \
|
|
/opt/conda/envs/quant/bin/pip install /tmp/wheels/*.whl && rm -rf /tmp/wheels
|
|
|
|
# torchcodec 运行时依赖: conda-forge ffmpeg 7.1 独立小环境 (debian 源走代理极不稳定, 弃 apt)
|
|
# SONAME 精确配对 libtorchcodec_core7: avcodec/avformat/avdevice 61, avutil 59, avfilter 10, swscale 8, swresample 5
|
|
# LD_LIBRARY_PATH 仅在 data env 进程注入 (entry/冒烟), 不污染 quant env
|
|
RUN conda create -y -n avlibs -c conda-forge "ffmpeg=7.1" \
|
|
&& ls /opt/conda/envs/avlibs/lib/libavcodec.so.61* /opt/conda/envs/avlibs/lib/libavdevice.so.61* /opt/conda/envs/avlibs/lib/libavfilter.so.10* >/dev/null
|
|
|
|
# 源码层 (最后)
|
|
COPY quant_pipeline /opt/quant_dev/quant_pipeline
|
|
COPY vendor /opt/quant_dev/vendor
|
|
COPY configs /opt/quant_dev/configs
|
|
COPY board /opt/quant_dev/board
|
|
COPY entry /app/entry
|
|
COPY report.sh /app/report.sh
|
|
# 运行树开放写权限: horizon_plugin 在 CWD 写 .horizon_plugin_pytorch_logs, 平台以 uid 10001 运行
|
|
RUN chmod -R a+rwX /opt/quant_dev \
|
|
&& chmod +x /app/entry/*.sh /app/report.sh \
|
|
&& useradd -u 10001 -M -d /tmp -s /usr/sbin/nologin ops
|
|
|
|
# 构建期冒烟: 双栈 import + torchcodec 链接验证 (data env 注入 avlibs 的 LD_LIBRARY_PATH)
|
|
RUN LD_LIBRARY_PATH=/opt/conda/envs/avlibs/lib \
|
|
/opt/conda/envs/data/bin/python -c "import torch, torchcodec, datasets, lerobot.datasets.lerobot_dataset as ld; print('data-stack ok:', torch.__version__, torchcodec.__version__, datasets.__version__)" \
|
|
&& /opt/conda/envs/quant/bin/python -c "import torch, datasets, transformers, sentencepiece; print('quant-stack ok:', torch.__version__, datasets.__version__, transformers.__version__)"
|
|
|
|
# 运行时默认走 quant env (run_quant 主场); stage0 由 entry 显式切 data env
|
|
ENV http_proxy= https_proxy=
|
|
ENV PATH=/opt/conda/envs/quant/bin:/opt/conda/condabin:$PATH
|
|
ENV PYTHONPATH=/opt/quant_dev/vendor:/opt/quant_dev/vendor/llm_compression/lightcompress
|
|
WORKDIR /app
|
|
# 平台覆盖 ENTRYPOINT, 靠 containerCommand 指定 /app/entry/quant_one_step.sh
|
|
ENTRYPOINT ["/bin/bash", "/app/entry/quant_one_step.sh"]
|