From b7ec03d2fae490d8c9a1a5bbe95605da485ccecc Mon Sep 17 00:00:00 2001 From: binwang Date: Thu, 11 Jun 2026 11:45:35 +0800 Subject: [PATCH] feat: add YOLO distributed training Docker deployment --- .gitignore | 1 + docker/distributed/.dockerignore | 8 +++ docker/distributed/.gitignore | 2 + docker/distributed/Dockerfile | 27 ++++++++++ docker/distributed/build_docker.sh | 80 ++++++++++++++++++++++++++++++ docker/distributed/entrypoint.sh | 58 ++++++++++++++++++++++ docker/distributed/train_ddp.py | 9 ++++ docker/distributed/yolo26.yaml | 52 +++++++++++++++++++ 8 files changed, 237 insertions(+) create mode 100644 docker/distributed/.dockerignore create mode 100644 docker/distributed/.gitignore create mode 100644 docker/distributed/Dockerfile create mode 100755 docker/distributed/build_docker.sh create mode 100755 docker/distributed/entrypoint.sh create mode 100644 docker/distributed/train_ddp.py create mode 100644 docker/distributed/yolo26.yaml diff --git a/.gitignore b/.gitignore index a1d17dc9..80fc3ff4 100644 --- a/.gitignore +++ b/.gitignore @@ -206,3 +206,4 @@ results.csv # CI test asset cache .ultralytics_cache/ +data/ diff --git a/docker/distributed/.dockerignore b/docker/distributed/.dockerignore new file mode 100644 index 00000000..b17a3187 --- /dev/null +++ b/docker/distributed/.dockerignore @@ -0,0 +1,8 @@ +* +!entrypoint.sh +!train_ddp.py +!Dockerfile +!yolo26.yaml +!yolo26n.pt +!Arial.ttf +!Arial.Unicode.ttf diff --git a/docker/distributed/.gitignore b/docker/distributed/.gitignore new file mode 100644 index 00000000..9e340f0d --- /dev/null +++ b/docker/distributed/.gitignore @@ -0,0 +1,2 @@ +*.ttf +*.pt diff --git a/docker/distributed/Dockerfile b/docker/distributed/Dockerfile new file mode 100644 index 00000000..245e61e2 --- /dev/null +++ b/docker/distributed/Dockerfile @@ -0,0 +1,27 @@ +FROM timm-train:latest + +ARG http_proxy +ARG https_proxy +ARG no_proxy + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + MKL_THREADING_LAYER=GNU \ + YOLO_OFFLINE=true + +RUN pip install --no-cache-dir ultralytics + +COPY entrypoint.sh /entrypoint.sh +COPY yolo26.yaml /workspace/models/yolo26.yaml +COPY yolo26n.pt /workspace/models/yolo26n.pt +COPY yolo26n.pt /workspace/yolo26n.pt +COPY train_ddp.py /workspace/train_ddp.py +COPY Arial.ttf /root/.config/Ultralytics/Arial.ttf +COPY Arial.Unicode.ttf /root/.config/Ultralytics/Arial.Unicode.ttf +RUN chmod +x /entrypoint.sh + +ENTRYPOINT [] +CMD ["bash"] + +WORKDIR /workspace diff --git a/docker/distributed/build_docker.sh b/docker/distributed/build_docker.sh new file mode 100755 index 00000000..ae365da2 --- /dev/null +++ b/docker/distributed/build_docker.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Build YOLO distributed training Docker image and push to registry +# CUDA 12.8 + PyTorch 2.8.0, supports RTX 5090 (sm_120 / compute capability 12.0) +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REGISTRY="registry.d-robotics.cc/fat-auwomo-fat-auwomo/yolo_distributed" +IMAGE_TAG="${IMAGE_TAG:-v$(date +%Y%m%d)_fix}" +LOCAL_TAG="yolo-train:${IMAGE_TAG}" + +echo "============================================" +echo " Building YOLO Distributed Training Image" +echo "============================================" +echo " Registry: ${REGISTRY}" +echo " Tag: ${IMAGE_TAG}" +echo " CUDA: 12.8 (supports RTX 5090 sm_120)" +echo " PyTorch: 2.8.0+cu128" +echo "============================================" + +# Pass proxy env vars if set +PROXY_ARGS=() +[ -n "$http_proxy" ] && PROXY_ARGS+=(--build-arg "http_proxy=$http_proxy") +[ -n "$https_proxy" ] && PROXY_ARGS+=(--build-arg "https_proxy=$https_proxy") +[ -n "$no_proxy" ] && PROXY_ARGS+=(--build-arg "no_proxy=$no_proxy") + +# Build +docker build \ + "${PROXY_ARGS[@]}" \ + -t "${LOCAL_TAG}" \ + -f "${SCRIPT_DIR}/Dockerfile" \ + "${SCRIPT_DIR}" + +echo "" +echo "Build complete: ${LOCAL_TAG}" + +# Tag for registry +REMOTE_TAG="${REGISTRY}:${IMAGE_TAG}" +echo "" +echo "Tagging for registry: ${REMOTE_TAG}" +docker tag "${LOCAL_TAG}" "${REMOTE_TAG}" + +# Push +echo "" +echo "Pushing to registry..." +docker push "${REMOTE_TAG}" + +echo "" +echo "============================================" +echo " Push complete: ${REMOTE_TAG}" +echo "============================================" +echo "" +echo "Usage examples:" +echo "" +echo " # Pull image" +echo " docker pull ${REMOTE_TAG}" +echo "" +echo " # Single-node training (all GPUs)" +echo " docker run --gpus all --ipc=host \\" +echo " -v \$(pwd)/datasets:/workspace/datasets \\" +echo " ${REMOTE_TAG} bash /entrypoint.sh train data=kitti.yaml model=yolo26n.pt epochs=100" +echo "" +echo " # Distributed multi-node (node 0 - master)" +echo " docker run --gpus all --ipc=host --network=host \\" +echo " -v /shared/datasets:/workspace/datasets \\" +echo " -e MASTER_ADDR=10.0.0.1 \\" +echo " -e MASTER_PORT=29500 \\" +echo " -e NNODES=2 \\" +echo " -e NPROC_PER_NODE=4 \\" +echo " -e RANK=0 \\" +echo " ${REMOTE_TAG} bash /entrypoint.sh train data=kitti.yaml model=yolo26n.pt epochs=100" +echo "" +echo " # Distributed multi-node (node 1 - worker)" +echo " docker run --gpus all --ipc=host --network=host \\" +echo " -v /shared/datasets:/workspace/datasets \\" +echo " -e MASTER_ADDR=10.0.0.1 \\" +echo " -e MASTER_PORT=29500 \\" +echo " -e NNODES=2 \\" +echo " -e NPROC_PER_NODE=4 \\" +echo " -e RANK=1 \\" +echo " ${REMOTE_TAG} bash /entrypoint.sh train data=kitti.yaml model=yolo26n.pt epochs=100" diff --git a/docker/distributed/entrypoint.sh b/docker/distributed/entrypoint.sh new file mode 100755 index 00000000..610af1e4 --- /dev/null +++ b/docker/distributed/entrypoint.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# YOLO Distributed Training Entrypoint +# +# Reads env vars at runtime: +# Distributed mode (multi-node/multi-GPU): +# MASTER_ADDR - master node IP (triggers distributed mode) +# MASTER_PORT - master port (default: 29500) +# NNODES - total number of nodes (default: 1) +# NPROC_PER_NODE - GPUs per node (default: 1) +# RANK - this node's rank, 0-indexed (default: 0) +# NCCL_* - NCCL env vars passed through to PyTorch +# +# Single-node mode (MASTER_ADDR not set): +# Runs yolo directly, auto-detecting available GPUs. + +set -e + +CMD_ARGS=("$@") +[ ${#CMD_ARGS[@]} -eq 0 ] && CMD_ARGS=("train") + +# --- Distributed mode --- +if [ -n "${MASTER_ADDR}" ]; then + MASTER_PORT="${MASTER_PORT:-29500}" + NNODES="${NNODES:-1}" + NPROC_PER_NODE="${NPROC_PER_NODE:-1}" + RANK="${RANK:-0}" + + # Build per-node GPU list for device=0,1,...,N-1 + GPU_LIST=$(seq -s, 0 $((NPROC_PER_NODE - 1))) + + echo "============================================" + echo " Distributed Training Mode" + echo "============================================" + echo " MASTER_ADDR = ${MASTER_ADDR}" + echo " MASTER_PORT = ${MASTER_PORT}" + echo " NNODES = ${NNODES}" + echo " NPROC_PER_NODE = ${NPROC_PER_NODE}" + echo " RANK (node) = ${RANK}" + echo " WORLD_SIZE = $((NNODES * NPROC_PER_NODE))" + echo " GPU_LIST = ${GPU_LIST}" + echo " CMD = yolo ${CMD_ARGS[*]}" + echo "============================================" + + exec torchrun \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + --node_rank="${RANK}" \ + /workspace/train_ddp.py device="${GPU_LIST}" "${CMD_ARGS[@]}" +else + echo "============================================" + echo " Single-Node Training Mode" + echo "============================================" + echo " CMD = yolo ${CMD_ARGS[*]}" + echo "============================================" + exec yolo "${CMD_ARGS[@]}" +fi diff --git a/docker/distributed/train_ddp.py b/docker/distributed/train_ddp.py new file mode 100644 index 00000000..545ad0d5 --- /dev/null +++ b/docker/distributed/train_ddp.py @@ -0,0 +1,9 @@ +import os +import sys + +local_rank = int(os.environ.get("LOCAL_RANK", 0)) +os.environ["RANK"] = str(local_rank) + +from ultralytics.cfg import entrypoint +sys.argv[0] = "yolo" +entrypoint() diff --git a/docker/distributed/yolo26.yaml b/docker/distributed/yolo26.yaml new file mode 100644 index 00000000..4fc9eb29 --- /dev/null +++ b/docker/distributed/yolo26.yaml @@ -0,0 +1,52 @@ +# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license + +# Ultralytics YOLO26 object detection model with P3/8 - P5/32 outputs +# Model docs: https://docs.ultralytics.com/models/yolo26 +# Task docs: https://docs.ultralytics.com/tasks/detect + +# Parameters +nc: 80 # number of classes +end2end: True # whether to use end-to-end mode +reg_max: 1 # DFL bins +scales: # model compound scaling constants, i.e. 'model=yolo26n.yaml' will call yolo26.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.50, 0.25, 1024] # summary: 260 layers, 2,572,280 parameters, 2,572,280 gradients, 6.1 GFLOPs + s: [0.50, 0.50, 1024] # summary: 260 layers, 10,009,784 parameters, 10,009,784 gradients, 22.8 GFLOPs + m: [0.50, 1.00, 512] # summary: 280 layers, 21,896,248 parameters, 21,896,248 gradients, 75.4 GFLOPs + l: [1.00, 1.00, 512] # summary: 392 layers, 26,299,704 parameters, 26,299,704 gradients, 93.8 GFLOPs + x: [1.00, 1.50, 512] # summary: 392 layers, 58,993,368 parameters, 58,993,368 gradients, 209.5 GFLOPs + +# YOLO26n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 2, C3k2, [256, False, 0.25]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 2, C3k2, [512, False, 0.25]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 2, C3k2, [512, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 2, C3k2, [1024, True]] + - [-1, 1, SPPF, [1024, 5, 3, True]] # 9 + - [-1, 2, C2PSA, [1024]] # 10 + +# YOLO26n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 2, C3k2, [512, True]] # 13 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 2, C3k2, [256, True]] # 16 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 13], 1, Concat, [1]] # cat head P4 + - [-1, 2, C3k2, [512, True]] # 19 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 10], 1, Concat, [1]] # cat head P5 + - [-1, 1, C3k2, [1024, True, 0.5, True]] # 22 (P5/32-large) + + - [[16, 19, 22], 1, Detect, [nc]] # Detect(P3, P4, P5)