#!/bin/bash # YOLO Distributed Training Entrypoint # # Reads env vars at runtime: # Distributed mode (multi-node/multi-GPU): # MASTER_ADDR - master node IP (triggers distributed mode) # MASTER_PORT - master port (default: 29500) # NNODES - total number of nodes (default: 1) # NPROC_PER_NODE - GPUs per node (default: 1) # RANK - this node's rank, 0-indexed (default: 0) # NCCL_* - NCCL env vars passed through to PyTorch # # Single-node mode (MASTER_ADDR not set): # Runs yolo directly, auto-detecting available GPUs. set -e CMD_ARGS=("$@") [ ${#CMD_ARGS[@]} -eq 0 ] && CMD_ARGS=("train") # --- Distributed mode --- if [ -n "${MASTER_ADDR}" ]; then MASTER_PORT="${MASTER_PORT:-29500}" NNODES="${NNODES:-1}" NPROC_PER_NODE="${NPROC_PER_NODE:-1}" RANK="${RANK:-0}" # Build per-node GPU list for device=0,1,...,N-1 GPU_LIST=$(seq -s, 0 $((NPROC_PER_NODE - 1))) echo "============================================" echo " Distributed Training Mode" echo "============================================" echo " MASTER_ADDR = ${MASTER_ADDR}" echo " MASTER_PORT = ${MASTER_PORT}" echo " NNODES = ${NNODES}" echo " NPROC_PER_NODE = ${NPROC_PER_NODE}" echo " RANK (node) = ${RANK}" echo " WORLD_SIZE = $((NNODES * NPROC_PER_NODE))" echo " GPU_LIST = ${GPU_LIST}" echo " CMD = yolo ${CMD_ARGS[*]}" echo "============================================" exec torchrun \ --nnodes="${NNODES}" \ --nproc_per_node="${NPROC_PER_NODE}" \ --master_addr="${MASTER_ADDR}" \ --master_port="${MASTER_PORT}" \ --node_rank="${RANK}" \ /workspace/train_ddp.py device="${GPU_LIST}" "${CMD_ARGS[@]}" else echo "============================================" echo " Single-Node Training Mode" echo "============================================" echo " CMD = yolo ${CMD_ARGS[*]}" echo "============================================" exec yolo "${CMD_ARGS[@]}" fi