yolo_distributed/entrypoint.sh

59 lines
2.0 KiB
Bash
Executable File

#!/bin/bash
# YOLO Distributed Training Entrypoint
#
# Reads env vars at runtime:
# Distributed mode (multi-node/multi-GPU):
# MASTER_ADDR - master node IP (triggers distributed mode)
# MASTER_PORT - master port (default: 29500)
# NNODES - total number of nodes (default: 1)
# NPROC_PER_NODE - GPUs per node (default: 1)
# RANK - this node's rank, 0-indexed (default: 0)
# NCCL_* - NCCL env vars passed through to PyTorch
#
# Single-node mode (MASTER_ADDR not set):
# Runs yolo directly, auto-detecting available GPUs.
set -e
CMD_ARGS=("$@")
[ ${#CMD_ARGS[@]} -eq 0 ] && CMD_ARGS=("train")
# --- Distributed mode ---
if [ -n "${MASTER_ADDR}" ]; then
MASTER_PORT="${MASTER_PORT:-29500}"
NNODES="${NNODES:-1}"
NPROC_PER_NODE="${NPROC_PER_NODE:-1}"
RANK="${RANK:-0}"
# Build per-node GPU list for device=0,1,...,N-1
GPU_LIST=$(seq -s, 0 $((NPROC_PER_NODE - 1)))
echo "============================================"
echo " Distributed Training Mode"
echo "============================================"
echo " MASTER_ADDR = ${MASTER_ADDR}"
echo " MASTER_PORT = ${MASTER_PORT}"
echo " NNODES = ${NNODES}"
echo " NPROC_PER_NODE = ${NPROC_PER_NODE}"
echo " RANK (node) = ${RANK}"
echo " WORLD_SIZE = $((NNODES * NPROC_PER_NODE))"
echo " GPU_LIST = ${GPU_LIST}"
echo " CMD = yolo ${CMD_ARGS[*]}"
echo "============================================"
exec torchrun \
--nnodes="${NNODES}" \
--nproc_per_node="${NPROC_PER_NODE}" \
--master_addr="${MASTER_ADDR}" \
--master_port="${MASTER_PORT}" \
--node_rank="${RANK}" \
/workspace/train_ddp.py device="${GPU_LIST}" "${CMD_ARGS[@]}"
else
echo "============================================"
echo " Single-Node Training Mode"
echo "============================================"
echo " CMD = yolo ${CMD_ARGS[*]}"
echo "============================================"
exec yolo "${CMD_ARGS[@]}"
fi