59 lines
2.0 KiB
Bash
Executable File
59 lines
2.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# YOLO Distributed Training Entrypoint
|
|
#
|
|
# Reads env vars at runtime:
|
|
# Distributed mode (multi-node/multi-GPU):
|
|
# MASTER_ADDR - master node IP (triggers distributed mode)
|
|
# MASTER_PORT - master port (default: 29500)
|
|
# NNODES - total number of nodes (default: 1)
|
|
# NPROC_PER_NODE - GPUs per node (default: 1)
|
|
# RANK - this node's rank, 0-indexed (default: 0)
|
|
# NCCL_* - NCCL env vars passed through to PyTorch
|
|
#
|
|
# Single-node mode (MASTER_ADDR not set):
|
|
# Runs yolo directly, auto-detecting available GPUs.
|
|
|
|
set -e
|
|
|
|
CMD_ARGS=("$@")
|
|
[ ${#CMD_ARGS[@]} -eq 0 ] && CMD_ARGS=("train")
|
|
|
|
# --- Distributed mode ---
|
|
if [ -n "${MASTER_ADDR}" ]; then
|
|
MASTER_PORT="${MASTER_PORT:-29500}"
|
|
NNODES="${NNODES:-1}"
|
|
NPROC_PER_NODE="${NPROC_PER_NODE:-1}"
|
|
RANK="${RANK:-0}"
|
|
|
|
# Build per-node GPU list for device=0,1,...,N-1
|
|
GPU_LIST=$(seq -s, 0 $((NPROC_PER_NODE - 1)))
|
|
|
|
echo "============================================"
|
|
echo " Distributed Training Mode"
|
|
echo "============================================"
|
|
echo " MASTER_ADDR = ${MASTER_ADDR}"
|
|
echo " MASTER_PORT = ${MASTER_PORT}"
|
|
echo " NNODES = ${NNODES}"
|
|
echo " NPROC_PER_NODE = ${NPROC_PER_NODE}"
|
|
echo " RANK (node) = ${RANK}"
|
|
echo " WORLD_SIZE = $((NNODES * NPROC_PER_NODE))"
|
|
echo " GPU_LIST = ${GPU_LIST}"
|
|
echo " CMD = yolo ${CMD_ARGS[*]}"
|
|
echo "============================================"
|
|
|
|
exec torchrun \
|
|
--nnodes="${NNODES}" \
|
|
--nproc_per_node="${NPROC_PER_NODE}" \
|
|
--master_addr="${MASTER_ADDR}" \
|
|
--master_port="${MASTER_PORT}" \
|
|
--node_rank="${RANK}" \
|
|
/workspace/train_ddp.py device="${GPU_LIST}" "${CMD_ARGS[@]}"
|
|
else
|
|
echo "============================================"
|
|
echo " Single-Node Training Mode"
|
|
echo "============================================"
|
|
echo " CMD = yolo ${CMD_ARGS[*]}"
|
|
echo "============================================"
|
|
exec yolo "${CMD_ARGS[@]}"
|
|
fi
|