56 lines
1.8 KiB
Bash
Executable File
56 lines
1.8 KiB
Bash
Executable File
#!/bin/sh
|
|
# Pi05 torch 训练入口 (design_v2 §5.3)。torch 起点权重经 PYTORCH_WEIGHT_PATH
|
|
# (j2t 产物或用户外部权重); 单机多卡经 NPROC_PER_NODE (torchrun, D14)。
|
|
set -eu
|
|
|
|
echo "[$(date -u +%FT%TZ)] node=${NODE_LABEL:-} run=${RUN_ID:-}"
|
|
|
|
if ! (
|
|
set -eu
|
|
|
|
echo "== container env self-check (log only, failures tolerated) =="
|
|
df -h || true
|
|
free -h || true
|
|
nvidia-smi || echo "[env-check] nvidia-smi unavailable, continue without GPU info"
|
|
|
|
: "${OUTPUT_DIR:?OUTPUT_DIR not set}"
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
# stdout/stderr 双写: 控制台 + $OUTPUT_DIR/stdout.log (PVC)。
|
|
# 平台对失败节点的容器 stdout 不持久化 (Job GC 即失), 落盘才能事后取证。
|
|
LOG_PIPE="$OUTPUT_DIR/.stdout.pipe"
|
|
rm -f "$LOG_PIPE"
|
|
if mkfifo "$LOG_PIPE" 2>/dev/null; then
|
|
tee "$OUTPUT_DIR/stdout.log" < "$LOG_PIPE" &
|
|
TEE_PID=$!
|
|
exec > "$LOG_PIPE" 2>&1
|
|
else
|
|
exec > "$OUTPUT_DIR/stdout.log" 2>&1
|
|
fi
|
|
|
|
|
|
# 幂等清理: 只清本功能产物
|
|
rm -rf "$OUTPUT_DIR/checkpoints" "$OUTPUT_DIR/assets" "$OUTPUT_DIR/train_report.json"
|
|
|
|
export PYTHONPATH="/app/src/openpi/src:/app/src${PYTHONPATH:+:$PYTHONPATH}"
|
|
export OPENPI_DATA_HOME="${OPENPI_DATA_HOME:-/app/src/openpi_cache}"
|
|
export HF_HUB_OFFLINE="${HF_HUB_OFFLINE:-1}"
|
|
|
|
NPROC="${NPROC_PER_NODE:-1}"
|
|
if [ "$NPROC" -gt 1 ]; then
|
|
# torchrun 走 venv 全路径 (镜像无系统级 torchrun, 不依赖 PATH)
|
|
/app/src/venv_train/bin/torchrun --standalone --nnodes=1 --nproc_per_node="$NPROC" \
|
|
/app/src/pi_pipeline/train_torch_runner.py
|
|
else
|
|
/app/src/venv_train/bin/python /app/src/pi_pipeline/train_torch_runner.py
|
|
fi
|
|
|
|
); then
|
|
echo "[entry] train_torch FAILED; env keys: $(env | cut -d= -f1 | sort | tr '\n' ' ')"
|
|
echo "[entry] keeping pod alive 900s for diagnosis"
|
|
sleep 900
|
|
exit 1
|
|
fi
|
|
|
|
sh /app/report.sh
|