zulifeng cb5786c319 feat: 多机 NCCL Scaling 融入 Python 测试框架
- modules/nccl_test.py: 新增 MultiNodeNCCLTest 类,通过 mpirun + hostfile
  跑跨节点 NCCL Scaling 测试;支持 scale_levels 逐级测试,AllToAll 自动
  用 1G 消息避免大消息 hang,NCCL 环境变量通过 -x 传入各节点
- gpu_tester.py: 菜单加 [m] Multi-node NCCL Scaling,CLI 加 --test multinode-nccl
- configs/default.yaml: 新增 multinode_nccl 配置段(hostfile/mpi_home/HCA/网卡等)
- scripts/multinode_nccl/: 保留 shell 脚本版(nccl_env.sh/run_scaling.sh/
  run_cross_switch_test.sh/hosts_all,含 gpu-51/55 共 34 台)
- README: 新增 5m 多机 NCCL 章节,更新 SOP-3 和项目结构说明

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-06 18:58:08 +08:00

64 lines
1.9 KiB
Bash

#!/bin/bash
# ===== 多机 NCCL Scaling 测试 =====
# 用法: bash run_scaling.sh [节点级别...]
# 默认: "2 4 8 16 32"
# 示例: bash run_scaling.sh 2 4 8
# 前置: source nccl_env.sh, hosts_all 文件与本脚本同目录或指定 HOSTS_ALL
# 主节点运行: gpu-20
set -euo pipefail
source "$(dirname "$0")/nccl_env.sh"
NT=/opt/nccl-tests/build
HOSTS_ALL="${HOSTS_ALL:-$(dirname "$0")/hosts_all}"
LEVELS="${*:-2 4 8 16 32}"
OUT=~/nccl_kit/results_$(date +%Y%m%d_%H%M%S)
mkdir -p "$OUT"
SUMMARY="$OUT/summary.csv"
echo "nodes,gpus,op,size,busbw_GBps" > "$SUMMARY"
MPI=(mpirun --allow-run-as-root
--mca plm_rsh_args "-oStrictHostKeyChecking=no"
-x PATH -x LD_LIBRARY_PATH
-x NCCL_IB_HCA -x NCCL_IB_DISABLE
-x NCCL_SOCKET_IFNAME -x NCCL_DEBUG)
echo "Scaling levels: $LEVELS"
echo "结果目录: $OUT"
for N in $LEVELS; do
hf="$OUT/hosts_$N"
head -"$N" "$HOSTS_ALL" > "$hf"
avail=$(wc -l < "$hf")
if [ "$avail" -lt "$N" ]; then
echo "!! hosts_all 只有 $avail 台, 跳过 $N 节点"
continue
fi
np=$((N * 8))
echo ""
echo "================ $N nodes / $np GPUs ================"
for op in all_reduce alltoall all_gather reduce_scatter; do
log="$OUT/${op}_${N}n.log"
"${MPI[@]}" -np "$np" -N 8 --hostfile "$hf" \
"$NT/${op}_perf" -b 8 -e 8G -f 2 -g 1 -w 5 -n 50 > "$log" 2>&1
# AllToAll 在大消息 (>2G) + 多节点会 hang, 优先取 1G 行
if [[ "$op" == "alltoall" ]]; then
row=$(awk '$1==1073741824{print}' "$log" | tail -1)
sz="1G"
else
row=$(awk '$1==8589934592{print}' "$log" | tail -1)
sz="8G"
fi
busbw=$(echo "$row" | awk '{print $(NF-1)}')
[ -z "$busbw" ] && busbw="ERR(见 $log)"
printf " %-14s %s busbw=%s GB/s\n" "$op" "$sz" "$busbw"
echo "$N,$np,$op,$sz,$busbw" >> "$SUMMARY"
done
done
echo ""
echo "==== 汇总 (AllReduce 应随节点数基本持平 ~317 GB/s) ===="
column -t -s, "$SUMMARY"
echo "结果目录: $OUT"