test_gpu_scripts/scripts/multinode_nccl/run_cross_switch_test.sh
zulifeng cb5786c319 feat: 多机 NCCL Scaling 融入 Python 测试框架
- modules/nccl_test.py: 新增 MultiNodeNCCLTest 类,通过 mpirun + hostfile
  跑跨节点 NCCL Scaling 测试;支持 scale_levels 逐级测试,AllToAll 自动
  用 1G 消息避免大消息 hang,NCCL 环境变量通过 -x 传入各节点
- gpu_tester.py: 菜单加 [m] Multi-node NCCL Scaling,CLI 加 --test multinode-nccl
- configs/default.yaml: 新增 multinode_nccl 配置段(hostfile/mpi_home/HCA/网卡等)
- scripts/multinode_nccl/: 保留 shell 脚本版(nccl_env.sh/run_scaling.sh/
  run_cross_switch_test.sh/hosts_all,含 gpu-51/55 共 34 台)
- README: 新增 5m 多机 NCCL 章节,更新 SOP-3 和项目结构说明

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-06 18:58:08 +08:00

77 lines
2.3 KiB
Bash

#!/bin/bash
# ===== 跨交换机多机 NCCL 通信测试 =====
# 适用场景: 验证不同交换机的节点加入集群后跨交换机通信是否正常
# 用法: bash run_cross_switch_test.sh
# 主节点运行: gpu-20 (需与目标节点已打通免密 SSH)
#
# 默认分三个阶段:
# old2node — 旧集群内 2 台基准 (gpu-20/21)
# new2node — 新机 2 台同交换机 (gpu-51/55)
# cross4node — 跨交换机 4 台联合
#
# 新机接入前置步骤:
# 1. 四台节点互相打通 authorized_keys
# 2. 新机上重编 nccl-tests: cd /opt/nccl-tests && make clean &&
# make MPI=1 MPI_HOME=/usr/mpi/gcc/openmpi-4.1.7rc1 CUDA_HOME=/usr/local/cuda -j
set -euo pipefail
source "$(dirname "$0")/nccl_env.sh"
NT=/opt/nccl-tests/build
OUT=~/nccl_kit/results_cross_$(date +%Y%m%d_%H%M%S)
mkdir -p "$OUT"
SUMMARY="$OUT/summary.csv"
echo "test,nodes,op,size,busbw_GBps" > "$SUMMARY"
MPI=(mpirun --allow-run-as-root
--mca plm_rsh_args "-oStrictHostKeyChecking=no"
-x PATH -x LD_LIBRARY_PATH
-x NCCL_IB_HCA -x NCCL_IB_DISABLE
-x NCCL_SOCKET_IFNAME -x NCCL_DEBUG)
run_test() {
local tag="$1" hostfile="$2"
local np=$(( $(wc -l < "$hostfile") * 8 ))
echo ""
echo "======== $tag ($np GPUs) ========"
for op in all_reduce alltoall all_gather reduce_scatter; do
log="$OUT/${tag}_${op}.log"
"${MPI[@]}" -np "$np" -N 8 --hostfile "$hostfile" \
"$NT/${op}_perf" -b 8 -e 4G -f 2 -g 1 -w 5 -n 30 > "$log" 2>&1
row=$(awk '$1==4294967296{print}' "$log" | tail -1)
busbw=$(echo "$row" | awk '{print $(NF-1)}')
[ -z "$busbw" ] && busbw="ERR(见 $log)"
printf " %-14s 4G busbw=%s GB/s\n" "$op" "$busbw"
echo "$tag,$(wc -l < "$hostfile"),$op,4G,$busbw" >> "$SUMMARY"
done
}
# === 阶段1: 旧集群内 2 台 (基准) ===
cat > "$OUT/hosts_old2" << HEOF
10.184.0.20 slots=8
10.184.0.21 slots=8
HEOF
run_test "old2node" "$OUT/hosts_old2"
# === 阶段2: 新机 2 台同交换机 ===
cat > "$OUT/hosts_new2" << HEOF
10.184.0.51 slots=8
10.184.0.55 slots=8
HEOF
run_test "new2node" "$OUT/hosts_new2"
# === 阶段3: 跨交换机 4 台 ===
cat > "$OUT/hosts_cross4" << HEOF
10.184.0.20 slots=8
10.184.0.21 slots=8
10.184.0.51 slots=8
10.184.0.55 slots=8
HEOF
run_test "cross4node" "$OUT/hosts_cross4"
echo ""
echo "==== 汇总 ===="
column -t -s, "$SUMMARY"
echo "结果目录: $OUT"