#!/bin/bash # ===== 跨交换机多机 NCCL 通信测试 ===== # 适用场景: 验证不同交换机的节点加入集群后跨交换机通信是否正常 # 用法: bash run_cross_switch_test.sh # 主节点运行: gpu-20 (需与目标节点已打通免密 SSH) # # 默认分三个阶段: # old2node — 旧集群内 2 台基准 (gpu-20/21) # new2node — 新机 2 台同交换机 (gpu-51/55) # cross4node — 跨交换机 4 台联合 # # 新机接入前置步骤: # 1. 四台节点互相打通 authorized_keys # 2. 新机上重编 nccl-tests: cd /opt/nccl-tests && make clean && # make MPI=1 MPI_HOME=/usr/mpi/gcc/openmpi-4.1.7rc1 CUDA_HOME=/usr/local/cuda -j set -euo pipefail source "$(dirname "$0")/nccl_env.sh" NT=/opt/nccl-tests/build OUT=~/nccl_kit/results_cross_$(date +%Y%m%d_%H%M%S) mkdir -p "$OUT" SUMMARY="$OUT/summary.csv" echo "test,nodes,op,size,busbw_GBps" > "$SUMMARY" MPI=(mpirun --allow-run-as-root --mca plm_rsh_args "-oStrictHostKeyChecking=no" -x PATH -x LD_LIBRARY_PATH -x NCCL_IB_HCA -x NCCL_IB_DISABLE -x NCCL_SOCKET_IFNAME -x NCCL_DEBUG) run_test() { local tag="$1" hostfile="$2" local np=$(( $(wc -l < "$hostfile") * 8 )) echo "" echo "======== $tag ($np GPUs) ========" for op in all_reduce alltoall all_gather reduce_scatter; do log="$OUT/${tag}_${op}.log" "${MPI[@]}" -np "$np" -N 8 --hostfile "$hostfile" \ "$NT/${op}_perf" -b 8 -e 4G -f 2 -g 1 -w 5 -n 30 > "$log" 2>&1 row=$(awk '$1==4294967296{print}' "$log" | tail -1) busbw=$(echo "$row" | awk '{print $(NF-1)}') [ -z "$busbw" ] && busbw="ERR(见 $log)" printf " %-14s 4G busbw=%s GB/s\n" "$op" "$busbw" echo "$tag,$(wc -l < "$hostfile"),$op,4G,$busbw" >> "$SUMMARY" done } # === 阶段1: 旧集群内 2 台 (基准) === cat > "$OUT/hosts_old2" << HEOF 10.184.0.20 slots=8 10.184.0.21 slots=8 HEOF run_test "old2node" "$OUT/hosts_old2" # === 阶段2: 新机 2 台同交换机 === cat > "$OUT/hosts_new2" << HEOF 10.184.0.51 slots=8 10.184.0.55 slots=8 HEOF run_test "new2node" "$OUT/hosts_new2" # === 阶段3: 跨交换机 4 台 === cat > "$OUT/hosts_cross4" << HEOF 10.184.0.20 slots=8 10.184.0.21 slots=8 10.184.0.51 slots=8 10.184.0.55 slots=8 HEOF run_test "cross4node" "$OUT/hosts_cross4" echo "" echo "==== 汇总 ====" column -t -s, "$SUMMARY" echo "结果目录: $OUT"