#!/bin/bash # ===== 多机 NCCL Scaling 测试 ===== # 用法: bash run_scaling.sh [节点级别...] # 默认: "2 4 8 16 32" # 示例: bash run_scaling.sh 2 4 8 # 前置: source nccl_env.sh, hosts_all 文件与本脚本同目录或指定 HOSTS_ALL # 主节点运行: gpu-20 set -euo pipefail source "$(dirname "$0")/nccl_env.sh" NT=/opt/nccl-tests/build HOSTS_ALL="${HOSTS_ALL:-$(dirname "$0")/hosts_all}" LEVELS="${*:-2 4 8 16 32}" OUT=~/nccl_kit/results_$(date +%Y%m%d_%H%M%S) mkdir -p "$OUT" SUMMARY="$OUT/summary.csv" echo "nodes,gpus,op,size,busbw_GBps" > "$SUMMARY" MPI=(mpirun --allow-run-as-root --mca plm_rsh_args "-oStrictHostKeyChecking=no" -x PATH -x LD_LIBRARY_PATH -x NCCL_IB_HCA -x NCCL_IB_DISABLE -x NCCL_SOCKET_IFNAME -x NCCL_DEBUG) echo "Scaling levels: $LEVELS" echo "结果目录: $OUT" for N in $LEVELS; do hf="$OUT/hosts_$N" head -"$N" "$HOSTS_ALL" > "$hf" avail=$(wc -l < "$hf") if [ "$avail" -lt "$N" ]; then echo "!! hosts_all 只有 $avail 台, 跳过 $N 节点" continue fi np=$((N * 8)) echo "" echo "================ $N nodes / $np GPUs ================" for op in all_reduce alltoall all_gather reduce_scatter; do log="$OUT/${op}_${N}n.log" "${MPI[@]}" -np "$np" -N 8 --hostfile "$hf" \ "$NT/${op}_perf" -b 8 -e 8G -f 2 -g 1 -w 5 -n 50 > "$log" 2>&1 # AllToAll 在大消息 (>2G) + 多节点会 hang, 优先取 1G 行 if [[ "$op" == "alltoall" ]]; then row=$(awk '$1==1073741824{print}' "$log" | tail -1) sz="1G" else row=$(awk '$1==8589934592{print}' "$log" | tail -1) sz="8G" fi busbw=$(echo "$row" | awk '{print $(NF-1)}') [ -z "$busbw" ] && busbw="ERR(见 $log)" printf " %-14s %s busbw=%s GB/s\n" "$op" "$sz" "$busbw" echo "$N,$np,$op,$sz,$busbw" >> "$SUMMARY" done done echo "" echo "==== 汇总 (AllReduce 应随节点数基本持平 ~317 GB/s) ====" column -t -s, "$SUMMARY" echo "结果目录: $OUT"