test_gpu_scripts/configs/default.yaml
zulifeng cb5786c319 feat: 多机 NCCL Scaling 融入 Python 测试框架
- modules/nccl_test.py: 新增 MultiNodeNCCLTest 类,通过 mpirun + hostfile
  跑跨节点 NCCL Scaling 测试;支持 scale_levels 逐级测试,AllToAll 自动
  用 1G 消息避免大消息 hang,NCCL 环境变量通过 -x 传入各节点
- gpu_tester.py: 菜单加 [m] Multi-node NCCL Scaling,CLI 加 --test multinode-nccl
- configs/default.yaml: 新增 multinode_nccl 配置段(hostfile/mpi_home/HCA/网卡等)
- scripts/multinode_nccl/: 保留 shell 脚本版(nccl_env.sh/run_scaling.sh/
  run_cross_switch_test.sh/hosts_all,含 gpu-51/55 共 34 台)
- README: 新增 5m 多机 NCCL 章节,更新 SOP-3 和项目结构说明

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-06 18:58:08 +08:00

122 lines
4.3 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# GPU type: auto-detect or override to a100/a800/h100/h800/h200/h20/b200/b300
gpu_type: auto
benchmark:
memory:
size_mb: 4096
iterations: 10
nvbandwidth_buffer_mb: 512
nvbandwidth_samples: 3
compute:
dtypes:
- fp32
- tf32
- fp16
- bf16
- fp8
# MAMF-style shape sweep: measure each dtype at every shape below and keep the max
# TFLOPS (the realistic achievable peak). A single fixed shape under-reports by
# ~7-12% and can't meet the MAMF-calibrated thresholds in gpu_specs.py.
# Each entry is either N (square N×N×N) or [M, N, K]. K-heavy non-square shapes
# (e.g. 2048×2048×13312) hit the true Hopper MAMF — bf16 ~790 vs ~755 square.
# Empty list => single matrix_size shape (legacy behaviour).
sweep_sizes:
- 3584
- 4608
- 5376
- 8192
- 11520
- [2048, 2048, 13312]
- [2048, 2048, 16384]
matrix_size: 8192 # fallback shape when sweep_sizes is empty
warmup: 20
iterations: 80
# NOTE: torch.compile was dropped — on H100 eager cuBLAS beats Triton for plain
# GEMM, and compiling would re-autotune per shape and make the sweep very slow.
health:
temp_warning: 75
temp_critical: 85
power_limit: null # null = auto-detect from GPU TDP per gpu_specs.py
nccl:
min_bandwidth_gbps: null # null = auto-detect (40% of GPU NVLink BW)
test_allreduce: true
test_alltoall: true
test_broadcast: true
test_reduce_scatter: false
test_allgather: false
test_sendrecv: false
stress:
duration_sec: 600 # 10 min — reaches thermal steady state, validates throttle/jitter beyond warmup
use_doubles: false
use_tensor_cores: true
memory_pct: 90
gpus: all
rdma:
min_bandwidth_gbps: 50
max_latency_us: 10
ib_iterations: 1000
msg_size: 65536
ib_device: null
ib_port: 1
# Cross-node (two-host) RDMA via perftest, orchestrated over SSH from the CLIENT
# node. Replaces the old scripts/rdma_cross_node.sh. Run on the client; it starts
# ib_write_bw/ib_write_lat servers on `server` over SSH (passwordless required),
# then drives the local client per device.
cross_node:
enabled: false # set true on the client node to run cross-node RDMA
server: null # peer ssh address, e.g. 172.72.8.12 (server node)
server_addr: null # OOB addr client connects to (default: = server)
ssh_user: root
devices: [] # e.g. [mlx5_0, mlx5_1, mlx5_6, mlx5_7]; [] = auto-detect active IB
ib_port: 1
gid_index: null # -x <n> for RoCE; null for pure InfiniBand
msg_size: 1048576 # 1 MiB — large enough to reach NDR400 peak
iters: 5000
base_oob_port: 18515 # per-device OOB port = base + device index
server_warmup_sec: 2.0
min_bandwidth_gbps: 350 # per-port PASS floor (NDR400 ≈ 0.9 × 400)
max_latency_us: 5
multinode_nccl:
# Path to OpenMPI hostfile: one line per node, format "<ip> slots=8"
hosts_file: ~/nccl_kit/hosts_all
# MPI installation prefix (non-standard path, set empty to use system mpirun)
mpi_home: /usr/mpi/gcc/openmpi-4.1.7rc1
# IB HCAs for NCCL — GPU-direct (PIX) cards only; mlx5_6/7 are storage NICs, exclude them
# GPU↔NIC map: GPU0↔mlx5_3, GPU1↔mlx5_2, GPU2↔mlx5_1, GPU3↔mlx5_0
# GPU4↔mlx5_5, GPU5↔mlx5_4, GPU6↔mlx5_9, GPU7↔mlx5_8
nccl_ib_hca: "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9"
# NCCL bootstrap NIC — must be explicit (k8s virtual NICs cause connection refused)
nccl_socket_ifname: bond0
# Node counts to run scaling test at
scale_levels: [2, 4, 8]
# Operations to test
ops: [all_reduce, alltoall, all_gather, reduce_scatter]
# Message size for allreduce/allgather/reducescatter (bytes)
msg_size_bytes: 8589934592 # 8G
# Message size for alltoall — cap at 1G to avoid hang at 16+ nodes
msg_size_alltoall_bytes: 1073741824 # 1G
warmup: 5
iterations: 30
timeout_sec: 600 # per-op timeout (10 min)
# AllReduce busbw PASS floor (GB/s) — 4-node+ stable state on NDR400 cluster
allreduce_min_busbw_gbps: 310
training:
model: gpt2
batch_size: 8
seq_length: 2048
num_steps: 50
dtype: bf16
report:
output_dir: ./reports
format: json
tools:
install_dir: /opt/gpu-test-tools