- modules/nccl_test.py: 新增 MultiNodeNCCLTest 类,通过 mpirun + hostfile 跑跨节点 NCCL Scaling 测试;支持 scale_levels 逐级测试,AllToAll 自动 用 1G 消息避免大消息 hang,NCCL 环境变量通过 -x 传入各节点 - gpu_tester.py: 菜单加 [m] Multi-node NCCL Scaling,CLI 加 --test multinode-nccl - configs/default.yaml: 新增 multinode_nccl 配置段(hostfile/mpi_home/HCA/网卡等) - scripts/multinode_nccl/: 保留 shell 脚本版(nccl_env.sh/run_scaling.sh/ run_cross_switch_test.sh/hosts_all,含 gpu-51/55 共 34 台) - README: 新增 5m 多机 NCCL 章节,更新 SOP-3 和项目结构说明 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
122 lines
4.3 KiB
YAML
122 lines
4.3 KiB
YAML
# GPU type: auto-detect or override to a100/a800/h100/h800/h200/h20/b200/b300
|
||
gpu_type: auto
|
||
|
||
benchmark:
|
||
memory:
|
||
size_mb: 4096
|
||
iterations: 10
|
||
nvbandwidth_buffer_mb: 512
|
||
nvbandwidth_samples: 3
|
||
compute:
|
||
dtypes:
|
||
- fp32
|
||
- tf32
|
||
- fp16
|
||
- bf16
|
||
- fp8
|
||
# MAMF-style shape sweep: measure each dtype at every shape below and keep the max
|
||
# TFLOPS (the realistic achievable peak). A single fixed shape under-reports by
|
||
# ~7-12% and can't meet the MAMF-calibrated thresholds in gpu_specs.py.
|
||
# Each entry is either N (square N×N×N) or [M, N, K]. K-heavy non-square shapes
|
||
# (e.g. 2048×2048×13312) hit the true Hopper MAMF — bf16 ~790 vs ~755 square.
|
||
# Empty list => single matrix_size shape (legacy behaviour).
|
||
sweep_sizes:
|
||
- 3584
|
||
- 4608
|
||
- 5376
|
||
- 8192
|
||
- 11520
|
||
- [2048, 2048, 13312]
|
||
- [2048, 2048, 16384]
|
||
matrix_size: 8192 # fallback shape when sweep_sizes is empty
|
||
warmup: 20
|
||
iterations: 80
|
||
# NOTE: torch.compile was dropped — on H100 eager cuBLAS beats Triton for plain
|
||
# GEMM, and compiling would re-autotune per shape and make the sweep very slow.
|
||
|
||
health:
|
||
temp_warning: 75
|
||
temp_critical: 85
|
||
power_limit: null # null = auto-detect from GPU TDP per gpu_specs.py
|
||
|
||
nccl:
|
||
min_bandwidth_gbps: null # null = auto-detect (40% of GPU NVLink BW)
|
||
test_allreduce: true
|
||
test_alltoall: true
|
||
test_broadcast: true
|
||
test_reduce_scatter: false
|
||
test_allgather: false
|
||
test_sendrecv: false
|
||
|
||
stress:
|
||
duration_sec: 600 # 10 min — reaches thermal steady state, validates throttle/jitter beyond warmup
|
||
use_doubles: false
|
||
use_tensor_cores: true
|
||
memory_pct: 90
|
||
gpus: all
|
||
|
||
rdma:
|
||
min_bandwidth_gbps: 50
|
||
max_latency_us: 10
|
||
ib_iterations: 1000
|
||
msg_size: 65536
|
||
ib_device: null
|
||
ib_port: 1
|
||
# Cross-node (two-host) RDMA via perftest, orchestrated over SSH from the CLIENT
|
||
# node. Replaces the old scripts/rdma_cross_node.sh. Run on the client; it starts
|
||
# ib_write_bw/ib_write_lat servers on `server` over SSH (passwordless required),
|
||
# then drives the local client per device.
|
||
cross_node:
|
||
enabled: false # set true on the client node to run cross-node RDMA
|
||
server: null # peer ssh address, e.g. 172.72.8.12 (server node)
|
||
server_addr: null # OOB addr client connects to (default: = server)
|
||
ssh_user: root
|
||
devices: [] # e.g. [mlx5_0, mlx5_1, mlx5_6, mlx5_7]; [] = auto-detect active IB
|
||
ib_port: 1
|
||
gid_index: null # -x <n> for RoCE; null for pure InfiniBand
|
||
msg_size: 1048576 # 1 MiB — large enough to reach NDR400 peak
|
||
iters: 5000
|
||
base_oob_port: 18515 # per-device OOB port = base + device index
|
||
server_warmup_sec: 2.0
|
||
min_bandwidth_gbps: 350 # per-port PASS floor (NDR400 ≈ 0.9 × 400)
|
||
max_latency_us: 5
|
||
|
||
multinode_nccl:
|
||
# Path to OpenMPI hostfile: one line per node, format "<ip> slots=8"
|
||
hosts_file: ~/nccl_kit/hosts_all
|
||
# MPI installation prefix (non-standard path, set empty to use system mpirun)
|
||
mpi_home: /usr/mpi/gcc/openmpi-4.1.7rc1
|
||
# IB HCAs for NCCL — GPU-direct (PIX) cards only; mlx5_6/7 are storage NICs, exclude them
|
||
# GPU↔NIC map: GPU0↔mlx5_3, GPU1↔mlx5_2, GPU2↔mlx5_1, GPU3↔mlx5_0
|
||
# GPU4↔mlx5_5, GPU5↔mlx5_4, GPU6↔mlx5_9, GPU7↔mlx5_8
|
||
nccl_ib_hca: "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9"
|
||
# NCCL bootstrap NIC — must be explicit (k8s virtual NICs cause connection refused)
|
||
nccl_socket_ifname: bond0
|
||
# Node counts to run scaling test at
|
||
scale_levels: [2, 4, 8]
|
||
# Operations to test
|
||
ops: [all_reduce, alltoall, all_gather, reduce_scatter]
|
||
# Message size for allreduce/allgather/reducescatter (bytes)
|
||
msg_size_bytes: 8589934592 # 8G
|
||
# Message size for alltoall — cap at 1G to avoid hang at 16+ nodes
|
||
msg_size_alltoall_bytes: 1073741824 # 1G
|
||
warmup: 5
|
||
iterations: 30
|
||
timeout_sec: 600 # per-op timeout (10 min)
|
||
# AllReduce busbw PASS floor (GB/s) — 4-node+ stable state on NDR400 cluster
|
||
allreduce_min_busbw_gbps: 310
|
||
|
||
training:
|
||
model: gpt2
|
||
batch_size: 8
|
||
seq_length: 2048
|
||
num_steps: 50
|
||
dtype: bf16
|
||
|
||
report:
|
||
output_dir: ./reports
|
||
format: json
|
||
|
||
tools:
|
||
install_dir: /opt/gpu-test-tools
|