zulifeng
dd77a882f1
- modules/rdma_test.py: 新增 SSH 编排的跨机 RDMA(run_cross_node / _cross_node_perftest / 解析器),从 client 端逐设备拉起对端 perftest server 跑本地 client,替代已删除的 scripts/rdma_cross_node.sh;两机 4×NDR400 实测全 PASS(~387-392 Gb/s,~2 µs)。 - configs/default.yaml: 新增 rdma.cross_node 配置块(默认 enabled:false)。 - modules/gpu_specs.py: H800 PASS 门槛对齐 H100 实测地板 (tf32 400->385, bf16 720->730, fp8 1400->1200);H800=H100 硅片, PyTorch tensorwise fp8 天花板 ~1310,原 1400 不可达。 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
97 lines
3.0 KiB
YAML
97 lines
3.0 KiB
YAML
# GPU type: auto-detect or override to a100/a800/h100/h800/h200/h20/b200/b300
|
||
gpu_type: auto
|
||
|
||
benchmark:
|
||
memory:
|
||
size_mb: 4096
|
||
iterations: 10
|
||
nvbandwidth_buffer_mb: 512
|
||
nvbandwidth_samples: 3
|
||
compute:
|
||
dtypes:
|
||
- fp32
|
||
- tf32
|
||
- fp16
|
||
- bf16
|
||
- fp8
|
||
# MAMF-style shape sweep: measure each dtype at every shape below and keep the max
|
||
# TFLOPS (the realistic achievable peak). A single fixed shape under-reports by
|
||
# ~7-12% and can't meet the MAMF-calibrated thresholds in gpu_specs.py.
|
||
# Each entry is either N (square N×N×N) or [M, N, K]. K-heavy non-square shapes
|
||
# (e.g. 2048×2048×13312) hit the true Hopper MAMF — bf16 ~790 vs ~755 square.
|
||
# Empty list => single matrix_size shape (legacy behaviour).
|
||
sweep_sizes:
|
||
- 3584
|
||
- 4608
|
||
- 5376
|
||
- 8192
|
||
- 11520
|
||
- [2048, 2048, 13312]
|
||
- [2048, 2048, 16384]
|
||
matrix_size: 8192 # fallback shape when sweep_sizes is empty
|
||
warmup: 20
|
||
iterations: 80
|
||
# NOTE: torch.compile was dropped — on H100 eager cuBLAS beats Triton for plain
|
||
# GEMM, and compiling would re-autotune per shape and make the sweep very slow.
|
||
|
||
health:
|
||
temp_warning: 75
|
||
temp_critical: 85
|
||
power_limit: null # null = auto-detect from GPU TDP per gpu_specs.py
|
||
|
||
nccl:
|
||
min_bandwidth_gbps: null # null = auto-detect (40% of GPU NVLink BW)
|
||
test_allreduce: true
|
||
test_alltoall: true
|
||
test_broadcast: true
|
||
test_reduce_scatter: false
|
||
test_allgather: false
|
||
test_sendrecv: false
|
||
|
||
stress:
|
||
duration_sec: 600 # 10 min — reaches thermal steady state, validates throttle/jitter beyond warmup
|
||
use_doubles: false
|
||
use_tensor_cores: true
|
||
memory_pct: 90
|
||
gpus: all
|
||
|
||
rdma:
|
||
min_bandwidth_gbps: 50
|
||
max_latency_us: 10
|
||
ib_iterations: 1000
|
||
msg_size: 65536
|
||
ib_device: null
|
||
ib_port: 1
|
||
# Cross-node (two-host) RDMA via perftest, orchestrated over SSH from the CLIENT
|
||
# node. Replaces the old scripts/rdma_cross_node.sh. Run on the client; it starts
|
||
# ib_write_bw/ib_write_lat servers on `server` over SSH (passwordless required),
|
||
# then drives the local client per device.
|
||
cross_node:
|
||
enabled: false # set true on the client node to run cross-node RDMA
|
||
server: null # peer ssh address, e.g. 172.72.8.12 (server node)
|
||
server_addr: null # OOB addr client connects to (default: = server)
|
||
ssh_user: root
|
||
devices: [] # e.g. [mlx5_0, mlx5_1, mlx5_6, mlx5_7]; [] = auto-detect active IB
|
||
ib_port: 1
|
||
gid_index: null # -x <n> for RoCE; null for pure InfiniBand
|
||
msg_size: 1048576 # 1 MiB — large enough to reach NDR400 peak
|
||
iters: 5000
|
||
base_oob_port: 18515 # per-device OOB port = base + device index
|
||
server_warmup_sec: 2.0
|
||
min_bandwidth_gbps: 350 # per-port PASS floor (NDR400 ≈ 0.9 × 400)
|
||
max_latency_us: 5
|
||
|
||
training:
|
||
model: gpt2
|
||
batch_size: 8
|
||
seq_length: 2048
|
||
num_steps: 50
|
||
dtype: bf16
|
||
|
||
report:
|
||
output_dir: ./reports
|
||
format: json
|
||
|
||
tools:
|
||
install_dir: /opt/gpu-test-tools
|