# GPU type: auto-detect or override to a100/a800/h100/h800/h200/h20/b200/b300 gpu_type: auto benchmark: memory: size_mb: 4096 iterations: 10 nvbandwidth_buffer_mb: 512 nvbandwidth_samples: 3 compute: dtypes: - fp32 - tf32 - fp16 - bf16 - fp8 # MAMF-style shape sweep: measure each dtype at every shape below and keep the max # TFLOPS (the realistic achievable peak). A single fixed shape under-reports by # ~7-12% and can't meet the MAMF-calibrated thresholds in gpu_specs.py. # Each entry is either N (square N×N×N) or [M, N, K]. K-heavy non-square shapes # (e.g. 2048×2048×13312) hit the true Hopper MAMF — bf16 ~790 vs ~755 square. # Empty list => single matrix_size shape (legacy behaviour). sweep_sizes: - 3584 - 4608 - 5376 - 8192 - 11520 - [2048, 2048, 13312] - [2048, 2048, 16384] matrix_size: 8192 # fallback shape when sweep_sizes is empty warmup: 20 iterations: 80 # NOTE: torch.compile was dropped — on H100 eager cuBLAS beats Triton for plain # GEMM, and compiling would re-autotune per shape and make the sweep very slow. health: temp_warning: 75 temp_critical: 85 power_limit: null # null = auto-detect from GPU TDP per gpu_specs.py nccl: min_bandwidth_gbps: null # null = auto-detect (40% of GPU NVLink BW) test_allreduce: true test_alltoall: true test_broadcast: true test_reduce_scatter: false test_allgather: false test_sendrecv: false stress: duration_sec: 600 # 10 min — reaches thermal steady state, validates throttle/jitter beyond warmup use_doubles: false use_tensor_cores: true memory_pct: 90 gpus: all rdma: min_bandwidth_gbps: 50 max_latency_us: 10 ib_iterations: 1000 msg_size: 65536 ib_device: null ib_port: 1 # Cross-node (two-host) RDMA via perftest, orchestrated over SSH from the CLIENT # node. Replaces the old scripts/rdma_cross_node.sh. Run on the client; it starts # ib_write_bw/ib_write_lat servers on `server` over SSH (passwordless required), # then drives the local client per device. cross_node: enabled: false # set true on the client node to run cross-node RDMA server: null # peer ssh address, e.g. 172.72.8.12 (server node) server_addr: null # OOB addr client connects to (default: = server) ssh_user: root devices: [] # e.g. [mlx5_0, mlx5_1, mlx5_6, mlx5_7]; [] = auto-detect active IB ib_port: 1 gid_index: null # -x for RoCE; null for pure InfiniBand msg_size: 1048576 # 1 MiB — large enough to reach NDR400 peak iters: 5000 base_oob_port: 18515 # per-device OOB port = base + device index server_warmup_sec: 2.0 min_bandwidth_gbps: 350 # per-port PASS floor (NDR400 ≈ 0.9 × 400) max_latency_us: 5 multinode_nccl: # Path to OpenMPI hostfile: one line per node, format " slots=8" hosts_file: ~/nccl_kit/hosts_all # MPI installation prefix (non-standard path, set empty to use system mpirun) mpi_home: /usr/mpi/gcc/openmpi-4.1.7rc1 # IB HCAs for NCCL — GPU-direct (PIX) cards only; mlx5_6/7 are storage NICs, exclude them # GPU↔NIC map: GPU0↔mlx5_3, GPU1↔mlx5_2, GPU2↔mlx5_1, GPU3↔mlx5_0 # GPU4↔mlx5_5, GPU5↔mlx5_4, GPU6↔mlx5_9, GPU7↔mlx5_8 nccl_ib_hca: "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9" # NCCL bootstrap NIC — must be explicit (k8s virtual NICs cause connection refused) nccl_socket_ifname: bond0 # Node counts to run scaling test at scale_levels: [2, 4, 8] # Operations to test ops: [all_reduce, alltoall, all_gather, reduce_scatter] # Message size for allreduce/allgather/reducescatter (bytes) msg_size_bytes: 8589934592 # 8G # Message size for alltoall — cap at 1G to avoid hang at 16+ nodes msg_size_alltoall_bytes: 1073741824 # 1G warmup: 5 iterations: 30 timeout_sec: 600 # per-op timeout (10 min) # AllReduce busbw PASS floor (GB/s) — 4-node+ stable state on NDR400 cluster allreduce_min_busbw_gbps: 310 training: model: gpt2 batch_size: 8 seq_length: 2048 num_steps: 50 dtype: bf16 report: output_dir: ./reports format: json tools: install_dir: /opt/gpu-test-tools