From 1f907e969177070089012155b833c12e801c9361 Mon Sep 17 00:00:00 2001 From: cs Date: Sat, 23 May 2026 15:58:21 +0800 Subject: [PATCH] Validate NCCL 2.27 multinode GDR performance --- configs/multinode_nccl_nccl227_16g.yaml | 62 +++++++++++ .../multinode_nccl_nccl227_diagnostic.yaml | 62 +++++++++++ configs/multinode_nccl_nccl227_sweep.yaml | 62 +++++++++++ modules/report.py | 7 +- reports_multinode_nccl_16g_2x8_nccl227.md | 66 ++++++++++++ reports_multinode_nccl_diagnosis_20260523.md | 101 +++++++++++++++--- ...ultinode_nccl_diagnostic_2x8_nccl227_v2.md | 66 ++++++++++++ reports_multinode_nccl_sweep_2x8_nccl227.md | 66 ++++++++++++ 8 files changed, 474 insertions(+), 18 deletions(-) create mode 100644 configs/multinode_nccl_nccl227_16g.yaml create mode 100644 configs/multinode_nccl_nccl227_diagnostic.yaml create mode 100644 configs/multinode_nccl_nccl227_sweep.yaml create mode 100644 reports_multinode_nccl_16g_2x8_nccl227.md create mode 100644 reports_multinode_nccl_diagnostic_2x8_nccl227_v2.md create mode 100644 reports_multinode_nccl_sweep_2x8_nccl227.md diff --git a/configs/multinode_nccl_nccl227_16g.yaml b/configs/multinode_nccl_nccl227_16g.yaml new file mode 100644 index 0000000..e7b718f --- /dev/null +++ b/configs/multinode_nccl_nccl227_16g.yaml @@ -0,0 +1,62 @@ +tools: + install_dir: /opt/gpu-test-tools + +report: + output_dir: ./reports + format: md + +multinode_nccl: + enabled: true + mode: large-message-nccl-2.27.7 + hosts: + - name: nccl-gpu-1 + addr: 172.72.8.12 + slots: 8 + - name: nccl-gpu-2 + addr: 172.72.8.16 + slots: 8 + ssh_user: root + ssh_preflight: true + mpirun_path: /usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun + mpi_ld_preload: null + extra_ld_library_path: + - /usr/mpi/gcc/openmpi-4.1.9a1/lib + - /tmp/nccl-2.27.7-cuda12.4/usr/lib/x86_64-linux-gnu + - /usr/local/cuda-12.4/targets/x86_64-linux/lib + nccl_tests_dir: null + tests: + - all_reduce_perf + - alltoall_perf + topologies: + - nodes: 2 + gpus_per_node: 8 + label: 2 nodes x 8 GPUs NCCL 2.27.7 16G + begin_size: 16G + end_size: 16G + step_factor: 2 + warmup_iters: 1 + iters: 3 + gpus_per_rank: 1 + timeout_sec: 1200 + debug: INFO + socket_ifname: bond0 + oob_tcp_ifname: bond0 + plm_rsh_args: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=30" + ib_gid_index: 3 + ib_sl: 5 + ib_tc: 136 + ib_hca: mlx5_0,mlx5_1,mlx5_6,mlx5_7 + ib_timeout: 22 + qps_per_connection: 4 + min_nchannels: 4 + net_plugin: none + nvls_enable: 1 + split_data_on_qps: 1 + extra_env: + NCCL_DEBUG_SUBSYS: INIT,NET + NCCL_NET_GDR_LEVEL: 5 + NCCL_NET_GDR_READ: 1 + NCCL_DMABUF_ENABLE: 0 + min_peak_busbw_gbps: + allreduce: 480 + alltoall: 75 diff --git a/configs/multinode_nccl_nccl227_diagnostic.yaml b/configs/multinode_nccl_nccl227_diagnostic.yaml new file mode 100644 index 0000000..8a769ad --- /dev/null +++ b/configs/multinode_nccl_nccl227_diagnostic.yaml @@ -0,0 +1,62 @@ +tools: + install_dir: /opt/gpu-test-tools + +report: + output_dir: ./reports + format: md + +multinode_nccl: + enabled: true + mode: diagnostic-nccl-2.27.7 + hosts: + - name: nccl-gpu-1 + addr: 172.72.8.12 + slots: 8 + - name: nccl-gpu-2 + addr: 172.72.8.16 + slots: 8 + ssh_user: root + ssh_preflight: true + mpirun_path: /usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun + mpi_ld_preload: null + extra_ld_library_path: + - /usr/mpi/gcc/openmpi-4.1.9a1/lib + - /tmp/nccl-2.27.7-cuda12.4/usr/lib/x86_64-linux-gnu + - /usr/local/cuda-12.4/targets/x86_64-linux/lib + nccl_tests_dir: null + tests: + - all_reduce_perf + - alltoall_perf + topologies: + - nodes: 2 + gpus_per_node: 8 + label: 2 nodes x 8 GPUs NCCL 2.27.7 + begin_size: 256M + end_size: 256M + step_factor: 2 + warmup_iters: 1 + iters: 3 + gpus_per_rank: 1 + timeout_sec: 600 + debug: INFO + socket_ifname: bond0 + oob_tcp_ifname: bond0 + plm_rsh_args: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=30" + ib_gid_index: 3 + ib_sl: 5 + ib_tc: 136 + ib_hca: mlx5_0,mlx5_1,mlx5_6,mlx5_7 + ib_timeout: 22 + qps_per_connection: 4 + min_nchannels: 4 + net_plugin: none + nvls_enable: 1 + split_data_on_qps: 1 + extra_env: + NCCL_DEBUG_SUBSYS: INIT,NET + NCCL_NET_GDR_LEVEL: 5 + NCCL_NET_GDR_READ: 1 + NCCL_DMABUF_ENABLE: 0 + min_peak_busbw_gbps: + allreduce: 480 + alltoall: 75 diff --git a/configs/multinode_nccl_nccl227_sweep.yaml b/configs/multinode_nccl_nccl227_sweep.yaml new file mode 100644 index 0000000..3dcbf36 --- /dev/null +++ b/configs/multinode_nccl_nccl227_sweep.yaml @@ -0,0 +1,62 @@ +tools: + install_dir: /opt/gpu-test-tools + +report: + output_dir: ./reports + format: md + +multinode_nccl: + enabled: true + mode: sweep-nccl-2.27.7 + hosts: + - name: nccl-gpu-1 + addr: 172.72.8.12 + slots: 8 + - name: nccl-gpu-2 + addr: 172.72.8.16 + slots: 8 + ssh_user: root + ssh_preflight: true + mpirun_path: /usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun + mpi_ld_preload: null + extra_ld_library_path: + - /usr/mpi/gcc/openmpi-4.1.9a1/lib + - /tmp/nccl-2.27.7-cuda12.4/usr/lib/x86_64-linux-gnu + - /usr/local/cuda-12.4/targets/x86_64-linux/lib + nccl_tests_dir: null + tests: + - all_reduce_perf + - alltoall_perf + topologies: + - nodes: 2 + gpus_per_node: 8 + label: 2 nodes x 8 GPUs NCCL 2.27.7 sweep + begin_size: 1M + end_size: 4G + step_factor: 4 + warmup_iters: 2 + iters: 5 + gpus_per_rank: 1 + timeout_sec: 1200 + debug: INFO + socket_ifname: bond0 + oob_tcp_ifname: bond0 + plm_rsh_args: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=30" + ib_gid_index: 3 + ib_sl: 5 + ib_tc: 136 + ib_hca: mlx5_0,mlx5_1,mlx5_6,mlx5_7 + ib_timeout: 22 + qps_per_connection: 4 + min_nchannels: 4 + net_plugin: none + nvls_enable: 1 + split_data_on_qps: 1 + extra_env: + NCCL_DEBUG_SUBSYS: INIT,NET + NCCL_NET_GDR_LEVEL: 5 + NCCL_NET_GDR_READ: 1 + NCCL_DMABUF_ENABLE: 0 + min_peak_busbw_gbps: + allreduce: 480 + alltoall: 75 diff --git a/modules/report.py b/modules/report.py index c9e1b8d..acca41e 100644 --- a/modules/report.py +++ b/modules/report.py @@ -498,13 +498,14 @@ class ReportGenerator: if net: diag_rows.append((topo, net)) if diag_rows: - lines.append("| Topology | NCCL Network | GPU Direct RDMA | GDR Disabled HCAs |") - lines.append("|----------|--------------|-----------------|-------------------|") + lines.append("| Topology | NCCL Network | GPU Direct RDMA | GDR Enabled HCAs | GDR Disabled HCAs |") + lines.append("|----------|--------------|-----------------|------------------|-------------------|") for topo, net in diag_rows: networks = ", ".join(net.get("networks") or []) or "unknown" gdr = net.get("gpu_direct_rdma", "UNKNOWN") + enabled = ", ".join(net.get("gdr_enabled_hcas") or []) or "-" disabled = ", ".join(net.get("gdr_disabled_hcas") or []) or "-" - lines.append(f"| {topo.get('label', '')} | {networks} | {gdr} | {disabled} |") + lines.append(f"| {topo.get('label', '')} | {networks} | {gdr} | {enabled} | {disabled} |") lines.append("") failed_topos = [topo for topo in data.get("topologies", []) if topo.get("status") == "FAIL"] if failed_topos: diff --git a/reports_multinode_nccl_16g_2x8_nccl227.md b/reports_multinode_nccl_16g_2x8_nccl227.md new file mode 100644 index 0000000..394f191 --- /dev/null +++ b/reports_multinode_nccl_16g_2x8_nccl227.md @@ -0,0 +1,66 @@ +# GPU Test Report + +- **Date:** 2026-05-23T07:56:26.791384 +- **Host:** aikubeworker0012 + +## Overall Acceptance Verdict + +**Result: FAIL** + +Missing required evidence: +- GPU Info +- Health Check +- Memory Bandwidth +- Compute Throughput +- NVLink/NVSwitch +- NCCL +- Stress Test +- RDMA +- DCGM +- Training + +## Summary + +| Test | Result | +|------|--------| +| Multi-node NCCL | FAIL | + +## Multi-node NCCL / Cross Leaf + +Source: nccl-tests-mpirun | Mode: large-message-nccl-2.27.7 + +- **Hosts:** nccl-gpu-1(172.72.8.12), nccl-gpu-2(172.72.8.16) +- **Preflight:** PASS + +### Multi-node NCCL allreduce + +| Topology | Peak Bus BW | Peak Size | Avg Bus BW | Threshold | Status | +|----------|-------------|-----------|------------|-----------|--------| +| 2 nodes x 8 GPUs NCCL 2.27.7 16G | 237.86 GB/s | 16G | 238.56 GB/s | >= 480 GB/s | FAIL | + +| Topology | NCCL Network | GPU Direct RDMA | GDR Enabled HCAs | GDR Disabled HCAs | +|----------|--------------|-----------------|------------------|-------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 16G | IB | ENABLED | mlx5_0, mlx5_1, mlx5_6, mlx5_7 | - | + +| Topology | Return Code | Error / Output Tail | +|----------|-------------|---------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 16G | 0 | aikubeworker0016:1019342:1020412 [4] NCCL INFO comm 0x559f14871c30 rank 12 nranks 16 cudaDev 4 busId 9a000 - Destroy COMPLETE # Out of bounds values : 0 OK # Avg bus bandwidth : 238.555 # # Collective test concluded: all_reduce_perf # | + +### Multi-node NCCL alltoall + +| Topology | Peak Bus BW | Peak Size | Avg Bus BW | Threshold | Status | +|----------|-------------|-----------|------------|-----------|--------| +| 2 nodes x 8 GPUs NCCL 2.27.7 16G | 28.62 GB/s | 16G | 28.62 GB/s | >= 75 GB/s | FAIL | + +| Topology | NCCL Network | GPU Direct RDMA | GDR Enabled HCAs | GDR Disabled HCAs | +|----------|--------------|-----------------|------------------|-------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 16G | IB | ENABLED | mlx5_0, mlx5_1, mlx5_6, mlx5_7 | - | + +| Topology | Return Code | Error / Output Tail | +|----------|-------------|---------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 16G | 0 | E aikubeworker0016:1020609:1021756 [5] NCCL INFO comm 0x55f920e55d90 rank 13 nranks 16 cudaDev 5 busId ab000 - Destroy COMPLETE # Out of bounds values : 0 OK # Avg bus bandwidth : 28.6222 # # Collective test concluded: alltoall_perf # | + +**Overall: FAIL** + +--- +*Generated by GPU Test Suite v0.2.0* \ No newline at end of file diff --git a/reports_multinode_nccl_diagnosis_20260523.md b/reports_multinode_nccl_diagnosis_20260523.md index 6468215..bc20b72 100644 --- a/reports_multinode_nccl_diagnosis_20260523.md +++ b/reports_multinode_nccl_diagnosis_20260523.md @@ -8,9 +8,11 @@ ## 当前结论 -这不是单纯 “IB 不通” 的问题。底层 CUDA RDMA perftest 可以跑到接近单端口 400Gb/s 的水平，但 NCCL 在实际 2 节点通信时把 GPU Direct RDMA 禁用了，导致 NCCL 带宽显著低于验收阈值。 +这不是单纯 “IB 不通” 的问题。底层 CUDA RDMA perftest 可以跑到接近单端口 400Gb/s 的水平；最初使用 pip 包里的 NCCL 2.21.5 时，NCCL 在实际 2 节点通信中把 GPU Direct RDMA 禁用了，导致带宽显著偏低。 -同时，`nccl-gpu-2` 的 SSH 入口曾因未认证连接过多触发 `MaxStartups` 随机拒绝，导致 `mpirun` 拉起远端 rank 失败。已经做了临时 SSHD 缓解并拿到有效的 2 节点 x 8 GPU allreduce/alltoall 报告；当前剩余核心问题是 NCCL GDR 仍被禁用。 +后续临时切换到 apt 包解压出的 NCCL 2.27.7+cuda12.4 后，NCCL GDR 已经恢复启用，2 节点 x 8 GPU allreduce 从 `67.42 GB/s` 提升到 `237.86 GB/s`，alltoall 从 `9.56 GB/s` 提升到 `28.62 GB/s`。当前剩余问题不再是 GDR disabled，而是 GDR enabled 后仍低于当前配置里的验收阈值。 + +同时，`nccl-gpu-2` 的 SSH 入口曾因未认证连接过多触发 `MaxStartups` 随机拒绝，导致 `mpirun` 拉起远端 rank 失败。已经做了临时 SSHD 缓解并拿到有效的 2 节点 x 8 GPU allreduce/alltoall 报告。 ## 已完成的修正 @@ -22,6 +24,8 @@ 6. 增加诊断配置 `configs/multinode_nccl_diagnostic.yaml`，固定跑 2 节点 x 8 GPU、256M、`NCCL_DEBUG=INFO` 和 `NCCL_DEBUG_SUBSYS=INIT,NET`。 7. 在 `nccl-gpu-2` 上临时提高 SSHD `MaxStartups` 并缩短 `LoginGraceTime`，缓解未认证连接过多导致的 SSH 随机拒绝。 8. 将 OpenMPI OOB TCP 控制通道固定到 `bond0`，并加入 `plm_rsh_args`，减少 `mpirun` 远端启动受 SSH/host key/接口选择影响的概率。 +9. 从 NVIDIA apt 源下载但不安装 `libnccl2=2.27.7-1+cuda12.4`，解压到两台机器 `/tmp/nccl-2.27.7-cuda12.4`，用 `LD_LIBRARY_PATH` 临时覆盖 NCCL 运行库验证。 +10. 增强报告解析，能够区分 `GPU Direct RDMA ENABLED` 和 `DISABLED`，并列出 enabled/disabled HCA。 ## 关键证据 @@ -59,6 +63,8 @@ CUDA_VISIBLE_DEVICES=0 ib_write_bw -d mlx5_0 -i 1 --use_cuda=0 --use_cuda_dmabuf ### 3. NCCL 单卡跨节点仍禁用 GDR +使用 pip NCCL 2.21.5 时， + 已经尝试： - `NCCL_NET_GDR_LEVEL=SYS` @@ -77,9 +83,27 @@ NCCL INFO NET/IB : GPU Direct RDMA Disabled for HCA 0 'mlx5_0' 256M allreduce 约 `13.4 GB/s`，明显低于 400Gb/s IB 端口能力。 +### 3.1 NCCL 2.27.7 恢复 GDR + +临时使用： + +```bash +LD_LIBRARY_PATH=/usr/mpi/gcc/openmpi-4.1.9a1/lib:/tmp/nccl-2.27.7-cuda12.4/usr/lib/x86_64-linux-gnu:/usr/local/cuda-12.4/targets/x86_64-linux/lib +``` + +2 节点 x 1 GPU 日志显示： + +```text +NCCL version 2.27.7+cuda12.4 +NET/IB : GPU Direct RDMA Enabled for HCA 0 'mlx5_0' +Channel ... via NET/IB/0/GDRDMA +``` + +256M allreduce 从 NCCL 2.21.5 的约 `13.4 GB/s` 提升到 `45.2 GB/s`。判断：NCCL 2.21.5 与当前 driver/OFED/H100 组合存在 GDR 判定或注册路径兼容问题；升级 NCCL 是有效修复方向。 + ### 4. 脚本 2 节点 x 8 GPU 诊断结果 -原始报告：`reports_multinode_nccl_diagnostic_2x8_sshfix.md` +原始报告：`reports_multinode_nccl_diagnostic_2x8_sshfix.md`，使用 pip NCCL 2.21.5。 | Operation | Topology | Peak Bus BW | Threshold | Status | NCCL Network | GPU Direct RDMA | |-----------|----------|-------------|-----------|--------|--------------|-----------------| @@ -94,6 +118,31 @@ allreduce 失败原因是带宽不达标，且报告捕获到 GDR 被 NCCL 禁 allreduce 和 alltoall 本轮均正常完成，`returncode=0`、`wrong=0`，失败原因是带宽低于阈值，不是正确性失败。 +### 4.1 NCCL 2.27.7 诊断结果 + +256M 诊断报告：`reports_multinode_nccl_diagnostic_2x8_nccl227_v2.md` + +| Operation | Topology | Peak Bus BW | Threshold | Status | NCCL Network | GPU Direct RDMA | +|-----------|----------|-------------|-----------|--------|--------------|-----------------| +| allreduce | 2 nodes x 8 GPUs | `212.19 GB/s` | `>= 480 GB/s` | FAIL | IB | ENABLED | +| alltoall | 2 nodes x 8 GPUs | `28.37 GB/s` | `>= 75 GB/s` | FAIL | IB | ENABLED | + +1M 到 4G sweep 报告：`reports_multinode_nccl_sweep_2x8_nccl227.md` + +| Operation | Peak Bus BW | Peak Size | Threshold | Status | GPU Direct RDMA | +|-----------|-------------|-----------|-----------|--------|-----------------| +| allreduce | `237.26 GB/s` | `4G` | `>= 480 GB/s` | FAIL | ENABLED | +| alltoall | `28.78 GB/s` | `1G` | `>= 75 GB/s` | FAIL | ENABLED | + +16G 大包报告：`reports_multinode_nccl_16g_2x8_nccl227.md` + +| Operation | Peak Bus BW | Peak Size | Threshold | Status | GPU Direct RDMA | +|-----------|-------------|-----------|-----------|--------|-----------------| +| allreduce | `237.86 GB/s` | `16G` | `>= 480 GB/s` | FAIL | ENABLED | +| alltoall | `28.62 GB/s` | `16G` | `>= 75 GB/s` | FAIL | ENABLED | + +解释：NCCL 2.27.7 已经修复 GDR 禁用问题，且性能提升明显；但在当前跨节点/跨 Leaf 环境和当前阈值下仍不达标。allreduce 约稳定在 `238 GB/s`，alltoall 约稳定在 `28-29 GB/s`。 + ### 5. SSHD MaxStartups 阻塞已临时缓解 `nccl-gpu-2` 曾显示： @@ -109,13 +158,20 @@ maxstartups 10:30:100 kex_exchange_identification: Connection closed by remote host ``` -已临时改为： +先临时改为： ```text MaxStartups 120:30:240 LoginGraceTime 20 ``` +后续外部未认证连接继续上涨到 `110 of 120-240 startups`，测试窗口进一步临时改为： + +```text +MaxStartups 500:30:1000 +LoginGraceTime 5 +``` + 改完后从 0012 连续 SSH 0016 5 次成功，2 节点 `mpirun hostname` 成功，2 节点 x 8 GPU allreduce/alltoall 也都能跑出有效结果。 ### 6. `nvidia_peermem` legacy 模式实验无效 @@ -139,18 +195,27 @@ NET/IB : GPU Direct RDMA Disabled for HCA 0 'mlx5_0' ## 当前阻塞 -### 阻塞 1：NCCL 禁用 GPU Direct RDMA +### 阻塞 1：当前生产 NCCL 版本过旧，GDR 被禁用 现象： -- IB 能被 NCCL 识别：`Using network IB` -- 400Gb/s HCA 被 NCCL 选中：`mlx5_0, mlx5_1, mlx5_6, mlx5_7` -- 但 NCCL 明确禁用 GDR：`GPU Direct RDMA Disabled` -- perftest 的经典 CUDA RDMA 又能跑到 `387.16 Gb/s` +- pip NCCL 2.21.5：`GPU Direct RDMA Disabled`，2x8 allreduce `67.42 GB/s` +- 临时 NCCL 2.27.7：`GPU Direct RDMA Enabled`，2x8 allreduce `237.86 GB/s` +- 因此，生产测试环境应避免继续使用 pip NCCL 2.21.5 作为多机 NCCL 验收运行库 -判断：底层 RDMA 能力存在，但 NCCL 的 GDR 判定/注册路径没有打通。优先排查 NCCL 与 NVIDIA driver、OFED、`nvidia_peermem`、NCCL net plugin/内部 IB 后端之间的兼容性。 +判断：底层 RDMA 能力存在，GDR 禁用主要由旧 NCCL 版本触发。建议正式安装并固定 NCCL 2.27.7+cuda12.4 或更新的已验证版本。 -### 阻塞 2：`nccl-gpu-2` SSH 存在外部连接压力 +### 阻塞 2：GDR enabled 后带宽仍低于当前阈值 + +现象： + +- 2x8 16G allreduce：`237.86 GB/s`，阈值 `>= 480 GB/s` +- 2x8 16G alltoall：`28.62 GB/s`，阈值 `>= 75 GB/s` +- 已使用 4 个 400Gb/s HCA：`mlx5_0, mlx5_1, mlx5_6, mlx5_7` + +判断：需要确认当前 PDF/config 阈值是否适用于跨 Leaf 两节点场景；如果阈值确实要求跨 Leaf 也达到这些数值，则还需要继续查链路聚合、多 rail 使用、交换网络、NCCL net plugin/SHARP 或 rail mapping。 + +### 阻塞 3：`nccl-gpu-2` SSH 存在外部连接压力 现象： @@ -164,13 +229,19 @@ NET/IB : GPU Direct RDMA Disabled for HCA 0 'mlx5_0' ## 建议下一步 1. 从网络/安全侧处理 `172.239.10.85` 等来源的 SSH 未认证连接压力，或者保留更高的 `MaxStartups` 配置作为测试窗口临时策略。 -2. 尝试安装或启用匹配当前 OFED/driver 的 NCCL net plugin；当前日志显示 `No plugin found (libnccl-net.so)`，NCCL 使用的是 internal network plugin。 -3. 用同版本软件栈补测 `nccl-tests` + NCCL net plugin 后的 GDR 状态，核心判据是报告里 `GPU Direct RDMA` 从 `DISABLED` 变成未禁用，且 2x8 带宽显著抬升。 -4. 如果仍禁用 GDR，再继续查 NVIDIA driver 580.159.03、OFED 26.01、NCCL 2.21.5 与 H100/IB NDR 组合的兼容矩阵。 -5. GDR 修复后重跑完整多机配置：2 节点 x 8 GPU，至少覆盖 `all_reduce_perf` 和 `alltoall_perf`，消息大小从 `1K` 到 `16G`。 +2. 正式安装并固定已验证的 NCCL 2.27.7+cuda12.4 或更新版本，不要依赖 pip NCCL 2.21.5；当前 `/tmp/nccl-2.27.7-cuda12.4` 只是临时解压验证。 +3. 尝试安装或启用匹配当前 OFED/driver 的 NCCL net plugin/SHARP；当前日志显示 `Could not find: libnccl-net.so`，NCCL 使用的是 internal IB plugin。 +4. 核对跨 Leaf 链路的 rail mapping、交换机端口速率、路由和拥塞计数，确认 4 个 400Gb/s HCA 是否都在跨节点通信中充分利用。 +5. 确认当前 `allreduce >= 480 GB/s`、`alltoall >= 75 GB/s` 阈值是否应直接用于跨 Leaf 两节点场景；如果是，继续按链路和 NCCL rail 聚合方向排查。 ## 当前可交付物 - `configs/multinode_nccl_diagnostic.yaml`：多机多卡诊断配置 +- `configs/multinode_nccl_nccl227_diagnostic.yaml`：NCCL 2.27.7 256M 诊断配置 +- `configs/multinode_nccl_nccl227_sweep.yaml`：NCCL 2.27.7 1M 到 4G sweep 配置 +- `configs/multinode_nccl_nccl227_16g.yaml`：NCCL 2.27.7 16G 大包配置 - `reports_multinode_nccl_diagnostic_2x8_sshfix.md`：脚本生成的原始 2x8 诊断报告 +- `reports_multinode_nccl_diagnostic_2x8_nccl227_v2.md`：NCCL 2.27.7 256M 诊断报告 +- `reports_multinode_nccl_sweep_2x8_nccl227.md`：NCCL 2.27.7 1M 到 4G sweep 报告 +- `reports_multinode_nccl_16g_2x8_nccl227.md`：NCCL 2.27.7 16G 大包报告 - `reports_multinode_nccl_diagnosis_20260523.md`：本中文诊断总结 diff --git a/reports_multinode_nccl_diagnostic_2x8_nccl227_v2.md b/reports_multinode_nccl_diagnostic_2x8_nccl227_v2.md new file mode 100644 index 0000000..1b188d5 --- /dev/null +++ b/reports_multinode_nccl_diagnostic_2x8_nccl227_v2.md @@ -0,0 +1,66 @@ +# GPU Test Report + +- **Date:** 2026-05-23T07:53:24.460277 +- **Host:** aikubeworker0012 + +## Overall Acceptance Verdict + +**Result: FAIL** + +Missing required evidence: +- GPU Info +- Health Check +- Memory Bandwidth +- Compute Throughput +- NVLink/NVSwitch +- NCCL +- Stress Test +- RDMA +- DCGM +- Training + +## Summary + +| Test | Result | +|------|--------| +| Multi-node NCCL | FAIL | + +## Multi-node NCCL / Cross Leaf + +Source: nccl-tests-mpirun | Mode: diagnostic-nccl-2.27.7 + +- **Hosts:** nccl-gpu-1(172.72.8.12), nccl-gpu-2(172.72.8.16) +- **Preflight:** PASS + +### Multi-node NCCL allreduce + +| Topology | Peak Bus BW | Peak Size | Avg Bus BW | Threshold | Status | +|----------|-------------|-----------|------------|-----------|--------| +| 2 nodes x 8 GPUs NCCL 2.27.7 | 212.19 GB/s | 256M | 211.75 GB/s | >= 480 GB/s | FAIL | + +| Topology | NCCL Network | GPU Direct RDMA | GDR Enabled HCAs | GDR Disabled HCAs | +|----------|--------------|-----------------|------------------|-------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 | IB | ENABLED | mlx5_0, mlx5_1, mlx5_6, mlx5_7 | - | + +| Topology | Return Code | Error / Output Tail | +|----------|-------------|---------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 | 0 | 0016:1009332:1009965 [2] NCCL INFO comm 0x56388eec2e40 rank 10 nranks 16 cudaDev 2 busId 3a000 - Destroy COMPLETE aikubeworker0012:2144366:2144531 [5] NCCL INFO comm 0x556e4fcf5280 rank 5 nranks 16 cudaDev 5 busId ab000 - Destroy COMPLETE | + +### Multi-node NCCL alltoall + +| Topology | Peak Bus BW | Peak Size | Avg Bus BW | Threshold | Status | +|----------|-------------|-----------|------------|-----------|--------| +| 2 nodes x 8 GPUs NCCL 2.27.7 | 28.37 GB/s | 256M | 28.32 GB/s | >= 75 GB/s | FAIL | + +| Topology | NCCL Network | GPU Direct RDMA | GDR Enabled HCAs | GDR Disabled HCAs | +|----------|--------------|-----------------|------------------|-------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 | IB | ENABLED | mlx5_0, mlx5_1, mlx5_6, mlx5_7 | - | + +| Topology | Return Code | Error / Output Tail | +|----------|-------------|---------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 | 0 | 0012:2144547:2144713 [4] NCCL INFO comm 0x55896a1dae20 rank 4 nranks 16 cudaDev 4 busId 9a000 - Destroy COMPLETE aikubeworker0016:1010164:1010881 [2] NCCL INFO comm 0x565344db7790 rank 10 nranks 16 cudaDev 2 busId 3a000 - Destroy COMPLETE | + +**Overall: FAIL** + +--- +*Generated by GPU Test Suite v0.2.0* \ No newline at end of file diff --git a/reports_multinode_nccl_sweep_2x8_nccl227.md b/reports_multinode_nccl_sweep_2x8_nccl227.md new file mode 100644 index 0000000..701492b --- /dev/null +++ b/reports_multinode_nccl_sweep_2x8_nccl227.md @@ -0,0 +1,66 @@ +# GPU Test Report + +- **Date:** 2026-05-23T07:54:48.990378 +- **Host:** aikubeworker0012 + +## Overall Acceptance Verdict + +**Result: FAIL** + +Missing required evidence: +- GPU Info +- Health Check +- Memory Bandwidth +- Compute Throughput +- NVLink/NVSwitch +- NCCL +- Stress Test +- RDMA +- DCGM +- Training + +## Summary + +| Test | Result | +|------|--------| +| Multi-node NCCL | FAIL | + +## Multi-node NCCL / Cross Leaf + +Source: nccl-tests-mpirun | Mode: sweep-nccl-2.27.7 + +- **Hosts:** nccl-gpu-1(172.72.8.12), nccl-gpu-2(172.72.8.16) +- **Preflight:** PASS + +### Multi-node NCCL allreduce + +| Topology | Peak Bus BW | Peak Size | Avg Bus BW | Threshold | Status | +|----------|-------------|-----------|------------|-----------|--------| +| 2 nodes x 8 GPUs NCCL 2.27.7 sweep | 237.26 GB/s | 4G | 150.62 GB/s | >= 480 GB/s | FAIL | + +| Topology | NCCL Network | GPU Direct RDMA | GDR Enabled HCAs | GDR Disabled HCAs | +|----------|--------------|-----------------|------------------|-------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 sweep | IB | ENABLED | mlx5_0, mlx5_1, mlx5_6, mlx5_7 | - | + +| Topology | Return Code | Error / Output Tail | +|----------|-------------|---------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 sweep | 0 | aikubeworker0012:2145024:2145189 [0] NCCL INFO comm 0x561f7dc1f780 rank 0 nranks 16 cudaDev 0 busId 18000 - Destroy COMPLETE # Out of bounds values : 0 OK # Avg bus bandwidth : 150.624 # # Collective test concluded: all_reduce_perf # | + +### Multi-node NCCL alltoall + +| Topology | Peak Bus BW | Peak Size | Avg Bus BW | Threshold | Status | +|----------|-------------|-----------|------------|-----------|--------| +| 2 nodes x 8 GPUs NCCL 2.27.7 sweep | 28.78 GB/s | 1G | 23.57 GB/s | >= 75 GB/s | FAIL | + +| Topology | NCCL Network | GPU Direct RDMA | GDR Enabled HCAs | GDR Disabled HCAs | +|----------|--------------|-----------------|------------------|-------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 sweep | IB | ENABLED | mlx5_0, mlx5_1, mlx5_6, mlx5_7 | - | + +| Topology | Return Code | Error / Output Tail | +|----------|-------------|---------------------| +| 2 nodes x 8 GPUs NCCL 2.27.7 sweep | 0 | r0012:2145213:2145384 [7] NCCL INFO comm 0x558d54228110 rank 7 nranks 16 cudaDev 7 busId db000 - Destroy COMPLETE aikubeworker0016:1014703:1015544 [0] NCCL INFO comm 0x55ed6d99d8e0 rank 8 nranks 16 cudaDev 0 busId 18000 - Destroy COMPLETE | + +**Overall: FAIL** + +--- +*Generated by GPU Test Suite v0.2.0* \ No newline at end of file