Compare commits

..

41 Commits

Author SHA1 Message Date
cs
017c981062 Remove remaining report docs from PR 2026-05-26 00:44:56 +08:00
cs
1c3c811254 Remove generated reports from PR 2026-05-26 00:44:39 +08:00
cs
7ec2da18bc Clean report whitespace 2026-05-26 00:15:48 +08:00
cs
4dddab27b3 Add FP8 GEMM path comparison reports 2026-05-26 00:13:33 +08:00
cs
4484c731b6 Add H100 acceptance PR summary 2026-05-26 00:12:59 +08:00
cs
f80a3b3636 Add H100 acceptance delivery manifest 2026-05-26 00:12:59 +08:00
cs
639651ef24 Add H100 network escalation request 2026-05-26 00:12:59 +08:00
cs
edb4612cc6 Add H100 acceptance closure checklist 2026-05-26 00:12:59 +08:00
cs
1203b025a0 Document H100 acceptance entrypoint 2026-05-26 00:12:59 +08:00
cs
5b022d5849 Summarize current H100 acceptance status 2026-05-26 00:12:59 +08:00
cs
90c46e40b3 Archive all-collectives NCCL artifacts 2026-05-26 00:12:59 +08:00
cs
c2db68f608 Add multinode NCCL all collectives run 2026-05-26 00:12:59 +08:00
cs
e0cb796b0c Analyze multinode NCCL artifact signals 2026-05-26 00:12:59 +08:00
cs
4d06639129 Record multinode NCCL artifacts run 2026-05-26 00:12:59 +08:00
cs
098d1715f2 Archive multinode NCCL raw artifacts 2026-05-26 00:12:59 +08:00
cs
7bc15742ea Clarify multinode NCCL report thresholds 2026-05-26 00:12:59 +08:00
cs
c73d738557 Record multinode NCCL PDF matrix run 2026-05-26 00:12:55 +08:00
cs
8923270ce0 Add multinode NCCL PDF matrix runner 2026-05-26 00:12:55 +08:00
cs
2c5c31e451 Add single-node H100 all runner 2026-05-26 00:12:55 +08:00
cs
cadfbcfaa3 Add NCCL environment snapshot script 2026-05-26 00:12:55 +08:00
cs
ef56e5f15a Add NCCL latest report index 2026-05-26 00:12:55 +08:00
cs
892f833ff4 Add NCCL network handoff plan 2026-05-26 00:12:55 +08:00
cs
f64e85efaf Document NCCL environment equivalence gaps 2026-05-26 00:12:55 +08:00
cs
c183f5a9d1 Document NCCL deep diagnosis rerun 2026-05-26 00:12:55 +08:00
cs
b55666948c Add multinode NCCL deep diagnosis tools 2026-05-26 00:12:55 +08:00
cs
24a7bd5c1b Document NCCL graph comparison 2026-05-26 00:12:55 +08:00
cs
82c6316716 Document NCCL alltoall secondary sweep 2026-05-26 00:12:55 +08:00
cs
1813c11bbf Compare NCCL allreduce alltoall counters 2026-05-26 00:12:55 +08:00
cs
edc469cee9 Document NCCL alltoall counter probe 2026-05-26 00:12:55 +08:00
cs
2e194ded14 Document PXN alltoall rail balancing 2026-05-26 00:12:55 +08:00
cs
619a471634 Tune multinode alltoall PXN behavior 2026-05-26 00:12:54 +08:00
cs
a64e964e3c Add raw RDMA rail bandwidth evidence 2026-05-26 00:12:54 +08:00
cs
ce363b2f7a Document missing NCCL network plugin 2026-05-26 00:12:54 +08:00
cs
e756f0b7b4 Document NCCL rail saturation evidence 2026-05-26 00:12:54 +08:00
cs
aa05ccab2e Add NCCL PDF matrix topology report 2026-05-26 00:12:54 +08:00
cs
6c9f049b71 Tune multinode NCCL auto parameters 2026-05-26 00:12:50 +08:00
cs
1f907e9691 Validate NCCL 2.27 multinode GDR performance 2026-05-26 00:12:50 +08:00
cs
c660e04c99 Stabilize multinode NCCL launch diagnostics 2026-05-26 00:12:50 +08:00
cs
4b93fc785f Add multinode NCCL diagnostic report 2026-05-26 00:12:43 +08:00
cs
4b17bafd53 Add multi-node NCCL sweep test 2026-05-26 00:12:25 +08:00
cs
86f15544d7 Add H100 acceptance test coverage and reports 2026-05-26 00:12:10 +08:00
53 changed files with 4255 additions and 15156 deletions

8
.gitignore vendored
View File

@ -6,6 +6,12 @@ __pycache__/
dist/ dist/
build/ build/
reports/ reports/
reports_*
H100*.md
test_all*.md
docs/h100_test_all_metrics_guide_cn.md
docs/multinode_nccl_concepts.md
docs/multinode_nccl_deep_diagnose_runbook.md
*.egg *.egg
.eggs/ .eggs/
*.log *.log
@ -14,4 +20,6 @@ reports/
.venv/ .venv/
venv/ venv/
.qoder/* .qoder/*
.playwright-mcp/
.claude/settings.local.json .claude/settings.local.json
.omx/

148
README.md
View File

@ -19,7 +19,6 @@
- [配置说明](#配置说明) - [配置说明](#配置说明)
- [测试 SOP(标准操作流程)](#测试-sop标准操作流程) - [测试 SOP(标准操作流程)](#测试-sop标准操作流程)
- [报告输出](#报告输出) - [报告输出](#报告输出)
- [GPU 节点画像工具](#gpu-节点画像工具)
- [故障排除](#故障排除) - [故障排除](#故障排除)
--- ---
@ -37,27 +36,13 @@ servertest/
│ ├── gpu_info.py # GPU 检测 & 信息 │ ├── gpu_info.py # GPU 检测 & 信息
│ ├── health_check.py # 健康诊断 │ ├── health_check.py # 健康诊断
│ ├── benchmark.py # 内存带宽 + 计算吞吐 │ ├── benchmark.py # 内存带宽 + 计算吞吐
│ ├── nccl_test.py # NCCL 单节点多卡通信 (NCCLTest) + 多机 Scaling (MultiNodeNCCLTest) │ ├── nccl_test.py # NCCL 多卡通信
│ ├── stress_test.py # GPU 压力/稳定性 │ ├── stress_test.py # GPU 压力/稳定性
│ ├── rdma_test.py # RDMA/InfiniBand │ ├── rdma_test.py # RDMA/InfiniBand
│ ├── training_sim.py # 训练模拟 │ ├── training_sim.py # 训练模拟
│ └── report.py # 报告生成 │ └── report.py # 报告生成
├── scripts/
│ ├── gpu_node_profile/ # JSON 驱动的运维画像与 3D CT
│ │ ├── gpu_node_profile.py # 只读采集与渲染入口
│ │ ├── templates/ # 两个独立 HTML 源模板
│ │ ├── examples/ # 脱敏真机 JSON 与自包含示例页面
│ │ └── tests/ # 解析与渲染回归测试
│ └── multinode_nccl/ # 独立 shell 脚本版 (历史备用)
│ ├── nccl_env.sh # 环境变量 (MPI路径/IB网卡/bond0)
│ ├── hosts_all # 全量 34 台节点列表
│ ├── run_scaling.sh # Scaling 测试 (2→32节点, 同交换机)
│ └── run_cross_switch_test.sh # 跨交换机新机接入验证
├── reports/ # 测试报告
│ ├── multinode_nccl_scaling_20260602.md
│ └── multinode_nccl_cross_switch_20260606.md
├── requirements.txt ├── requirements.txt
└── doc/ # 验收标准文档 └── 调研.md # 行业框架调研
``` ```
--- ---
@ -297,7 +282,7 @@ python3 gpu_tester.py --config /path/to/config.yaml --test all
默认配置:4096×4096 矩阵,10 次 warmup,100 次迭代。 默认配置:4096×4096 矩阵,10 次 warmup,100 次迭代。
### 5. NCCL Multi-GPU Test(单节点多卡通信) ### 5. NCCL Multi-GPU Test(多卡通信)
优先使用官方 nccl-tests(通过 mpirun 调用),不可用时 torchrun fallback。 优先使用官方 nccl-tests(通过 mpirun 调用),不可用时 torchrun fallback。
@ -314,34 +299,6 @@ python3 gpu_tester.py --config /path/to/config.yaml --test all
**NVLink 参考带宽:** A100/A800 ≥ 240 GB/s | H100/H200 ≥ 360 GB/s | B200/B300 ≥ 720 GB/s(40% NVLink 峰值) **NVLink 参考带宽:** A100/A800 ≥ 240 GB/s | H100/H200 ≥ 360 GB/s | B200/B300 ≥ 720 GB/s(40% NVLink 峰值)
### 5m. Multi-node NCCL Scaling(多机跨节点通信)
通过 mpirun + hostfile 在集群多台节点上运行 NCCL Scaling 测试,验证 IB fabric 健康和带宽随节点数的线性扩展。**菜单选 `m`,CLI 用 `--test multinode-nccl`。**
```bash
# 交互式菜单选 m
python3 gpu_tester.py
# 或 CLI 模式
python3 gpu_tester.py --test multinode-nccl
# 自定义节点数和 hostfile
python3 gpu_tester.py --test multinode-nccl --config configs/h20_cluster.yaml
```
配置(`configs/default.yaml` 的 `multinode_nccl` 段):
| 参数 | 默认值 | 说明 |
|---|---|---|
| `hosts_file` | `~/nccl_kit/hosts_all` | OpenMPI hostfile 路径 |
| `mpi_home` | `/usr/mpi/gcc/openmpi-4.1.7rc1` | MPI 安装路径 |
| `nccl_ib_hca` | `mlx5_0~5,8,9` | GPU 直连 IB 卡(排除存储 NIC) |
| `nccl_socket_ifname` | `bond0` | NCCL bootstrap 网卡 |
| `scale_levels` | `[2, 4, 8]` | 要测试的节点数列表 |
| `allreduce_min_busbw_gbps` | `310` | AllReduce PASS 阈值(GB/s) |
**验收标准:** AllReduce busbw ≥ 310 GB/s(4 节点起稳态 ~317 GB/s,32 节点无衰减)
### 6. GPU Stress Test(压力测试) ### 6. GPU Stress Test(压力测试)
使用 gpu-burn 进行长时满载测试,验证热稳定性和内存正确性。 使用 gpu-burn 进行长时满载测试,验证热稳定性和内存正确性。
@ -522,10 +479,9 @@ report:
--- ---
### SOP-3: 多节点集群验收 (H20-3e, 34 台) ### SOP-3: 多节点集群验收
**适用场景:** 多台 GPU 服务器组成训练集群,验证节点间 NCCL 通信及 IB fabric 健康。 **适用场景:** 多台 GPU 服务器组成训练集群,验证节点间通信。
**工具脚本位置:** `scripts/multinode_nccl/`,主节点 `~/nccl_kit/`(gpu-20)
``` ```
前置条件: 每台单节点已通过 SOP-1 前置条件: 每台单节点已通过 SOP-1
@ -534,61 +490,32 @@ report:
├── 在每台节点上执行 SOP-1 ├── 在每台节点上执行 SOP-1
└── 确保所有单节点测试通过 └── 确保所有单节点测试通过
步骤 2: 确认 IB 网卡拓扑 步骤 2: RDMA 网络测试
├── nvidia-smi topo -m ├── python3 gpu_tester.py --test rdma
├── 确认 GPU 直连 NIC (PIX): mlx5_0/1/2/3/4/5/8/9 (每节点 8 张) ├── 确认: IB 设备被识别
├── mlx5_6/7 为存储 NIC (NODE 级别), 不参与 NCCL ├── 确认: 端口状态 Active
└── 全集群 34 台拓扑一致: md5 均为 65a5f1bd43922dc879adadc5cce523b2 ├── 确认: 写带宽 ≥ 50 GB/s
├── 确认: 延迟 ≤ 10 μs
└── 异常: 检查 IB 线缆、交换机配置、子网管理器
步骤 3: 准备主节点环境 步骤 3: 多节点 NCCL 测试
├── # 在 gpu-20 上: ├── 在每个节点上配置:
├── mkdir -p ~/nccl_kit && cd ~/nccl_kit │ export MASTER_ADDR=<主节点IP>
├── # 上传 scripts/multinode_nccl/ 下的所有文件到 ~/nccl_kit/ │ export MASTER_PORT=29500
├── # 若 nccl-tests 为 MPI=0 编译, 需重编: │ export NCCL_SOCKET_IFNAME=ib0 # IB 网卡名
│ cd /opt/nccl-tests && make clean │ export NCCL_DEBUG=INFO
│ make MPI=1 MPI_HOME=/usr/mpi/gcc/openmpi-4.1.7rc1 CUDA_HOME=/usr/local/cuda -j ├── 运行 nccl-tests 手动测试:
│ ldd build/all_reduce_perf | grep -i mpi # 确认 libmpi 链接 │ mpirun -np <总GPU数> -hostfile hosts \
└── # 确认各节点免密 SSH 已打通 (集群预置共享 key) │ /opt/gpu-test-tools/nccl-tests/build/all_reduce_perf \
│ -b 8 -e 256M -f 2 -g 1 -w 5 -n 20
└── 确认: 多节点 AllReduce 带宽正常
步骤 4: Scaling 测试 (同交换机, 2→32 节点) 步骤 4: 训练验证
├── # 在 gpu-20 上: ├── python3 gpu_tester.py --test training
├── source ~/nccl_kit/nccl_env.sh ├── 可选: 加载更大模型(如 llama 模型)
├── bash ~/nccl_kit/run_scaling.sh 2 4 8 16 32 └── 确认: 训练 loss 正常下降
├── 验收标准:
│ AllReduce busbw: 2节点 ~480 GB/s, 4节点起稳定 ~317 GB/s
│ AllGather / ReduceScatter: ~315-340 GB/s (随节点数平稳)
│ AllToAll (1G): ~48-70 GB/s (随节点数轻微下降属正常)
└── 结果: ~/nccl_kit/results_*/summary.csv
步骤 5: 新机 / 跨交换机验证 (按需)
├── # 新机接入时需额外操作:
│ 1. SSH 互信: 在新机和现有节点间互相追加 authorized_keys
│ 2. 新机重编 nccl-tests (预装版通常为 MPI=0):
│ cd /opt/nccl-tests && make clean
│ make MPI=1 MPI_HOME=/usr/mpi/gcc/openmpi-4.1.7rc1 CUDA_HOME=/usr/local/cuda -j
│ 3. 确认新机 nvidia-smi topo -m 拓扑与集群一致
├── bash ~/nccl_kit/run_cross_switch_test.sh
├── 验收标准: 跨交换机 AllReduce ~317 GB/s (与同交换机持平即正常)
└── AllToAll 跨交换机较同交换机下降 ≤20% 属正常 (多一跳 IB spine)
``` ```
**关键环境变量说明 (nccl_env.sh):**
| 变量 | 值 | 原因 |
|---|---|---|
| `NCCL_IB_HCA` | `mlx5_0,1,2,3,4,5,8,9` | 仅 GPU 直连的 8 张 IB 卡;mlx5_6/7 为存储 NIC,不参与 NCCL |
| `NCCL_SOCKET_IFNAME` | `bond0` | 节点有 k8s 虚拟网卡,不指定会 bootstrap 选错导致 connection refused |
| `MPI_HOME` | `/usr/mpi/gcc/openmpi-4.1.7rc1` | 非系统默认路径,必须显式指定 |
**常见问题:**
| 现象 | 原因 | 解决 |
|---|---|---|
| busbw=0.00,每进程各打印一行结果 | nccl-tests 是 MPI=0 编译的 | `make clean && make MPI=1 ...` 重编 |
| `socketPollConnect: Connection refused` | NCCL 选了 k8s 虚拟网卡 | 确认 `NCCL_SOCKET_IFNAME=bond0` |
| AllToAll 在 8G 消息时 hang | 已知问题,16+ 节点大消息 hang | AllToAll 消息上限用 ≤2G |
| 新机 mpirun 连接失败 | SSH 互信未打通 | 互相追加 authorized_keys |
--- ---
### SOP-4: 故障诊断 ### SOP-4: 故障诊断
@ -683,27 +610,6 @@ python3 gpu_tester.py --test all --format html --output report.html
--- ---
## GPU 节点画像工具
`scripts/gpu_node_profile/` 提供独立、只读的 GPU 节点画像工具。采集器可在
一次执行中生成结构化 JSON,并由同一份 JSON 分别渲染运维画像和可逐层钻取
的 3D CT 页面。两个 HTML 源模板位于 `templates/`;`examples/` 包含脱敏
真机 JSON 和由其生成的两张自包含示例页面。支持 NUMA、PCIe、GPU、
NVLink/NVSwitch、RoCE Bond、无 Bond 的独立 RoCE/InfiniBand 端口、
物理网卡、CPU 亲和性和功耗信息。GPU 到 RDMA 端点的亲和关系来自
`nvidia-smi topo -m`,页面不会为矩阵未暴露的设备推测连线;详细命令与
采集边界见该目录的 `README.md`。
```bash
cd scripts/gpu_node_profile
python3 gpu_node_profile.py \
--snapshot-out gpu_node_snapshot.json \
--output gpu_node_profile.html \
--topology-output gpu_topology_3d.html
```
---
## 故障排除 ## 故障排除
| 问题 | 原因 | 解决方案 | | 问题 | 原因 | 解决方案 |

View File

@ -1,62 +0,0 @@
gpu_type: b300
tools:
install_dir: /root/test_gpu_scripts/.gpu-test-env/tools
benchmark:
memory:
size_mb: 4096
iterations: 10
nvbandwidth_buffer_mb: 512
nvbandwidth_samples: 3
compute:
dtypes: [fp32, tf32, fp16, bf16, fp8]
sweep_sizes:
- 3584
- 4608
- 5376
- 8192
- 11520
- [2048, 2048, 13312]
- [2048, 2048, 16384]
matrix_size: 8192
warmup: 20
iterations: 80
health:
temp_warning: 75
temp_critical: 85
power_limit: null
nccl:
min_bandwidth_gbps: null
test_allreduce: true
test_alltoall: true
test_broadcast: true
test_reduce_scatter: false
test_allgather: false
test_sendrecv: false
stress:
duration_sec: 600
use_doubles: false
use_tensor_cores: true
memory_pct: 90
gpus: all
rdma:
min_bandwidth_gbps: 50
max_latency_us: 10
cross_node:
enabled: false
training:
model: synthetic
batch_size: 8
seq_length: 2048
num_steps: 50
dtype: bf16
report:
output_dir: /root/test_gpu_scripts/reports/b300_single_node_20260827
format: json

View File

@ -1,86 +0,0 @@
gpu_type: bw1000
runtime:
name: DTK
version: 25.04.2
benchmark:
memory:
size_mb: 4096
iterations: 10
nvbandwidth_buffer_mb: 512
nvbandwidth_samples: 3
compute:
dtypes: [fp32, tf32, fp16, bf16, fp8]
sweep_sizes: [4096, 6144, 8192, [2048, 2048, 13312]]
matrix_size: 8192
warmup: 10
iterations: 30
health:
temp_warning: 75
temp_critical: 85
power_limit: 800
nccl:
min_bandwidth_gbps: 10
test_allreduce: true
test_alltoall: true
test_broadcast: true
test_reduce_scatter: true
test_allgather: true
test_sendrecv: false
torch_backend: nccl
torchrun_size_mb: 64
torchrun_warmup: 5
torchrun_iterations: 20
stress:
duration_sec: 600
use_doubles: false
use_tensor_cores: true
memory_pct: 80
gpus: all
rdma:
min_bandwidth_gbps: 200
max_latency_us: 6
ib_iterations: 5000
msg_size: 1048576
ib_device: null
ib_port: 1
cross_node:
enabled: false
allow_roce: true
server: null
server_addr: null
ssh_user: root
devices: [mlx5_bond_0, mlx5_bond_1, mlx5_bond_2, mlx5_bond_3]
ib_port: 1
gid_index: 3
msg_size: 1048576
iters: 5000
duration_sec: 10
queue_pairs: 4
base_oob_port: 18515
server_warmup_sec: 2.0
min_bandwidth_gbps: 200
max_latency_us: 6
training:
model: synthetic
batch_size: 2
seq_length: 1024
num_steps: 20
dtype: bf16
hidden_size: 2048
num_layers: 4
num_heads: 16
vocab_size: 8192
report:
output_dir: ./reports
format: json
tools:
install_dir: /opt/gpu-test-tools

View File

@ -1,15 +0,0 @@
gpu_type: bw1000
runtime:
name: DTK
version: 25.04.2
stress:
duration_sec: 60
use_doubles: false
use_tensor_cores: true
memory_pct: 80
gpus: all
tools:
install_dir: /opt/gpu-test-tools

View File

@ -48,6 +48,55 @@ nccl:
test_allgather: false test_allgather: false
test_sendrecv: false test_sendrecv: false
multinode_nccl:
enabled: false
mode: sweep
hosts:
- name: nccl-gpu-1
addr: 172.72.8.12
slots: 8
- name: nccl-gpu-2
addr: 172.72.8.16
slots: 8
ssh_user: root
ssh_preflight: true
mpirun_path: /usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun
mpi_ld_preload: null
extra_ld_library_path:
- /usr/mpi/gcc/openmpi-4.1.9a1/lib
- /root/gpu-test-venv/lib/python3.10/site-packages/nvidia/nccl/lib
- /usr/local/cuda-12.4/targets/x86_64-linux/lib
nccl_tests_dir: null # null = tools.install_dir/nccl-tests/build
tests:
- all_reduce_perf
- alltoall_perf
topologies:
- nodes: 2
gpus_per_node: 8
begin_size: 1k
end_size: 16g
step_factor: 2
warmup_iters: 10
gpus_per_rank: 1
timeout_sec: 1800
socket_ifname: bond0
oob_tcp_ifname: bond0
plm_rsh_args: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=30"
ib_gid_index: 3
ib_sl: 5
ib_tc: 136
ib_hca: mlx5_0,mlx5_1,mlx5_6,mlx5_7
ib_timeout: 22
qps_per_connection: null
min_nchannels: null
net_plugin: none
nvls_enable: 1
split_data_on_qps: null
extra_env: {}
min_peak_busbw_gbps:
allreduce: 480
alltoall: 75
stress: stress:
duration_sec: 600 # 10 min — reaches thermal steady state, validates throttle/jitter beyond warmup duration_sec: 600 # 10 min — reaches thermal steady state, validates throttle/jitter beyond warmup
use_doubles: false use_doubles: false
@ -81,31 +130,6 @@ rdma:
min_bandwidth_gbps: 350 # per-port PASS floor (NDR400 ≈ 0.9 × 400) min_bandwidth_gbps: 350 # per-port PASS floor (NDR400 ≈ 0.9 × 400)
max_latency_us: 5 max_latency_us: 5
multinode_nccl:
# Path to OpenMPI hostfile: one line per node, format "<ip> slots=8"
hosts_file: ~/nccl_kit/hosts_all
# MPI installation prefix (non-standard path, set empty to use system mpirun)
mpi_home: /usr/mpi/gcc/openmpi-4.1.7rc1
# IB HCAs for NCCL — GPU-direct (PIX) cards only; mlx5_6/7 are storage NICs, exclude them
# GPU↔NIC map: GPU0↔mlx5_3, GPU1↔mlx5_2, GPU2↔mlx5_1, GPU3↔mlx5_0
# GPU4↔mlx5_5, GPU5↔mlx5_4, GPU6↔mlx5_9, GPU7↔mlx5_8
nccl_ib_hca: "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9"
# NCCL bootstrap NIC — must be explicit (k8s virtual NICs cause connection refused)
nccl_socket_ifname: bond0
# Node counts to run scaling test at
scale_levels: [2, 4, 8]
# Operations to test
ops: [all_reduce, alltoall, all_gather, reduce_scatter]
# Message size for allreduce/allgather/reducescatter (bytes)
msg_size_bytes: 8589934592 # 8G
# Message size for alltoall — cap at 1G to avoid hang at 16+ nodes
msg_size_alltoall_bytes: 1073741824 # 1G
warmup: 5
iterations: 30
timeout_sec: 600 # per-op timeout (10 min)
# AllReduce busbw PASS floor (GB/s) — 4-node+ stable state on NDR400 cluster
allreduce_min_busbw_gbps: 310
training: training:
model: gpt2 model: gpt2
batch_size: 8 batch_size: 8

View File

@ -0,0 +1,62 @@
tools:
install_dir: /opt/gpu-test-tools
report:
output_dir: ./reports
format: md
multinode_nccl:
enabled: true
mode: diagnostic
hosts:
- name: nccl-gpu-1
addr: 172.72.8.12
slots: 8
- name: nccl-gpu-2
addr: 172.72.8.16
slots: 8
ssh_user: root
ssh_preflight: true
mpirun_path: /usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun
mpi_ld_preload: null
extra_ld_library_path:
- /usr/mpi/gcc/openmpi-4.1.9a1/lib
- /root/gpu-test-venv/lib/python3.10/site-packages/nvidia/nccl/lib
- /usr/local/cuda-12.4/targets/x86_64-linux/lib
nccl_tests_dir: null
tests:
- all_reduce_perf
- alltoall_perf
topologies:
- nodes: 2
gpus_per_node: 8
label: 2 nodes x 8 GPUs diagnostic
begin_size: 256M
end_size: 256M
step_factor: 2
warmup_iters: 1
iters: 3
gpus_per_rank: 1
timeout_sec: 600
debug: INFO
socket_ifname: bond0
oob_tcp_ifname: bond0
plm_rsh_args: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=30"
ib_gid_index: 3
ib_sl: 5
ib_tc: 136
ib_hca: mlx5_0,mlx5_1,mlx5_6,mlx5_7
ib_timeout: 22
qps_per_connection: null
min_nchannels: null
net_plugin: none
nvls_enable: 1
split_data_on_qps: null
extra_env:
NCCL_DEBUG_SUBSYS: INIT,NET
NCCL_NET_GDR_LEVEL: 5
NCCL_NET_GDR_READ: 1
NCCL_DMABUF_ENABLE: 0
min_peak_busbw_gbps:
allreduce: 480
alltoall: 75

View File

@ -0,0 +1,62 @@
tools:
install_dir: /opt/gpu-test-tools
report:
output_dir: ./reports
format: md
multinode_nccl:
enabled: true
mode: large-message-nccl-2.27.7
hosts:
- name: nccl-gpu-1
addr: 172.72.8.12
slots: 8
- name: nccl-gpu-2
addr: 172.72.8.16
slots: 8
ssh_user: root
ssh_preflight: true
mpirun_path: /usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun
mpi_ld_preload: null
extra_ld_library_path:
- /usr/mpi/gcc/openmpi-4.1.9a1/lib
- /tmp/nccl-2.27.7-cuda12.4/usr/lib/x86_64-linux-gnu
- /usr/local/cuda-12.4/targets/x86_64-linux/lib
nccl_tests_dir: /data/nccl-tests-latest/build
tests:
- all_reduce_perf
- alltoall_perf
topologies:
- nodes: 2
gpus_per_node: 8
label: 2 nodes x 8 GPUs NCCL 2.27.7 16G
begin_size: 16G
end_size: 16G
step_factor: 2
warmup_iters: 1
iters: 3
gpus_per_rank: 1
timeout_sec: 1200
debug: INFO
socket_ifname: bond0
oob_tcp_ifname: bond0
plm_rsh_args: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=30"
ib_gid_index: 3
ib_sl: 5
ib_tc: 136
ib_hca: mlx5_0,mlx5_1,mlx5_6,mlx5_7
ib_timeout: 22
qps_per_connection: null
min_nchannels: null
net_plugin: none
nvls_enable: 1
split_data_on_qps: null
extra_env:
NCCL_DEBUG_SUBSYS: INIT,NET
NCCL_NET_GDR_LEVEL: 5
NCCL_NET_GDR_READ: 1
NCCL_DMABUF_ENABLE: 0
min_peak_busbw_gbps:
allreduce: 480
alltoall: 75

View File

@ -0,0 +1,72 @@
tools:
install_dir: /opt/gpu-test-tools
report:
output_dir: ./reports
format: md
multinode_nccl:
enabled: true
mode: cross-leaf-all-collectives-nccl-2.27.7
hosts:
- name: nccl-gpu-1
addr: 172.72.8.12
slots: 8
- name: nccl-gpu-2
addr: 172.72.8.16
slots: 8
ssh_user: root
ssh_preflight: true
mpirun_path: /usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun
mpi_ld_preload: null
extra_ld_library_path:
- /usr/mpi/gcc/openmpi-4.1.9a1/lib
- /tmp/nccl-2.27.7-cuda12.4/usr/lib/x86_64-linux-gnu
- /usr/local/cuda-12.4/targets/x86_64-linux/lib
nccl_tests_dir: /data/nccl-tests-latest/build
tests:
- all_reduce_perf
- alltoall_perf
- broadcast_perf
- reduce_scatter_perf
- all_gather_perf
- sendrecv_perf
topologies:
- nodes: 2
gpus_per_node: 8
label: 2 nodes x 8 GPUs (all collectives evidence run)
op_env:
alltoall:
NCCL_PXN_DISABLE: 1
begin_size: 16G
end_size: 16G
step_factor: 2
warmup_iters: 10
gpus_per_rank: 1
timeout_sec: 1800
debug: INFO
socket_ifname: bond0
oob_tcp_ifname: bond0
plm_rsh_args: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=30"
ib_gid_index: 3
ib_sl: 5
ib_tc: 136
ib_hca: mlx5_0,mlx5_1,mlx5_6,mlx5_7
ib_timeout: 22
qps_per_connection: null
min_nchannels: null
net_plugin: none
nvls_enable: 1
split_data_on_qps: null
extra_env:
NCCL_DEBUG_SUBSYS: INIT,NET
NCCL_NET_GDR_LEVEL: 5
NCCL_NET_GDR_READ: 1
NCCL_DMABUF_ENABLE: 0
min_peak_busbw_gbps:
allreduce: 491.84
alltoall: 76.54
broadcast: 0
reducescatter: 0
allgather: 0
sendrecv: 0

View File

@ -0,0 +1,62 @@
tools:
install_dir: /opt/gpu-test-tools
report:
output_dir: ./reports
format: md
multinode_nccl:
enabled: true
mode: large-message-nccl-2.27.7-auto
hosts:
- name: nccl-gpu-1
addr: 172.72.8.12
slots: 8
- name: nccl-gpu-2
addr: 172.72.8.16
slots: 8
ssh_user: root
ssh_preflight: true
mpirun_path: /usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun
mpi_ld_preload: null
extra_ld_library_path:
- /usr/mpi/gcc/openmpi-4.1.9a1/lib
- /tmp/nccl-2.27.7-cuda12.4/usr/lib/x86_64-linux-gnu
- /usr/local/cuda-12.4/targets/x86_64-linux/lib
nccl_tests_dir: /data/nccl-tests-latest/build
tests:
- all_reduce_perf
- alltoall_perf
topologies:
- nodes: 2
gpus_per_node: 8
label: 2 nodes x 8 GPUs NCCL 2.27.7 auto 16G
begin_size: 16G
end_size: 16G
step_factor: 2
warmup_iters: 1
iters: 3
gpus_per_rank: 1
timeout_sec: 1200
debug: INFO
socket_ifname: bond0
oob_tcp_ifname: bond0
plm_rsh_args: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=30"
ib_gid_index: 3
ib_sl: 5
ib_tc: 136
ib_hca: mlx5_0,mlx5_1,mlx5_6,mlx5_7
ib_timeout: 22
qps_per_connection: null
min_nchannels: null
net_plugin: none
nvls_enable: 1
split_data_on_qps: null
extra_env:
NCCL_DEBUG_SUBSYS: INIT,NET
NCCL_NET_GDR_LEVEL: 5
NCCL_NET_GDR_READ: 1
NCCL_DMABUF_ENABLE: 0
min_peak_busbw_gbps:
allreduce: 480
alltoall: 75

View File

@ -0,0 +1,62 @@
tools:
install_dir: /opt/gpu-test-tools
report:
output_dir: ./reports
format: md
multinode_nccl:
enabled: true
mode: diagnostic-nccl-2.27.7
hosts:
- name: nccl-gpu-1
addr: 172.72.8.12
slots: 8
- name: nccl-gpu-2
addr: 172.72.8.16
slots: 8
ssh_user: root
ssh_preflight: true
mpirun_path: /usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun
mpi_ld_preload: null
extra_ld_library_path:
- /usr/mpi/gcc/openmpi-4.1.9a1/lib
- /tmp/nccl-2.27.7-cuda12.4/usr/lib/x86_64-linux-gnu
- /usr/local/cuda-12.4/targets/x86_64-linux/lib
nccl_tests_dir: /data/nccl-tests-latest/build
tests:
- all_reduce_perf
- alltoall_perf
topologies:
- nodes: 2
gpus_per_node: 8
label: 2 nodes x 8 GPUs NCCL 2.27.7
begin_size: 256M
end_size: 256M
step_factor: 2
warmup_iters: 1
iters: 3
gpus_per_rank: 1
timeout_sec: 600
debug: INFO
socket_ifname: bond0
oob_tcp_ifname: bond0
plm_rsh_args: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=30"
ib_gid_index: 3
ib_sl: 5
ib_tc: 136
ib_hca: mlx5_0,mlx5_1,mlx5_6,mlx5_7
ib_timeout: 22
qps_per_connection: null
min_nchannels: null
net_plugin: none
nvls_enable: 1
split_data_on_qps: null
extra_env:
NCCL_DEBUG_SUBSYS: INIT,NET
NCCL_NET_GDR_LEVEL: 5
NCCL_NET_GDR_READ: 1
NCCL_DMABUF_ENABLE: 0
min_peak_busbw_gbps:
allreduce: 480
alltoall: 75

View File

@ -0,0 +1,91 @@
tools:
install_dir: /opt/gpu-test-tools
report:
output_dir: ./reports
format: md
multinode_nccl:
enabled: true
mode: cross-leaf-pdf-matrix-nccl-2.27.7
hosts:
- name: nccl-gpu-1
addr: 172.72.8.12
slots: 8
- name: nccl-gpu-2
addr: 172.72.8.16
slots: 8
ssh_user: root
ssh_preflight: true
mpirun_path: /usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun
mpi_ld_preload: null
extra_ld_library_path:
- /usr/mpi/gcc/openmpi-4.1.9a1/lib
- /tmp/nccl-2.27.7-cuda12.4/usr/lib/x86_64-linux-gnu
- /usr/local/cuda-12.4/targets/x86_64-linux/lib
nccl_tests_dir: /data/nccl-tests-latest/build
tests:
- all_reduce_perf
- alltoall_perf
topologies:
- nodes: 2
gpus_per_node: 1
label: 2 nodes x 1 GPU (PDF 2 machines 2 GPUs)
min_peak_busbw_gbps:
allreduce: 48.90
alltoall: 27.25
- nodes: 2
gpus_per_node: 2
label: 2 nodes x 2 GPUs (PDF 2 machines 4 GPUs)
min_peak_busbw_gbps:
allreduce: 136.93
alltoall: 54.41
- nodes: 2
gpus_per_node: 4
label: 2 nodes x 4 GPUs (PDF 2 machines 8 GPUs)
cuda_visible_devices: 0,1,4,5
op_env:
alltoall:
NCCL_IB_QPS_PER_CONNECTION: 4
NCCL_MIN_NCHANNELS: 4
NCCL_IB_SPLIT_DATA_ON_QPS: 1
min_peak_busbw_gbps:
allreduce: 335.48
alltoall: 73.73
- nodes: 2
gpus_per_node: 8
label: 2 nodes x 8 GPUs (PDF 2 machines 16 GPUs)
op_env:
alltoall:
NCCL_PXN_DISABLE: 1
min_peak_busbw_gbps:
allreduce: 491.84
alltoall: 76.54
begin_size: 16G
end_size: 16G
step_factor: 2
warmup_iters: 10
gpus_per_rank: 1
timeout_sec: 1800
debug: INFO
socket_ifname: bond0
oob_tcp_ifname: bond0
plm_rsh_args: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=30"
ib_gid_index: 3
ib_sl: 5
ib_tc: 136
ib_hca: mlx5_0,mlx5_1,mlx5_6,mlx5_7
ib_timeout: 22
qps_per_connection: null
min_nchannels: null
net_plugin: none
nvls_enable: 1
split_data_on_qps: null
extra_env:
NCCL_DEBUG_SUBSYS: INIT,NET
NCCL_NET_GDR_LEVEL: 5
NCCL_NET_GDR_READ: 1
NCCL_DMABUF_ENABLE: 0
min_peak_busbw_gbps:
allreduce: 0
alltoall: 0

View File

@ -0,0 +1,62 @@
tools:
install_dir: /opt/gpu-test-tools
report:
output_dir: ./reports
format: md
multinode_nccl:
enabled: true
mode: sweep-nccl-2.27.7
hosts:
- name: nccl-gpu-1
addr: 172.72.8.12
slots: 8
- name: nccl-gpu-2
addr: 172.72.8.16
slots: 8
ssh_user: root
ssh_preflight: true
mpirun_path: /usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun
mpi_ld_preload: null
extra_ld_library_path:
- /usr/mpi/gcc/openmpi-4.1.9a1/lib
- /tmp/nccl-2.27.7-cuda12.4/usr/lib/x86_64-linux-gnu
- /usr/local/cuda-12.4/targets/x86_64-linux/lib
nccl_tests_dir: /data/nccl-tests-latest/build
tests:
- all_reduce_perf
- alltoall_perf
topologies:
- nodes: 2
gpus_per_node: 8
label: 2 nodes x 8 GPUs NCCL 2.27.7 sweep
begin_size: 1M
end_size: 4G
step_factor: 4
warmup_iters: 2
iters: 5
gpus_per_rank: 1
timeout_sec: 1200
debug: INFO
socket_ifname: bond0
oob_tcp_ifname: bond0
plm_rsh_args: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=30"
ib_gid_index: 3
ib_sl: 5
ib_tc: 136
ib_hca: mlx5_0,mlx5_1,mlx5_6,mlx5_7
ib_timeout: 22
qps_per_connection: null
min_nchannels: null
net_plugin: none
nvls_enable: 1
split_data_on_qps: null
extra_env:
NCCL_DEBUG_SUBSYS: INIT,NET
NCCL_NET_GDR_LEVEL: 5
NCCL_NET_GDR_READ: 1
NCCL_DMABUF_ENABLE: 0
min_peak_busbw_gbps:
allreduce: 480
alltoall: 75

View File

@ -5,6 +5,7 @@ import argparse
import json import json
import os import os
import signal import signal
import socket
import sys import sys
import time import time
from datetime import datetime from datetime import datetime
@ -22,69 +23,138 @@ from modules.gpu_info import GPUInfo
from modules.health_check import HealthCheck from modules.health_check import HealthCheck
from modules.benchmark import Benchmark from modules.benchmark import Benchmark
from modules.nccl_test import NCCLTest from modules.nccl_test import NCCLTest
from modules.multinode_nccl_test import MultiNodeNCCLTest
from modules.training_sim import TrainingSim from modules.training_sim import TrainingSim
from modules.stress_test import StressTest from modules.stress_test import StressTest
from modules.rdma_test import RDMATest from modules.rdma_test import RDMATest
from modules.nvlink_test import NVLinkTest
from modules.dcgm_test import DCGMTest
from modules.multinode_nccl_test import MultiNodeNCCLTest
from modules.report import ReportGenerator from modules.report import ReportGenerator
from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label, get_supported_gpus, validate_driver_compatibility from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label, get_supported_gpus, validate_driver_compatibility
from modules.hygon_smi import find_hy_smi
DEFAULT_CONFIG = { DEFAULT_CONFIG = {
"benchmark": { "benchmark": {
"memory": {"size_mb": 4096, "iterations": 10, "nvbandwidth_buffer_mb": 512, "nvbandwidth_samples": 3}, "memory": {"size_mb": 4096, "iterations": 10, "nvbandwidth_buffer_mb": 512, "nvbandwidth_samples": 3},
"compute": { "compute": {
"dtypes": ["fp32", "tf32", "fp16", "bf16", "fp8"], "dtypes": ["fp32", "tf32", "fp16", "bf16", "fp8", "fp64", "int8"],
"matrix_size": 4096, "matrix_size": 8192,
"warmup": 10, "warmup": 50,
"iterations": 100, "iterations": 500,
"use_compile": True,
}, },
}, },
"health": {"temp_warning": 80, "temp_critical": 90, "power_limit": None}, "health": {"temp_warning": 75, "temp_critical": 85, "power_limit": None},
"nccl": { "nccl": {
"min_bandwidth_gbps": None, "min_bandwidth_gbps": None,
"test_allreduce": True, "test_allreduce": True,
"test_alltoall": True, "test_alltoall": True,
"test_broadcast": True, "test_broadcast": True,
"test_reduce_scatter": False, "test_reduce_scatter": True,
"test_allgather": False, "test_allgather": True,
"test_sendrecv": False, "test_sendrecv": True,
"message_sizes": ["1M", "256M", "2G"],
"repeats": 3,
"max_stddev_pct": 3,
}, },
"multinode_nccl": { "multinode_nccl": {
"hosts_file": "~/nccl_kit/hosts_all", "enabled": False,
"mpi_home": "/usr/mpi/gcc/openmpi-4.1.7rc1", "mode": "sweep",
"nccl_ib_hca": "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9", "hosts": [
"nccl_socket_ifname": "bond0", {"name": "nccl-gpu-1", "addr": "172.72.8.12", "slots": 8},
"scale_levels": [2, 4, 8], {"name": "nccl-gpu-2", "addr": "172.72.8.16", "slots": 8},
"ops": ["all_reduce", "alltoall", "all_gather", "reduce_scatter"], ],
"msg_size_bytes": 8589934592, "ssh_user": "root",
"msg_size_alltoall_bytes": 1073741824, "ssh_preflight": True,
"warmup": 5, "mpirun_path": "/usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun",
"iterations": 30, "mpi_ld_preload": None,
"timeout_sec": 600, "extra_ld_library_path": [
"allreduce_min_busbw_gbps": 310, "/usr/mpi/gcc/openmpi-4.1.9a1/lib",
"/root/gpu-test-venv/lib/python3.10/site-packages/nvidia/nccl/lib",
"/usr/local/cuda-12.4/targets/x86_64-linux/lib",
],
"nccl_tests_dir": None,
"tests": ["all_reduce_perf", "alltoall_perf"],
"topologies": [{"nodes": 2, "gpus_per_node": 8}],
"begin_size": "1k",
"end_size": "16g",
"step_factor": 2,
"warmup_iters": 10,
"gpus_per_rank": 1,
"timeout_sec": 1800,
"socket_ifname": "bond0",
"ib_gid_index": 3,
"ib_sl": 5,
"ib_tc": 136,
"ib_hca": "mlx5_0,mlx5_1,mlx5_6,mlx5_7",
"ib_timeout": 22,
"qps_per_connection": 4,
"min_nchannels": 4,
"net_plugin": "none",
"nvls_enable": 1,
"split_data_on_qps": 1,
"min_peak_busbw_gbps": {"allreduce": 480, "alltoall": 75},
}, },
"stress": { "stress": {
"duration_sec": 60, "duration_sec": 1800,
"production_duration_sec": 1800,
"use_gpu_burn": False,
"use_doubles": False, "use_doubles": False,
"use_tensor_cores": True, "use_tensor_cores": True,
"memory_pct": 90, "memory_pct": 90,
"gpus": "all", "gpus": "all",
"dtype": "bf16",
"matrix_size": 24576,
"telemetry_interval_sec": 1,
"warmup_sec": 60,
"min_steady_samples": 10,
"max_temp_c": 80,
"max_temp_delta_c": 5,
"min_power_watts": 630,
"max_tflops_jitter_pct": 5,
"require_tflops_jitter": True,
}, },
"rdma": { "rdma": {
"min_bandwidth_gbps": 50, "min_bandwidth_gbps": 47,
"max_latency_us": 10, "min_port_rate_gbps": 400,
"max_latency_us": 3.5,
"max_write_latency_us": 2.0,
"max_read_latency_us": 3.5,
"ib_iterations": 1000, "ib_iterations": 1000,
"msg_size": 65536, "msg_size": 4194304,
"latency_msg_size": 8,
"ib_device": None, "ib_device": None,
"ib_port": 1, "ib_port": 1,
"server_addr": None,
"ibping_target": None,
"ibping_count": 5,
"role": "auto",
"pfc_ecn_counters": True,
},
"nvlink": {
"expected_links_per_gpu": 18,
"expected_link_speed_gbps": 25,
"require_zero_errors": True,
},
"dcgm": {
"diag_level": 3,
"timeout_sec": 1200,
"expected_num_gpus": 8,
"json_output": True,
"require_subtests": True,
}, },
"training": { "training": {
"model": "gpt2", "model": "synthetic_1.5b",
"batch_size": 8, "batch_size": 8,
"seq_length": 2048, "seq_length": 2048,
"num_steps": 50, "num_steps": 50,
"warmup_steps": 5,
"dtype": "bf16", "dtype": "bf16",
"mode": "ddp",
"synthetic_params_b": 1.5,
"min_tokens_per_sec": 45000,
"max_step_jitter_pct": 3,
"max_peak_memory_gb": 70,
"require_distributed": True,
}, },
"report": {"output_dir": "./reports", "format": "json"}, "report": {"output_dir": "./reports", "format": "json"},
"tools": {"install_dir": "/opt/gpu-test-tools"}, "tools": {"install_dir": "/opt/gpu-test-tools"},
@ -119,9 +189,9 @@ def check_prerequisites(console: Console) -> bool:
import shutil import shutil
ok = True ok = True
if not shutil.which("nvidia-smi") and not find_hy_smi(): if not shutil.which("nvidia-smi"):
console.print("[bold red]ERROR: neither nvidia-smi nor hy-smi was found![/bold red]") console.print("[bold red]ERROR: nvidia-smi not found![/bold red]")
console.print(" Please install the NVIDIA driver or Hygon DCU driver first.") console.print(" Please install NVIDIA drivers first.")
ok = False ok = False
return ok return ok
@ -147,7 +217,7 @@ def interactive_menu(config: dict):
if not check_prerequisites(console): if not check_prerequisites(console):
return return
results_store: dict = {"timestamp": datetime.now().isoformat(), "tests": {}} results_store: dict = {"timestamp": datetime.now().isoformat(), "hostname": socket.gethostname(), "tests": {}}
menu_items = [ menu_items = [
("1", "GPU Information", "gpu_info"), ("1", "GPU Information", "gpu_info"),
@ -155,11 +225,13 @@ def interactive_menu(config: dict):
("3", "Memory Benchmark (nvbandwidth)", "memory_bench"), ("3", "Memory Benchmark (nvbandwidth)", "memory_bench"),
("4", "Compute Benchmark", "compute_bench"), ("4", "Compute Benchmark", "compute_bench"),
("5", "NCCL Multi-GPU Test", "nccl"), ("5", "NCCL Multi-GPU Test", "nccl"),
("6", "GPU Stress Test (gpu-burn)", "stress"), ("6", "GPU Stress Test (PyTorch/gpu-burn)", "stress"),
("7", "RDMA/IB Test", "rdma"), ("7", "RDMA/IB Test", "rdma"),
("8", "Training Simulation", "training"), ("8", "NVLink/NVSwitch Test", "nvlink"),
("m", "Multi-node NCCL Scaling", "multinode_nccl"), ("9", "DCGM Diagnostic", "dcgm"),
("9", "Full Test Suite (All Tests)", "all"), ("10", "Training Simulation", "training"),
("11", "Multi-node NCCL Test", "multinode_nccl"),
("12", "Full Test Suite (All Tests)", "all"),
("0", "Generate Report", "report"), ("0", "Generate Report", "report"),
] ]
@ -181,10 +253,12 @@ def interactive_menu(config: dict):
"memory_bench": "HBM bandwidth via nvbandwidth", "memory_bench": "HBM bandwidth via nvbandwidth",
"compute_bench": "GEMM TFLOPS across FP32/TF32/FP16/BF16/FP8", "compute_bench": "GEMM TFLOPS across FP32/TF32/FP16/BF16/FP8",
"nccl": "AllReduce, AllToAll, Broadcast via nccl-tests", "nccl": "AllReduce, AllToAll, Broadcast via nccl-tests",
"multinode_nccl": "跨节点 NCCL Scaling via mpirun + hostfile", "stress": "Long-running high-power GEMM stress with telemetry",
"stress": "Long-running GPU stress via gpu-burn",
"rdma": "InfiniBand bandwidth & latency (ib_write_bw)", "rdma": "InfiniBand bandwidth & latency (ib_write_bw)",
"nvlink": "NVLink links, speed, and error counters",
"dcgm": "DCGM diag -r 3 production diagnostic",
"training": "Simulate LLM training with PyTorch", "training": "Simulate LLM training with PyTorch",
"multinode_nccl": "Cross-node NCCL via mpirun/nccl-tests",
"all": "Run all tests sequentially", "all": "Run all tests sequentially",
"report": "Export results to JSON/HTML", "report": "Export results to JSON/HTML",
} }
@ -263,12 +337,6 @@ def _run_test(test_name: str, config: dict, console: Console) -> dict:
m.print_results(result) m.print_results(result)
return result return result
elif test_name == "multinode_nccl":
m = MultiNodeNCCLTest(config)
result = m.run()
m.print_results(result)
return result
elif test_name == "stress": elif test_name == "stress":
m = StressTest(config) m = StressTest(config)
result = m.run() result = m.run()
@ -281,12 +349,30 @@ def _run_test(test_name: str, config: dict, console: Console) -> dict:
m.print_results(result) m.print_results(result)
return result return result
elif test_name == "nvlink":
m = NVLinkTest(config)
result = m.run()
m.print_results(result)
return result
elif test_name == "dcgm":
m = DCGMTest(config)
result = m.run()
m.print_results(result)
return result
elif test_name == "training": elif test_name == "training":
m = TrainingSim(config) m = TrainingSim(config)
result = m.run() result = m.run()
m.print_results(result) m.print_results(result)
return result return result
elif test_name == "multinode_nccl":
m = MultiNodeNCCLTest(config)
result = m.run()
m.print_results(result)
return result
elif test_name == "all": elif test_name == "all":
return _run_full_suite(config, console) return _run_full_suite(config, console)
@ -304,17 +390,21 @@ def _run_test(test_name: str, config: dict, console: Console) -> dict:
def _run_full_suite(config: dict, console: Console) -> dict: def _run_full_suite(config: dict, console: Console) -> dict:
"""Run all tests sequentially.""" """Run all tests sequentially."""
console.print(Panel("[bold cyan]Running Full Test Suite[/bold cyan]", box=box.DOUBLE)) console.print(Panel("[bold cyan]Running Full Test Suite[/bold cyan]", box=box.DOUBLE))
all_results: dict = {"timestamp": datetime.now().isoformat()} all_results: dict = {"timestamp": datetime.now().isoformat(), "hostname": socket.gethostname()}
tests = [ tests = [
("gpu_info", "GPU Information", GPUInfo), ("gpu_info", "GPU Information", GPUInfo),
("health", "Health Check", HealthCheck), ("health", "Health Check", HealthCheck),
("memory_bench", "Memory Benchmark", lambda c: Benchmark(c)), ("memory_bench", "Memory Benchmark", lambda c: Benchmark(c)),
("compute_bench", "Compute Benchmark", lambda c: Benchmark(c)), ("compute_bench", "Compute Benchmark", lambda c: Benchmark(c)),
("nvlink", "NVLink/NVSwitch Test", NVLinkTest),
("nccl", "NCCL Test", NCCLTest), ("nccl", "NCCL Test", NCCLTest),
("stress", "GPU Stress Test", StressTest), ("stress", "GPU Stress Test", StressTest),
("rdma", "RDMA/IB Test", RDMATest), ("rdma", "RDMA/IB Test", RDMATest),
("dcgm", "DCGM Diagnostic", DCGMTest),
("training", "Training Simulation", TrainingSim), ("training", "Training Simulation", TrainingSim),
] ]
if (config.get("multinode_nccl", {}) or {}).get("enabled"):
tests.append(("multinode_nccl", "Multi-node NCCL Test", MultiNodeNCCLTest))
for i, (key, name, mod_cls) in enumerate(tests, 1): for i, (key, name, mod_cls) in enumerate(tests, 1):
console.print(f"\n[bold cyan][{i}/{len(tests)}] {name}[/bold cyan]") console.print(f"\n[bold cyan][{i}/{len(tests)}] {name}[/bold cyan]")
@ -334,42 +424,52 @@ def _run_full_suite(config: dict, console: Console) -> dict:
console.print(f"[bold red]{name} FAILED: {e}[/bold red]") console.print(f"[bold red]{name} FAILED: {e}[/bold red]")
all_results[key] = {"error": str(e)} all_results[key] = {"error": str(e)}
# Summary: classify actual verdicts. The old implementation counted every # Summary
# result without an exception as PASS, even when passed=False or status=FAIL.
console.print("\n" + "=" * 60) console.print("\n" + "=" * 60)
test_results = {k: v for k, v in all_results.items() if k != "timestamp"} # Only count test results, exclude metadata like timestamp
statuses = {} test_results = {k: v for k, v in all_results.items() if k not in ("timestamp", "hostname")}
for key, value in test_results.items(): passed = sum(1 for v in test_results.values() if _test_result_passed(v))
status = "PASS" total = len(test_results)
if isinstance(value, dict) and value.get("error"): color = "green" if passed == total else ("yellow" if passed > 0 else "red")
status = "FAIL" console.print(f"[bold {color}]Suite complete: {passed}/{total} tests passed[/bold {color}]")
elif key == "health":
status = "PASS" if value.get("passed") else "FAIL"
elif key == "memory_bench":
mem = value.get("memory", value)
if mem.get("error"):
status = "FAIL"
else:
eff = mem.get("d2d_efficiency_pct") or mem.get("efficiency_pct") or 0
status = "PASS" if eff >= 80 else ("WARN" if eff >= 60 else "FAIL")
elif key == "compute_bench":
status = Benchmark.judge_compute(value)["verdict"]
elif key in ("nccl", "rdma"):
status = value.get("status") or ("PASS" if value.get("passed") else "FAIL")
elif key == "stress":
status = "PASS" if value.get("passed") else "FAIL"
elif key == "training":
status = value.get("performance_verdict", "MEASURED")
statuses[key] = status
counts = {name: list(statuses.values()).count(name) for name in ("PASS", "WARN", "FAIL", "SKIP", "MEASURED")}
color = "red" if counts["FAIL"] else ("yellow" if counts["WARN"] else "green")
summary = ", ".join(f"{name} {count}" for name, count in counts.items() if count)
console.print(f"[bold {color}]Suite complete: {summary}[/bold {color}]")
all_results["suite_status"] = statuses
return all_results return all_results
def _test_result_passed(result) -> bool:
"""Strict production verdict helper for full-suite exit status."""
if not isinstance(result, dict):
return True
if result.get("error"):
return False
if result.get("skipped") or result.get("status") == "SKIP":
return False
if result.get("source") == "torchrun_fallback":
return False
if "passed" in result:
return bool(result.get("passed"))
if "memory" in result:
mem = result["memory"]
if isinstance(mem, dict) and "passed" in mem:
return bool(mem.get("passed"))
if mem.get("error") or mem.get("source") == "pytorch":
return False
eff = mem.get("d2d_efficiency_pct") or mem.get("efficiency_pct") or 0
return eff >= 80
if "compute" in result:
comp = result["compute"]
if isinstance(comp, dict) and "passed" in comp:
return bool(comp.get("passed"))
thresholds = comp.get("pass_thresholds_tflops", {}) or {}
per_dtype = comp.get("per_dtype_tflops", {})
for dt, threshold in thresholds.items():
val = per_dtype.get(dt)
if not isinstance(val, (int, float)) or val < threshold:
return False
consistency = comp.get("consistency", {})
return not any(not c.get("passed", False) for c in consistency.values())
return True
def main(): def main():
gpu_list_str = " / ".join(g.upper() for g in get_supported_gpus()) gpu_list_str = " / ".join(g.upper() for g in get_supported_gpus())
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
@ -384,15 +484,18 @@ Examples:
python gpu_tester.py --test benchmark --type memory python gpu_tester.py --test benchmark --type memory
python gpu_tester.py --test benchmark --type compute --dtype fp16 python gpu_tester.py --test benchmark --type compute --dtype fp16
python gpu_tester.py --test nccl # NCCL test python gpu_tester.py --test nccl # NCCL test
python gpu_tester.py --test multinode-nccl # Cross-node NCCL test
python gpu_tester.py --test nvlink # NVLink/NVSwitch test
python gpu_tester.py --test dcgm # DCGM diagnostic
python gpu_tester.py --test training # Training sim python gpu_tester.py --test training # Training sim
python gpu_tester.py --test all # Full suite python gpu_tester.py --test all # Full suite
python gpu_tester.py --report --format json --output report.json python gpu_tester.py --report --format json --output report.json
""", """,
) )
parser.add_argument("--test", choices=["gpu-info", "health", "benchmark", "nccl", "multinode-nccl", "stress", "rdma", "training", "all"], parser.add_argument("--test", choices=["gpu-info", "health", "benchmark", "nccl", "multinode-nccl", "stress", "rdma", "nvlink", "dcgm", "training", "all"],
help="Run a specific test") help="Run a specific test")
parser.add_argument("--type", choices=["memory", "compute"], help="Benchmark type (with --test benchmark)") parser.add_argument("--type", choices=["memory", "compute"], help="Benchmark type (with --test benchmark)")
parser.add_argument("--dtype", choices=["fp32", "tf32", "fp16", "bf16", "fp8"], parser.add_argument("--dtype", choices=["fp32", "tf32", "fp16", "bf16", "fp8", "fp64", "int8"],
help="Compute benchmark dtype (with --test benchmark --type compute)") help="Compute benchmark dtype (with --test benchmark --type compute)")
parser.add_argument("--interactive", action="store_true", help="Force interactive mode") parser.add_argument("--interactive", action="store_true", help="Force interactive mode")
parser.add_argument("--report", action="store_true", help="Generate report from last results") parser.add_argument("--report", action="store_true", help="Generate report from last results")
@ -449,6 +552,8 @@ Examples:
"multinode-nccl": "multinode_nccl", "multinode-nccl": "multinode_nccl",
"stress": "stress", "stress": "stress",
"rdma": "rdma", "rdma": "rdma",
"nvlink": "nvlink",
"dcgm": "dcgm",
"training": "training", "training": "training",
"all": "all", "all": "all",
} }
@ -465,22 +570,30 @@ Examples:
result = bench.run() result = bench.run()
Benchmark.print_results(result) Benchmark.print_results(result)
if args.report: if args.report:
ReportGenerator(config).generate({"benchmark": result, "timestamp": datetime.now().isoformat()}, ReportGenerator(config).generate({
"benchmark": result,
"timestamp": datetime.now().isoformat(),
"hostname": socket.gethostname(),
},
fmt=args.format, output=args.output) fmt=args.format, output=args.output)
sys.exit(0 if _test_result_passed(result) else 1)
elif args.test == "all": elif args.test == "all":
results = _run_full_suite(config, console) results = _run_full_suite(config, console)
if args.report: if args.report:
ReportGenerator(config).generate(results, fmt=args.format, output=args.output) ReportGenerator(config).generate(results, fmt=args.format, output=args.output)
has_errors = any("error" in v for v in results.values() if isinstance(v, dict)) failed = any(not _test_result_passed(v) for k, v in results.items() if k not in ("timestamp", "hostname"))
has_failures = any( sys.exit(1 if failed else 0)
status == "FAIL" for status in results.get("suite_status", {}).values()
)
sys.exit(1 if has_errors or has_failures else 0)
else: else:
result = _run_test(test_map[args.test], config, console) result = _run_test(test_map[args.test], config, console)
if args.report and result: if args.report and result:
ReportGenerator(config).generate({test_map[args.test]: result, "timestamp": datetime.now().isoformat()}, report_key = test_map[args.test] or args.test
ReportGenerator(config).generate({
report_key: result,
"timestamp": datetime.now().isoformat(),
"hostname": socket.gethostname(),
},
fmt=args.format, output=args.output) fmt=args.format, output=args.output)
sys.exit(0 if _test_result_passed(result) else 1)
if __name__ == "__main__": if __name__ == "__main__":

View File

@ -17,10 +17,6 @@ from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label, res
TORCH_AVAILABLE = False TORCH_AVAILABLE = False
try: try:
import torch import torch
try:
import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim
except ImportError:
pass
if torch.cuda.is_available(): if torch.cuda.is_available():
TORCH_AVAILABLE = True TORCH_AVAILABLE = True
except ImportError: except ImportError:
@ -67,14 +63,12 @@ class Benchmark:
return None return None
def run_memory_benchmark(self) -> dict: def run_memory_benchmark(self) -> dict:
# nvbandwidth is NVIDIA-specific. DTK PyTorch provides the portable nvbw = self._find_nvbandwidth()
# transfer/copy path used below for Hygon DCUs.
nvbw = None if self.gpu_type == "bw1000" else self._find_nvbandwidth()
if nvbw: if nvbw:
return self._run_nvbandwidth(nvbw) return self._run_nvbandwidth(nvbw)
self.console.print("[yellow]Using PyTorch memory benchmark fallback[/yellow]") self.console.print("[yellow]nvbandwidth not found, falling back to PyTorch[/yellow]")
return self._run_memory_pytorch() return self._run_memory_pytorch()
def _run_nvbandwidth(self, nvbw_path: str) -> dict: def _run_nvbandwidth(self, nvbw_path: str) -> dict:
@ -150,14 +144,11 @@ class Benchmark:
progress.advance(task) progress.advance(task)
# Compare like with like: the 900 GB/s reference is per direction, so do
# not select the bidirectional SM result as the acceptance value. Keep it
# as a separate informational metric.
d2d_bw = max( d2d_bw = max(
results_by_test.get("d2d_write", 0), results_by_test.get("d2d_write", 0),
results_by_test.get("d2d_read", 0), results_by_test.get("d2d_read", 0),
results_by_test.get("d2d_bidir", 0),
) )
d2d_bidir_bw = results_by_test.get("d2d_bidir", 0)
h2d_bw = results_by_test.get("h2d", 0) h2d_bw = results_by_test.get("h2d", 0)
d2h_bw = results_by_test.get("d2h", 0) d2h_bw = results_by_test.get("d2h", 0)
@ -188,7 +179,6 @@ class Benchmark:
"h2d_bandwidth_gbps": round(h2d_bw, 1), "h2d_bandwidth_gbps": round(h2d_bw, 1),
"d2h_bandwidth_gbps": round(d2h_bw, 1), "d2h_bandwidth_gbps": round(d2h_bw, 1),
"d2d_bandwidth_gbps": round(d2d_bw, 1), "d2d_bandwidth_gbps": round(d2d_bw, 1),
"d2d_bidirectional_gbps": round(d2d_bidir_bw, 1),
"h2d_peak_gbps": pcie_peak if pcie_peak else None, "h2d_peak_gbps": pcie_peak if pcie_peak else None,
"d2h_peak_gbps": pcie_peak if pcie_peak else None, "d2h_peak_gbps": pcie_peak if pcie_peak else None,
"d2d_peak_gbps": round(d2d_peak, 1) if d2d_peak else None, "d2d_peak_gbps": round(d2d_peak, 1) if d2d_peak else None,
@ -199,8 +189,6 @@ class Benchmark:
"efficiency_pct": d2d_efficiency, "efficiency_pct": d2d_efficiency,
"results_by_test": results_by_test, "results_by_test": results_by_test,
"per_gpu": [], "per_gpu": [],
"scope_note": "GPU peer/PCIe transfer bandwidth; local HBM bandwidth not measured",
"accelerator_vendor": self.specs.get("vendor", "nvidia"),
} }
} }
@ -304,31 +292,20 @@ class Benchmark:
progress.advance(task) progress.advance(task)
best_d2d = max(v["d2d_gbps"] for v in bandwidth_by_size.values()) best_d2d = max(v["d2d_gbps"] for v in bandwidth_by_size.values())
best_h2d = max(v["h2d_gbps"] for v in bandwidth_by_size.values())
best_d2h = max(v["d2h_gbps"] for v in bandwidth_by_size.values())
peak_bw = self.specs["memory_bandwidth_gbps"] peak_bw = self.specs["memory_bandwidth_gbps"]
efficiency = round((best_d2d / peak_bw) * 100, 1) if peak_bw else None efficiency = round((best_d2d / peak_bw) * 100, 1) if peak_bw else None
pcie_gen = self.specs.get("pcie_gen", 0)
pcie_peak = {3: 16, 4: 32, 5: 64, 6: 128}.get(pcie_gen) if pcie_gen else None
return { return {
"memory": { "memory": {
"source": "pytorch", "source": "pytorch",
"h2d_bandwidth_gbps": round(best_h2d, 1), "h2d_bandwidth_gbps": round(max(v["h2d_gbps"] for v in bandwidth_by_size.values()), 1),
"d2h_bandwidth_gbps": round(best_d2h, 1), "d2h_bandwidth_gbps": round(max(v["d2h_gbps"] for v in bandwidth_by_size.values()), 1),
"d2d_bandwidth_gbps": round(best_d2d, 1), "d2d_bandwidth_gbps": round(best_d2d, 1),
"h2d_peak_gbps": pcie_peak,
"d2h_peak_gbps": pcie_peak,
"d2d_peak_gbps": None,
"h2d_efficiency_pct": round(best_h2d / pcie_peak * 100, 1) if pcie_peak else None,
"d2h_efficiency_pct": round(best_d2h / pcie_peak * 100, 1) if pcie_peak else None,
"d2d_efficiency_pct": efficiency,
"peak_bandwidth_gbps": self.specs["memory_bandwidth_gbps"], "peak_bandwidth_gbps": self.specs["memory_bandwidth_gbps"],
"efficiency_pct": efficiency, "efficiency_pct": efficiency,
"test_sizes_mb": test_sizes_mb, "test_sizes_mb": test_sizes_mb,
"bandwidth_by_size": bandwidth_by_size, "bandwidth_by_size": bandwidth_by_size,
"per_gpu": [], "per_gpu": [],
"accelerator_vendor": self.specs.get("vendor", "nvidia"),
} }
} }
@ -378,7 +355,7 @@ class Benchmark:
"tf32": ("tf32", self.specs["tf32_tflops"]), "tf32": ("tf32", self.specs["tf32_tflops"]),
"fp16": (torch.float16, self.specs["fp16_tflops"]), "fp16": (torch.float16, self.specs["fp16_tflops"]),
"bf16": (torch.bfloat16, self.specs["bf16_tflops"]), "bf16": (torch.bfloat16, self.specs["bf16_tflops"]),
"fp8": (getattr(torch, "float8_e4m3fn", None), self.specs["fp8_tflops"]), "fp8": (torch.float8_e4m3fn, self.specs["fp8_tflops"]),
} }
results_by_dtype = {} results_by_dtype = {}
@ -390,23 +367,19 @@ class Benchmark:
BarColumn(), TextColumn("{task.completed}/{task.total}"), BarColumn(), TextColumn("{task.completed}/{task.total}"),
TimeElapsedColumn(), console=self.console, TimeElapsedColumn(), console=self.console,
) as progress: ) as progress:
task = progress.add_task( task = progress.add_task("Testing dtypes...", total=len(configured_dtypes))
"Testing dtypes on every GPU...", total=len(configured_dtypes) * gpu_count
)
for dtype_name in configured_dtypes: for dtype_name in configured_dtypes:
if dtype_name not in dtype_map: if dtype_name not in dtype_map:
progress.advance(task, advance=gpu_count) progress.advance(task)
continue continue
# Skip FP8 if GPU architecture doesn't support it # Skip FP8 if GPU architecture doesn't support it
if dtype_name == "fp8" and ( if dtype_name == "fp8" and self.specs.get("fp8_tflops", 0) == 0:
self.specs.get("fp8_tflops", 0) == 0 or dtype_map["fp8"][0] is None
):
arch = self.specs.get("architecture", "unknown") arch = self.specs.get("architecture", "unknown")
results_by_dtype["fp8"] = f"skipped ({arch} does not support FP8)" results_by_dtype["fp8"] = f"skipped ({arch} does not support FP8)"
self.console.print(f"[dim] fp8: skipped - {arch} architecture has no FP8 support[/dim]") self.console.print(f"[dim] fp8: skipped - {arch} architecture has no FP8 support[/dim]")
progress.advance(task, advance=gpu_count) progress.advance(task)
continue continue
dtype_val, peak_tflops = dtype_map[dtype_name] dtype_val, peak_tflops = dtype_map[dtype_name]
@ -420,48 +393,32 @@ class Benchmark:
elif dtype_name == "fp32": elif dtype_name == "fp32":
torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cuda.matmul.allow_tf32 = False
gpu_values = [] best_tflops, best_shape, last_err = 0.0, None, None
gpu_shapes = {} for (M, N, K) in shapes:
last_err = None try:
fastest = (0.0, None) t = self._bench_matmul_once(dtype_name, dtype_val, M, N, K, warmup, iterations)
for gpu_idx in range(gpu_count): if t > best_tflops:
best_tflops, best_shape = 0.0, None best_tflops, best_shape = t, (M, N, K)
for (M, N, K) in shapes: except Exception as e: # noqa: BLE001 - record and try the next shape
try: last_err = e
t = self._bench_matmul_once(
dtype_name, dtype_val, M, N, K,
warmup, iterations, gpu_idx,
)
if t > best_tflops:
best_tflops, best_shape = t, (M, N, K)
except Exception as e: # noqa: BLE001 - record and try next shape
last_err = e
if best_shape is not None:
value = round(best_tflops, 1)
per_gpu_results[gpu_idx][dtype_name] = value
gpu_values.append(value)
gpu_shapes[str(gpu_idx)] = "x".join(str(d) for d in best_shape)
if best_tflops > fastest[0]:
fastest = (best_tflops, best_shape)
else:
per_gpu_results[gpu_idx][dtype_name] = f"error: {last_err}"
progress.advance(task)
torch.backends.cuda.matmul.allow_tf32 = old_tf32 torch.backends.cuda.matmul.allow_tf32 = old_tf32
if not gpu_values: if best_shape is None:
results_by_dtype[dtype_name] = f"error: {last_err}" results_by_dtype[dtype_name] = f"error: {last_err}"
self.console.print(f"[yellow] {dtype_name}: {last_err}[/yellow]") self.console.print(f"[yellow] {dtype_name}: {last_err}[/yellow]")
else: else:
avg_value = round(sum(gpu_values) / len(gpu_values), 1) shape_str = "x".join(str(d) for d in best_shape)
results_by_dtype[dtype_name] = avg_value results_by_dtype[dtype_name] = round(best_tflops, 1)
best_shapes[dtype_name] = "x".join(str(d) for d in fastest[1]) best_shapes[dtype_name] = shape_str
for pg in per_gpu_results:
pg[dtype_name] = round(best_tflops, 1)
if len(shapes) > 1: if len(shapes) > 1:
self.console.print( self.console.print(
f"[dim] {dtype_name}: avg {avg_value:.1f}, " f"[dim] {dtype_name}: {best_tflops:.1f} TFLOPS @ {shape_str}[/dim]"
f"min {min(gpu_values):.1f}, max {max(gpu_values):.1f} TFLOPS[/dim]"
) )
best_shapes.setdefault("per_gpu", {})[dtype_name] = gpu_shapes
progress.advance(task)
efficiency = {} efficiency = {}
for dt, achieved in results_by_dtype.items(): for dt, achieved in results_by_dtype.items():
@ -470,18 +427,6 @@ class Benchmark:
if peak_tp: if peak_tp:
efficiency[dt] = round((achieved / peak_tp) * 100, 1) efficiency[dt] = round((achieved / peak_tp) * 100, 1)
worst_by_dtype = {}
best_by_dtype = {}
spread_by_dtype = {}
for dt in results_by_dtype:
values = [pg.get(dt) for pg in per_gpu_results]
values = [v for v in values if isinstance(v, (int, float))]
if values:
worst_by_dtype[dt] = min(values)
best_by_dtype[dt] = max(values)
avg = sum(values) / len(values)
spread_by_dtype[dt] = round((max(values) - min(values)) / avg * 100, 2) if avg else 0
return { return {
"compute": { "compute": {
"per_dtype_tflops": results_by_dtype, "per_dtype_tflops": results_by_dtype,
@ -494,20 +439,16 @@ class Benchmark:
self.specs.get("compute_pass_thresholds_tflops") or {} self.specs.get("compute_pass_thresholds_tflops") or {}
), ),
"per_gpu": per_gpu_results, "per_gpu": per_gpu_results,
"worst_per_dtype_tflops": worst_by_dtype,
"best_per_dtype_tflops": best_by_dtype,
"spread_pct": spread_by_dtype,
"sweep_sizes": list(sweep_sizes), "sweep_sizes": list(sweep_sizes),
"best_shapes": best_shapes, "best_shapes": best_shapes,
"matrix_size": matrix_size, "matrix_size": matrix_size,
"warmup": warmup, "warmup": warmup,
"iterations": iterations, "iterations": iterations,
"accelerator_backend": "dtk" if self.gpu_type == "bw1000" else "cuda",
} }
} }
def _bench_matmul_once(self, dtype_name: str, dtype_val, M: int, N: int, K: int, def _bench_matmul_once(self, dtype_name: str, dtype_val, M: int, N: int, K: int,
warmup: int, iterations: int, gpu_index: int = 0) -> float: warmup: int, iterations: int) -> float:
"""Time one (M×K)·(K×N) matmul for a dtype and return achieved TFLOPS. """Time one (M×K)·(K×N) matmul for a dtype and return achieved TFLOPS.
Uses an L2-cache-busting pool of matrix pairs (total > 256 MB) so operands Uses an L2-cache-busting pool of matrix pairs (total > 256 MB) so operands
@ -516,37 +457,34 @@ class Benchmark:
cuBLAS, which on H100 beats torch.compile/Triton for plain GEMM and avoids the cuBLAS, which on H100 beats torch.compile/Triton for plain GEMM and avoids the
per-shape recompile cost that would make a sweep pathologically slow. per-shape recompile cost that would make a sweep pathologically slow.
""" """
device = f"cuda:{gpu_index}" elem_bytes = 1 if dtype_name == "fp8" else torch.tensor([], dtype=dtype_val).element_size()
with torch.cuda.device(gpu_index):
elem_bytes = 1 if dtype_name == "fp8" else torch.tensor([], dtype=dtype_val).element_size()
pair_bytes = (M * K + K * N) * elem_bytes pair_bytes = (M * K + K * N) * elem_bytes
num_pools = max(4, -(-256 * 1024 * 1024 // pair_bytes)) # ceil(256MB / pair) num_pools = max(4, -(-256 * 1024 * 1024 // pair_bytes)) # ceil(256MB / pair)
if dtype_name == "fp8": if dtype_name == "fp8":
if not hasattr(torch, "_scaled_mm"): if not hasattr(torch, "_scaled_mm"):
raise RuntimeError("torch._scaled_mm unavailable — upgrade to PyTorch >= 2.1") raise RuntimeError("torch._scaled_mm unavailable — upgrade to PyTorch >= 2.1")
pools_a = [torch.randn(M, K, device=device, dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)] pools_a = [torch.randn(M, K, device="cuda", dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)]
pools_b = [torch.randn(N, K, device=device, dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)] pools_b = [torch.randn(N, K, device="cuda", dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)]
scale_a = torch.tensor(1.0, device=device) scale_a = torch.tensor(1.0, device="cuda")
scale_b = torch.tensor(1.0, device=device) scale_b = torch.tensor(1.0, device="cuda")
def op(i): def op(i):
return torch._scaled_mm(pools_a[i], pools_b[i].T, scale_a=scale_a, scale_b=scale_b, out_dtype=torch.bfloat16) return torch._scaled_mm(pools_a[i], pools_b[i].T, scale_a=scale_a, scale_b=scale_b, out_dtype=torch.bfloat16)
else: else:
pools_a = [torch.randn(M, K, device=device, dtype=dtype_val) for _ in range(num_pools)] pools_a = [torch.randn(M, K, device="cuda", dtype=dtype_val) for _ in range(num_pools)]
pools_b = [torch.randn(K, N, device=device, dtype=dtype_val) for _ in range(num_pools)] pools_b = [torch.randn(K, N, device="cuda", dtype=dtype_val) for _ in range(num_pools)]
def op(i): def op(i):
return torch.matmul(pools_a[i], pools_b[i]) return torch.matmul(pools_a[i], pools_b[i])
try: try:
torch.cuda.set_device(gpu_index)
# Probe once so a broken/unsupported kernel raises before the timed loop. # Probe once so a broken/unsupported kernel raises before the timed loop.
_probe = op(0) _probe = op(0)
torch.cuda.synchronize(gpu_index) torch.cuda.synchronize()
del _probe del _probe
for i in range(warmup): for i in range(warmup):
op(i % num_pools) op(i % num_pools)
torch.cuda.synchronize(gpu_index) torch.cuda.synchronize()
start_event = torch.cuda.Event(enable_timing=True) start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True) end_event = torch.cuda.Event(enable_timing=True)
@ -554,7 +492,7 @@ class Benchmark:
for i in range(iterations): for i in range(iterations):
op(i % num_pools) op(i % num_pools)
end_event.record() end_event.record()
torch.cuda.synchronize(gpu_index) torch.cuda.synchronize()
elapsed_ms = start_event.elapsed_time(end_event) elapsed_ms = start_event.elapsed_time(end_event)
finally: finally:
del pools_a, pools_b del pools_a, pools_b
@ -625,39 +563,24 @@ class Benchmark:
table = Table(box=None, padding=(0, 1)) table = Table(box=None, padding=(0, 1))
table.add_column("DType", style="bold") table.add_column("DType", style="bold")
table.add_column("Avg", justify="right") table.add_column("Achieved (TFLOPS)", justify="right")
table.add_column("Min", justify="right")
table.add_column("Max", justify="right")
table.add_column("Spread", justify="right")
table.add_column("Peak", justify="right") table.add_column("Peak", justify="right")
table.add_column("Efficiency", justify="right") table.add_column("Efficiency", justify="right")
peak = comp.get("peak_tflops", {}) peak = comp.get("peak_tflops", {})
per_dtype = comp.get("per_dtype_tflops", {}) per_dtype = comp.get("per_dtype_tflops", {})
eff = comp.get("efficiency_pct", {}) eff = comp.get("efficiency_pct", {})
worst = comp.get("worst_per_dtype_tflops", {})
best = comp.get("best_per_dtype_tflops", {})
spread = comp.get("spread_pct", {})
for dt in per_dtype: for dt in per_dtype:
achieved = per_dtype[dt] achieved = per_dtype[dt]
if isinstance(achieved, str): if isinstance(achieved, str):
table.add_row(dt, f"[red]{achieved}[/red]", "-", "-", "-", str(peak.get(dt, "N/A")), "N/A") table.add_row(dt, f"[red]{achieved}[/red]", str(peak.get(dt, "N/A")), "N/A")
continue continue
pk = peak.get(dt, 0) pk = peak.get(dt, 0)
ef = eff.get(dt, 0) ef = eff.get(dt, 0)
if pk: ec = "green" if ef >= 80 else ("yellow" if ef >= 50 else "red")
ec = "green" if ef >= 80 else ("yellow" if ef >= 50 else "red") table.add_row(dt.upper(), f"{achieved:.1f}", f"{pk:.0f}",
peak_text = f"{pk:.0f}" f"[{ec}]{ef:.1f}%[/{ec}]")
eff_text = f"[{ec}]{ef:.1f}%[/{ec}]"
else:
peak_text = "N/A"
eff_text = "[cyan]MEASURED[/cyan]"
table.add_row(
dt.upper(), f"{achieved:.1f}", f"{worst.get(dt, achieved):.1f}",
f"{best.get(dt, achieved):.1f}", f"{spread.get(dt, 0):.2f}%",
peak_text, eff_text,
)
c.print(table) c.print(table)
@staticmethod @staticmethod
@ -667,40 +590,28 @@ class Benchmark:
Single source of truth for the PASS/WARN/FAIL rule (same one report.py uses): Single source of truth for the PASS/WARN/FAIL rule (same one report.py uses):
achieved >= thr -> PASS; >= 0.9*thr -> WARN; else FAIL. A string achieved value achieved >= thr -> PASS; >= 0.9*thr -> WARN; else FAIL. A string achieved value
(skipped/error) -> SKIP. A dtype without a threshold falls back to efficiency (skipped/error) -> SKIP. A dtype without a threshold falls back to efficiency
(>=80 PASS / >=50 WARN / else FAIL). If neither threshold nor peak is (>=80 PASS / >=50 WARN / else FAIL).
published, the numeric result is retained as MEASURED without a verdict.
Returns {"rows": [(dtype, achieved, threshold, status), ...], "verdict": str}. Returns {"rows": [(dtype, achieved, threshold, status), ...], "verdict": str}.
""" """
comp = results.get("compute", results) comp = results.get("compute", results)
per_dtype = comp.get("per_dtype_tflops", {}) per_dtype = comp.get("per_dtype_tflops", {})
worst_per_dtype = comp.get("worst_per_dtype_tflops", {}) or {}
thresholds = comp.get("pass_thresholds_tflops", {}) or {} thresholds = comp.get("pass_thresholds_tflops", {}) or {}
eff = comp.get("efficiency_pct", {}) eff = comp.get("efficiency_pct", {})
rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0} rank = {"PASS": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
rows, verdict = [], "PASS" rows, verdict = [], "PASS"
evaluated = False
measured = False
for dt, val in per_dtype.items(): for dt, val in per_dtype.items():
judged_val = worst_per_dtype.get(dt, val)
thr = thresholds.get(dt) thr = thresholds.get(dt)
if isinstance(val, str): if isinstance(val, str):
status = "SKIP" status = "SKIP"
elif thr: elif thr:
status = "PASS" if judged_val >= thr else ("WARN" if judged_val >= thr * 0.9 else "FAIL") status = "PASS" if val >= thr else ("WARN" if val >= thr * 0.9 else "FAIL")
evaluated = True
elif dt not in eff:
status = "MEASURED"
measured = True
else: else:
e = eff.get(dt, 0) e = eff.get(dt, 0)
status = "PASS" if e >= 80 else ("WARN" if e >= 50 else "FAIL") status = "PASS" if e >= 80 else ("WARN" if e >= 50 else "FAIL")
evaluated = True rows.append((dt, val, thr, status))
rows.append((dt, judged_val, thr, status))
if rank[status] > rank[verdict]: if rank[status] > rank[verdict]:
verdict = status verdict = status
if measured and not evaluated:
verdict = "MEASURED"
return {"rows": rows, "verdict": verdict} return {"rows": rows, "verdict": verdict}
@staticmethod @staticmethod
@ -708,7 +619,7 @@ class Benchmark:
"""Print the PASS/WARN/FAIL table for compute results; return the verdict.""" """Print the PASS/WARN/FAIL table for compute results; return the verdict."""
c = console or Console() c = console or Console()
judged = Benchmark.judge_compute(results) judged = Benchmark.judge_compute(results)
color = {"PASS": "green", "MEASURED": "cyan", "WARN": "yellow", "FAIL": "red", "SKIP": "dim"} color = {"PASS": "green", "WARN": "yellow", "FAIL": "red", "SKIP": "dim"}
c.print("\n[bold cyan]Compute Verdict (vs thresholds)[/bold cyan]") c.print("\n[bold cyan]Compute Verdict (vs thresholds)[/bold cyan]")
for dt, val, thr, status in judged["rows"]: for dt, val, thr, status in judged["rows"]:
val_s = f"{val:.1f}" if isinstance(val, (int, float)) else str(val) val_s = f"{val:.1f}" if isinstance(val, (int, float)) else str(val)

231
modules/dcgm_test.py Normal file
View File

@ -0,0 +1,231 @@
"""DCGM diagnostic acceptance wrapper."""
import json
import os
import re
import shutil
import signal
import subprocess
from datetime import datetime
from typing import Optional
from rich.console import Console
from rich.table import Table
class DCGMTest:
def __init__(self, config: dict):
self.config = config
self.console = Console()
self.cfg = config.get("dcgm", {})
def run(self) -> dict:
dcgmi = shutil.which("dcgmi")
if not dcgmi:
return {
"passed": False,
"error": "dcgmi not found",
"timestamp": datetime.now().isoformat(),
}
level = str(self.cfg.get("diag_level", 3))
timeout = int(self.cfg.get("timeout_sec", 1200))
cmd = [dcgmi, "diag", "-r", level]
expected_gpus = self.cfg.get("expected_num_gpus")
if expected_gpus:
cmd.extend(["-n", f"gpu:{int(expected_gpus)}"])
if self.cfg.get("json_output", True):
cmd.append("-j")
try:
r = self._run_with_process_group_timeout(cmd, timeout)
except subprocess.TimeoutExpired as e:
output = ((e.output or "") + "\n" + (e.stderr or "")).strip()
return {
"passed": False,
"error": f"dcgmi diag -r {level} timeout after {timeout}s",
"command": cmd,
"raw_output_tail": output[-8000:],
"timestamp": datetime.now().isoformat(),
}
output = r.stdout + "\n" + r.stderr
subtests = self._parse_json_output(output) or self._parse_output(output)
strict_statuses = {"PASS"}
failed = [s for s in subtests if s["status"] not in strict_statuses]
require_subtests = bool(self.cfg.get("require_subtests", True))
passed = r.returncode == 0 and not failed and (bool(subtests) or not require_subtests)
return {
"passed": passed,
"returncode": r.returncode,
"level": int(level),
"command": cmd,
"expected_num_gpus": int(expected_gpus) if expected_gpus else None,
"subtests": subtests,
"raw_output_tail": output[-8000:],
"timestamp": datetime.now().isoformat(),
}
@staticmethod
def _run_with_process_group_timeout(cmd: list[str], timeout: int) -> subprocess.CompletedProcess:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
start_new_session=True,
)
try:
stdout, stderr = proc.communicate(timeout=timeout)
except subprocess.TimeoutExpired as e:
try:
os.killpg(proc.pid, signal.SIGTERM)
stdout, stderr = proc.communicate(timeout=10)
except subprocess.TimeoutExpired:
os.killpg(proc.pid, signal.SIGKILL)
stdout, stderr = proc.communicate(timeout=10)
raise subprocess.TimeoutExpired(cmd, timeout, output=stdout, stderr=stderr) from e
return subprocess.CompletedProcess(cmd, proc.returncode, stdout, stderr)
@classmethod
def _parse_json_output(cls, output: str) -> list[dict]:
text = output.strip()
if not text:
return []
try:
payload = json.loads(text)
except json.JSONDecodeError:
m = re.search(r"(\{.*\})", text, re.S)
if not m:
return []
try:
payload = json.loads(m.group(1))
except json.JSONDecodeError:
return []
dcgm_payload = payload.get("DCGM Diagnostic") if isinstance(payload, dict) else None
if isinstance(dcgm_payload, dict):
parsed = cls._parse_dcgm_diagnostic_json(dcgm_payload)
if parsed:
return parsed
subtests = []
def walk(node, path: list[str]):
if isinstance(node, dict):
node_name = (
node.get("name")
or node.get("testName")
or node.get("test_name")
or node.get("category")
or node.get("category_name")
)
child_path = [*path, str(node_name)] if node_name else path
status = node.get("status") or node.get("result") or node.get("Result")
if isinstance(status, str):
name = (
node_name
or " / ".join(path[-3:])
)
normalized = cls._normalize_status(status)
if normalized:
subtests.append({
"name": str(name)[:160],
"status": normalized,
"raw": json.dumps(node, default=str)[:1000],
})
for key, value in node.items():
walk(value, [*child_path, str(key)])
elif isinstance(node, list):
for idx, item in enumerate(node):
walk(item, [*path, str(idx)])
walk(payload, [])
return subtests
@classmethod
def _parse_dcgm_diagnostic_json(cls, payload: dict) -> list[dict]:
subtests = []
for category in payload.get("test_categories", []) or []:
category_name = str(category.get("category") or "DCGM")
for test in category.get("tests", []) or []:
test_name = str(test.get("name") or "unnamed")
for result in test.get("results", []) or []:
status = cls._normalize_status(str(result.get("status", "")))
if not status:
continue
entity_group = result.get("entity_group") or "entity"
entity_id = result.get("entity_id", "unknown")
name = f"{category_name}/{test_name}/{entity_group}{entity_id}"
subtests.append({
"name": name[:160],
"status": status,
"raw": json.dumps(result, default=str)[:1000],
})
summary = test.get("test_summary") or {}
status = cls._normalize_status(str(summary.get("status", "")))
if status:
subtests.append({
"name": f"{category_name}/{test_name}/summary"[:160],
"status": status,
"raw": json.dumps(summary, default=str)[:1000],
})
return subtests
@staticmethod
def _normalize_status(status: str) -> str:
s = status.strip().upper()
aliases = {
"PASS": "PASS",
"PASSED": "PASS",
"OK": "PASS",
"FAIL": "FAIL",
"FAILED": "FAIL",
"ERROR": "ERROR",
"WARN": "WARN",
"WARNING": "WARN",
"SKIP": "SKIP",
"SKIPPED": "SKIP",
"NOT_RUN": "SKIP",
"NOT RUN": "SKIP",
}
return aliases.get(s, s if s in {"PASS", "FAIL", "ERROR", "WARN", "SKIP"} else "")
@staticmethod
def _parse_output(output: str) -> list[dict]:
subtests = []
for line in output.splitlines():
stripped = line.strip()
if not stripped:
continue
m = re.search(r"(.+?)\s*[:|]\s*(PASS|FAIL|WARN|ERROR|SKIP)\b", stripped, re.I)
if not m:
m = re.search(r"\b(PASS|FAIL|WARN|ERROR|SKIP)\b\s*[-:|]\s*(.+)", stripped, re.I)
if m:
status = DCGMTest._normalize_status(m.group(1))
name = m.group(2).strip()
else:
continue
else:
name = m.group(1).strip(" .|-")
status = DCGMTest._normalize_status(m.group(2))
if name and len(name) < 160:
subtests.append({"name": name, "status": status, "raw": stripped})
return subtests
@staticmethod
def print_results(results: dict, console: Optional[Console] = None):
c = console or Console()
if results.get("error"):
c.print(f"[bold red]DCGM error: {results['error']}[/bold red]")
return
passed = results.get("passed", False)
c.print("[bold green]✓ DCGM diag PASSED[/bold green]" if passed else "[bold red]✗ DCGM diag FAILED[/bold red]")
subtests = results.get("subtests", [])
if subtests:
table = Table(box=None, padding=(0, 1))
table.add_column("Subtest")
table.add_column("Status", style="bold")
for s in subtests:
table.add_row(s.get("name", ""), s.get("status", ""))
c.print(table)

View File

@ -1,4 +1,4 @@
"""Accelerator information detection for NVIDIA GPUs and Hygon DCUs.""" """GPU information detection module for NVIDIA datacenter GPUs (A100/A800/H100/H200/B200/B300)."""
import subprocess import subprocess
import shutil import shutil
@ -11,7 +11,6 @@ from rich.panel import Panel
from rich.text import Text from rich.text import Text
from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label
from modules.hygon_smi import collect_hygon_info, find_hy_smi
class GPUInfo: class GPUInfo:
@ -19,8 +18,7 @@ class GPUInfo:
def __init__(self, config: dict): def __init__(self, config: dict):
self.config = config self.config = config
self.console = Console() self.console = Console()
configured = config.get("gpu_type", "auto") self.gpu_type = detect_gpu_type()
self.gpu_type = configured if configured != "auto" else detect_gpu_type()
self.specs = get_gpu_specs(self.gpu_type) self.specs = get_gpu_specs(self.gpu_type)
self.gpu_label = get_gpu_label(self.gpu_type) self.gpu_label = get_gpu_label(self.gpu_type)
@ -37,19 +35,6 @@ class GPUInfo:
return None return None
def run(self) -> dict: def run(self) -> dict:
if self.gpu_type == "bw1000" or (not shutil.which("nvidia-smi") and find_hy_smi()):
result = collect_hygon_info(include_topology=True)
runtime_cfg = self.config.get("runtime", {}) or {}
result.update({
"runtime_name": runtime_cfg.get("name", result.get("runtime_name", "DTK")),
"runtime_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")),
"cuda_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")),
"timestamp": datetime.now().isoformat(),
"detected_gpu_type": "bw1000",
"gpu_label": get_gpu_label("bw1000"),
})
return result
if not shutil.which("nvidia-smi"): if not shutil.which("nvidia-smi"):
self.console.print("[bold red]nvidia-smi not found![/bold red]") self.console.print("[bold red]nvidia-smi not found![/bold red]")
return {"error": "nvidia-smi not found", "gpu_count": 0} return {"error": "nvidia-smi not found", "gpu_count": 0}
@ -174,9 +159,7 @@ class GPUInfo:
c.print(f"\n[bold cyan]GPU Information[/bold cyan]") c.print(f"\n[bold cyan]GPU Information[/bold cyan]")
c.print(f" Driver Version : {results.get('driver_version', 'N/A')}") c.print(f" Driver Version : {results.get('driver_version', 'N/A')}")
runtime_name = results.get("runtime_name", "CUDA") c.print(f" CUDA Version : {results.get('cuda_version', 'N/A')}")
runtime_version = results.get("runtime_version", results.get("cuda_version", "N/A"))
c.print(f" {runtime_name} Version{' ' * max(0, 4-len(runtime_name))}: {runtime_version}")
c.print(f" GPU Count : {results.get('gpu_count', 0)}") c.print(f" GPU Count : {results.get('gpu_count', 0)}")
c.print(f" Detected GPU : {results.get('gpu_label', 'Unknown')} ({results.get('detected_gpu_type', 'unknown')})") c.print(f" Detected GPU : {results.get('gpu_label', 'Unknown')} ({results.get('detected_gpu_type', 'unknown')})")
c.print(f" Timestamp : {results.get('timestamp', 'N/A')}") c.print(f" Timestamp : {results.get('timestamp', 'N/A')}")
@ -198,7 +181,7 @@ class GPUInfo:
for g in gpus: for g in gpus:
name = g["name"] name = g["name"]
if any(k in name for k in ("A100", "A800", "H100", "H200", "B200", "B300", "BW1000")): if any(k in name for k in ("A100", "A800", "H100", "H200", "B200", "B300")):
name = f"[bold green]{name}[/bold green]" name = f"[bold green]{name}[/bold green]"
vram = f"{g['vram_used_mb']}/{g['vram_total_mb']} MB" vram = f"{g['vram_used_mb']}/{g['vram_total_mb']} MB"
temp = f"{g['temperature']}°C" temp = f"{g['temperature']}°C"
@ -221,8 +204,7 @@ class GPUInfo:
topology = results.get("topology", "") topology = results.get("topology", "")
if topology and topology != "Unavailable": if topology and topology != "Unavailable":
title = "HSW/XHCL Topology" if results.get("vendor") == "hygon" else "NVLink/NVSwitch Topology" c.print(Panel(topology, title="[bold]NVLink/NVSwitch Topology[/bold]", border_style="cyan"))
c.print(Panel(topology, title=f"[bold]{title}[/bold]", border_style="cyan"))
warnings = [] warnings = []
for g in gpus: for g in gpus:

View File

@ -1,4 +1,4 @@
"""Accelerator specifications for NVIDIA GPUs and Hygon DCUs.""" """GPU specifications database for NVIDIA datacenter GPUs."""
import os import os
import shutil import shutil
@ -8,7 +8,6 @@ from typing import List, Optional
# GPU name patterns -> internal key mapping # GPU name patterns -> internal key mapping
# Order matters: longer/more-specific patterns must come before shorter ones. # Order matters: longer/more-specific patterns must come before shorter ones.
GPU_NAME_PATTERNS = { GPU_NAME_PATTERNS = {
"BW1000": "bw1000",
"A100": "a100", "A100": "a100",
"A800": "a800", "A800": "a800",
"H100": "h100", "H100": "h100",
@ -16,37 +15,11 @@ GPU_NAME_PATTERNS = {
"H200": "h200", "H200": "h200",
"H20": "h20", # H20 / H20-3e is the China-compliance export variant, REDUCED peaks "H20": "h20", # H20 / H20-3e is the China-compliance export variant, REDUCED peaks
"B200": "b200", "B200": "b200",
"B30-E": "b300", # nvidia-smi product name used by the B300 systems under test
"B300": "b300", "B300": "b300",
} }
# Specs database — ALL values are DENSE (non-sparse) TFLOPS # Specs database — ALL values are DENSE (non-sparse) TFLOPS
GPU_SPECS = { GPU_SPECS = {
"bw1000": {
# Public Hygon material identifies the supported precisions but does not
# publish stable dense peak figures for BW1000_H. Keep unknown peaks at
# zero so reports show measured throughput without inventing efficiency.
"full_name": "Hygon BW1000_H DCU",
"vendor": "hygon",
"architecture": "C-3000 GPGPU",
"compute_capability": 0.0,
"hbm_capacity_gb": 64,
"hbm_type": "HBM",
"memory_bandwidth_gbps": 0,
"fp32_tflops": 0,
"tf32_tflops": 0,
"fp16_tflops": 0,
"bf16_tflops": 0,
"fp8_tflops": 0,
"compute_pass_thresholds_tflops": {},
"tdp_watts": 800,
"nvlink_gen": 0,
"nvlink_bandwidth_gbps": 0,
"interconnect": "HSW/XHCL",
"pcie_gen": 5,
"min_driver_version": "",
"min_cuda_version": "",
},
"h100": { "h100": {
# Peaks below are NVIDIA marketing dense peaks (theoretical Tensor Core max). # Peaks below are NVIDIA marketing dense peaks (theoretical Tensor Core max).
# `compute_pass_thresholds_tflops` carries the absolute PASS thresholds used # `compute_pass_thresholds_tflops` carries the absolute PASS thresholds used
@ -229,33 +202,15 @@ GPU_SPECS = {
"hbm_capacity_gb": 288, "hbm_capacity_gb": 288,
"hbm_type": "HBM3e", "hbm_type": "HBM3e",
"memory_bandwidth_gbps": 8000, # GB/s (8 TB/s) "memory_bandwidth_gbps": 8000, # GB/s (8 TB/s)
# Dense, per-GPU reference peaks. NVIDIA publishes DGX B300 FP8 as "fp32_tflops": 125,
# 72 PFLOPS sparse for eight GPUs and explicitly states dense is half, "tf32_tflops": 1750, # dense (estimated)
# i.e. 4.5 PFLOPS/GPU. The previous 7 PFLOPS dense value mixed product "fp16_tflops": 3500, # dense
# variants/modes and made healthy B300 cards fail by construction. "bf16_tflops": 3500, # dense
"fp32_tflops": 75, "fp8_tflops": 7000, # dense
"tf32_tflops": 1100,
"fp16_tflops": 2250,
"bf16_tflops": 2250,
"fp8_tflops": 4500,
# Acceptance floors are achievable eager-PyTorch GEMM values rather than
# a blanket percentage of marketing peak. BF16/FP8 are calibrated against
# NVIDIA Transformer Engine's public B300 GEMM examples; the remaining
# modes use conservative floors below sustained measurements on this path.
"compute_pass_thresholds_tflops": {
"fp32": 60, "tf32": 750, "fp16": 1450, "bf16": 1450, "fp8": 2800,
},
# Single-node 8-GPU floors derived from public B300 nccl-tests results.
# These are operation-specific because collectives do not reach the same
# fraction of raw NVLink bandwidth.
"nccl_pass_thresholds_gbps": {
"allreduce": 750, "alltoall": 620, "broadcast": 600,
"allgather": 700, "reducescatter": 700, "sendrecv": 600,
},
"tdp_watts": 1200, "tdp_watts": 1200,
"nvlink_gen": 5, "nvlink_gen": 5,
"nvlink_bandwidth_gbps": 1800, "nvlink_bandwidth_gbps": 1800,
"pcie_gen": 6, "pcie_gen": 5,
"min_driver_version": "550", "min_driver_version": "550",
"min_cuda_version": "12.4", "min_cuda_version": "12.4",
}, },
@ -285,41 +240,30 @@ _UNKNOWN_SPECS = {
def detect_gpu_type() -> str: def detect_gpu_type() -> str:
"""Detect accelerator type via nvidia-smi or hy-smi. """Detect GPU type via nvidia-smi and return the internal key (e.g. 'h200').
Returns 'unknown' if nvidia-smi is unavailable or the GPU is not recognized. Returns 'unknown' if nvidia-smi is unavailable or the GPU is not recognized.
""" """
nvidia_smi = shutil.which("nvidia-smi") nvidia_smi = shutil.which("nvidia-smi")
if nvidia_smi: if not nvidia_smi:
try: return "unknown"
r = subprocess.run(
[nvidia_smi, "--query-gpu=name", "--format=csv,noheader"],
capture_output=True, text=True, timeout=10,
)
if r.returncode == 0 and r.stdout.strip():
first_line = r.stdout.strip().splitlines()[0].strip().upper()
# Iterate longest-pattern-first so "H200" doesn't match "H20".
for pattern, key in sorted(GPU_NAME_PATTERNS.items(), key=lambda kv: -len(kv[0])):
if pattern in first_line:
return key
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
pass
# Hygon DTK hosts expose hy-smi on PATH or under /opt/hyhal.
try: try:
from modules.hygon_smi import find_hy_smi r = subprocess.run(
hy_smi = find_hy_smi() [nvidia_smi, "--query-gpu=name", "--format=csv,noheader"],
if hy_smi: capture_output=True, text=True, timeout=10,
r = subprocess.run( )
[hy_smi, "--showproductname"], capture_output=True, if r.returncode != 0:
text=True, timeout=10, return "unknown"
)
product = (r.stdout or "").upper() first_line = r.stdout.strip().splitlines()[0].strip().upper()
if r.returncode == 0 and "BW1000" in product: # Iterate longest-pattern-first so "H200" doesn't get matched by "H20".
return "bw1000" for pattern, key in sorted(GPU_NAME_PATTERNS.items(), key=lambda kv: -len(kv[0])):
except (ImportError, subprocess.TimeoutExpired, FileNotFoundError, OSError): if pattern in first_line:
pass return key
return "unknown" return "unknown"
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
return "unknown"
def get_gpu_specs(gpu_type: str = None) -> dict: def get_gpu_specs(gpu_type: str = None) -> dict:
@ -342,7 +286,8 @@ def get_gpu_label(gpu_type: str) -> str:
specs = GPU_SPECS.get(gpu_type) specs = GPU_SPECS.get(gpu_type)
if specs: if specs:
full = specs["full_name"] full = specs["full_name"]
return full.replace("NVIDIA ", "").replace("Hygon ", "") # Strip the "NVIDIA " prefix for display
return full.replace("NVIDIA ", "")
return "Unknown GPU" return "Unknown GPU"

View File

@ -1,4 +1,4 @@
"""Hardware health monitoring for NVIDIA GPUs and Hygon DCUs.""" """Hardware health monitoring module for NVIDIA datacenter GPUs."""
import subprocess import subprocess
import shutil import shutil
@ -12,7 +12,6 @@ from rich.panel import Panel
from rich.text import Text from rich.text import Text
from modules.gpu_specs import detect_gpu_type, get_gpu_specs from modules.gpu_specs import detect_gpu_type, get_gpu_specs
from modules.hygon_smi import collect_hygon_info, find_hy_smi
class HealthCheck: class HealthCheck:
@ -21,8 +20,7 @@ class HealthCheck:
self.config = config self.config = config
self.console = Console() self.console = Console()
self.health_cfg = config.get("health", {}) self.health_cfg = config.get("health", {})
configured = config.get("gpu_type", "auto") self.gpu_type = detect_gpu_type()
self.gpu_type = configured if configured != "auto" else detect_gpu_type()
self.specs = get_gpu_specs(self.gpu_type) self.specs = get_gpu_specs(self.gpu_type)
def _run_smi(self, query: str) -> Optional[str]: def _run_smi(self, query: str) -> Optional[str]:
@ -57,9 +55,6 @@ class HealthCheck:
return default return default
def run(self) -> dict: def run(self) -> dict:
if self.gpu_type == "bw1000" or (not shutil.which("nvidia-smi") and find_hy_smi()):
return self._run_hygon()
if not shutil.which("nvidia-smi"): if not shutil.which("nvidia-smi"):
return {"error": "nvidia-smi not found", "passed": False} return {"error": "nvidia-smi not found", "passed": False}
@ -176,6 +171,10 @@ class HealthCheck:
gpu_health.append({"index": i, "status": worst, "checks": checks}) gpu_health.append({"index": i, "status": worst, "checks": checks})
system_health = self._check_system() system_health = self._check_system()
for key in ("fabricmanager", "retired_pages", "kernel_errors"):
item = system_health.get(key, {})
if isinstance(item, dict) and item.get("status") == "FAIL":
overall_pass = False
return { return {
"passed": overall_pass, "passed": overall_pass,
@ -185,102 +184,6 @@ class HealthCheck:
"detected_gpu_type": self.gpu_type, "detected_gpu_type": self.gpu_type,
} }
def _run_hygon(self) -> dict:
info = collect_hygon_info(include_topology=False)
if info.get("error"):
return {"error": info["error"], "passed": False}
temp_warn = self.health_cfg.get("temp_warning", 80)
temp_crit = self.health_cfg.get("temp_critical", 90)
configured_power = self.health_cfg.get("power_limit")
gpu_health = []
overall_pass = True
for gpu in info.get("gpus", []):
checks = {}
temp = gpu.get("temperature", 0)
if temp >= temp_crit:
temp_status = "FAIL"
elif temp >= temp_warn:
temp_status = "WARN"
else:
temp_status = "PASS"
checks["temperature"] = {
"value": temp, "status": temp_status,
"threshold": temp_crit if temp_status == "FAIL" else temp_warn,
}
draw = gpu.get("power_draw", 0.0)
limit = configured_power or gpu.get("power_limit") or self.specs.get("tdp_watts", 800)
checks["power"] = {
"value": draw, "limit": limit,
"status": "PASS" if draw <= limit * 1.05 else "WARN",
}
ecc_enabled = bool(gpu.get("ecc_enabled"))
checks["ecc_errors"] = {
"single": gpu.get("ecc_errors_single", 0),
"double": gpu.get("ecc_errors_double", 0),
"enabled": ecc_enabled,
"status": "PASS" if ecc_enabled else "FAIL",
}
xgmi = int(gpu.get("xgmi_errors", 0))
replay = int(gpu.get("pcie_replay_count", 0))
healthy = bool(gpu.get("healthy"))
checks["memory_errors"] = {
"xgmi_errors": xgmi,
"pcie_replay_count": replay,
"device_healthy": healthy,
"status": "PASS" if healthy and xgmi == 0 and replay == 0 else "FAIL",
}
generation = int(gpu.get("pcie_link_gen", 0))
width = int(gpu.get("pcie_link_width", 0))
expected = self.specs.get("pcie_gen", 0)
pcie_ok = generation >= expected and width >= 16
checks["pcie_link"] = {
"gen": generation, "width": width,
"status": "PASS" if pcie_ok else "WARN",
}
sm = int(gpu.get("clock_sm", 0))
mem = int(gpu.get("clock_mem", 0))
checks["clock_speed"] = {
"sm": sm, "mem": mem,
"status": "PASS" if sm > 0 and mem > 0 else "WARN",
}
throttling = bool(gpu.get("throttling_active"))
checks["throttling"] = {
"status": "FAIL" if throttling else "PASS",
"reasons": [gpu.get("throttling_text")] if throttling else [],
}
# Hygon has no NVIDIA-style persistence mode; a loaded /dev/kfd
# driver and successful hy-smi health query are the equivalent check.
checks["persistence_mode"] = {
"enabled": healthy, "status": "PASS" if healthy else "FAIL"
}
worst = "PASS"
for check in checks.values():
if check["status"] == "FAIL":
worst = "FAIL"
break
if check["status"] == "WARN":
worst = "WARN"
if worst == "FAIL":
overall_pass = False
gpu_health.append({"index": gpu["index"], "status": worst, "checks": checks})
return {
"passed": overall_pass,
"gpu_health": gpu_health,
"system_health": self._check_system(),
"timestamp": datetime.now().isoformat(),
"detected_gpu_type": "bw1000",
"driver_version": info.get("driver_version", "N/A"),
"vendor": "hygon",
}
def _check_system(self) -> dict: def _check_system(self) -> dict:
persistd = shutil.which("nvidia-persistenced") is not None persistd = shutil.which("nvidia-persistenced") is not None
persistd_running = False persistd_running = False
@ -328,10 +231,10 @@ class HealthCheck:
if os.path.isdir("/sys/class/infiniband_verbs"): if os.path.isdir("/sys/class/infiniband_verbs"):
rdma_devs = os.listdir("/sys/class/infiniband_verbs") rdma_devs = os.listdir("/sys/class/infiniband_verbs")
nccl_env = { nccl_env = {k: v for k, v in os.environ.items() if k.startswith("NCCL_")}
k: v for k, v in os.environ.items() fabric = self._check_fabricmanager()
if k.startswith(("NCCL_", "RCCL_", "HCCL_")) retired = self._check_retired_pages()
} kernel_errors = self._check_kernel_errors()
return { return {
"nvidia_persistenced": {"installed": persistd, "running": persistd_running}, "nvidia_persistenced": {"installed": persistd, "running": persistd_running},
@ -342,6 +245,41 @@ class HealthCheck:
"infiniband_devices": ib_devs, "infiniband_devices": ib_devs,
"rdma_devices": rdma_devs, "rdma_devices": rdma_devs,
"nccl_env_vars": nccl_env, "nccl_env_vars": nccl_env,
"fabricmanager": fabric,
"retired_pages": retired,
"kernel_errors": kernel_errors,
}
def _check_fabricmanager(self) -> dict:
r = self._run_cmd(["systemctl", "is-active", "nvidia-fabricmanager"], timeout=5)
active = r == "active"
logs = self._run_cmd(["journalctl", "-u", "nvidia-fabricmanager", "-n", "200", "--no-pager"], timeout=10) or ""
has_error = "ERROR" in logs.upper() or "FAILED" in logs.upper()
return {
"active": active,
"has_error_logs": has_error,
"status": "PASS" if active and not has_error else "FAIL",
}
def _check_retired_pages(self) -> dict:
raw = self._run_cmd(["nvidia-smi", "-q", "-d", "PAGE_RETIREMENT"], timeout=30) or ""
nums = [int(x) for x in __import__("re").findall(r"Retired Pages.*?:\s*(\d+)", raw, flags=__import__("re").I)]
pending = "Pending Page Blacklist" in raw and "Yes" in raw
total = sum(nums)
return {
"retired_pages": total,
"pending_blacklist": pending,
"status": "PASS" if total == 0 and not pending else "FAIL",
}
def _check_kernel_errors(self) -> dict:
raw = self._run_cmd(["dmesg", "--ctime", "--level=err,crit,alert,emerg"], timeout=10) or ""
upper = raw.upper()
hits = [line for line in raw.splitlines() if any(k in line.upper() for k in ("XID", "AER", "PCIE", "NVRM"))]
return {
"count": len(hits),
"tail": hits[-20:],
"status": "PASS" if not hits else "FAIL",
} }
@staticmethod @staticmethod

View File

@ -1,190 +0,0 @@
"""Helpers for querying Hygon DCU devices through ``hy-smi``."""
import os
import re
import shutil
import subprocess
from typing import Dict, List, Optional
def find_hy_smi() -> Optional[str]:
"""Return the hy-smi executable path when available."""
path = shutil.which("hy-smi")
if path:
return path
path = "/opt/hyhal/bin/hy-smi"
return path if os.path.isfile(path) and os.access(path, os.X_OK) else None
def run_hy_smi(*args: str, timeout: int = 30) -> Optional[str]:
"""Run hy-smi and return stdout, or ``None`` on failure."""
binary = find_hy_smi()
if not binary:
return None
try:
result = subprocess.run(
[binary, *args], capture_output=True, text=True, timeout=timeout
)
except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
return None
return result.stdout.strip() if result.returncode == 0 else None
def _indexed_values(text: Optional[str], pattern: str) -> Dict[int, str]:
if not text:
return {}
values: Dict[int, str] = {}
for match in re.finditer(pattern, text, re.MULTILINE | re.IGNORECASE):
values[int(match.group(1))] = match.group(2).strip()
return values
def _number(value, default=0.0):
try:
return float(value)
except (TypeError, ValueError):
return default
def _pcie_gen(gt_per_second: float) -> int:
if gt_per_second >= 32:
return 5
if gt_per_second >= 16:
return 4
if gt_per_second >= 8:
return 3
if gt_per_second >= 5:
return 2
return 1 if gt_per_second > 0 else 0
def collect_hygon_info(include_topology: bool = True) -> dict:
"""Collect normalized per-device information for GPUInfo and HealthCheck."""
summary = run_hy_smi()
if not summary:
return {"error": "hy-smi query failed", "gpu_count": 0, "gpus": []}
rows: Dict[int, dict] = {}
row_re = re.compile(
r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W\s+(\S+)\s+"
r"([\d.]+)W\s+(\d+)%\s+([\d.]+)%",
re.MULTILINE,
)
for match in row_re.finditer(summary):
index = int(match.group(1))
rows[index] = {
"index": index,
"temperature": int(round(float(match.group(2)))),
"power_draw": float(match.group(3)),
"performance_level": match.group(4),
"power_limit": float(match.group(5)),
"vram_used_pct": int(match.group(6)),
"utilization_pct": float(match.group(7)),
}
product = run_hy_smi("--showproductname")
driver = run_hy_smi("--showdriverversion")
memory = run_hy_smi("--showmemavailable")
bus = run_hy_smi("--showbus")
unique = run_hy_smi("--showuniqueid")
serial = run_hy_smi("--showserial")
clocks = run_hy_smi("--showclocks")
ecc = run_hy_smi("--showmemeccinfo")
xgmi = run_hy_smi("--showxgmierr")
replay = run_hy_smi("--showreplaycount")
health = run_hy_smi("--healthcheck")
throttle = run_hy_smi("--showfdr")
names = _indexed_values(product, r"HCU\[(\d+)\].*Card Series:\s*([^\s]+)")
memories = _indexed_values(memory, r"HCU\[(\d+)\].*Available memory size \(MiB\):\s*(\d+)")
buses = _indexed_values(bus, r"HCU\[(\d+)\].*PCI Bus:\s*([0-9a-fA-F:.]+)")
unique_ids = _indexed_values(unique, r"HCU\[(\d+)\].*Unique ID:\s*(\S+)")
serials = _indexed_values(serial, r"HCU\[(\d+)\].*Serial Number:\s*(\S+)")
sclk = _indexed_values(clocks, r"HCU\[(\d+)\].*sclk clock level:.*\((\d+)Mhz\)")
mclk = _indexed_values(clocks, r"HCU\[(\d+)\].*mclk clock level:.*\((\d+)Mhz\)")
pcie = {}
if clocks:
for match in re.finditer(
r"HCU\[(\d+)\].*pcie clock level:.*\(([\d.]+)GT/s,\s*x(\d+)", clocks
):
pcie[int(match.group(1))] = (float(match.group(2)), int(match.group(3)))
ecc_enabled = {
idx: value.upper() == "ON"
for idx, value in _indexed_values(
ecc, r"HCU\[(\d+)\].*Vram memory ECC status:\s*(\S+)"
).items()
}
xgmi_errors = {
idx: int(value)
for idx, value in _indexed_values(
xgmi, r"HCU\[(\d+)\].*XGMI Error count\s+(\d+)"
).items()
}
replay_counts = {
idx: int(value)
for idx, value in _indexed_values(
replay, r"HCU\[(\d+)\].*PCIe Replay Count:\s*(\d+)"
).items()
}
healthy = {
idx: value.lower() == "healthy"
for idx, value in _indexed_values(
health, r"HCU\[(\d+)\].*:\s*(Healthy|Unhealthy)\s*$"
).items()
}
throttle_status = _indexed_values(
throttle, r"HCU\[(\d+)\].*Throttler Status:\s*(.+?)\s*$"
)
driver_match = re.search(r"Driver Version:\s*(\S+)", driver or "")
driver_version = driver_match.group(1) if driver_match else "N/A"
gpu_count = max(rows.keys(), default=-1) + 1
gpus: List[dict] = []
for index in range(gpu_count):
row = rows.get(index, {"index": index})
total_mb = int(memories.get(index, 0))
used_mb = int(total_mb * int(row.get("vram_used_pct", 0)) / 100)
gt_s, width = pcie.get(index, (0.0, 0))
throttle_text = throttle_status.get(index, "")
real_throttle = bool(
throttle_text
and any(token not in {"IDLE", "ACTIVE", "NONE"}
for token in re.findall(r"[A-Z_]+", throttle_text.upper()))
)
gpus.append({
**row,
"name": names.get(index, "Hygon DCU"),
"uuid": unique_ids.get(index, "N/A"),
"pci_bus_id": buses.get(index, "N/A"),
"pcie_link_gen": _pcie_gen(gt_s),
"pcie_link_width": width,
"pcie_gt_s": gt_s,
"vram_total_mb": total_mb,
"vram_used_mb": used_mb,
"vram_free_mb": max(total_mb - used_mb, 0),
"clock_sm": int(_number(sclk.get(index))),
"clock_mem": int(_number(mclk.get(index))),
"fan_speed": 0,
"persistence_mode": True,
"compute_mode": row.get("performance_level", "auto"),
"serial_number": serials.get(index, "N/A"),
"ecc_enabled": ecc_enabled.get(index, False),
"ecc_errors_single": 0,
"ecc_errors_double": 0,
"xgmi_errors": xgmi_errors.get(index, 0),
"pcie_replay_count": replay_counts.get(index, 0),
"healthy": healthy.get(index, False),
"throttling_active": real_throttle,
"throttling_text": throttle_text,
})
return {
"driver_version": driver_version,
"runtime_name": "DTK",
"runtime_version": os.environ.get("DTK_VERSION", "N/A"),
"gpu_count": gpu_count,
"gpus": gpus,
"topology": run_hy_smi("--showtopo") if include_topology else "",
"vendor": "hygon",
}

View File

@ -3,7 +3,6 @@
import json import json
import os import os
import re import re
import shlex
import shutil import shutil
import subprocess import subprocess
from datetime import datetime from datetime import datetime
@ -211,11 +210,9 @@ class MultiNodeNCCLTest:
"-o", "BatchMode=yes", "-o", "BatchMode=yes",
"-o", "ConnectTimeout=5", "-o", "ConnectTimeout=5",
"-o", "StrictHostKeyChecking=accept-new", "-o", "StrictHostKeyChecking=accept-new",
target,
"hostname",
] ]
plm_rsh_args = self.cfg.get("plm_rsh_args")
if plm_rsh_args:
cmd.extend(shlex.split(str(plm_rsh_args)))
cmd.extend([target, "hostname"])
try: try:
r = subprocess.run(cmd, capture_output=True, text=True, timeout=8, env=self._runtime_env()) r = subprocess.run(cmd, capture_output=True, text=True, timeout=8, env=self._runtime_env())
detail = r.stdout.strip() or r.stderr.strip()[:120] detail = r.stdout.strip() or r.stderr.strip()[:120]
@ -289,27 +286,10 @@ class MultiNodeNCCLTest:
"--mca", "btl_openib_warn_no_device_params_found", "0", "--mca", "btl_openib_warn_no_device_params_found", "0",
"--mca", "btl_tcp_if_include", str(self.cfg.get("socket_ifname", "bond0")), "--mca", "btl_tcp_if_include", str(self.cfg.get("socket_ifname", "bond0")),
"--mca", "oob_tcp_if_include", str(self.cfg.get("oob_tcp_ifname", self.cfg.get("socket_ifname", "bond0"))), "--mca", "oob_tcp_if_include", str(self.cfg.get("oob_tcp_ifname", self.cfg.get("socket_ifname", "bond0"))),
]
# Keep the MPI launcher/control plane separately configurable from the
# NCCL data plane. For example, PML/UCX can be restricted to TCP/eth0
# while NCCL uses the configured RoCE HCAs for benchmark traffic.
mpi_pml = self.cfg.get("mpi_pml", "ob1")
if mpi_pml:
cmd.extend(["--mca", "pml", str(mpi_pml)])
mpi_btl = self.cfg.get("mpi_btl", "self,tcp,vader")
if mpi_btl:
cmd.extend(["--mca", "btl", str(mpi_btl)])
# HCOLL does an eager UCX IB wireup during MPI_Init that deadlocks on nodes
# where UCX only sees mlx5_2/mlx5_3 instead of mlx5_0/mlx5_1. HCOLL
# collective optimisations are irrelevant for NCCL tests (NCCL handles its
# own collectives), so disable it unconditionally unless the caller opts in.
if self.cfg.get("disable_hcoll", True):
cmd.extend(["--mca", "coll", "^hcoll"])
cmd.extend([
"-H", host_arg, "-H", host_arg,
"--map-by", f"ppr:{gpus_per_node}:node", "--map-by", f"ppr:{gpus_per_node}:node",
"-np", str(ranks), "-np", str(ranks),
]) ]
plm_rsh_args = self.cfg.get("plm_rsh_args") plm_rsh_args = self.cfg.get("plm_rsh_args")
if plm_rsh_args: if plm_rsh_args:
cmd.extend(["--mca", "plm_rsh_args", str(plm_rsh_args)]) cmd.extend(["--mca", "plm_rsh_args", str(plm_rsh_args)])

View File

@ -5,12 +5,11 @@ import os
import re import re
import shutil import shutil
import subprocess import subprocess
import sysconfig import statistics
import tempfile import sys
from datetime import datetime from datetime import datetime
from typing import Optional from typing import Optional
from rich import box
from rich.console import Console from rich.console import Console
from rich.table import Table from rich.table import Table
from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
@ -20,10 +19,6 @@ from modules.gpu_specs import detect_gpu_type, get_gpu_specs, resolve_tools_dir
TORCH_AVAILABLE = False TORCH_AVAILABLE = False
try: try:
import torch import torch
try:
import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim
except ImportError:
pass
if torch.cuda.is_available(): if torch.cuda.is_available():
TORCH_AVAILABLE = True TORCH_AVAILABLE = True
except ImportError: except ImportError:
@ -51,8 +46,7 @@ class NCCLTest:
self.console = Console() self.console = Console()
self.nccl_cfg = config.get("nccl", {}) self.nccl_cfg = config.get("nccl", {})
self.tools_dir = resolve_tools_dir(config) self.tools_dir = resolve_tools_dir(config)
configured = config.get("gpu_type", "auto") self.gpu_type = detect_gpu_type()
self.gpu_type = configured if configured != "auto" else detect_gpu_type()
self.specs = get_gpu_specs(self.gpu_type) self.specs = get_gpu_specs(self.gpu_type)
def _find_nccl_test(self, name: str) -> Optional[str]: def _find_nccl_test(self, name: str) -> Optional[str]:
@ -78,28 +72,36 @@ class NCCLTest:
return p return p
return None return None
@staticmethod def _message_sizes(self) -> list[str]:
def _runtime_env() -> dict: return list(self.nccl_cfg.get("message_sizes") or ["1M", "256M", "2G"])
"""Return an environment that can load NCCL bundled in the active venv.
def _repeats(self) -> int:
return int(self.nccl_cfg.get("repeats", 3))
def _max_stddev_pct(self) -> float:
return float(self.nccl_cfg.get("max_stddev_pct", 3))
def _runtime_env(self) -> dict:
env = {**os.environ, "NCCL_DEBUG": "WARN"}
lib_dirs = []
nccl_home = env.get("NCCL_HOME") or self.nccl_cfg.get("nccl_home")
if nccl_home:
lib_dirs.append(os.path.join(str(nccl_home), "lib"))
for path in sys.path:
lib_dirs.append(os.path.join(path, "nvidia", "nccl", "lib"))
venv_root = os.path.dirname(os.path.dirname(sys.executable))
lib_dirs.extend(glob.glob(os.path.join(venv_root, "lib", "python*", "site-packages", "nvidia", "nccl", "lib")))
CUDA PyTorch wheels install libnccl under site-packages/nvidia/nccl/lib,
which is intentionally isolated from the system linker path. nccl-tests
is an external executable, so it needs that directory explicitly even
though importing torch works in the same virtual environment.
"""
env = os.environ.copy()
purelib = sysconfig.get_paths().get("purelib", "")
candidates = [
os.path.join(purelib, "nvidia", "nccl", "lib"),
os.path.join(purelib, "torch", "lib"),
]
library_dirs = [path for path in candidates if os.path.isdir(path)]
existing = env.get("LD_LIBRARY_PATH", "") existing = env.get("LD_LIBRARY_PATH", "")
if existing: valid_dirs = []
library_dirs.append(existing) for d in lib_dirs:
if library_dirs: if d and os.path.isdir(d) and d not in valid_dirs:
env["LD_LIBRARY_PATH"] = os.pathsep.join(library_dirs) valid_dirs.append(d)
env["NCCL_DEBUG"] = "WARN" if valid_dirs:
env["LD_LIBRARY_PATH"] = ":".join(valid_dirs + ([existing] if existing else []))
return env return env
def run(self) -> dict: def run(self) -> dict:
@ -126,7 +128,6 @@ class NCCLTest:
tests.append(("sendrecv_perf", "SendRecv")) tests.append(("sendrecv_perf", "SendRecv"))
nvlink_bw = self.specs.get("nvlink_bandwidth_gbps", 0) nvlink_bw = self.specs.get("nvlink_bandwidth_gbps", 0)
calibrated_thresholds = self.specs.get("nccl_pass_thresholds_gbps", {}) or {}
# User-provided override applies uniformly across all ops; otherwise # User-provided override applies uniformly across all ops; otherwise
# each op gets its own threshold from _OP_BW_FRACTIONS. # each op gets its own threshold from _OP_BW_FRACTIONS.
user_override = self.nccl_cfg.get("min_bandwidth_gbps") user_override = self.nccl_cfg.get("min_bandwidth_gbps")
@ -134,9 +135,6 @@ class NCCLTest:
def threshold_for(label: str) -> float: def threshold_for(label: str) -> float:
if user_override: if user_override:
return float(user_override) return float(user_override)
calibrated = calibrated_thresholds.get(label.lower())
if calibrated:
return float(calibrated)
if nvlink_bw <= 0: if nvlink_bw <= 0:
return 10.0 # conservative floor return 10.0 # conservative floor
frac = _OP_BW_FRACTIONS.get(label.lower(), 0.45) frac = _OP_BW_FRACTIONS.get(label.lower(), 0.45)
@ -181,19 +179,14 @@ class NCCLTest:
self.console.print("[yellow]nccl-tests binaries failed, falling back to torchrun[/yellow]") self.console.print("[yellow]nccl-tests binaries failed, falling back to torchrun[/yellow]")
return self._run_torchrun_fallback(gpu_count) return self._run_torchrun_fallback(gpu_count)
statuses = [ all_passed = all(
r.get("status") for r in results.values() r.get("status") == "PASS"
if isinstance(r, dict) and r.get("status") for r in results.values()
] if isinstance(r, dict) and "status" in r
overall_status = (
"FAIL" if "FAIL" in statuses else
"WARN" if "WARN" in statuses else
"PASS" if statuses else "SKIP"
) )
return { return {
"passed": overall_status in ("PASS", "WARN"), "passed": all_passed,
"status": overall_status,
"source": "nccl-tests", "source": "nccl-tests",
"min_bandwidth_gbps": { "min_bandwidth_gbps": {
lbl.lower(): threshold_for(lbl) for _, lbl in tests lbl.lower(): threshold_for(lbl) for _, lbl in tests
@ -211,38 +204,7 @@ class NCCLTest:
if not binary: if not binary:
return {"status": "SKIP", "error": f"{binary_name} not found"} return {"status": "SKIP", "error": f"{binary_name} not found"}
cmd = [ return self._run_nccl_matrix([binary, "-g", str(gpu_count)], min_bw)
binary,
"-b", "8M",
"-e", "8G",
"-f", "2",
"-g", str(gpu_count),
"-w", "5",
"-n", "20",
]
try:
env = self._runtime_env()
r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env)
combined = r.stdout + r.stderr
# Check for NCCL/CUDA compatibility errors
if "CUDA driver version is insufficient" in combined or \
"Test NCCL failure" in combined:
error_msg = "NCCL/CUDA driver version mismatch" \
if "CUDA driver version" in combined \
else "NCCL test failure (library incompatibility)"
return {"status": "FAIL", "error": error_msg}
if r.returncode != 0:
return {"status": "FAIL", "error": r.stderr[:300]}
return self._parse_nccl_output(r.stdout, min_bw)
except subprocess.TimeoutExpired:
return {"status": "FAIL", "error": "timeout"}
except Exception as e:
return {"status": "FAIL", "error": str(e)}
def _run_one_nccl_test_mpirun(self, binary_name: str, label: str, def _run_one_nccl_test_mpirun(self, binary_name: str, label: str,
gpu_count: int, mpirun: str, min_bw: float) -> dict: gpu_count: int, mpirun: str, min_bw: float) -> dict:
@ -258,36 +220,64 @@ class NCCLTest:
"-x", "NCCL_DEBUG=WARN", "-x", "NCCL_DEBUG=WARN",
"-x", "CUDA_VISIBLE_DEVICES=" + ",".join(str(i) for i in range(gpu_count)), "-x", "CUDA_VISIBLE_DEVICES=" + ",".join(str(i) for i in range(gpu_count)),
binary, binary,
"-b", "8",
"-e", "256M",
"-f", "2",
"-g", "1", "-g", "1",
"-w", "5",
"-n", "20",
] ]
return self._run_nccl_matrix(cmd, min_bw)
def _run_nccl_matrix(self, base_cmd: list[str], min_bw: float) -> dict:
size_results = []
failures = []
env = self._runtime_env()
try: try:
env = self._runtime_env() for size in self._message_sizes():
r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env) runs = []
for _ in range(self._repeats()):
combined = r.stdout + r.stderr cmd = [*base_cmd, "-b", size, "-e", size, "-f", "2", "-w", "5", "-n", "20"]
if "CUDA driver version is insufficient" in combined or \ r = subprocess.run(cmd, capture_output=True, text=True, timeout=300, env=env)
"Test NCCL failure" in combined: combined = r.stdout + r.stderr
error_msg = "NCCL/CUDA driver version mismatch" \ if "CUDA driver version is insufficient" in combined or "Test NCCL failure" in combined:
if "CUDA driver version" in combined \ failures.append({"size": size, "error": "NCCL/CUDA/library failure"})
else "NCCL test failure (library incompatibility)" continue
return {"status": "FAIL", "error": error_msg} if r.returncode != 0:
failures.append({"size": size, "error": r.stderr[:300]})
if r.returncode != 0: continue
return {"status": "FAIL", "error": r.stderr[:300]} parsed = self._parse_nccl_output(r.stdout, min_bw)
runs.append(parsed.get("best_busbw_gbps", 0))
return self._parse_nccl_output(r.stdout, min_bw) if runs:
worst = min(runs)
mean = sum(runs) / len(runs)
std_pct = (statistics.pstdev(runs) / mean * 100) if len(runs) > 1 and mean else 0
size_results.append({
"size": size,
"runs_busbw_gbps": [round(v, 1) for v in runs],
"worst_busbw_gbps": round(worst, 1),
"mean_busbw_gbps": round(mean, 1),
"stddev_pct": round(std_pct, 2),
"status": "PASS" if worst >= min_bw and std_pct <= self._max_stddev_pct() else "FAIL",
})
else:
size_results.append({"size": size, "status": "FAIL", "runs_busbw_gbps": []})
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
return {"status": "FAIL", "error": "timeout"} return {"status": "FAIL", "error": "timeout"}
except Exception as e: except Exception as e:
return {"status": "FAIL", "error": str(e)} return {"status": "FAIL", "error": str(e)}
best_bus = max((r.get("mean_busbw_gbps", 0) for r in size_results), default=0)
worst_bus = min((r.get("worst_busbw_gbps", 0) for r in size_results if r.get("runs_busbw_gbps")), default=0)
passed = bool(size_results) and all(r.get("status") == "PASS" for r in size_results) and not failures
return {
"status": "PASS" if passed else "FAIL",
"best_busbw_gbps": round(best_bus, 1),
"worst_busbw_gbps": round(worst_bus, 1),
"min_required_gbps": min_bw,
"max_stddev_pct": self._max_stddev_pct(),
"by_size": size_results,
"failures": failures,
}
@staticmethod @staticmethod
def _parse_nccl_output(stdout: str, min_bw: float) -> dict: def _parse_nccl_output(stdout: str, min_bw: float) -> dict:
"""Parse nccl-tests tabular output and extract bandwidth results.""" """Parse nccl-tests tabular output and extract bandwidth results."""
@ -332,78 +322,60 @@ class NCCLTest:
} }
def _run_torchrun_fallback(self, gpu_count: int) -> dict: def _run_torchrun_fallback(self, gpu_count: int) -> dict:
"""Benchmark collectives through PyTorch when nccl-tests is unavailable.""" """Basic NCCL connectivity test via torchrun — verifies NCCL works but does not benchmark performance."""
self.console.print("[yellow]nccl-tests not available, benchmarking collectives via torchrun[/yellow]") self.console.print("[yellow]nccl-tests not available, running basic NCCL connectivity check[/yellow]")
size_mb = int(self.nccl_cfg.get("torchrun_size_mb", 64))
warmup = int(self.nccl_cfg.get("torchrun_warmup", 5))
iterations = int(self.nccl_cfg.get("torchrun_iterations", 20))
backend = self.nccl_cfg.get("torch_backend", "nccl")
enabled_ops = []
for key, enabled in [
("allreduce", self.nccl_cfg.get("test_allreduce", True)),
("alltoall", self.nccl_cfg.get("test_alltoall", True)),
("broadcast", self.nccl_cfg.get("test_broadcast", True)),
("reducescatter", self.nccl_cfg.get("test_reduce_scatter", False)),
("allgather", self.nccl_cfg.get("test_allgather", False)),
]:
if enabled:
enabled_ops.append(key)
code = f""" code = f"""
import os, time import torch, torch.distributed as dist, os
import torch
try:
import torch_dcu
except ImportError:
pass
import torch.distributed as dist
os.environ.setdefault("MASTER_ADDR","127.0.0.1") os.environ.setdefault("MASTER_ADDR","127.0.0.1")
os.environ.setdefault("MASTER_PORT","29500") os.environ.setdefault("MASTER_PORT","29500")
rank=int(os.environ.get("LOCAL_RANK",0)) rank=int(os.environ.get("LOCAL_RANK",0))
ws={gpu_count} ws={gpu_count}
dist.init_process_group({backend!r},rank=rank,world_size=ws) dist.init_process_group("nccl",rank=rank,world_size=ws)
torch.cuda.set_device(rank) torch.cuda.set_device(rank)
size_bytes={size_mb}*1024*1024
numel=size_bytes//4
x=torch.ones(numel,device=f"cuda:{{rank}}",dtype=torch.float32)
allgather_out=torch.empty(numel*ws,device=f"cuda:{{rank}}",dtype=torch.float32)
rs_out=torch.empty(numel//ws,device=f"cuda:{{rank}}",dtype=torch.float32)
a2a_out=torch.empty_like(x)
def run_op(name): x=torch.randn(1024*1024,device=f"cuda:{{rank}}",dtype=torch.float32)
if name == "allreduce":
return dist.all_reduce(x) # Test AllReduce
if name == "broadcast": try:
return dist.broadcast(x, src=0) dist.all_reduce(x.clone())
if name == "allgather": if rank==0: print("allreduce:ok")
return dist.all_gather_into_tensor(allgather_out, x) except Exception as e:
if name == "reducescatter": if rank==0: print(f"allreduce:fail:{{e}}")
return dist.reduce_scatter_tensor(rs_out, x)
if name == "alltoall": # Test Broadcast
return dist.all_to_all_single(a2a_out, x) try:
raise ValueError(name) dist.broadcast(x.clone(),src=0)
if rank==0: print("broadcast:ok")
except Exception as e:
if rank==0: print(f"broadcast:fail:{{e}}")
# Test AllGather
try:
tensor_list=[torch.empty_like(x) for _ in range(ws)]
dist.all_gather(tensor_list,x.clone())
if rank==0: print("allgather:ok")
except Exception as e:
if rank==0: print(f"allgather:fail:{{e}}")
# Test ReduceScatter
try:
chunks=list(x.chunk(ws))
output=torch.empty_like(chunks[0])
dist.reduce_scatter(output,chunks)
if rank==0: print("reducescatter:ok")
except Exception as e:
if rank==0: print(f"reducescatter:fail:{{e}}")
# Test AllToAll
try:
chunks=list(x.chunk(ws))
output_list=[torch.empty_like(c) for c in chunks]
dist.all_to_all(output_list,chunks)
if rank==0: print("alltoall:ok")
except Exception as e:
if rank==0: print(f"alltoall:fail:{{e}}")
for name in {enabled_ops!r}:
try:
for _ in range({warmup}):
run_op(name)
torch.cuda.synchronize()
dist.barrier()
start=time.perf_counter()
for _ in range({iterations}):
run_op(name)
torch.cuda.synchronize()
elapsed=(time.perf_counter()-start)/{iterations}
algbw=size_bytes/elapsed/1e9
factor={{"allreduce":2*(ws-1)/ws,"allgather":(ws-1)/ws,
"reducescatter":(ws-1)/ws,"alltoall":(ws-1)/ws,
"broadcast":1.0}}[name]
busbw=algbw*factor
if rank==0:
print(f"METRIC|{{name}}|{{elapsed*1e6:.3f}}|{{algbw:.3f}}|{{busbw:.3f}}", flush=True)
except Exception as exc:
if rank==0:
print(f"ERROR|{{name}}|{{exc}}", flush=True)
dist.destroy_process_group() dist.destroy_process_group()
""" """
import tempfile import tempfile
@ -419,50 +391,46 @@ dist.destroy_process_group()
r = subprocess.run( r = subprocess.run(
[torchrun_cmd, f"--nproc_per_node={gpu_count}", tmp.name], [torchrun_cmd, f"--nproc_per_node={gpu_count}", tmp.name],
capture_output=True, text=True, timeout=300, capture_output=True, text=True, timeout=120,
env={**os.environ, "NCCL_DEBUG": "WARN", "RCCL_DEBUG": "WARN"}, env=self._runtime_env(),
) )
os.unlink(tmp.name) os.unlink(tmp.name)
# Parse connectivity results — format: op_name:ok or op_name:fail:error
tests = {} tests = {}
all_passed = True
for line in r.stdout.split("\n"): for line in r.stdout.split("\n"):
line = line.strip() line = line.strip()
if line.startswith("METRIC|"): if not line:
_, name, time_us, algbw, busbw = line.split("|", 4) continue
threshold = self.nccl_cfg.get("min_bandwidth_gbps") or 10.0 parts = line.split(":")
busbw_f = float(busbw) op_name = parts[0]
tests[name] = { result = parts[1] if len(parts) > 1 else "unknown"
"status": "PASS" if busbw_f >= float(threshold) else "WARN",
"best_algbw_gbps": round(float(algbw), 2), if result == "ok":
"best_busbw_gbps": round(busbw_f, 2), status = "PASS"
"min_required_gbps": float(threshold), else:
"by_size": [{ status = "FAIL"
"size": size_mb * 1024 * 1024, all_passed = False
"time_us": round(float(time_us), 2),
"algbw_gbps": round(float(algbw), 2), tests[op_name] = {
"busbw_gbps": round(busbw_f, 2), "status": status,
}], "error": ":".join(parts[2:]) if len(parts) > 2 and result == "fail" else None,
} }
elif line.startswith("ERROR|"):
_, name, error = line.split("|", 2)
tests[name] = {"status": "FAIL", "error": error}
if not tests:
error = (r.stderr or r.stdout or "torchrun produced no metrics")[-1000:]
return {"passed": False, "source": "torchrun_collective", "error": error}
all_passed = all(value.get("status") == "PASS" for value in tests.values())
return { return {
"passed": all_passed, # torchrun fallback is a functional smoke only. It never proves
"source": "torchrun_collective", # production bus bandwidth, so it must not satisfy acceptance.
"passed": False,
"functional_passed": all_passed,
"source": "torchrun_fallback",
"tests": tests, "tests": tests,
"gpu_count": gpu_count, "gpu_count": gpu_count,
"message_size_mb": size_mb, "error": None if all_passed else "torchrun functional NCCL smoke failed",
"warmup": warmup, "acceptance_gap": "nccl-tests bus bandwidth was not measured",
"iterations": iterations,
"backend": backend,
"timestamp": datetime.now().isoformat(),
} }
except Exception as e: except Exception as e:
return {"passed": False, "source": "torchrun_collective", "error": str(e)} return {"passed": False, "source": "torchrun_fallback", "error": str(e)}
@staticmethod @staticmethod
def print_results(results: dict, console: Console = None): def print_results(results: dict, console: Console = None):
@ -472,12 +440,12 @@ dist.destroy_process_group()
return return
passed = results.get("passed", False) passed = results.get("passed", False)
overall_status = results.get("status") or ("PASS" if passed else "FAIL")
source = results.get("source", "unknown") source = results.get("source", "unknown")
if source == "torchrun_fallback": if source == "torchrun_fallback":
# Connectivity check mode # Connectivity check mode
verdict = "[bold green]✓ NCCL Connectivity OK[/bold green]" if passed else "[bold red]✗ NCCL Connectivity FAILED[/bold red]" functional = results.get("functional_passed", passed)
verdict = "[bold yellow]⚠ NCCL bus BW NOT VERIFIED[/bold yellow]" if functional else "[bold red]✗ NCCL Connectivity FAILED[/bold red]"
c.print(f"{verdict} [dim](basic check via torchrun)[/dim]") c.print(f"{verdict} [dim](basic check via torchrun)[/dim]")
tests = results.get("tests", {}) tests = results.get("tests", {})
@ -494,15 +462,10 @@ dist.destroy_process_group()
else: else:
c.print(f" [{s_color}]{op_name}[/{s_color}]") c.print(f" [{s_color}]{op_name}[/{s_color}]")
c.print("\n[yellow]Note: functional connectivity test only (no performance data)[/yellow]") c.print("\n[yellow]Note: functional connectivity test only (no bus bandwidth data; acceptance FAIL)[/yellow]")
else: else:
# nccl-tests mode # nccl-tests mode
if overall_status == "PASS": verdict = "[bold green]✓ NCCL tests PASSED[/bold green]" if passed else "[bold yellow]⚠ NCCL tests WARNING[/bold yellow]"
verdict = "[bold green]✓ NCCL tests PASSED[/bold green]"
elif overall_status == "WARN":
verdict = "[bold yellow]⚠ NCCL tests WARNING[/bold yellow]"
else:
verdict = "[bold red]✗ NCCL tests FAILED[/bold red]"
c.print(f"{verdict} [dim](via {source})[/dim]") c.print(f"{verdict} [dim](via {source})[/dim]")
tests = results.get("tests", {}) tests = results.get("tests", {})
@ -520,260 +483,16 @@ dist.destroy_process_group()
if by_size: if by_size:
t = Table(box=None, padding=(0, 1)) t = Table(box=None, padding=(0, 1))
t.add_column("Size", style="bold", justify="right") t.add_column("Size", style="bold", justify="right")
t.add_column("Time (us)", justify="right") t.add_column("Worst Bus BW", justify="right")
t.add_column("Alg BW (GB/s)", justify="right") t.add_column("Mean Bus BW", justify="right")
t.add_column("Bus BW (GB/s)", justify="right") t.add_column("StdDev", justify="right")
t.add_column("Status", justify="right")
for r in by_size: for r in by_size:
sz = r.get("size", 0) t.add_row(
sz_str = f"{sz/1024:.0f}K" if sz < 1048576 else f"{sz/1048576:.0f}M" str(r.get("size", "")),
t.add_row(sz_str, f"{r.get('time_us',0):.1f}", f"{r.get('worst_busbw_gbps', 0):.1f}",
f"{r.get('algbw_gbps',0):.1f}", f"{r.get('busbw_gbps',0):.1f}") f"{r.get('mean_busbw_gbps', 0):.1f}",
f"{r.get('stddev_pct', 0):.2f}%",
r.get("status", "?"),
)
c.print(t) c.print(t)
class MultiNodeNCCLTest:
"""Multi-node NCCL scaling test — runs nccl-tests via mpirun across a cluster hostfile.
Config section: multinode_nccl
hosts_file path to OpenMPI hostfile (one line: "<ip> slots=8")
mpi_home MPI installation prefix (non-default location)
nccl_ib_hca comma-separated IB HCAs (GPU-direct PIX cards only)
nccl_socket_ifname bootstrap NIC name (e.g. bond0)
scale_levels list of node counts to test
ops list of operations (all_reduce, alltoall, all_gather, reduce_scatter)
msg_size_bytes message size for most ops (default 8G)
msg_size_alltoall_bytes message size for alltoall (default 1G, avoids hang at 16+ nodes)
warmup / iterations nccl-tests -w / -n
timeout_sec per-op subprocess timeout
allreduce_min_busbw_gbps PASS threshold for AllReduce
"""
_OP_BINARY = {
"all_reduce": "all_reduce_perf",
"alltoall": "alltoall_perf",
"all_gather": "all_gather_perf",
"reduce_scatter": "reduce_scatter_perf",
}
_OP_LABEL = {
"all_reduce": "AllReduce",
"alltoall": "AllToAll",
"all_gather": "AllGather",
"reduce_scatter": "ReduceScatter",
}
def __init__(self, config: dict):
self.config = config
self.console = Console()
self.mn_cfg = config.get("multinode_nccl", {})
self.tools_dir = resolve_tools_dir(config)
def _find_mpirun(self) -> Optional[str]:
mpi_home = os.path.expanduser(self.mn_cfg.get("mpi_home", ""))
if mpi_home:
candidate = os.path.join(mpi_home, "bin", "mpirun")
if os.path.isfile(candidate):
return candidate
return shutil.which("mpirun") or shutil.which("mpiexec")
def _find_nccl_build_dir(self) -> Optional[str]:
candidates = [
"/opt/nccl-tests/build",
os.path.join(self.tools_dir, "nccl-tests", "build"),
]
for d in candidates:
probe = os.path.join(d, "all_reduce_perf")
if os.path.isfile(probe) and os.access(probe, os.X_OK):
return d
return None
def _build_env(self) -> dict:
env = os.environ.copy()
mpi_home = os.path.expanduser(self.mn_cfg.get("mpi_home", ""))
if mpi_home:
env["PATH"] = f"{mpi_home}/bin:{env.get('PATH', '')}"
env["LD_LIBRARY_PATH"] = f"{mpi_home}/lib:{env.get('LD_LIBRARY_PATH', '')}"
env["NCCL_IB_HCA"] = self.mn_cfg.get(
"nccl_ib_hca", "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9"
)
env["NCCL_IB_DISABLE"] = "0"
env["NCCL_SOCKET_IFNAME"] = self.mn_cfg.get("nccl_socket_ifname", "bond0")
env["NCCL_DEBUG"] = "WARN"
return env
def _load_hosts(self, hosts_file: str) -> list:
path = os.path.expanduser(hosts_file)
if not os.path.isfile(path):
return []
lines = []
with open(path) as f:
for line in f:
line = line.strip()
if line and not line.startswith("#"):
lines.append(line)
return lines
def _run_op(self, mpirun: str, nt_dir: str, op: str, n_nodes: int,
host_lines: list, env: dict) -> dict:
binary = os.path.join(nt_dir, self._OP_BINARY[op])
if not os.path.isfile(binary):
return {"busbw_gbps": None, "status": "SKIP", "error": f"{op} binary not found"}
if op == "alltoall":
msg_bytes = self.mn_cfg.get("msg_size_alltoall_bytes", 1073741824)
else:
msg_bytes = self.mn_cfg.get("msg_size_bytes", 8589934592)
with tempfile.NamedTemporaryFile(mode="w", suffix=".hosts", delete=False) as tf:
tf.write("\n".join(host_lines[:n_nodes]) + "\n")
hf_path = tf.name
cmd = [
mpirun,
"--allow-run-as-root",
"--mca", "plm_rsh_args", "-oStrictHostKeyChecking=no",
"-np", str(n_nodes * 8),
"-N", "8",
"--hostfile", hf_path,
"-x", "PATH",
"-x", "LD_LIBRARY_PATH",
"-x", "NCCL_IB_HCA",
"-x", "NCCL_IB_DISABLE",
"-x", "NCCL_SOCKET_IFNAME",
"-x", "NCCL_DEBUG",
binary,
"-b", "8",
"-e", str(msg_bytes),
"-f", "2",
"-g", "1",
"-w", str(self.mn_cfg.get("warmup", 5)),
"-n", str(self.mn_cfg.get("iterations", 30)),
]
timeout = self.mn_cfg.get("timeout_sec", 600)
try:
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env)
os.unlink(hf_path)
if r.returncode != 0:
return {"busbw_gbps": None, "status": "ERR", "error": r.stderr[:200]}
return self._parse_op_output(r.stdout, msg_bytes)
except subprocess.TimeoutExpired:
os.unlink(hf_path)
return {"busbw_gbps": None, "status": "ERR", "error": "timeout"}
except Exception as e:
os.unlink(hf_path)
return {"busbw_gbps": None, "status": "ERR", "error": str(e)}
@staticmethod
def _parse_op_output(stdout: str, target_bytes: int) -> dict:
for line in stdout.split("\n"):
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split()
if len(parts) < 9:
continue
try:
if int(parts[0]) == target_bytes:
return {"busbw_gbps": round(float(parts[7]), 2), "status": "OK"}
except (ValueError, IndexError):
continue
return {"busbw_gbps": None, "status": "ERR", "error": "size not found in output"}
def run(self) -> dict:
hosts_file = self.mn_cfg.get("hosts_file", "~/nccl_kit/hosts_all")
scale_levels = self.mn_cfg.get("scale_levels", [2, 4, 8])
ops = self.mn_cfg.get("ops", ["all_reduce", "alltoall", "all_gather", "reduce_scatter"])
min_busbw = self.mn_cfg.get("allreduce_min_busbw_gbps", 310)
mpirun = self._find_mpirun()
if not mpirun:
return {"error": "mpirun not found — set multinode_nccl.mpi_home in config"}
nt_dir = self._find_nccl_build_dir()
if not nt_dir:
return {"error": "nccl-tests not found (need MPI=1 build at /opt/nccl-tests/build)"}
host_lines = self._load_hosts(hosts_file)
if not host_lines:
return {"error": f"hostfile not found or empty: {hosts_file}"}
env = self._build_env()
results = {}
runnable = [n for n in scale_levels if n <= len(host_lines)]
skipped = [n for n in scale_levels if n > len(host_lines)]
if skipped:
self.console.print(f"[yellow]⚠ 跳过节点数 {skipped}:hostfile 只有 {len(host_lines)} 台[/yellow]")
with Progress(
SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
TimeElapsedColumn(), console=self.console,
) as progress:
task = progress.add_task("Multi-node NCCL...", total=len(runnable) * len(ops))
for n in runnable:
results[n] = {}
for op in ops:
progress.update(task, description=f"{n} nodes / {self._OP_LABEL.get(op, op)}...")
results[n][op] = self._run_op(mpirun, nt_dir, op, n, host_lines, env)
progress.advance(task)
passed = all(
(results.get(n, {}).get("all_reduce", {}).get("busbw_gbps") or 0) >= min_busbw
for n in runnable
)
return {
"passed": passed,
"hosts_file": hosts_file,
"available_nodes": len(host_lines),
"scale_levels": scale_levels,
"ops": ops,
"allreduce_min_busbw_gbps": min_busbw,
"results": results,
"timestamp": datetime.now().isoformat(),
}
def print_results(self, results: dict):
c = self.console
if "error" in results:
c.print(f"[bold red]Multi-node NCCL Error: {results['error']}[/bold red]")
return
passed = results.get("passed", False)
verdict = "[bold green]✓ PASS[/bold green]" if passed else "[bold red]✗ FAIL[/bold red]"
c.print(f"\n[bold cyan]Multi-node NCCL Scaling[/bold cyan] — {verdict}")
c.print(f" Hostfile: {results['hosts_file']} ({results['available_nodes']} nodes available)")
c.print(f" AllReduce PASS threshold: ≥ {results['allreduce_min_busbw_gbps']} GB/s")
data = results.get("results", {})
ops = results.get("ops", [])
scale_levels = [n for n in results.get("scale_levels", []) if n in data]
min_busbw = results.get("allreduce_min_busbw_gbps", 310)
t = Table(box=box.SIMPLE_HEAD, border_style="cyan")
t.add_column("Nodes / GPUs", style="bold", justify="center")
for op in ops:
t.add_column(self._OP_LABEL.get(op, op) + " (GB/s)", justify="right")
t.add_column("", justify="center")
for n in scale_levels:
row_data = data.get(n, {})
cells = []
ok = True
for op in ops:
r = row_data.get(op, {})
bw = r.get("busbw_gbps")
if bw is None:
err = r.get("error", "ERR")[:8]
cells.append(f"[red]{err}[/red]")
ok = False
else:
is_alltoall = op == "alltoall"
fail = (op == "all_reduce" and bw < min_busbw)
color = "red" if fail else "green"
suffix = "¹" if is_alltoall else ""
cells.append(f"[{color}]{bw:.1f}[/{color}]{suffix}")
status_str = "[green]PASS[/green]" if ok else "[red]FAIL[/red]"
t.add_row(f"{n} / {n * 8}", *cells, status_str)
c.print(t)
c.print("[dim]¹ AllToAll 使用 1G 消息(避免大消息 hang) | 其余操作使用 8G 消息[/dim]")

188
modules/nvlink_test.py Normal file
View File

@ -0,0 +1,188 @@
"""NVLink / NVSwitch production acceptance checks."""
import re
import shutil
import subprocess
from datetime import datetime
from typing import Optional
from rich.console import Console
from rich.table import Table
class NVLinkTest:
def __init__(self, config: dict):
self.config = config
self.console = Console()
self.cfg = config.get("nvlink", {})
def _run(self, args: list[str], timeout: int = 60) -> tuple[int, str, str]:
if not shutil.which("nvidia-smi"):
return 127, "", "nvidia-smi not found"
r = subprocess.run(["nvidia-smi", *args], capture_output=True, text=True, timeout=timeout)
return r.returncode, r.stdout, r.stderr
def run(self) -> dict:
expected_links = int(self.cfg.get("expected_links_per_gpu", 18))
expected_speed = float(self.cfg.get("expected_link_speed_gbps", 25))
require_zero_errors = bool(self.cfg.get("require_zero_errors", True))
rc_s, out_s, err_s = self._run(["nvlink", "-s"])
rc_c, out_c, err_c = self._run(["nvlink", "-c"])
rc_e, out_e, err_e = self._run(["nvlink", "-e"])
if rc_s != 0:
return {
"passed": False,
"error": (err_s or out_s or "nvidia-smi nvlink -s failed")[:1000],
"timestamp": datetime.now().isoformat(),
}
links = self._parse_status(out_s)
if not links:
return {
"passed": False,
"error": "no NVLink status entries parsed from nvidia-smi nvlink -s",
"raw_status": out_s[-4000:],
"timestamp": datetime.now().isoformat(),
}
speeds = self._parse_speeds(out_c) if rc_c == 0 else {}
status_speeds = self._parse_speeds(out_s)
for gpu, gpu_speeds in status_speeds.items():
speeds.setdefault(gpu, {}).update({k: v for k, v in gpu_speeds.items() if k not in speeds.get(gpu, {})})
errors = self._parse_errors(out_e) if rc_e == 0 else {}
gpu_results = []
overall = True
for gpu, gpu_links in sorted(links.items(), key=lambda x: int(x[0])):
active = sum(1 for l in gpu_links.values() if l.get("active"))
inactive = [lid for lid, l in gpu_links.items() if not l.get("active")]
speed_bad = []
for lid in gpu_links:
speed = speeds.get(gpu, {}).get(lid)
if speed is not None and speed < expected_speed:
speed_bad.append({"link": lid, "speed_gbps": speed})
err_bad = []
if require_zero_errors:
for lid, counters in errors.get(gpu, {}).items():
total = sum(v for v in counters.values() if isinstance(v, int))
if total:
err_bad.append({"link": lid, "counters": counters})
passed = active == expected_links and not inactive and not speed_bad and not err_bad
if not passed:
overall = False
gpu_results.append({
"gpu": int(gpu),
"active_links": active,
"expected_links": expected_links,
"inactive_links": inactive,
"speed_issues": speed_bad,
"error_issues": err_bad,
"passed": passed,
})
return {
"passed": overall,
"expected_links_per_gpu": expected_links,
"expected_link_speed_gbps": expected_speed,
"require_zero_errors": require_zero_errors,
"gpus": gpu_results,
"raw_status": out_s[-4000:],
"raw_speed": out_c[-4000:] if out_c else "",
"raw_errors": out_e[-4000:] if out_e else "",
"timestamp": datetime.now().isoformat(),
}
@staticmethod
def _parse_status(text: str) -> dict[str, dict[str, dict]]:
result: dict[str, dict[str, dict]] = {}
gpu = None
for line in text.splitlines():
m_gpu = re.search(r"GPU\s+(\d+)", line, re.I)
if m_gpu:
gpu = m_gpu.group(1)
result.setdefault(gpu, {})
continue
if gpu is None:
continue
m_link = re.search(r"Link\s+(\d+).*?(Active|Inactive|Disabled|Off|Down)", line, re.I)
if m_link:
state = m_link.group(2)
result[gpu][m_link.group(1)] = {
"state": state,
"active": state.lower() == "active",
"raw": line.strip(),
}
continue
m_speed = re.search(r"Link\s+(\d+).*?([0-9.]+)\s*GB/s", line, re.I)
if m_speed:
result[gpu][m_speed.group(1)] = {
"state": "Active",
"active": True,
"raw": line.strip(),
}
return result
@staticmethod
def _parse_speeds(text: str) -> dict[str, dict[str, float]]:
result: dict[str, dict[str, float]] = {}
gpu = None
for line in text.splitlines():
m_gpu = re.search(r"GPU\s+(\d+)", line, re.I)
if m_gpu:
gpu = m_gpu.group(1)
result.setdefault(gpu, {})
continue
if gpu is None:
continue
m_link = re.search(r"Link\s+(\d+).*?([0-9.]+)\s*GB/s", line, re.I)
if m_link:
result[gpu][m_link.group(1)] = float(m_link.group(2))
return result
@staticmethod
def _parse_errors(text: str) -> dict[str, dict[str, dict[str, int]]]:
result: dict[str, dict[str, dict[str, int]]] = {}
gpu = None
link = None
for line in text.splitlines():
m_gpu = re.search(r"GPU\s+(\d+)", line, re.I)
if m_gpu:
gpu = m_gpu.group(1)
result.setdefault(gpu, {})
continue
m_link = re.search(r"Link\s+(\d+)", line, re.I)
if m_link and gpu is not None:
link = m_link.group(1)
result[gpu].setdefault(link, {})
if gpu is None or link is None:
continue
for name in ("CRC", "Replay", "Recovery"):
m = re.search(rf"{name}[^0-9]*(\d+)", line, re.I)
if m:
result[gpu][link][name.lower()] = int(m.group(1))
return result
@staticmethod
def print_results(results: dict, console: Optional[Console] = None):
c = console or Console()
if results.get("error"):
c.print(f"[bold red]NVLink error: {results['error']}[/bold red]")
return
passed = results.get("passed", False)
c.print("[bold green]✓ NVLink PASSED[/bold green]" if passed else "[bold red]✗ NVLink FAILED[/bold red]")
table = Table(box=None, padding=(0, 1))
table.add_column("GPU", style="bold")
table.add_column("Active Links", justify="right")
table.add_column("Issues")
for g in results.get("gpus", []):
issues = []
if g.get("inactive_links"):
issues.append("inactive=" + ",".join(g["inactive_links"]))
if g.get("speed_issues"):
issues.append(f"speed={len(g['speed_issues'])}")
if g.get("error_issues"):
issues.append(f"errors={len(g['error_issues'])}")
table.add_row(str(g["gpu"]), f"{g['active_links']}/{g['expected_links']}", "; ".join(issues) or "OK")
c.print(table)

View File

@ -62,7 +62,6 @@ class RDMATest:
# bond/management interfaces (e.g. mlx5_bond_0) can show ACTIVE state # bond/management interfaces (e.g. mlx5_bond_0) can show ACTIVE state
# without actually providing IB fabric connectivity. # without actually providing IB fabric connectivity.
ib_devices = [] ib_devices = []
active_ib_devices = []
active_ib_port = False active_ib_port = False
for dev in devices: for dev in devices:
for port in self._get_ib_ports(dev): for port in self._get_ib_ports(dev):
@ -75,24 +74,8 @@ class RDMATest:
f"/sys/class/infiniband/{dev}/ports/{port}/state") f"/sys/class/infiniband/{dev}/ports/{port}/state")
if "ACTIVE" in state.upper(): if "ACTIVE" in state.upper():
active_ib_port = True active_ib_port = True
active_ib_devices.append(dev)
device_info = self._collect_device_info(devices) device_info = self._collect_device_info(devices)
cross_enabled = (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled")
if not ib_devices and cross_enabled:
# RoCE ports report link_layer=Ethernet. They are valid for configured
# cross-node perftest even though local InfiniBand loopback is skipped.
cross = self.run_cross_node()
return {
"passed": cross.get("status") == "PASS",
"status": cross.get("status", "FAIL"),
"reason": "RoCE cross-node mode",
"devices": device_info,
"bandwidth_tests": [],
"latency_tests": [],
"cross_node": cross,
"timestamp": datetime.now().isoformat(),
}
if not ib_devices: if not ib_devices:
self.console.print( self.console.print(
"[yellow]No InfiniBand-link_layer ports present — " "[yellow]No InfiniBand-link_layer ports present — "
@ -116,35 +99,28 @@ class RDMATest:
"timestamp": datetime.now().isoformat(), "timestamp": datetime.now().isoformat(),
} }
# perftest is a two-endpoint benchmark. Running both endpoints on self.console.print(f"[cyan]RDMA Test - Devices: {', '.join(devices)}[/cyan]")
# localhost can select a DOWN HCA, bypass the fabric, or return a zero
# result that looks like hardware failure. Inventory-only is objective
# until a peer is explicitly configured.
if not cross_enabled:
self.console.print(
"[yellow]Active IB ports detected, but no cross-node peer is configured — "
"skipping bandwidth/latency benchmarks[/yellow]"
)
return {
"status": "SKIP", "skipped": True,
"reason": "cross-node RDMA peer not configured; local loopback is not an acceptance test",
"devices": device_info,
"active_ib_devices": sorted(set(active_ib_devices)),
"timestamp": datetime.now().isoformat(),
}
cross = self.run_cross_node() bw_results = self._run_bandwidth_tests(devices)
return { latency_results = self._run_latency_tests(devices)
"passed": cross.get("status") == "PASS",
"status": cross.get("status", "FAIL"), all_passed = all(
"reason": "cross-node RDMA benchmark", r.get("status") == "PASS"
for r in bw_results + latency_results
if isinstance(r, dict)
)
result = {
"passed": all_passed,
"devices": device_info, "devices": device_info,
"active_ib_devices": sorted(set(active_ib_devices)), "bandwidth_tests": bw_results,
"bandwidth_tests": [], "latency_tests": latency_results,
"latency_tests": [],
"cross_node": cross,
"timestamp": datetime.now().isoformat(), "timestamp": datetime.now().isoformat(),
} }
# Cross-node (two-host) RDMA, run only when a peer is configured.
if (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled"):
result["cross_node"] = self.run_cross_node()
return result
def _collect_device_info(self, devices: List[str]) -> List[dict]: def _collect_device_info(self, devices: List[str]) -> List[dict]:
info = [] info = []
@ -290,14 +266,13 @@ class RDMATest:
# ------------------------------------------------------------------ # ------------------------------------------------------------------
def _active_ib_devices(self) -> List[str]: def _active_ib_devices(self) -> List[str]:
"""Active IB devices, optionally including Ethernet/RoCE devices.""" """IB devices whose port 1 is InfiniBand link_layer and ACTIVE."""
out = [] out = []
allow_roce = bool((self.rdma_cfg.get("cross_node", {}) or {}).get("allow_roce"))
for dev in self._get_ib_devices(): for dev in self._get_ib_devices():
for port in self._get_ib_ports(dev): for port in self._get_ib_ports(dev):
ll = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/link_layer") ll = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/link_layer")
st = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/state") st = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/state")
if (ll == "InfiniBand" or allow_roce) and "ACTIVE" in st.upper(): if ll == "InfiniBand" and "ACTIVE" in st.upper():
out.append(dev) out.append(dev)
break break
return out return out
@ -321,8 +296,6 @@ class RDMATest:
gid_index = cn.get("gid_index") gid_index = cn.get("gid_index")
msg_size = cn.get("msg_size", 1048576) msg_size = cn.get("msg_size", 1048576)
iters = cn.get("iters", 5000) iters = cn.get("iters", 5000)
duration_sec = cn.get("duration_sec")
queue_pairs = int(cn.get("queue_pairs", 1))
base_port = cn.get("base_oob_port", 18515) base_port = cn.get("base_oob_port", 18515)
warmup = cn.get("server_warmup_sec", 2.0) warmup = cn.get("server_warmup_sec", 2.0)
min_bw = cn.get("min_bandwidth_gbps", 350) min_bw = cn.get("min_bandwidth_gbps", 350)
@ -349,16 +322,10 @@ class RDMATest:
entry = {"device": dev} entry = {"device": dev}
if has_bw: if has_bw:
bw_extra = ["--report_gbits", "-s", str(msg_size),
"-q", str(queue_pairs)]
if duration_sec:
bw_extra += ["-D", str(duration_sec)]
else:
bw_extra += ["-n", str(iters)]
bw = self._cross_node_perftest( bw = self._cross_node_perftest(
"ib_write_bw", dev, server_target, server_addr, ib_port, "ib_write_bw", dev, server_target, server_addr, ib_port,
oob, gid_index, warmup, oob, gid_index, warmup,
extra=bw_extra, extra=["--report_gbits", "-s", str(msg_size), "-n", str(iters)],
parse="bw") parse="bw")
entry["bandwidth_gbps"] = bw entry["bandwidth_gbps"] = bw
if isinstance(bw, (int, float)): if isinstance(bw, (int, float)):

View File

@ -2,7 +2,6 @@
import json import json
import os import os
import re
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
@ -94,17 +93,13 @@ class ReportGenerator:
def _generate_html(self, results: dict, output: str) -> str: def _generate_html(self, results: dict, output: str) -> str:
import socket import socket
hostname = socket.gethostname() hostname = results.get("hostname") or socket.gethostname()
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") timestamp = results.get("timestamp") or datetime.now().strftime("%Y-%m-%d %H:%M:%S")
sections = [] sections = []
if "gpu_info" in results: if "gpu_info" in results:
gpus = results["gpu_info"].get("gpus", []) gpus = results["gpu_info"].get("gpus", [])
runtime_name = results["gpu_info"].get("runtime_name", "CUDA")
runtime_version = results["gpu_info"].get(
"runtime_version", results["gpu_info"].get("cuda_version", "N/A")
)
rows = "" rows = ""
for g in gpus: for g in gpus:
rows += f"<tr><td>GPU {g['index']}</td><td>{g['name']}</td>" rows += f"<tr><td>GPU {g['index']}</td><td>{g['name']}</td>"
@ -114,7 +109,7 @@ class ReportGenerator:
sections.append( sections.append(
f'<div class="section"><h2>GPU Information</h2>' f'<div class="section"><h2>GPU Information</h2>'
f'<p>Driver: {results["gpu_info"].get("driver_version", "N/A")} | ' f'<p>Driver: {results["gpu_info"].get("driver_version", "N/A")} | '
f'{runtime_name}: {runtime_version} | ' f'CUDA: {results["gpu_info"].get("cuda_version", "N/A")} | '
f'Count: {len(gpus)}</p>' f'Count: {len(gpus)}</p>'
f'<table><tr><th>GPU</th><th>Model</th><th>VRAM</th><th>Temp</th><th>SM Clock</th></tr>' f'<table><tr><th>GPU</th><th>Model</th><th>VRAM</th><th>Temp</th><th>SM Clock</th></tr>'
f'{rows}</table></div>' f'{rows}</table></div>'
@ -183,8 +178,8 @@ class ReportGenerator:
def _generate_markdown(self, results: dict, output: str) -> str: def _generate_markdown(self, results: dict, output: str) -> str:
import socket import socket
hostname = socket.gethostname() hostname = results.get("hostname") or socket.gethostname()
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") timestamp = results.get("timestamp") or datetime.now().strftime("%Y-%m-%d %H:%M:%S")
lines: list[str] = [] lines: list[str] = []
@ -199,15 +194,28 @@ class ReportGenerator:
gpus = gpu_info.get("gpus", []) gpus = gpu_info.get("gpus", [])
gpu_name = gpus[0]["name"] if gpus else "Unknown" gpu_name = gpus[0]["name"] if gpus else "Unknown"
lines.append(f"- **GPU:** {gpu_name} x{gpu_info.get('gpu_count', len(gpus))}") lines.append(f"- **GPU:** {gpu_name} x{gpu_info.get('gpu_count', len(gpus))}")
runtime_name = gpu_info.get("runtime_name", "CUDA")
runtime_version = gpu_info.get("runtime_version", gpu_info.get("cuda_version", "N/A"))
lines.append(f"- **Driver:** {gpu_info.get('driver_version', 'N/A')} | " lines.append(f"- **Driver:** {gpu_info.get('driver_version', 'N/A')} | "
f"**{runtime_name}:** {runtime_version}") f"**CUDA:** {gpu_info.get('cuda_version', 'N/A')}")
lines.append("") lines.append("")
# --- Summary table --- # --- Summary table ---
summary_items = self._build_summary(results) summary_items = self._build_summary(results)
if summary_items: if summary_items:
verdict, failures, missing = self._overall_acceptance_verdict(summary_items)
lines.append("## Overall Acceptance Verdict\n")
lines.append(f"**Result: {verdict}**")
lines.append("")
if failures:
lines.append("Failed or unverified items:")
for name, status in failures:
lines.append(f"- {name}: {status}")
lines.append("")
if missing:
lines.append("Missing required evidence:")
for name in missing:
lines.append(f"- {name}")
lines.append("")
lines.append("## Summary\n") lines.append("## Summary\n")
lines.append("| Test | Result |") lines.append("| Test | Result |")
lines.append("|------|--------|") lines.append("|------|--------|")
@ -259,10 +267,8 @@ class ReportGenerator:
# --- Memory Bandwidth --- # --- Memory Bandwidth ---
mem_data = self._extract_memory_results(results) mem_data = self._extract_memory_results(results)
if mem_data and not mem_data.get("error"): if mem_data and not mem_data.get("error"):
lines.append("## GPU Interconnect and PCIe Bandwidth\n") lines.append("## Memory Bandwidth\n")
lines.append(f"Source: {mem_data.get('source', 'unknown')}\n") lines.append(f"Source: {mem_data.get('source', 'unknown')}\n")
if mem_data.get("scope_note"):
lines.append(f"> Scope: {mem_data['scope_note']}.\n")
lines.append("| Metric | Value | Peak | Efficiency |") lines.append("| Metric | Value | Peak | Efficiency |")
lines.append("|--------|-------|------|------------|") lines.append("|--------|-------|------|------------|")
d2d = mem_data.get("d2d_bandwidth_gbps") or 0 d2d = mem_data.get("d2d_bandwidth_gbps") or 0
@ -279,21 +285,9 @@ class ReportGenerator:
if not d2d_peak: if not d2d_peak:
d2d_peak = mem_data.get("peak_bandwidth_gbps") or 0 d2d_peak = mem_data.get("peak_bandwidth_gbps") or 0
d2d_eff = mem_data.get("efficiency_pct") or 0 d2d_eff = mem_data.get("efficiency_pct") or 0
def peak_text(value): lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {h2d_peak:.0f} GB/s | {h2d_eff:.1f}% |")
return f"{value:.0f} GB/s" if value else "N/A" lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {d2h_peak:.0f} GB/s | {d2h_eff:.1f}% |")
lines.append(f"| D2D (NVLink) | {d2d:.1f} GB/s | {d2d_peak:.0f} GB/s | {d2d_eff:.1f}% |")
def efficiency_text(value):
return f"{value:.1f}%" if value else "N/A"
lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {peak_text(h2d_peak)} | {efficiency_text(h2d_eff)} |")
lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {peak_text(d2h_peak)} | {efficiency_text(d2h_eff)} |")
d2d_label = "D2D (HBM copy)" if mem_data.get("accelerator_vendor") == "hygon" else "D2D (NVLink, unidirectional)"
lines.append(f"| {d2d_label} | {d2d:.1f} GB/s | {peak_text(d2d_peak)} | {efficiency_text(d2d_eff)} |")
if mem_data.get("d2d_bidirectional_gbps"):
lines.append(
f"| D2D (NVLink, bidirectional informational) | "
f"{mem_data['d2d_bidirectional_gbps']:.1f} GB/s | N/A | N/A |"
)
lines.append("") lines.append("")
# PyTorch fallback can't accurately measure HBM peak (intra-GPU copy_() # PyTorch fallback can't accurately measure HBM peak (intra-GPU copy_()
# only reaches ~20% of HBM bandwidth). When fallback is used, report # only reaches ~20% of HBM bandwidth). When fallback is used, report
@ -319,67 +313,107 @@ class ReportGenerator:
# When present, override the legacy 80%-of-peak rule on a per-dtype basis. # When present, override the legacy 80%-of-peak rule on a per-dtype basis.
pass_thresholds = comp_data.get("pass_thresholds_tflops", {}) or {} pass_thresholds = comp_data.get("pass_thresholds_tflops", {}) or {}
use_abs = bool(pass_thresholds) use_abs = bool(pass_thresholds)
has_peak_reference = any(
isinstance(value, (int, float)) and value > 0
for value in peak_tflops.values()
)
worst_values = comp_data.get("worst_per_dtype_tflops", {}) or {}
best_values = comp_data.get("best_per_dtype_tflops", {}) or {}
spreads = comp_data.get("spread_pct", {}) or {}
if use_abs: if use_abs:
lines.append("| DType | Avg | Min | Max | Spread | Dense Peak | Pass Floor | Status |") lines.append("| DType | Achieved (TFLOPS) | Peak | Threshold | Status |")
else: else:
lines.append("| DType | Avg | Min | Max | Spread | Dense Peak | Efficiency | Status |") lines.append("| DType | Achieved (TFLOPS) | Peak | Efficiency | Status |")
lines.append("|-------|-----|-----|-----|--------|------------|------------|--------|") lines.append("|-------|-------------------|------|------------|--------|")
worst_eff = 100.0 worst_eff = 100.0
overall_status = "PASS" overall_status = "PASS"
rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0} rank = {"PASS": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
for dt, val in per_dtype.items(): for dt, val in per_dtype.items():
if isinstance(val, str): if isinstance(val, str):
# skipped or error # skipped or error
lines.append(f"| {dt.upper()} | {val} | - | - | - | - | N/A | SKIP |") lines.append(f"| {dt.upper()} | {val} | - | N/A | SKIP |")
else: else:
pk = peak_tflops.get(dt, 0) pk = peak_tflops.get(dt, 0)
ef = eff_pct.get(dt, 0) ef = eff_pct.get(dt, 0)
worst = worst_values.get(dt, val)
best = best_values.get(dt, val)
spread = spreads.get(dt, 0)
if isinstance(ef, (int, float)) and ef > 0: if isinstance(ef, (int, float)) and ef > 0:
worst_eff = min(worst_eff, ef) worst_eff = min(worst_eff, ef)
thr = pass_thresholds.get(dt) thr = pass_thresholds.get(dt)
if use_abs and thr: if use_abs and thr:
if worst >= thr: if val >= thr:
status = "PASS" status = "PASS"
elif worst >= thr * 0.9:
status = "WARN"
else: else:
status = "FAIL" status = "FAIL"
lines.append( lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | >= {thr} | {status} |")
f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | "
f"{spread:.2f}% | {pk:.0f} | >= {thr} | {status} |"
)
elif has_peak_reference:
status = "PASS" if ef >= 80 else ("WARN" if ef >= 50 else "FAIL")
lines.append(
f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | "
f"{spread:.2f}% | {pk:.0f} | {ef:.1f}% | {status} |"
)
else: else:
status = "MEASURED" status = "PASS" if ef >= 80 else ("WARN" if ef >= 50 else "FAIL")
lines.append( lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | {ef:.1f}% | {status} |")
f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | "
f"{spread:.2f}% | N/A | N/A | {status} |"
)
if rank.get(status, 0) > rank.get(overall_status, 0): if rank.get(status, 0) > rank.get(overall_status, 0):
overall_status = status overall_status = status
lines.append("") lines.append("")
if use_abs: if use_abs:
lines.append(f"**Verdict: {overall_status}** (each dtype is judged by the slowest GPU against an achievable B300 floor)\n") if any(not row.get("passed", False) for row in (comp_data.get("consistency", {}) or {}).values()):
elif has_peak_reference: overall_status = "FAIL"
lines.append(f"**Verdict: {overall_status}** (absolute TFLOPS thresholds; worst efficiency {worst_eff:.1f}%)\n")
else:
overall_status = "PASS" if worst_eff >= 80 else ("WARN" if worst_eff >= 50 else "FAIL") overall_status = "PASS" if worst_eff >= 80 else ("WARN" if worst_eff >= 50 else "FAIL")
lines.append(f"**Verdict: {overall_status}** (worst efficiency {worst_eff:.1f}%)\n") lines.append(f"**Verdict: {overall_status}** (worst efficiency {worst_eff:.1f}%)\n")
else:
lines.append("**Verdict: MEASURED** (no published peak/acceptance threshold configured)\n") consistency = comp_data.get("consistency", {}) or {}
if consistency:
lines.append("### Compute Consistency\n")
lines.append("| DType | Min | Mean | Max | Spread | Limit | Status |")
lines.append("|-------|-----|------|-----|--------|-------|--------|")
for dt, row in consistency.items():
status = "PASS" if row.get("passed") else "FAIL"
lines.append(
f"| {dt.upper()} | {row.get('min_tflops', 0):.1f} | "
f"{row.get('mean_tflops', 0):.1f} | {row.get('max_tflops', 0):.1f} | "
f"{row.get('spread_pct', 0):.2f}% | <= {row.get('max_allowed_pct', 3)}% | {status} |"
)
lines.append("")
per_gpu = comp_data.get("per_gpu", []) or []
dtype_order = [dt for dt in per_dtype.keys() if not isinstance(per_dtype.get(dt), str)]
if per_gpu and dtype_order:
lines.append("### Compute Per-GPU TFLOPS\n")
headers = ["GPU", *[dt.upper() for dt in dtype_order]]
lines.append("| " + " | ".join(headers) + " |")
lines.append("|" + "|".join(["---"] * len(headers)) + "|")
for row in per_gpu:
cells = [str(row.get("index", ""))]
for dt in dtype_order:
val = row.get(dt, "")
cells.append(f"{val:.1f}" if isinstance(val, (int, float)) else str(val))
lines.append("| " + " | ".join(cells) + " |")
lines.append("")
# --- NCCL ---
nvlink = results.get("nvlink")
if nvlink and not nvlink.get("error"):
lines.append("## NVLink/NVSwitch\n")
lines.append(f"**Overall: {'PASS' if nvlink.get('passed') else 'FAIL'}**\n")
lines.append("| GPU | Active Links | Issues |")
lines.append("|-----|--------------|--------|")
for g in nvlink.get("gpus", []):
issues = []
if g.get("inactive_links"):
issues.append("inactive=" + ",".join(g["inactive_links"]))
if g.get("speed_issues"):
issues.append(f"speed issues={len(g['speed_issues'])}")
if g.get("error_issues"):
issues.append(f"errors={len(g['error_issues'])}")
lines.append(f"| {g.get('gpu')} | {g.get('active_links')}/{g.get('expected_links')} | {', '.join(issues) or 'OK'} |")
lines.append("")
elif nvlink and nvlink.get("error"):
lines.append("## NVLink/NVSwitch\n")
lines.append(f"**Overall: FAIL** ({nvlink.get('error')})\n")
dcgm = results.get("dcgm")
if dcgm and not dcgm.get("error"):
lines.append("## DCGM Diagnostic\n")
lines.append(f"**Overall: {'PASS' if dcgm.get('passed') else 'FAIL'}**\n")
if dcgm.get("subtests"):
lines.append("| Subtest | Status |")
lines.append("|---------|--------|")
for s in dcgm.get("subtests", []):
lines.append(f"| {s.get('name', '')} | {s.get('status', '')} |")
lines.append("")
elif dcgm and dcgm.get("error"):
lines.append("## DCGM Diagnostic\n")
lines.append(f"**Overall: FAIL** ({dcgm.get('error')})\n")
# --- NCCL --- # --- NCCL ---
nccl = results.get("nccl") nccl = results.get("nccl")
@ -387,74 +421,115 @@ class ReportGenerator:
lines.append("## NCCL Multi-GPU\n") lines.append("## NCCL Multi-GPU\n")
lines.append(f"Source: {nccl.get('source', 'unknown')} | " lines.append(f"Source: {nccl.get('source', 'unknown')} | "
f"GPUs: {nccl.get('gpu_count', '?')}\n") f"GPUs: {nccl.get('gpu_count', '?')}\n")
if nccl.get("source") == "torchrun_fallback":
lines.append("> Functional NCCL smoke only: nccl-tests bus bandwidth was not measured, so this does not satisfy production acceptance.\n")
tests = nccl.get("tests", {}) tests = nccl.get("tests", {})
if tests: if tests:
lines.append("| Operation | Bus BW (GB/s) | Threshold | Status |") lines.append("> Summary reports the best Bus BW observed for each operation. PASS/FAIL is evaluated across every tested message size and repeat run shown in the detail table below.\n")
lines.append("|-----------|---------------|-----------|--------|") lines.append("| Operation | Best Bus BW (GB/s) | Failed Sizes | Threshold | Status |")
lines.append("|-----------|--------------------|--------------|-----------|--------|")
for op, data in tests.items(): for op, data in tests.items():
if isinstance(data, dict) and not data.get("error"): if isinstance(data, dict) and not data.get("error"):
bw = data.get("best_busbw_gbps", 0) bw = data.get("best_busbw_gbps", 0)
req = data.get("min_required_gbps", 0) req = data.get("min_required_gbps", 0)
status = data.get("status", "?") status = data.get("status", "?")
lines.append(f"| {op} | {bw:.1f} | >= {req:.0f} | {status} |") failed_sizes = [
str(row.get("size", "?"))
for row in data.get("by_size", [])
if row.get("status") != "PASS"
]
failed_sizes_text = ", ".join(failed_sizes) if failed_sizes else "-"
lines.append(f"| {op} | {bw:.1f} | {failed_sizes_text} | >= {_format_gbps(req)} | {status} |")
elif isinstance(data, dict) and data.get("error"): elif isinstance(data, dict) and data.get("error"):
lines.append(f"| {op} | - | - | ERROR: {data['error']} |") lines.append(f"| {op} | - | - | - | ERROR: {data['error']} |")
lines.append("") lines.append("")
nccl_status = nccl.get("status") or ("PASS" if nccl.get("passed") else "FAIL") for op, data in tests.items():
lines.append(f"**Overall: {nccl_status}**\n") by_size = data.get("by_size", []) if isinstance(data, dict) else []
if not by_size:
# --- Multi-node NCCL --- continue
multinode = results.get("multinode_nccl") lines.append(f"### NCCL {op} by size\n")
if multinode: lines.append("| Size | Runs Bus BW (GB/s) | Worst | Mean | StdDev | Threshold | Status |")
lines.append("## Multi-node NCCL\n") lines.append("|------|---------------------|-------|------|--------|-----------|--------|")
lines.append(f"Source: {multinode.get('source', 'unknown')} | " for row in by_size:
f"Mode: {multinode.get('mode', 'unknown')}\n") runs = ", ".join(str(v) for v in row.get("runs_busbw_gbps", []))
hosts = multinode.get("hosts", []) or []
if hosts:
host_text = ", ".join(
f"{host.get('name', host.get('addr', '?'))} "
f"({host.get('addr', '?')}, {host.get('slots', '?')} GPUs)"
for host in hosts
)
lines.append(f"- **Hosts:** {host_text}")
lines.append(f"- **Overall: {'PASS' if multinode.get('passed') else 'FAIL'}**")
if multinode.get("artifact_dir"):
lines.append(f"- **Artifacts:** `{multinode['artifact_dir']}`")
lines.append("")
tests = multinode.get("tests", {}) or {}
if tests:
lines.append("| Operation | Topology | Peak Bus BW | Avg Bus BW | Peak Size | Threshold | Wrong | Network | GDR | Status |")
lines.append("|-----------|----------|-------------|------------|-----------|-----------|-------|---------|-----|--------|")
for op, test_data in tests.items():
for topo in test_data.get("topologies", []):
network = topo.get("network", {}) or {}
networks = ", ".join(network.get("networks", [])) or "N/A"
ib_using = " ".join(network.get("ib_using", []))
roce_hcas = sorted(set(re.findall(r"mlx5_bond_\d+", ib_using)))
if roce_hcas:
networks = f"{networks} ({len(roce_hcas)}x RoCE)"
threshold = topo.get("min_required_gbps", 0)
lines.append( lines.append(
f"| {op} | {topo.get('label', '')} | " f"| {row.get('size', '')} | {runs} | "
f"{topo.get('peak_busbw_gbps', 0):.2f} GB/s | " f"{row.get('worst_busbw_gbps', 0):.1f} | "
f"{topo.get('avg_busbw_gbps', 0):.2f} GB/s | " f"{row.get('mean_busbw_gbps', 0):.1f} | "
f"{topo.get('peak_size', '')} | >= {threshold:g} GB/s | " f"{row.get('stddev_pct', 0):.2f}% | "
f"{topo.get('wrong_count', 0)} | {networks} | " f">= {_format_gbps(data.get('min_required_gbps', 0))} | "
f"{network.get('gpu_direct_rdma', 'UNKNOWN')} | {topo.get('status', '?')} |" f"{row.get('status', '?')} |"
) )
lines.append("") lines.append("")
passed = nccl.get("passed", False)
lines.append(f"**Overall: {'PASS' if passed else 'FAIL'}**\n")
preflight = multinode.get("preflight", {}) or {} multinode = results.get("multinode_nccl")
checks = preflight.get("checks", []) or [] if multinode and not multinode.get("error"):
if checks: lines.append("## Multi-node NCCL / Cross Leaf\n")
lines.append("### Preflight\n") lines.append(f"Source: {multinode.get('source', 'unknown')} | Mode: {multinode.get('mode', 'unknown')}\n")
if multinode.get("artifact_dir"):
lines.append(f"- **Artifacts:** `{multinode.get('artifact_dir')}`")
hosts = multinode.get("hosts", [])
if hosts:
host_text = ", ".join(f"{h.get('name') or h.get('addr')}({h.get('addr')})" for h in hosts)
lines.append(f"- **Hosts:** {host_text}")
preflight = multinode.get("preflight", {})
if preflight.get("checks"):
failed_checks = [c for c in preflight["checks"] if c.get("status") == "FAIL"]
warn_checks = [c for c in preflight["checks"] if c.get("status") == "WARN"]
lines.append(f"- **Preflight:** {'PASS' if not failed_checks else 'FAIL'}"
f"{f' ({len(warn_checks)} warnings)' if warn_checks else ''}")
lines.append("")
for op, data in (multinode.get("tests") or {}).items():
lines.append(f"### Multi-node NCCL {op}\n")
lines.append("| Topology | CUDA Visible Devices | Peak Bus BW | Peak Size | Avg Bus BW | Threshold | Status |")
lines.append("|----------|----------------------|-------------|-----------|------------|-----------|--------|")
for topo in data.get("topologies", []):
threshold = topo.get("min_required_gbps", 0) or 0
threshold_text = f">= {_format_gbps(threshold)} GB/s" if threshold else "-"
cuda_visible = topo.get("cuda_visible_devices") or "-"
lines.append(
f"| {topo.get('label', '')} | {cuda_visible} | {topo.get('peak_busbw_gbps', 0):.2f} GB/s | "
f"{topo.get('peak_size', '')} | {topo.get('avg_busbw_gbps', 0):.2f} GB/s | "
f"{threshold_text} | {topo.get('status', '?')} |"
)
lines.append("")
diag_rows = []
for topo in data.get("topologies", []):
net = topo.get("network") or {}
if net:
diag_rows.append((topo, net))
if diag_rows:
lines.append("| Topology | NCCL Network | GPU Direct RDMA | GDR Enabled HCAs | GDR Disabled HCAs |")
lines.append("|----------|--------------|-----------------|------------------|-------------------|")
for topo, net in diag_rows:
networks = ", ".join(net.get("networks") or []) or "unknown"
gdr = net.get("gpu_direct_rdma", "UNKNOWN")
enabled = ", ".join(net.get("gdr_enabled_hcas") or []) or "-"
disabled = ", ".join(net.get("gdr_disabled_hcas") or []) or "-"
lines.append(f"| {topo.get('label', '')} | {networks} | {gdr} | {enabled} | {disabled} |")
lines.append("")
failed_topos = [topo for topo in data.get("topologies", []) if topo.get("status") == "FAIL"]
if failed_topos:
lines.append("| Topology | Return Code | Error / Output Tail |")
lines.append("|----------|-------------|---------------------|")
for topo in failed_topos:
tail = topo.get("error") or topo.get("stderr_tail") or topo.get("stdout_tail") or ""
tail = str(tail).replace("\n", " ").replace("|", "\\|")[-240:]
lines.append(f"| {topo.get('label', '')} | {topo.get('returncode', '')} | {tail} |")
lines.append("")
lines.append(f"**Overall: {'PASS' if multinode.get('passed') else 'FAIL'}**\n")
elif multinode and multinode.get("error"):
lines.append("## Multi-node NCCL / Cross Leaf\n")
lines.append(f"**Overall: FAIL** ({multinode.get('error')})\n")
preflight = multinode.get("preflight", {})
if preflight.get("checks"):
lines.append("| Check | Status | Detail |") lines.append("| Check | Status | Detail |")
lines.append("|-------|--------|--------|") lines.append("|-------|--------|--------|")
for check in checks: for check in preflight["checks"]:
detail = str(check.get("detail", "")).replace("|", "\\|").replace("\n", " ") detail = str(check.get("detail", "")).replace("\n", " ")
lines.append(f"| {check.get('name', '')} | {check.get('status', '?')} | {detail} |") lines.append(f"| {check.get('name', '')} | {check.get('status', '')} | {detail} |")
lines.append("") lines.append("")
# --- Stress Test --- # --- Stress Test ---
@ -467,24 +542,22 @@ class ReportGenerator:
source = stress.get("source", "unknown") source = stress.get("source", "unknown")
lines.append(f"- **Source:** {source}") lines.append(f"- **Source:** {source}")
lines.append(f"- **Duration:** {elapsed:.0f}s (requested {duration}s)") lines.append(f"- **Duration:** {elapsed:.0f}s (requested {duration}s)")
telemetry = stress.get("telemetry") or {}
if telemetry:
lines.append(f"- **Telemetry samples:** {telemetry.get('samples', 0)}")
lines.append(f"- **Max temp:** {telemetry.get('max_temp_c', {})}")
lines.append(f"- **Avg power:** {telemetry.get('avg_power_w', {})}")
lines.append(f"- **Temp delta:** {telemetry.get('temp_delta_c', 'N/A')} C")
lines.append(f"- **TFLOPS jitter:** {telemetry.get('tflops_jitter_pct', 'N/A')}%")
lines.append(f"- **Steady TFLOPS samples:** {telemetry.get('steady_tflops_samples', 0)}")
lines.append(f"- **Throttle events:** {telemetry.get('throttle_event_count', len(telemetry.get('throttle_events', [])))}")
lines.append(f"- **XID events:** {len(telemetry.get('xid_events', []))}")
failures = telemetry.get("failures") or []
if failures:
lines.append("- **Failure reasons:**")
for reason in failures:
lines.append(f" - {reason}")
lines.append(f"- **Result: {'PASS' if passed else 'FAIL'}**") lines.append(f"- **Result: {'PASS' if passed else 'FAIL'}**")
lines.append(f"- **Validation scope:** {stress.get('validation_scope', 'runtime completion')}")
if stress.get("numerical_validation") is False:
lines.append("- **Numerical result validation:** Not performed by PyTorch fallback")
ecc_delta = stress.get("uncorrected_ecc_delta", {}) or {}
if ecc_delta:
lines.append(f"- **Uncorrected ECC delta:** {sum(ecc_delta.values())}")
max_temp = stress.get("max_temperature_c", {}) or {}
max_power = stress.get("max_power_w", {}) or {}
if max_temp or max_power:
lines.append("")
lines.append("| GPU | Peak Temp | Peak Power |")
lines.append("|-----|-----------|------------|")
for gpu in sorted(set(max_temp) | set(max_power), key=lambda value: int(value)):
lines.append(
f"| {gpu} | {max_temp.get(gpu, 'N/A')} C | "
f"{max_power.get(gpu, 'N/A')} W |"
)
lines.append("") lines.append("")
# --- RDMA --- # --- RDMA ---
@ -494,26 +567,70 @@ class ReportGenerator:
lines.append(f"**Overall: SKIP** [{rdma.get('reason', 'no IB hardware detected')}]\n") lines.append(f"**Overall: SKIP** [{rdma.get('reason', 'no IB hardware detected')}]\n")
elif rdma and not rdma.get("error"): elif rdma and not rdma.get("error"):
lines.append("## RDMA/InfiniBand\n") lines.append("## RDMA/InfiniBand\n")
rdma_legacy_note = self._rdma_legacy_note(rdma)
if rdma_legacy_note:
lines.append(f"> {rdma_legacy_note}\n")
port_checks = rdma.get("port_checks", [])
if port_checks:
lines.append("### RDMA Port Checks\n")
lines.append("| Device | Port | State | Rate | Required | Status |")
lines.append("|--------|------|-------|------|----------|--------|")
for p in port_checks:
lines.append(
f"| {p.get('device', '')} | {p.get('port', '')} | "
f"{p.get('state', '')} | {p.get('rate', '')} | "
f">= {p.get('min_rate_gbps', 400):.0f}Gbps ACTIVE | {p.get('status', '?')} |"
)
lines.append("")
bw_tests = rdma.get("bandwidth_tests", []) bw_tests = rdma.get("bandwidth_tests", [])
lat_tests = rdma.get("latency_tests", []) lat_tests = rdma.get("latency_tests", [])
if bw_tests or lat_tests: ibping_tests = rdma.get("ibping_tests", [])
if bw_tests or lat_tests or ibping_tests:
lines.append("| Test | Value | Threshold | Status |") lines.append("| Test | Value | Threshold | Status |")
lines.append("|------|-------|-----------|--------|") lines.append("|------|-------|-----------|--------|")
for bt in bw_tests: for bt in bw_tests:
if not bt.get("error"): if bt.get("error"):
lines.append(f"| {bt.get('test', 'ib_bw')} | {bt.get('error')} | required runnable test | {bt.get('status', 'FAIL')} |")
else:
threshold, status = self._rdma_bandwidth_verdict(bt)
lines.append(f"| {bt['test']} | {bt.get('bandwidth_gbps', 0):.1f} GB/s | " lines.append(f"| {bt['test']} | {bt.get('bandwidth_gbps', 0):.1f} GB/s | "
f">= {bt.get('min_required_gbps', 0)} GB/s | {bt.get('status', '?')} |") f">= {threshold:g} GB/s | {status} |")
for lt in lat_tests: for lt in lat_tests:
if not lt.get("error"): if lt.get("error"):
lines.append(f"| {lt.get('test', 'ib_lat')} | {lt.get('error')} | required runnable test | {lt.get('status', 'FAIL')} |")
else:
threshold, status = self._rdma_latency_verdict(lt)
lines.append(f"| {lt['test']} | {lt.get('latency_us', 0):.2f} us | " lines.append(f"| {lt['test']} | {lt.get('latency_us', 0):.2f} us | "
f"<= {lt.get('max_allowed_us', 0)} us | {lt.get('status', '?')} |") f"<= {threshold:g} us | {status} |")
for it in ibping_tests:
direction = it.get("direction") or it.get("role", "N/A")
if it.get("error"):
lines.append(f"| {it.get('test', 'ibping')} | {it.get('error')} | bidirectional peer evidence | {it.get('status', 'FAIL')} |")
else:
lines.append(f"| {it['test']} | {direction} target={it.get('target', 'N/A')} count={it.get('count', 'N/A')} | "
f"0% packet loss | {it.get('status', '?')} |")
lines.append("") lines.append("")
fabric = rdma.get("fabric_counters") or {}
if fabric:
counters = fabric.get("counters", {})
lines.append(f"- **PFC/ECN/CNP/congestion counters checked:** {len(counters)}")
lines.append(f"- **PFC/ECN/CNP/congestion non-zero:** {'yes' if fabric.get('failed') else 'no'}")
if not counters:
lines.append("- **PFC/ECN/CNP/congestion evidence:** missing")
failures = rdma.get("failures") or []
if not failures:
failures = self._rdma_failure_reasons(rdma)
if failures:
lines.append("- **Failure reasons:**")
for reason in failures:
lines.append(f" - {reason}")
passed = rdma.get("passed", False) passed = rdma.get("passed", False)
lines.append(f"**Overall: {'PASS' if passed else 'FAIL'}**\n") lines.append(f"**Overall: {'PASS' if passed else 'FAIL'}**\n")
# --- Training --- # --- Training ---
training = results.get("training") training = results.get("training")
if training and not training.get("error"): if training and not training.get("error"):
training_status, training_detail, training_missing = self._training_verdict(training)
lines.append("## Training Simulation\n") lines.append("## Training Simulation\n")
lines.append("| Metric | Value |") lines.append("| Metric | Value |")
lines.append("|--------|-------|") lines.append("|--------|-------|")
@ -521,12 +638,15 @@ class ReportGenerator:
lines.append(f"| Params | {training.get('total_params_m', 0):.1f}M |") lines.append(f"| Params | {training.get('total_params_m', 0):.1f}M |")
lines.append(f"| Throughput | {training.get('throughput_tokens_per_sec', 0):.0f} tokens/sec |") lines.append(f"| Throughput | {training.get('throughput_tokens_per_sec', 0):.0f} tokens/sec |")
lines.append(f"| Avg Step Time | {training.get('avg_step_time_ms', 0):.1f} ms |") lines.append(f"| Avg Step Time | {training.get('avg_step_time_ms', 0):.1f} ms |")
lines.append(f"| Warmup Steps | {training.get('warmup_steps', 'N/A')} |")
lines.append(f"| Peak Memory | {training.get('peak_memory_gb', 0):.1f} GB |") lines.append(f"| Peak Memory | {training.get('peak_memory_gb', 0):.1f} GB |")
lines.append(f"| Final Loss | {training.get('final_loss', 'N/A')} |") lines.append(f"| Final Loss | {training.get('final_loss', 'N/A')} |")
lines.append(f"| GPU Scope | {training.get('gpu_scope', 'framework-defined')} |") lines.append(f"| Step Jitter | {training.get('step_jitter_pct', 'N/A')}% |")
lines.append(f"| SDPA Policy | {training.get('sdpa_backend_policy', 'framework_default')} |") lines.append(f"| Distributed Mode | {training.get('distributed_mode', 'N/A')} |")
if training_missing:
lines.append(f"| Acceptance Gaps | missing {', '.join(training_missing)} |")
lines.append(f"| Verdict | {training_status} ({training_detail}) |")
lines.append("") lines.append("")
lines.append("**Verdict: MEASURED** (synthetic workload; not an 8-GPU scaling acceptance test)\n")
# --- Footer --- # --- Footer ---
lines.append("---") lines.append("---")
@ -560,6 +680,114 @@ class ReportGenerator:
return bench["compute"] return bench["compute"]
return {} return {}
@staticmethod
def _training_verdict(training: dict) -> tuple[str, str, list[str]]:
"""Return report status for both current and legacy training result schemas."""
tps = float(training.get("throughput_tokens_per_sec", 0) or 0)
if "passed" in training:
status = "PASS" if training.get("passed") else "FAIL"
return status, f"{tps:.0f} tokens/sec", []
required = ["passed", "step_jitter_pct", "distributed_mode", "loss_finite"]
missing = [k for k in required if k not in training]
return "UNVERIFIED", f"{tps:.0f} tokens/sec; legacy result lacks explicit acceptance verdict", missing
def _rdma_cfg_value(self, key: str, default: float) -> float:
try:
return float((self.config.get("rdma", {}) or {}).get(key, default))
except (TypeError, ValueError):
return default
def _rdma_bandwidth_verdict(self, row: dict) -> tuple[float, str]:
threshold = self._rdma_cfg_value("min_bandwidth_gbps", 47.0)
value = float(row.get("bandwidth_gbps", 0) or 0)
return threshold, "PASS" if value >= threshold else "FAIL"
def _rdma_latency_verdict(self, row: dict) -> tuple[float, str]:
name = row.get("test", "")
if name == "ib_write_lat":
threshold = self._rdma_cfg_value("max_write_latency_us", 2.0)
elif name == "ib_read_lat":
threshold = self._rdma_cfg_value("max_read_latency_us", 3.5)
else:
threshold = self._rdma_cfg_value("max_latency_us", 3.5)
value = float(row.get("latency_us", 0) or 0)
return threshold, "PASS" if 0 < value <= threshold else "FAIL"
def _rdma_legacy_note(self, rdma: dict) -> str:
"""Flag old RDMA result schemas whose embedded thresholds were looser."""
for row in rdma.get("bandwidth_tests", []) or []:
if row.get("min_required_gbps") != self._rdma_cfg_value("min_bandwidth_gbps", 47.0):
return (
"Legacy RDMA result re-evaluated with current PDF acceptance thresholds; "
"old WARN statuses and old 50GB/s/10us limits are not used for verdict."
)
for row in rdma.get("latency_tests", []) or []:
threshold, _ = self._rdma_latency_verdict(row)
if row.get("max_allowed_us") != threshold:
return (
"Legacy RDMA result re-evaluated with current PDF acceptance thresholds; "
"old WARN statuses and old 50GB/s/10us limits are not used for verdict."
)
return ""
def _rdma_failure_reasons(self, rdma: dict) -> list[str]:
failures = []
for row in rdma.get("bandwidth_tests", []) or []:
threshold, status = self._rdma_bandwidth_verdict(row)
if status != "PASS":
failures.append(
f"{row.get('test')} bandwidth {row.get('bandwidth_gbps', 0)}GB/s < {threshold:g}GB/s"
)
for row in rdma.get("latency_tests", []) or []:
threshold, status = self._rdma_latency_verdict(row)
if status != "PASS":
failures.append(
f"{row.get('test')} latency {row.get('latency_us', 0)}us > {threshold:g}us"
)
for row in rdma.get("ibping_tests", []) or []:
if row.get("status") != "PASS":
failures.append(f"{row.get('test')} failed")
return failures
@staticmethod
def _overall_acceptance_verdict(summary_items: list[tuple[str, str]]) -> tuple[str, list[tuple[str, str]], list[str]]:
"""PDF-style verdict for the report scope.
Full-suite reports require every single-node acceptance item. Standalone
reports, such as `--test multinode-nccl`, should only judge the items
that were actually requested instead of reporting unrelated evidence as
missing.
"""
single_node_required = [
"GPU Info",
"Health Check",
"Memory Bandwidth",
"Compute Throughput",
"NVLink/NVSwitch",
"NCCL",
"Stress Test",
"RDMA",
"DCGM",
"Training",
]
status_by_name = dict(summary_items)
present_single_node = [name for name in single_node_required if name in status_by_name]
if len(present_single_node) >= 3:
required = list(single_node_required)
if "Multi-node NCCL" in status_by_name:
required.append("Multi-node NCCL")
else:
required = list(status_by_name)
missing = [name for name in required if name not in status_by_name]
failures = [
(name, status)
for name, status in summary_items
if name in required and not str(status).startswith("PASS")
]
verdict = "PASS" if not missing and not failures else "FAIL"
return verdict, failures, missing
def _build_summary(self, results: dict) -> list[tuple[str, str]]: def _build_summary(self, results: dict) -> list[tuple[str, str]]:
"""Build summary verdict list from results.""" """Build summary verdict list from results."""
items = [] items = []
@ -586,15 +814,15 @@ class ReportGenerator:
mem = self._extract_memory_results(results) mem = self._extract_memory_results(results)
if mem: if mem:
if mem.get("error"): if mem.get("error"):
items.append(("GPU Interconnect / PCIe", f"ERROR: {mem['error']}")) items.append(("Memory Bandwidth", f"ERROR: {mem['error']}"))
elif mem.get("source") == "pytorch": elif mem.get("source") == "pytorch":
# PyTorch fallback can't reach HBM peak — report as WARN, not FAIL. # PyTorch fallback can't reach HBM peak — report as WARN, not FAIL.
d2d = mem.get("d2d_bandwidth_gbps") or 0 d2d = mem.get("d2d_bandwidth_gbps") or 0
items.append(("GPU Interconnect / PCIe", f"WARN ({d2d:.0f} GB/s via PyTorch fallback)")) items.append(("Memory Bandwidth", f"WARN ({d2d:.0f} GB/s via PyTorch fallback)"))
else: else:
eff = mem.get("efficiency_pct") or 0 eff = mem.get("d2d_efficiency_pct") or mem.get("efficiency_pct") or 0
verdict = "PASS" if eff >= 80 else ("WARN" if eff >= 60 else "FAIL") verdict = "PASS" if eff >= 80 else ("WARN" if eff >= 60 else "FAIL")
items.append(("GPU Interconnect / PCIe", f"{verdict} ({eff:.1f}% unidirectional D2D)")) items.append(("Memory Bandwidth", f"{verdict} ({eff:.1f}%)"))
# Compute # Compute
comp = self._extract_compute_results(results) comp = self._extract_compute_results(results)
@ -603,7 +831,6 @@ class ReportGenerator:
items.append(("Compute Throughput", f"ERROR: {comp['error']}")) items.append(("Compute Throughput", f"ERROR: {comp['error']}"))
else: else:
per_dtype = comp.get("per_dtype_tflops", {}) per_dtype = comp.get("per_dtype_tflops", {})
worst_per_dtype = comp.get("worst_per_dtype_tflops", {}) or {}
eff_pct = comp.get("efficiency_pct", {}) eff_pct = comp.get("efficiency_pct", {})
pass_thresholds = comp.get("pass_thresholds_tflops", {}) or {} pass_thresholds = comp.get("pass_thresholds_tflops", {}) or {}
if pass_thresholds: if pass_thresholds:
@ -611,27 +838,43 @@ class ReportGenerator:
rank = {"PASS": 0, "WARN": 1, "FAIL": 2} rank = {"PASS": 0, "WARN": 1, "FAIL": 2}
worst_status = "PASS" worst_status = "PASS"
worst_dt = None worst_dt = None
lowest_margin = None
for dt, thr in pass_thresholds.items(): for dt, thr in pass_thresholds.items():
avg_val = per_dtype.get(dt) val = per_dtype.get(dt)
val = worst_per_dtype.get(dt, avg_val)
if not isinstance(val, (int, float)): if not isinstance(val, (int, float)):
continue continue
if val >= thr: if val >= thr:
st = "PASS" st = "PASS"
elif val >= thr * 0.9:
st = "WARN"
else: else:
st = "FAIL" st = "FAIL"
margin = val / thr if thr else 0
if lowest_margin is None or margin < lowest_margin:
lowest_margin = margin
worst_dt = dt
if rank[st] > rank[worst_status]: if rank[st] > rank[worst_status]:
worst_status = st worst_status = st
worst_dt = dt
if worst_dt: if worst_dt:
items.append(( consistency = comp.get("consistency", {}) or {}
"Compute Throughput", failed_consistency = [
f"{worst_status} (worst {worst_dt.upper()} " (dt, row)
f"min {worst_per_dtype.get(worst_dt, per_dtype[worst_dt]):.0f} " for dt, row in consistency.items()
f"vs >= {pass_thresholds[worst_dt]})" if not row.get("passed", False)
)) ]
if failed_consistency:
worst_status = "FAIL"
fail_dt, fail_row = failed_consistency[0]
items.append((
"Compute Throughput",
f"FAIL ({fail_dt.upper()} spread "
f"{fail_row.get('spread_pct', 0):.2f}% > "
f"{fail_row.get('max_allowed_pct', 3)}%)"
))
else:
items.append((
"Compute Throughput",
f"{worst_status} (worst {worst_dt.upper()} "
f"{per_dtype[worst_dt]:.0f} vs >= {pass_thresholds[worst_dt]})"
))
else: else:
items.append(("Compute Throughput", f"{worst_status}")) items.append(("Compute Throughput", f"{worst_status}"))
else: else:
@ -641,24 +884,44 @@ class ReportGenerator:
verdict = "PASS" if worst >= 80 else ("WARN" if worst >= 50 else "FAIL") verdict = "PASS" if worst >= 80 else ("WARN" if worst >= 50 else "FAIL")
items.append(("Compute Throughput", f"{verdict} (worst {worst:.1f}%)")) items.append(("Compute Throughput", f"{verdict} (worst {worst:.1f}%)"))
else: else:
measured = [v for v in per_dtype.values() if isinstance(v, (int, float))] items.append(("Compute Throughput", "N/A"))
items.append(("Compute Throughput", "MEASURED (no published threshold)" if measured else "N/A"))
# NCCL
if "nvlink" in results:
nvl = results["nvlink"]
if nvl.get("error"):
items.append(("NVLink/NVSwitch", f"ERROR: {nvl['error']}"))
elif nvl.get("passed"):
items.append(("NVLink/NVSwitch", "PASS"))
else:
items.append(("NVLink/NVSwitch", "FAIL"))
if "dcgm" in results:
d = results["dcgm"]
if d.get("error"):
items.append(("DCGM", f"ERROR: {d['error']}"))
elif d.get("passed"):
items.append(("DCGM", "PASS"))
else:
items.append(("DCGM", "FAIL"))
# NCCL # NCCL
if "nccl" in results: if "nccl" in results:
n = results["nccl"] n = results["nccl"]
if n.get("error"): if n.get("error"):
items.append(("NCCL", f"ERROR: {n['error']}")) items.append(("NCCL", f"ERROR: {n['error']}"))
else: elif n.get("source") == "torchrun_fallback":
status = n.get("status") or ("PASS" if n.get("passed") else "FAIL") items.append(("NCCL", "FAIL (no nccl-tests bus BW)"))
items.append(("NCCL", status))
# Multi-node NCCL
if "multinode_nccl" in results:
n = results["multinode_nccl"]
if n.get("error"):
items.append(("Multi-node NCCL", f"ERROR: {n['error']}"))
elif n.get("passed"): elif n.get("passed"):
items.append(("NCCL", "PASS"))
else:
items.append(("NCCL", "FAIL"))
if "multinode_nccl" in results:
mn = results["multinode_nccl"]
if mn.get("error"):
items.append(("Multi-node NCCL", f"ERROR: {mn['error']}"))
elif mn.get("passed"):
items.append(("Multi-node NCCL", "PASS")) items.append(("Multi-node NCCL", "PASS"))
else: else:
items.append(("Multi-node NCCL", "FAIL")) items.append(("Multi-node NCCL", "FAIL"))
@ -691,8 +954,17 @@ class ReportGenerator:
if t.get("error"): if t.get("error"):
items.append(("Training", f"ERROR: {t['error']}")) items.append(("Training", f"ERROR: {t['error']}"))
else: else:
tps = t.get("throughput_tokens_per_sec", 0) status, detail, _missing = self._training_verdict(t)
scope = t.get("gpu_scope", "framework-defined") items.append(("Training", f"{status} ({detail})"))
items.append(("Training", f"MEASURED ({tps:.0f} tokens/sec; {scope})"))
return items return items
def _format_gbps(value) -> str:
try:
numeric = float(value)
except (TypeError, ValueError):
return str(value)
if numeric.is_integer():
return f"{numeric:.0f}"
return f"{numeric:.2f}"

View File

@ -1,10 +1,10 @@
"""GPU stress test module — wraps gpu-burn for long-running stability tests.""" """GPU stress test module — gpu-burn or PyTorch GEMM with telemetry."""
import glob import glob
import os import os
import re
import shutil import shutil
import subprocess import subprocess
import threading
import time import time
from datetime import datetime from datetime import datetime
@ -14,7 +14,6 @@ from rich.live import Live
from rich.text import Text from rich.text import Text
from modules.gpu_specs import resolve_tools_dir from modules.gpu_specs import resolve_tools_dir
from modules.hygon_smi import run_hy_smi
class StressTest: class StressTest:
@ -48,7 +47,7 @@ class StressTest:
memory_pct = cfg.get("memory_pct", 90) memory_pct = cfg.get("memory_pct", 90)
target_gpus = cfg.get("gpus", "all") target_gpus = cfg.get("gpus", "all")
gpu_burn = self._find_gpu_burn() gpu_burn = self._find_gpu_burn() if cfg.get("use_gpu_burn", False) else ""
if gpu_burn: if gpu_burn:
# Try gpu-burn first # Try gpu-burn first
@ -58,12 +57,12 @@ class StressTest:
if not result.get("passed") and result.get("elapsed_sec", 0) < duration_sec * 0.5: if not result.get("passed") and result.get("elapsed_sec", 0) < duration_sec * 0.5:
self.console.print("\n[yellow]gpu-burn exited early (possible OOM), switching to PyTorch stress test[/yellow]") self.console.print("\n[yellow]gpu-burn exited early (possible OOM), switching to PyTorch stress test[/yellow]")
self.console.print("[dim]PyTorch mode dynamically adapts to available memory[/dim]\n") self.console.print("[dim]PyTorch mode dynamically adapts to available memory[/dim]\n")
return self._run_pytorch_stress(duration_sec, memory_pct, use_tensor_cores) return self._run_pytorch_stress(duration_sec, memory_pct)
return result return result
self.console.print("[yellow]gpu_burn not found, using PyTorch stress test[/yellow]") self.console.print("[yellow]Using PyTorch stress test[/yellow]")
return self._run_pytorch_stress(duration_sec, memory_pct, use_tensor_cores) return self._run_pytorch_stress(duration_sec, memory_pct)
def _run_gpu_burn(self, gpu_burn: str, duration: int, def _run_gpu_burn(self, gpu_burn: str, duration: int,
doubles: bool, tensor_cores: bool, target_gpus: str) -> dict: doubles: bool, tensor_cores: bool, target_gpus: str) -> dict:
@ -79,12 +78,26 @@ class StressTest:
cmd.append(str(duration)) cmd.append(str(duration))
t0 = time.time() t0 = time.time()
xid_before = self._collect_xid_events()
interval = int(self.stress_cfg.get("telemetry_interval_sec", 1))
telemetry = []
stop_sampling = threading.Event()
sampler = threading.Thread(
target=self._sample_telemetry,
args=(telemetry, stop_sampling, interval),
daemon=True,
)
sampler.start()
try: try:
r = subprocess.run(cmd, capture_output=True, text=True, timeout=duration + 120) r = subprocess.run(cmd, capture_output=True, text=True, timeout=duration + 120)
elapsed = round(time.time() - t0, 1) elapsed = round(time.time() - t0, 1)
stop_sampling.set()
sampler.join(timeout=interval + 1)
output = r.stdout + r.stderr output = r.stdout + r.stderr
passed = r.returncode == 0 xid_events = self._new_xid_events(xid_before, self._collect_xid_events())
telemetry_summary = self._evaluate_telemetry(telemetry, [], xid_events)
passed = r.returncode == 0 and telemetry_summary.get("passed", False)
gpu_results = [] gpu_results = []
for line in output.split("\n"): for line in output.split("\n"):
@ -98,70 +111,40 @@ class StressTest:
"duration_sec": duration, "duration_sec": duration,
"elapsed_sec": elapsed, "elapsed_sec": elapsed,
"gpu_results": gpu_results, "gpu_results": gpu_results,
"telemetry": telemetry_summary,
"raw_output_tail": output[-500:] if output else "", "raw_output_tail": output[-500:] if output else "",
"timestamp": datetime.now().isoformat(), "timestamp": datetime.now().isoformat(),
} }
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
stop_sampling.set()
return { return {
"source": "gpu-burn", "source": "gpu-burn",
"passed": False, "passed": False,
"duration_sec": duration, "duration_sec": duration,
"error": "timeout", "error": "timeout",
"telemetry": self._evaluate_telemetry(
telemetry, [], self._new_xid_events(xid_before, self._collect_xid_events())
),
"timestamp": datetime.now().isoformat(), "timestamp": datetime.now().isoformat(),
} }
except Exception as e: except Exception as e:
stop_sampling.set()
return { return {
"source": "gpu-burn", "source": "gpu-burn",
"passed": False, "passed": False,
"error": str(e), "error": str(e),
"telemetry": self._evaluate_telemetry(
telemetry, [], self._new_xid_events(xid_before, self._collect_xid_events())
),
"timestamp": datetime.now().isoformat(), "timestamp": datetime.now().isoformat(),
} }
finally:
stop_sampling.set()
@staticmethod def _run_pytorch_stress(self, duration: int, memory_pct: int = 90) -> dict:
def _accelerator_snapshot() -> dict:
if shutil.which("nvidia-smi"):
try:
r = subprocess.run(
["nvidia-smi", "--query-gpu=index,temperature.gpu,power.draw,"
"ecc.errors.uncorrected.volatile.total", "--format=csv,noheader,nounits"],
capture_output=True, text=True, timeout=15,
)
temps, powers, ecc = {}, {}, {}
if r.returncode == 0:
for line in r.stdout.splitlines():
parts = [part.strip() for part in line.split(",")]
if len(parts) < 4:
continue
idx = int(parts[0])
temps[idx] = float(parts[1])
powers[idx] = float(parts[2])
ecc[idx] = int(parts[3]) if parts[3].isdigit() else 0
return {
"temperatures_c": temps, "power_w": powers,
"uncorrected_ecc": ecc,
}
except (OSError, ValueError, subprocess.SubprocessError):
pass
output = run_hy_smi()
temps, powers = {}, {}
if output:
pattern = re.compile(r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W", re.MULTILINE)
for match in pattern.finditer(output):
idx = int(match.group(1))
temps[idx] = float(match.group(2))
powers[idx] = float(match.group(3))
return {"temperatures_c": temps, "power_w": powers, "uncorrected_ecc": {}}
def _run_pytorch_stress(self, duration: int, memory_pct: int = 90,
tensor_cores: bool = True) -> dict:
try: try:
import torch import torch
try:
import torch_dcu # noqa: F401
except ImportError:
pass
if not torch.cuda.is_available(): if not torch.cuda.is_available():
return {"error": "pytorch_not_available"} return {"error": "pytorch_not_available"}
except ImportError: except ImportError:
@ -170,71 +153,81 @@ class StressTest:
gpu_count = torch.cuda.device_count() gpu_count = torch.cuda.device_count()
self.console.print(f"[cyan]PyTorch Stress Test ({duration}s, {gpu_count} GPUs, target {memory_pct}% memory)[/cyan]") self.console.print(f"[cyan]PyTorch Stress Test ({duration}s, {gpu_count} GPUs, target {memory_pct}% memory)[/cyan]")
dtype_name = self.stress_cfg.get("dtype", "bf16")
matrix_size = int(self.stress_cfg.get("matrix_size", 8192))
interval = int(self.stress_cfg.get("telemetry_interval_sec", 1))
dtype_map = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}
dtype = dtype_map.get(dtype_name, torch.bfloat16)
gpu_status = {} gpu_status = {}
telemetry = []
stop_sampling = threading.Event()
t0 = time.time() t0 = time.time()
snapshots = [self._accelerator_snapshot()] xid_before = self._collect_xid_events()
try: try:
work_a, work_b, outputs, ballast = {}, {}, {}, {} sampler = threading.Thread(
target=self._sample_telemetry,
args=(telemetry, stop_sampling, interval),
daemon=True,
)
sampler.start()
tensors = {}
ballast = {}
pass_tflops = []
for i in range(gpu_count): for i in range(gpu_count):
with torch.cuda.device(i): with torch.cuda.device(i):
# Get actual free memory (accounting for other processes)
free_mem, total_mem = torch.cuda.mem_get_info(i) free_mem, total_mem = torch.cuda.mem_get_info(i)
side = matrix_size
# Calculate allocation from configured memory_pct elem = torch.tensor([], dtype=dtype).element_size()
target_mem = int(total_mem * memory_pct / 100) compute_bytes = side * side * elem * 3
target_mem = min(int(total_mem * memory_pct / 100), int(free_mem * 0.90))
# Cap at actual free memory with 5% safety margin ballast_bytes = max(0, target_mem - compute_bytes)
alloc_bytes = min(target_mem, int(free_mem * 0.95)) if ballast_bytes:
ballast_elems = ballast_bytes // 2
dtype = torch.bfloat16 if tensor_cores else torch.float32 ballast[i] = torch.empty(ballast_elems, device=f"cuda:{i}", dtype=torch.float16)
elem_size = torch.tensor([], dtype=dtype).element_size() actual_mem_mb = (compute_bytes + ballast_bytes) / 1024 / 1024
mem_side = int((alloc_bytes / elem_size / 3) ** 0.5)
max_compute_side = 8192 if tensor_cores else 4096
side = min(mem_side, max_compute_side)
compute_bytes = side * side * elem_size * 3
ballast_target = max(0, alloc_bytes - compute_bytes - 1024**3)
ballast[i] = []
chunk_size = 1024**3
while ballast_target > 0:
this_chunk = min(chunk_size, ballast_target)
tensor = torch.empty(this_chunk, device=f"cuda:{i}", dtype=torch.uint8)
tensor.fill_(0x5A)
ballast[i].append(tensor)
ballast_target -= this_chunk
actual_mem_mb = (alloc_bytes - max(0, ballast_target)) / 1024 / 1024
total_mem_mb = total_mem / 1024 / 1024 total_mem_mb = total_mem / 1024 / 1024
free_mem_mb = free_mem / 1024 / 1024 free_mem_mb = free_mem / 1024 / 1024
self.console.print( self.console.print(
f" [dim]GPU {i}: total {total_mem_mb:.0f}MB, free {free_mem_mb:.0f}MB, " f" [dim]GPU {i}: total {total_mem_mb:.0f}MB, free {free_mem_mb:.0f}MB, "
f"alloc {actual_mem_mb:.0f}MB ({actual_mem_mb/total_mem_mb*100:.0f}%) - " f"alloc {actual_mem_mb:.0f}MB ({actual_mem_mb/total_mem_mb*100:.0f}%) - "
f"matrix {side}x{side}[/dim]" f"{dtype_name} matrix {side}x{side}[/dim]"
)
tensors[i] = (
torch.randn(side, side, device=f"cuda:{i}", dtype=dtype),
torch.randn(side, side, device=f"cuda:{i}", dtype=dtype),
torch.empty(side, side, device=f"cuda:{i}", dtype=dtype),
) )
work_a[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=dtype)
work_b[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=dtype)
outputs[i] = torch.empty(side, side, device=f"cuda:{i}", dtype=dtype)
self.console.print(f"\n[cyan]Starting stress test for {duration} seconds...[/cyan]") self.console.print(f"\n[cyan]Starting stress test for {duration} seconds...[/cyan]")
elapsed_check = 0 elapsed_check = 0
while time.time() - t0 < duration: while time.time() - t0 < duration:
loop_start = time.perf_counter()
# Dispatch matmul on all GPUs in parallel — do NOT synchronize between # Dispatch matmul on all GPUs in parallel — do NOT synchronize between
# GPUs, otherwise the 8 GPUs run serially and overshoot the duration. # GPUs, otherwise the 8 GPUs run serially and overshoot the duration.
for i in range(gpu_count): for i in range(gpu_count):
with torch.cuda.device(i): with torch.cuda.device(i):
outputs[i] = torch.matmul(work_a[i], work_b[i]) a, b, out = tensors[i]
torch.matmul(a, b, out=out)
# Single sync per pass — waits for all 8 streams concurrently # Single sync per pass — waits for all 8 streams concurrently
for i in range(gpu_count): for i in range(gpu_count):
with torch.cuda.device(i): with torch.cuda.device(i):
torch.cuda.synchronize() torch.cuda.synchronize()
loop_elapsed = time.perf_counter() - loop_start
current_elapsed = time.time() - t0
if loop_elapsed > 0:
flops = gpu_count * 2 * (matrix_size ** 3)
pass_tflops.append({
"elapsed_sec": current_elapsed,
"tflops": flops / loop_elapsed / 1e12,
})
# Show progress every 10 seconds # Show progress every 10 seconds
current_elapsed = time.time() - t0
if int(current_elapsed) != int(elapsed_check) and int(current_elapsed) % 10 == 0: if int(current_elapsed) != int(elapsed_check) and int(current_elapsed) % 10 == 0:
self.console.print(f" [dim]Running {int(current_elapsed)}s / {duration}s[/dim]") self.console.print(f" [dim]Running {int(current_elapsed)}s / {duration}s[/dim]")
snapshots.append(self._accelerator_snapshot())
elapsed_check = current_elapsed elapsed_check = current_elapsed
for i in range(gpu_count): for i in range(gpu_count):
@ -252,47 +245,196 @@ class StressTest:
"duration_sec": duration, "duration_sec": duration,
"error": error_msg, "error": error_msg,
"gpu_status": gpu_status, "gpu_status": gpu_status,
"telemetry": self._evaluate_telemetry(
telemetry, pass_tflops if "pass_tflops" in locals() else [],
self._new_xid_events(xid_before, self._collect_xid_events()),
),
} }
finally: finally:
for mapping in (work_a, work_b, outputs, ballast): stop_sampling.set()
mapping.clear() tensors.clear()
ballast.clear()
torch.cuda.empty_cache() torch.cuda.empty_cache()
elapsed = round(time.time() - t0, 1) elapsed = round(time.time() - t0, 1)
snapshots.append(self._accelerator_snapshot()) xid_events = self._new_xid_events(xid_before, self._collect_xid_events())
max_temp = {} telemetry_summary = self._evaluate_telemetry(telemetry, pass_tflops, xid_events)
max_power = {} passed = all(v == "PASS" for v in gpu_status.values()) and telemetry_summary.get("passed", False)
for snapshot in snapshots:
for idx, value in snapshot.get("temperatures_c", {}).items():
max_temp[idx] = max(max_temp.get(idx, value), value)
for idx, value in snapshot.get("power_w", {}).items():
max_power[idx] = max(max_power.get(idx, value), value)
first_ecc = snapshots[0].get("uncorrected_ecc", {}) if snapshots else {}
last_ecc = snapshots[-1].get("uncorrected_ecc", {}) if snapshots else {}
ecc_delta = {
idx: max(0, last_ecc.get(idx, 0) - first_ecc.get(idx, 0))
for idx in set(first_ecc) | set(last_ecc)
}
passed = not any(ecc_delta.values())
if not passed:
for idx, delta in ecc_delta.items():
if delta:
gpu_status[idx] = "FAIL"
return { return {
"source": "pytorch", "source": "pytorch",
"passed": passed, "passed": passed,
"duration_sec": duration, "duration_sec": duration,
"elapsed_sec": elapsed, "elapsed_sec": elapsed,
"gpu_status": gpu_status, "gpu_status": gpu_status,
"memory_pct": memory_pct, "telemetry": telemetry_summary,
"max_temperature_c": max_temp,
"max_power_w": max_power,
"uncorrected_ecc_delta": ecc_delta,
"numerical_validation": False,
"validation_scope": "runtime stability, allocation, temperature, power, and ECC delta",
"timestamp": datetime.now().isoformat(), "timestamp": datetime.now().isoformat(),
} }
def _sample_telemetry(self, telemetry: list, stop_event: threading.Event, interval: int):
query = "index,temperature.gpu,power.draw,clocks_throttle_reasons.active"
while not stop_event.is_set():
try:
r = subprocess.run(
["nvidia-smi", f"--query-gpu={query}", "--format=csv,noheader,nounits"],
capture_output=True, text=True, timeout=10,
)
if r.returncode == 0:
sample = {"time": time.time(), "gpus": []}
for line in r.stdout.splitlines():
parts = [p.strip() for p in line.split(",")]
if len(parts) >= 4:
sample["gpus"].append({
"index": int(parts[0]),
"temp_c": float(parts[1]),
"power_w": float(parts[2]),
"throttle": parts[3],
})
telemetry.append(sample)
except Exception:
pass
stop_event.wait(interval)
def _collect_xid_events(self) -> list[str]:
try:
r = subprocess.run(
["dmesg", "--color=never"],
capture_output=True, text=True, timeout=10,
)
if r.returncode != 0:
return []
return [
line.strip()
for line in r.stdout.splitlines()
if any(token in line.upper() for token in ("XID", "NVRM: XID"))
]
except Exception:
return []
@staticmethod
def _new_xid_events(before: list[str], after: list[str]) -> list[str]:
seen = set(before)
return [line for line in after if line not in seen]
def _evaluate_telemetry(self, telemetry: list, pass_tflops: list, xid_events: list[str] | None = None) -> dict:
cfg = self.stress_cfg
max_temp = float(cfg.get("max_temp_c", 80))
max_delta = float(cfg.get("max_temp_delta_c", 5))
min_power = float(cfg.get("min_power_watts", 630))
max_jitter = float(cfg.get("max_tflops_jitter_pct", 5))
require_jitter = bool(cfg.get("require_tflops_jitter", True))
duration = float(cfg.get("duration_sec", 60))
requested_warmup = float(cfg.get("warmup_sec", 60))
warmup_sec = min(requested_warmup, max(0.0, duration * 0.2))
min_steady_samples = int(cfg.get("min_steady_samples", 10))
temps = {}
powers = {}
throttle_bad = []
xid_events = xid_events or []
steady_telemetry = [
sample for sample in telemetry
if sample.get("time", 0) - telemetry[0].get("time", 0) >= warmup_sec
] if telemetry else []
evaluation_samples = steady_telemetry if len(steady_telemetry) >= min_steady_samples else telemetry
for sample in evaluation_samples:
for g in sample.get("gpus", []):
idx = g["index"]
temps.setdefault(idx, []).append(g["temp_c"])
powers.setdefault(idx, []).append(g["power_w"])
try:
bitmask = int(str(g["throttle"]), 16)
except ValueError:
bitmask = 0
real_throttle = bitmask & ~0x1
if real_throttle:
throttle_bad.append({
"gpu": idx,
"throttle": g["throttle"],
"real_throttle": f"0x{real_throttle:x}",
})
max_temps = {idx: max(vals) for idx, vals in temps.items() if vals}
avg_powers = {idx: sum(vals) / len(vals) for idx, vals in powers.items() if vals}
temp_delta = (max(max_temps.values()) - min(max_temps.values())) if len(max_temps) >= 2 else 0
jitter = 0
steady_tflops = []
for item in pass_tflops:
if isinstance(item, dict):
if float(item.get("elapsed_sec", 0)) >= warmup_sec:
steady_tflops.append(float(item.get("tflops", 0)))
else:
steady_tflops.append(float(item))
if len(steady_tflops) < 2 and pass_tflops:
steady_tflops = [
float(item.get("tflops", 0)) if isinstance(item, dict) else float(item)
for item in pass_tflops
]
if steady_tflops:
mean = sum(steady_tflops) / len(steady_tflops)
jitter = max(abs(v - mean) / mean * 100 for v in steady_tflops) if mean else 0
failures = []
temp_failures = {idx: v for idx, v in max_temps.items() if v > max_temp}
power_failures = {idx: v for idx, v in avg_powers.items() if v < min_power}
if not evaluation_samples:
failures.append("no telemetry samples available for evaluation")
if temp_failures:
failures.append(
"max temperature above threshold: "
+ ", ".join(f"GPU {idx} {val:.1f}C" for idx, val in sorted(temp_failures.items()))
)
if temp_delta > max_delta:
failures.append(f"GPU temperature delta {temp_delta:.1f}C exceeds {max_delta:.1f}C")
if power_failures:
failures.append(
"average steady-state power below threshold: "
+ ", ".join(f"GPU {idx} {val:.1f}W" for idx, val in sorted(power_failures.items()))
)
if throttle_bad:
failures.append(
f"non-idle throttle reasons observed in {len(throttle_bad)} samples "
f"(first: GPU {throttle_bad[0]['gpu']} {throttle_bad[0]['real_throttle']})"
)
if xid_events:
failures.append(f"{len(xid_events)} new XID/NVRM XID events observed")
if require_jitter and len(steady_tflops) < 2:
failures.append(
f"insufficient steady TFLOPS samples for jitter evaluation: {len(steady_tflops)} < 2"
)
if jitter > max_jitter:
failures.append(f"TFLOPS jitter {jitter:.2f}% exceeds {max_jitter:.2f}%")
passed = (
bool(evaluation_samples)
and all(v <= max_temp for v in max_temps.values())
and temp_delta <= max_delta
and all(v >= min_power for v in avg_powers.values())
and not throttle_bad
and not xid_events
and (not require_jitter or len(steady_tflops) >= 2)
and jitter <= max_jitter
)
return {
"passed": passed,
"samples": len(telemetry),
"steady_samples": len(evaluation_samples),
"warmup_sec": round(warmup_sec, 1),
"max_temp_c": {k: round(v, 1) for k, v in max_temps.items()},
"avg_power_w": {k: round(v, 1) for k, v in avg_powers.items()},
"temp_delta_c": round(temp_delta, 1),
"throttle_events": throttle_bad[:20],
"throttle_event_count": len(throttle_bad),
"xid_events": xid_events[-20:],
"tflops_jitter_pct": round(jitter, 2),
"steady_tflops_samples": len(steady_tflops),
"failures": failures,
"thresholds": {
"max_temp_c": max_temp,
"max_temp_delta_c": max_delta,
"min_power_w": min_power,
"max_tflops_jitter_pct": max_jitter,
"require_tflops_jitter": require_jitter,
"warmup_sec": requested_warmup,
"min_steady_samples": min_steady_samples,
},
}
@staticmethod @staticmethod
def print_results(results: dict, console: Console = None): def print_results(results: dict, console: Console = None):
c = console or Console() c = console or Console()
@ -325,15 +467,21 @@ class StressTest:
color = "green" if status == "PASS" else "red" color = "green" if status == "PASS" else "red"
c.print(f" GPU {gid}: [{color}]{status}[/{color}]") c.print(f" GPU {gid}: [{color}]{status}[/{color}]")
max_temp = results.get("max_temperature_c", {}) telemetry = results.get("telemetry") or {}
max_power = results.get("max_power_w", {}) if telemetry:
if max_temp or max_power: c.print("\n Telemetry:")
c.print("\n Peak telemetry:") c.print(f" Samples: {telemetry.get('samples', 0)} total, {telemetry.get('steady_samples', 0)} evaluated after {telemetry.get('warmup_sec', 0)}s warmup")
for gid in sorted(set(max_temp) | set(max_power)): c.print(f" Avg steady power: {telemetry.get('avg_power_w', {})}")
c.print( c.print(f" Max steady temp: {telemetry.get('max_temp_c', {})}")
f" GPU {gid}: {max_temp.get(gid, 'N/A')} C, " c.print(f" Temp delta: {telemetry.get('temp_delta_c', 'N/A')} C")
f"{max_power.get(gid, 'N/A')} W" c.print(f" TFLOPS jitter: {telemetry.get('tflops_jitter_pct', 'N/A')}%")
) c.print(f" Throttle events: {telemetry.get('throttle_event_count', len(telemetry.get('throttle_events', [])))}")
c.print(f" XID events: {len(telemetry.get('xid_events', []))}")
failures = telemetry.get("failures", [])
if failures:
c.print(" [red]Failure reasons:[/red]")
for reason in failures:
c.print(f" [red]- {reason}[/red]")
if results.get("error"): if results.get("error"):
c.print(f" [red]Error: {results['error']}[/red]") c.print(f" [red]Error: {results['error']}[/red]")

View File

@ -1,9 +1,13 @@
"""Training simulation module - LLM training workload with PyTorch.""" """Training simulation module - LLM training workload with PyTorch."""
import json
import os
import sys
import tempfile
import time import time
import subprocess import subprocess
import shutil import shutil
from contextlib import nullcontext import math
from datetime import datetime from datetime import datetime
from typing import Optional from typing import Optional
@ -14,10 +18,6 @@ from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeEl
TORCH_AVAILABLE = False TORCH_AVAILABLE = False
try: try:
import torch import torch
try:
import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim
except ImportError:
pass
if torch.cuda.is_available(): if torch.cuda.is_available():
TORCH_AVAILABLE = True TORCH_AVAILABLE = True
except ImportError: except ImportError:
@ -31,73 +31,6 @@ class TrainingSim:
self.console = Console() self.console = Console()
self.train_cfg = config.get("training", {}) self.train_cfg = config.get("training", {})
@staticmethod
def _requires_b300_sdpa_workaround(
configured_type: str, capability, cudnn_version: int | None,
) -> bool:
is_b300 = str(configured_type).lower() == "b300" or capability == (10, 3)
# cuDNN 9.20 restores SM103 plan generation and PyTorch 2.13 correctly
# rejects unsupported cuDNN shapes (for example dropout=0.1) before
# dispatch, allowing another SDPA backend to run. Keep the workaround
# only for the affected older stack.
return is_b300 and (not cudnn_version or cudnn_version < 92000)
def _sdpa_context(self):
"""Select a reliable SDPA backend policy for B300/SM103.
PyTorch 2.11 + cuDNN 9.19 can dispatch BF16 attention on B300 to the
cuDNN frontend even though that backend cannot build an execution plan.
Keep the workaround scoped to the training operation and retain ordered
Flash/Efficient/Math fallbacks instead of changing global process state.
"""
configured_type = str(self.config.get("gpu_type", "")).lower()
capability = None
cudnn_version = None
if torch.cuda.is_available():
try:
capability = tuple(torch.cuda.get_device_capability(0))
except Exception: # noqa: BLE001 - backend discovery must not abort training
capability = None
try:
cudnn_version = torch.backends.cudnn.version()
except Exception: # noqa: BLE001 - backend discovery must not abort training
cudnn_version = None
requires_workaround = self._requires_b300_sdpa_workaround(
configured_type, capability, cudnn_version,
)
if not requires_workaround:
if (configured_type == "b300" or capability == (10, 3)) and cudnn_version:
return nullcontext(), (
f"framework_default (cuDNN {cudnn_version}; "
"unsupported SDPA shapes may fall back automatically)"
)
return nullcontext(), "framework_default"
try:
from torch.nn.attention import SDPBackend, sdpa_kernel
context = sdpa_kernel(
[
SDPBackend.FLASH_ATTENTION,
SDPBackend.EFFICIENT_ATTENTION,
SDPBackend.MATH,
],
set_priority=True,
)
return context, "flash>efficient>math (cuDNN SDPA excluded on B300)"
except (ImportError, AttributeError, TypeError):
# Compatibility path for older PyTorch versions that do not expose
# the context-manager API. Restore the global flag when leaving.
class _DisableCudnnSDPA:
def __enter__(self_inner):
self_inner.old = torch.backends.cuda.cudnn_sdp_enabled()
torch.backends.cuda.enable_cudnn_sdp(False)
def __exit__(self_inner, exc_type, exc, traceback):
torch.backends.cuda.enable_cudnn_sdp(self_inner.old)
return _DisableCudnnSDPA(), "framework fallback (cuDNN SDPA disabled on B300)"
def run(self) -> dict: def run(self) -> dict:
if not TORCH_AVAILABLE: if not TORCH_AVAILABLE:
self.console.print("[yellow]PyTorch not available - skipping training simulation[/yellow]") self.console.print("[yellow]PyTorch not available - skipping training simulation[/yellow]")
@ -108,6 +41,7 @@ class TrainingSim:
batch_size = self.train_cfg.get("batch_size", 8) batch_size = self.train_cfg.get("batch_size", 8)
seq_length = self.train_cfg.get("seq_length", 2048) seq_length = self.train_cfg.get("seq_length", 2048)
num_steps = self.train_cfg.get("num_steps", 50) num_steps = self.train_cfg.get("num_steps", 50)
warmup_steps = int(self.train_cfg.get("warmup_steps", 5))
dtype_str = self.train_cfg.get("dtype", "bf16") dtype_str = self.train_cfg.get("dtype", "bf16")
dtype_map = { dtype_map = {
@ -119,10 +53,13 @@ class TrainingSim:
self.console.print(f"[cyan]Training Simulation[/cyan]") self.console.print(f"[cyan]Training Simulation[/cyan]")
self.console.print(f" Model: {model_name} | Batch: {batch_size} | Seq: {seq_length} | " self.console.print(f" Model: {model_name} | Batch: {batch_size} | Seq: {seq_length} | "
f"DType: {dtype_str} | Steps: {num_steps} | GPUs: {gpu_count}") f"DType: {dtype_str} | Steps: {num_steps} | Warmup: {warmup_steps} | GPUs: {gpu_count}")
if model_name == "synthetic": if self.train_cfg.get("mode", "ddp") == "ddp" and gpu_count > 1:
return self._run_synthetic(gpu_count, batch_size, seq_length, num_steps, dtype) ddp_result = self._run_synthetic_ddp(gpu_count, batch_size, seq_length, num_steps, dtype_str)
if ddp_result.get("passed") or not self.train_cfg.get("allow_fallback", False):
return ddp_result
self.console.print("[yellow]DDP synthetic training failed, falling back to single-process synthetic path[/yellow]")
try: try:
from transformers import AutoModelForCausalLM, AutoTokenizer from transformers import AutoModelForCausalLM, AutoTokenizer
@ -157,48 +94,53 @@ class TrainingSim:
step_times = [] step_times = []
mem_usage = [] mem_usage = []
sdpa_context, sdpa_policy = self._sdpa_context()
self.console.print(f" SDPA policy: {sdpa_policy}")
with Progress( with Progress(
SpinnerColumn(), TextColumn("[progress.description]{task.description}"), SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
BarColumn(), TextColumn("{task.completed}/{task.total}"), BarColumn(), TextColumn("{task.completed}/{task.total}"),
TimeElapsedColumn(), console=self.console, TimeElapsedColumn(), console=self.console,
) as progress: ) as progress:
task = progress.add_task("Training steps...", total=num_steps) total_steps = num_steps + warmup_steps
task = progress.add_task("Training steps...", total=total_steps)
with sdpa_context: for step in range(total_steps):
for step in range(num_steps): torch.cuda.synchronize()
torch.cuda.synchronize() t0 = time.perf_counter()
t0 = time.perf_counter()
input_ids = input_ids.to(model.device) input_ids = input_ids.to(model.device)
attention_mask = attention_mask.to(model.device) attention_mask = attention_mask.to(model.device)
if dtype in (torch.float16, torch.bfloat16): if dtype in (torch.float16, torch.bfloat16):
with torch.amp.autocast("cuda", dtype=dtype): with torch.amp.autocast("cuda", dtype=dtype):
outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
loss = outputs.loss
else:
outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids) outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
loss = outputs.loss loss = outputs.loss
else:
outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
loss = outputs.loss
loss.backward() loss.backward()
optimizer.step() optimizer.step()
optimizer.zero_grad() optimizer.zero_grad()
torch.cuda.synchronize() torch.cuda.synchronize()
elapsed = time.perf_counter() - t0 elapsed = time.perf_counter() - t0
step_times.append(elapsed) step_times.append(elapsed)
if torch.cuda.is_available(): if torch.cuda.is_available():
mem_used = torch.cuda.max_memory_allocated() / 1024**3 mem_used = torch.cuda.max_memory_allocated() / 1024**3
mem_usage.append(mem_used) mem_usage.append(mem_used)
torch.cuda.reset_peak_memory_stats() torch.cuda.reset_peak_memory_stats()
progress.advance(task) progress.advance(task)
avg_step_time = sum(step_times) / len(step_times) measured_steps = step_times[warmup_steps:] if len(step_times) > warmup_steps else step_times
avg_step_time = sum(measured_steps) / len(measured_steps)
throughput = batch_size * seq_length / avg_step_time throughput = batch_size * seq_length / avg_step_time
jitter = self._jitter_pct(measured_steps)
peak_mem = round(max(mem_usage) if mem_usage else 0, 2)
final_loss = float(loss.item()) if hasattr(loss, "item") else float("nan")
passed = self._acceptance_pass(throughput, jitter, peak_mem, final_loss)
if self.train_cfg.get("require_distributed", True):
passed = False
return { return {
"model": model_name, "model": model_name,
@ -208,12 +150,18 @@ class TrainingSim:
"batch_size": batch_size, "batch_size": batch_size,
"seq_length": seq_length, "seq_length": seq_length,
"num_steps": num_steps, "num_steps": num_steps,
"warmup_steps": warmup_steps,
"total_steps": total_steps,
"avg_step_time_ms": round(avg_step_time * 1000, 1), "avg_step_time_ms": round(avg_step_time * 1000, 1),
"throughput_tokens_per_sec": round(throughput, 0), "throughput_tokens_per_sec": round(throughput, 0),
"throughput_samples_per_sec": round(batch_size / avg_step_time, 2), "throughput_samples_per_sec": round(batch_size / avg_step_time, 2),
"peak_memory_gb": round(max(mem_usage) if mem_usage else 0, 2), "peak_memory_gb": peak_mem,
"final_loss": round(loss.item(), 4) if hasattr(loss, 'item') else None, "final_loss": round(final_loss, 4),
"sdpa_backend_policy": sdpa_policy, "step_jitter_pct": round(jitter, 2),
"distributed_mode": "device_map",
"loss_finite": math.isfinite(final_loss),
"passed": passed,
"acceptance_gap": "8-GPU DDP was not used" if self.train_cfg.get("require_distributed", True) else "",
"timestamp": datetime.now().isoformat(), "timestamp": datetime.now().isoformat(),
} }
@ -221,13 +169,203 @@ class TrainingSim:
self.console.print(f"[yellow]Model loading failed: {e}[/yellow]") self.console.print(f"[yellow]Model loading failed: {e}[/yellow]")
return self._run_synthetic(gpu_count, batch_size, seq_length, num_steps, dtype) return self._run_synthetic(gpu_count, batch_size, seq_length, num_steps, dtype)
def _run_synthetic_ddp(self, gpu_count: int, batch_size: int, seq_length: int,
num_steps: int, dtype_str: str) -> dict:
"""Run the 1.5B synthetic Transformer with one process per GPU."""
torchrun = os.path.join(os.path.dirname(sys.executable), "torchrun")
if not os.path.isfile(torchrun):
torchrun = shutil.which("torchrun") or ""
if not torchrun:
return {
"model": "synthetic_transformer_1.5b",
"gpu_count": gpu_count,
"distributed_mode": "ddp",
"passed": False,
"error": "torchrun not found",
"timestamp": datetime.now().isoformat(),
}
script = r'''
import json
import math
import os
import time
import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel as DDP
def main():
local_rank = int(os.environ["LOCAL_RANK"])
world_size = int(os.environ["WORLD_SIZE"])
torch.cuda.set_device(local_rank)
dist.init_process_group("nccl")
global_batch = int(os.environ["TRAIN_BATCH_SIZE"])
local_batch = max(1, global_batch // world_size)
seq_length = int(os.environ["TRAIN_SEQ_LENGTH"])
num_steps = int(os.environ["TRAIN_NUM_STEPS"])
warmup_steps = int(os.environ.get("TRAIN_WARMUP_STEPS", "5"))
total_steps = num_steps + warmup_steps
dtype_name = os.environ.get("TRAIN_DTYPE", "bf16")
dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}.get(dtype_name, torch.bfloat16)
hidden_size = 4096
num_layers = 6
num_heads = 32
vocab_size = 32000
class SyntheticTransformer(torch.nn.Module):
def __init__(self):
super().__init__()
self.embed = torch.nn.Embedding(vocab_size, hidden_size)
self.layers = torch.nn.ModuleList([
torch.nn.TransformerEncoderLayer(
d_model=hidden_size,
nhead=num_heads,
dim_feedforward=hidden_size * 4,
batch_first=True,
dtype=dtype,
) for _ in range(num_layers)
])
self.head = torch.nn.Linear(hidden_size, vocab_size, dtype=dtype)
def forward(self, x):
h = self.embed(x).to(dtype)
for layer in self.layers:
h = layer(h)
return self.head(h)
model = SyntheticTransformer().cuda()
total_params = sum(p.numel() for p in model.parameters())
model = DDP(model, device_ids=[local_rank], output_device=local_rank)
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)
input_ids = torch.randint(0, vocab_size, (local_batch, seq_length), device="cuda")
step_times = []
last_loss = torch.tensor(float("nan"), device="cuda")
torch.cuda.reset_peak_memory_stats(local_rank)
for _ in range(total_steps):
torch.cuda.synchronize()
t0 = time.perf_counter()
with torch.amp.autocast("cuda", dtype=dtype, enabled=dtype in (torch.float16, torch.bfloat16)):
logits = model(input_ids)
loss = torch.nn.functional.cross_entropy(logits.reshape(-1, vocab_size), input_ids.reshape(-1))
loss.backward()
optimizer.step()
optimizer.zero_grad(set_to_none=True)
torch.cuda.synchronize()
step_times.append(time.perf_counter() - t0)
last_loss = loss.detach()
peak_mem = torch.tensor(torch.cuda.max_memory_allocated(local_rank) / 1024**3, device="cuda")
dist.all_reduce(peak_mem, op=dist.ReduceOp.MAX)
finite = torch.tensor(1 if math.isfinite(float(last_loss.item())) else 0, device="cuda")
dist.all_reduce(finite, op=dist.ReduceOp.MIN)
if dist.get_rank() == 0:
measured_steps = step_times[warmup_steps:] if len(step_times) > warmup_steps else step_times
avg_step = sum(measured_steps) / len(measured_steps)
mean = avg_step
jitter = max(abs(v - mean) / mean * 100 for v in measured_steps) if mean else 0.0
throughput = global_batch * seq_length / avg_step if avg_step else 0.0
print("TRAINING_DDP_JSON=" + json.dumps({
"model": "synthetic_transformer_1.5b",
"total_params_m": round(total_params / 1e6, 1),
"num_layers": num_layers,
"hidden_size": hidden_size,
"gpu_count": world_size,
"dtype": dtype_name,
"batch_size": global_batch,
"local_batch_size": local_batch,
"seq_length": seq_length,
"num_steps": num_steps,
"warmup_steps": warmup_steps,
"total_steps": total_steps,
"avg_step_time_ms": round(avg_step * 1000, 1),
"throughput_tokens_per_sec": round(throughput, 0),
"throughput_samples_per_sec": round(global_batch / avg_step, 2) if avg_step else 0,
"peak_memory_gb": round(float(peak_mem.item()), 2),
"final_loss": round(float(last_loss.item()), 4),
"step_jitter_pct": round(jitter, 2),
"distributed_mode": "ddp",
"loss_finite": bool(int(finite.item())),
}), flush=True)
dist.destroy_process_group()
if __name__ == "__main__":
main()
'''
tmp = tempfile.NamedTemporaryFile("w", suffix="_training_ddp.py", delete=False)
tmp.write(script)
tmp.close()
env = {
**os.environ,
"TRAIN_BATCH_SIZE": str(batch_size),
"TRAIN_SEQ_LENGTH": str(seq_length),
"TRAIN_NUM_STEPS": str(num_steps),
"TRAIN_WARMUP_STEPS": str(int(self.train_cfg.get("warmup_steps", 5))),
"TRAIN_DTYPE": dtype_str,
"NCCL_DEBUG": os.environ.get("NCCL_DEBUG", "WARN"),
}
cmd = [torchrun, f"--nproc_per_node={gpu_count}", tmp.name]
self.console.print(f" Running synthetic 1.5B DDP via torchrun ({gpu_count} processes)...")
try:
timeout = int(self.train_cfg.get("timeout_sec", max(600, num_steps * 180)))
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env)
except subprocess.TimeoutExpired:
os.unlink(tmp.name)
return {
"model": "synthetic_transformer_1.5b",
"gpu_count": gpu_count,
"distributed_mode": "ddp",
"passed": False,
"error": "training_ddp_timeout",
"timestamp": datetime.now().isoformat(),
}
finally:
if os.path.exists(tmp.name):
try:
os.unlink(tmp.name)
except OSError:
pass
marker = "TRAINING_DDP_JSON="
payload = None
for line in (r.stdout + "\n" + r.stderr).splitlines():
if marker in line:
payload = line.split(marker, 1)[1].strip()
if r.returncode != 0 or not payload:
return {
"model": "synthetic_transformer_1.5b",
"gpu_count": gpu_count,
"distributed_mode": "ddp",
"passed": False,
"error": (r.stderr or r.stdout or "training_ddp_failed")[-1000:],
"timestamp": datetime.now().isoformat(),
}
result = json.loads(payload)
loss_value = float(result.get("final_loss", "nan"))
passed = self._acceptance_pass(
float(result.get("throughput_tokens_per_sec", 0)),
float(result.get("step_jitter_pct", 999)),
float(result.get("peak_memory_gb", 999)),
loss_value,
) and bool(result.get("loss_finite", False)) and result.get("gpu_count") == gpu_count
result.update({
"passed": passed,
"timestamp": datetime.now().isoformat(),
})
return result
def _run_synthetic(self, gpu_count, batch_size, seq_length, num_steps, dtype) -> dict: def _run_synthetic(self, gpu_count, batch_size, seq_length, num_steps, dtype) -> dict:
self.console.print(" Running synthetic training benchmark...") self.console.print(" Running synthetic training benchmark...")
hidden_size = int(self.train_cfg.get("hidden_size", 4096)) hidden_size = 4096
num_layers = int(self.train_cfg.get("num_layers", 6)) num_layers = 6
num_heads = int(self.train_cfg.get("num_heads", 32)) num_heads = 32
vocab_size = int(self.train_cfg.get("vocab_size", 32000)) vocab_size = 32000
class SyntheticTransformer(torch.nn.Module): class SyntheticTransformer(torch.nn.Module):
def __init__(self): def __init__(self):
@ -249,11 +387,17 @@ class TrainingSim:
h = layer(h) h = layer(h)
return self.head(h) return self.head(h)
model = SyntheticTransformer().cuda() model = SyntheticTransformer()
total_params = sum(p.numel() for p in model.parameters()) total_params = sum(p.numel() for p in model.parameters())
self.console.print(f" Synthetic params: {total_params / 1e6:.1f}M") self.console.print(f" Synthetic params: {total_params / 1e6:.1f}M")
distributed_mode = "single_gpu"
if gpu_count > 1:
model = torch.nn.DataParallel(model).cuda()
distributed_mode = "data_parallel"
else:
model = model.cuda()
model.train() model.train()
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4) optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)
@ -262,40 +406,48 @@ class TrainingSim:
step_times = [] step_times = []
mem_usage = [] mem_usage = []
sdpa_context, sdpa_policy = self._sdpa_context() warmup_steps = int(self.train_cfg.get("warmup_steps", 5))
self.console.print(f" SDPA policy: {sdpa_policy}") total_steps = num_steps + warmup_steps
with Progress( with Progress(
SpinnerColumn(), TextColumn("[progress.description]{task.description}"), SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
BarColumn(), TextColumn("{task.completed}/{task.total}"), BarColumn(), TextColumn("{task.completed}/{task.total}"),
TimeElapsedColumn(), console=self.console, TimeElapsedColumn(), console=self.console,
) as progress: ) as progress:
task = progress.add_task("Synthetic training...", total=num_steps) task = progress.add_task("Synthetic training...", total=total_steps)
with sdpa_context: for step in range(total_steps):
for step in range(num_steps): torch.cuda.synchronize()
torch.cuda.synchronize() t0 = time.perf_counter()
t0 = time.perf_counter()
logits = model(input_ids) logits = model(input_ids)
loss = torch.nn.functional.cross_entropy( loss = torch.nn.functional.cross_entropy(
logits.view(-1, vocab_size), input_ids.view(-1) logits.view(-1, vocab_size), input_ids.view(-1)
) )
loss.backward() loss.backward()
optimizer.step() optimizer.step()
optimizer.zero_grad() optimizer.zero_grad()
torch.cuda.synchronize() torch.cuda.synchronize()
elapsed = time.perf_counter() - t0 elapsed = time.perf_counter() - t0
step_times.append(elapsed) step_times.append(elapsed)
mem_used = torch.cuda.max_memory_allocated() / 1024**3 mem_used = max(torch.cuda.max_memory_allocated(i) for i in range(gpu_count)) / 1024**3
mem_usage.append(mem_used) mem_usage.append(mem_used)
torch.cuda.reset_peak_memory_stats() for i in range(gpu_count):
torch.cuda.reset_peak_memory_stats(i)
progress.advance(task) progress.advance(task)
avg_step_time = sum(step_times) / len(step_times) measured_steps = step_times[warmup_steps:] if len(step_times) > warmup_steps else step_times
avg_step_time = sum(measured_steps) / len(measured_steps)
throughput = batch_size * seq_length / avg_step_time throughput = batch_size * seq_length / avg_step_time
jitter = self._jitter_pct(measured_steps)
peak_mem = round(max(mem_usage) if mem_usage else 0, 2)
final_loss = float(loss.item())
passed = self._acceptance_pass(throughput, jitter, peak_mem, final_loss)
if self.train_cfg.get("require_distributed", True):
passed = False
return { return {
"model": "synthetic_transformer", "model": "synthetic_transformer",
@ -303,21 +455,40 @@ class TrainingSim:
"num_layers": num_layers, "num_layers": num_layers,
"hidden_size": hidden_size, "hidden_size": hidden_size,
"gpu_count": gpu_count, "gpu_count": gpu_count,
"gpu_scope": "single_gpu:0",
"performance_verdict": "MEASURED",
"sdpa_backend_policy": sdpa_policy,
"dtype": str(dtype).replace("torch.", ""), "dtype": str(dtype).replace("torch.", ""),
"batch_size": batch_size, "batch_size": batch_size,
"seq_length": seq_length, "seq_length": seq_length,
"num_steps": num_steps, "num_steps": num_steps,
"warmup_steps": warmup_steps,
"total_steps": total_steps,
"avg_step_time_ms": round(avg_step_time * 1000, 1), "avg_step_time_ms": round(avg_step_time * 1000, 1),
"throughput_tokens_per_sec": round(throughput, 0), "throughput_tokens_per_sec": round(throughput, 0),
"throughput_samples_per_sec": round(batch_size / avg_step_time, 2), "throughput_samples_per_sec": round(batch_size / avg_step_time, 2),
"peak_memory_gb": round(max(mem_usage) if mem_usage else 0, 2), "peak_memory_gb": peak_mem,
"final_loss": round(loss.item(), 4), "final_loss": round(final_loss, 4),
"step_jitter_pct": round(jitter, 2),
"distributed_mode": distributed_mode,
"loss_finite": math.isfinite(final_loss),
"passed": passed,
"acceptance_gap": "8-GPU DDP was not used" if self.train_cfg.get("require_distributed", True) else "",
"timestamp": datetime.now().isoformat(), "timestamp": datetime.now().isoformat(),
} }
@staticmethod
def _jitter_pct(step_times: list[float]) -> float:
if not step_times:
return 0.0
mean = sum(step_times) / len(step_times)
return max(abs(v - mean) / mean * 100 for v in step_times) if mean else 0.0
def _acceptance_pass(self, throughput: float, jitter: float, peak_mem: float, loss_value: float) -> bool:
return (
throughput >= float(self.train_cfg.get("min_tokens_per_sec", 45000))
and jitter <= float(self.train_cfg.get("max_step_jitter_pct", 3))
and peak_mem <= float(self.train_cfg.get("max_peak_memory_gb", 70))
and math.isfinite(loss_value)
)
@staticmethod @staticmethod
def print_results(results: dict, console: Console = None): def print_results(results: dict, console: Console = None):
c = console or Console() c = console or Console()
@ -339,11 +510,15 @@ class TrainingSim:
("Batch Size", str(results.get("batch_size", "N/A"))), ("Batch Size", str(results.get("batch_size", "N/A"))),
("Seq Length", str(results.get("seq_length", "N/A"))), ("Seq Length", str(results.get("seq_length", "N/A"))),
("Steps", str(results.get("num_steps", "N/A"))), ("Steps", str(results.get("num_steps", "N/A"))),
("Warmup Steps", str(results.get("warmup_steps", "N/A"))),
("Avg Step Time", f"{results.get('avg_step_time_ms', 'N/A')} ms"), ("Avg Step Time", f"{results.get('avg_step_time_ms', 'N/A')} ms"),
("Throughput", f"{results.get('throughput_tokens_per_sec', 'N/A')} tokens/s"), ("Throughput", f"{results.get('throughput_tokens_per_sec', 'N/A')} tokens/s"),
("Samples/sec", f"{results.get('throughput_samples_per_sec', 'N/A')}"), ("Samples/sec", f"{results.get('throughput_samples_per_sec', 'N/A')}"),
("Peak Memory", f"{results.get('peak_memory_gb', 'N/A')} GB"), ("Peak Memory", f"{results.get('peak_memory_gb', 'N/A')} GB"),
("Final Loss", str(results.get("final_loss", "N/A"))), ("Final Loss", str(results.get("final_loss", "N/A"))),
("Step Jitter", f"{results.get('step_jitter_pct', 'N/A')}%"),
("Distributed Mode", results.get("distributed_mode", "N/A")),
("Verdict", "PASS" if results.get("passed") else "FAIL"),
] ]
for label, val in metrics: for label, val in metrics:
table.add_row(label, str(val)) table.add_row(label, str(val))

View File

@ -17,12 +17,10 @@ dependencies = [
# uv sync --extra torch-cu121 # uv sync --extra torch-cu121
# uv sync --extra torch-cu124 (recommended for CUDA 12.x + Driver 535+) # uv sync --extra torch-cu124 (recommended for CUDA 12.x + Driver 535+)
# uv sync --extra torch-cu128 (for CUDA 12.8+ / Driver 570+) # uv sync --extra torch-cu128 (for CUDA 12.8+ / Driver 570+)
# uv sync --extra torch-cu130 (recommended for B300 / Driver 580+)
torch-cu118 = ["torch>=2.1.0"] torch-cu118 = ["torch>=2.1.0"]
torch-cu121 = ["torch>=2.1.0"] torch-cu121 = ["torch>=2.1.0"]
torch-cu124 = ["torch>=2.1.0"] torch-cu124 = ["torch>=2.1.0"]
torch-cu128 = ["torch>=2.1.0"] torch-cu128 = ["torch>=2.1.0"]
torch-cu130 = ["torch>=2.13.0"]
[build-system] [build-system]
requires = ["hatchling"] requires = ["hatchling"]
@ -49,11 +47,6 @@ name = "pytorch-cu128"
url = "https://download.pytorch.org/whl/cu128" url = "https://download.pytorch.org/whl/cu128"
explicit = true explicit = true
[[tool.uv.index]]
name = "pytorch-cu130"
url = "https://download.pytorch.org/whl/cu130"
explicit = true
# Default: resolve torch from cu124 index (most broadly compatible with 535+ drivers) # Default: resolve torch from cu124 index (most broadly compatible with 535+ drivers)
# To use a different CUDA version, override: # To use a different CUDA version, override:
# uv pip install torch --index-url https://download.pytorch.org/whl/cu128 # uv pip install torch --index-url https://download.pytorch.org/whl/cu128

View File

@ -1,46 +0,0 @@
#!/usr/bin/env python3
"""Minimal DTK/PyTorch smoke test used before the full BW1000 suite."""
import json
import torch
try:
import torch_dcu # noqa: F401
except ImportError:
pass
def main() -> None:
count = torch.cuda.device_count()
if not torch.cuda.is_available() or count < 1:
raise SystemExit("DTK PyTorch reports no available accelerator")
devices = []
for index in range(count):
props = torch.cuda.get_device_properties(index)
devices.append({
"index": index,
"name": torch.cuda.get_device_name(index),
"memory_gib": round(props.total_memory / 1024**3, 2),
})
a = torch.randn(2048, 2048, device="cuda:0", dtype=torch.bfloat16)
b = torch.randn(2048, 2048, device="cuda:0", dtype=torch.bfloat16)
c = a @ b
torch.cuda.synchronize()
if not torch.isfinite(c).all().item():
raise SystemExit("BF16 matmul produced non-finite output")
print(json.dumps({
"torch_version": torch.__version__,
"cuda_api_available": torch.cuda.is_available(),
"device_count": count,
"devices": devices,
"bf16_matmul": "PASS",
"checksum": round(float(c.float().mean().item()), 6),
}, indent=2))
if __name__ == "__main__":
main()

View File

@ -0,0 +1,291 @@
#include <cublasLt.h>
#include <cuda_bf16.h>
#include <cuda_fp8.h>
#include <cuda_runtime.h>
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <numeric>
#include <string>
#include <vector>
#define CHECK_CUDA(call) \
do { \
cudaError_t status = (call); \
if (status != cudaSuccess) { \
std::fprintf(stderr, "CUDA error %s:%d: %s\n", __FILE__, __LINE__, \
cudaGetErrorString(status)); \
std::exit(1); \
} \
} while (0)
#define CHECK_CUBLAS(call) \
do { \
cublasStatus_t status = (call); \
if (status != CUBLAS_STATUS_SUCCESS) { \
std::fprintf(stderr, "cuBLASLt error %s:%d: status=%d\n", __FILE__, \
__LINE__, static_cast<int>(status)); \
std::exit(1); \
} \
} while (0)
__global__ void fill_fp8(__nv_fp8_e4m3 *ptr, size_t count, float value) {
size_t tid = blockIdx.x * blockDim.x + threadIdx.x;
size_t stride = blockDim.x * gridDim.x;
for (size_t i = tid; i < count; i += stride) {
ptr[i] = __nv_fp8_e4m3(value);
}
}
struct Args {
int matrix_size = 8192;
int warmup = 20;
int iterations = 200;
int first_gpu = 0;
int gpu_count = -1;
size_t workspace_mb = 256;
int fast_accum = 1;
};
static Args parse_args(int argc, char **argv) {
Args args;
for (int i = 1; i < argc; ++i) {
auto need = [&](const char *name) {
if (i + 1 >= argc) {
std::fprintf(stderr, "Missing value for %s\n", name);
std::exit(2);
}
return argv[++i];
};
if (!std::strcmp(argv[i], "--matrix-size")) {
args.matrix_size = std::atoi(need(argv[i]));
} else if (!std::strcmp(argv[i], "--warmup")) {
args.warmup = std::atoi(need(argv[i]));
} else if (!std::strcmp(argv[i], "--iterations")) {
args.iterations = std::atoi(need(argv[i]));
} else if (!std::strcmp(argv[i], "--first-gpu")) {
args.first_gpu = std::atoi(need(argv[i]));
} else if (!std::strcmp(argv[i], "--gpu-count")) {
args.gpu_count = std::atoi(need(argv[i]));
} else if (!std::strcmp(argv[i], "--workspace-mb")) {
args.workspace_mb = static_cast<size_t>(std::atoll(need(argv[i])));
} else if (!std::strcmp(argv[i], "--fast-accum")) {
args.fast_accum = std::atoi(need(argv[i]));
} else if (!std::strcmp(argv[i], "--help") || !std::strcmp(argv[i], "-h")) {
std::puts("Usage: cublaslt_fp8_gemm_bench [--matrix-size N] [--warmup N] "
"[--iterations N] [--first-gpu N] [--gpu-count N] "
"[--workspace-mb N] [--fast-accum 0|1]");
std::exit(0);
} else {
std::fprintf(stderr, "Unknown argument: %s\n", argv[i]);
std::exit(2);
}
}
return args;
}
static double run_one_gpu(int gpu, const Args &args) {
CHECK_CUDA(cudaSetDevice(gpu));
const int64_t m = args.matrix_size;
const int64_t n = args.matrix_size;
const int64_t k = args.matrix_size;
const size_t a_elems = static_cast<size_t>(m) * k;
const size_t b_elems = static_cast<size_t>(k) * n;
const size_t d_elems = static_cast<size_t>(m) * n;
__nv_fp8_e4m3 *d_a = nullptr;
__nv_fp8_e4m3 *d_b = nullptr;
__nv_bfloat16 *d_d = nullptr;
void *workspace = nullptr;
float *d_scale_a = nullptr;
float *d_scale_b = nullptr;
const float scale = 1.0f;
const size_t workspace_bytes = args.workspace_mb * 1024ULL * 1024ULL;
CHECK_CUDA(cudaMalloc(&d_a, a_elems * sizeof(__nv_fp8_e4m3)));
CHECK_CUDA(cudaMalloc(&d_b, b_elems * sizeof(__nv_fp8_e4m3)));
CHECK_CUDA(cudaMalloc(&d_d, d_elems * sizeof(__nv_bfloat16)));
CHECK_CUDA(cudaMalloc(&workspace, workspace_bytes));
CHECK_CUDA(cudaMalloc(&d_scale_a, sizeof(float)));
CHECK_CUDA(cudaMalloc(&d_scale_b, sizeof(float)));
CHECK_CUDA(cudaMemcpy(d_scale_a, &scale, sizeof(scale), cudaMemcpyHostToDevice));
CHECK_CUDA(cudaMemcpy(d_scale_b, &scale, sizeof(scale), cudaMemcpyHostToDevice));
const int threads = 256;
const int blocks = 4096;
fill_fp8<<<blocks, threads>>>(d_a, a_elems, 0.01f);
fill_fp8<<<blocks, threads>>>(d_b, b_elems, 0.01f);
CHECK_CUDA(cudaMemset(d_d, 0, d_elems * sizeof(__nv_bfloat16)));
CHECK_CUDA(cudaGetLastError());
CHECK_CUDA(cudaDeviceSynchronize());
cublasLtHandle_t lt;
cublasLtMatmulDesc_t op_desc;
cublasLtMatrixLayout_t a_desc, b_desc, d_desc;
cublasLtMatmulPreference_t preference;
CHECK_CUBLAS(cublasLtCreate(&lt));
CHECK_CUBLAS(cublasLtMatmulDescCreate(&op_desc, CUBLAS_COMPUTE_32F, CUDA_R_32F));
// cuBLASLt FP8 kernels require TN format: A is transposed, B is non-transposed.
// With square GEMMs this keeps the benchmark FLOP count identical to the PDF
// acceptance shape while satisfying the library's FP8 kernel constraints.
cublasOperation_t transa = CUBLAS_OP_T;
cublasOperation_t transb = CUBLAS_OP_N;
CHECK_CUBLAS(cublasLtMatmulDescSetAttribute(
op_desc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)));
CHECK_CUBLAS(cublasLtMatmulDescSetAttribute(
op_desc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transb)));
CHECK_CUBLAS(cublasLtMatmulDescSetAttribute(
op_desc, CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, &d_scale_a,
sizeof(d_scale_a)));
CHECK_CUBLAS(cublasLtMatmulDescSetAttribute(
op_desc, CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, &d_scale_b,
sizeof(d_scale_b)));
int8_t fast_accum = args.fast_accum ? 1 : 0;
CHECK_CUBLAS(cublasLtMatmulDescSetAttribute(
op_desc, CUBLASLT_MATMUL_DESC_FAST_ACCUM, &fast_accum,
sizeof(fast_accum)));
CHECK_CUBLAS(cublasLtMatrixLayoutCreate(&a_desc, CUDA_R_8F_E4M3, k, m, k));
CHECK_CUBLAS(cublasLtMatrixLayoutCreate(&b_desc, CUDA_R_8F_E4M3, k, n, k));
CHECK_CUBLAS(cublasLtMatrixLayoutCreate(&d_desc, CUDA_R_16BF, m, n, m));
CHECK_CUBLAS(cublasLtMatmulPreferenceCreate(&preference));
CHECK_CUBLAS(cublasLtMatmulPreferenceSetAttribute(
preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspace_bytes,
sizeof(workspace_bytes)));
cublasLtMatmulHeuristicResult_t heuristic;
int returned = 0;
CHECK_CUBLAS(cublasLtMatmulAlgoGetHeuristic(
lt, op_desc, a_desc, b_desc, d_desc, d_desc, preference, 1, &heuristic,
&returned));
if (returned == 0) {
std::fprintf(stderr, "No cuBLASLt heuristic returned for GPU %d\n", gpu);
std::exit(1);
}
auto get_algo_attr_i32 = [&](cublasLtMatmulAlgoConfigAttributes_t attr) {
int32_t value = -1;
size_t written = 0;
CHECK_CUBLAS(cublasLtMatmulAlgoConfigGetAttribute(
&heuristic.algo, attr, &value, sizeof(value), &written));
return static_cast<int>(value);
};
auto get_algo_attr_u32 = [&](cublasLtMatmulAlgoConfigAttributes_t attr) {
uint32_t value = 0;
size_t written = 0;
CHECK_CUBLAS(cublasLtMatmulAlgoConfigGetAttribute(
&heuristic.algo, attr, &value, sizeof(value), &written));
return static_cast<int>(value);
};
auto get_algo_attr_u16 = [&](cublasLtMatmulAlgoConfigAttributes_t attr) {
uint16_t value = 0;
size_t written = 0;
CHECK_CUBLAS(cublasLtMatmulAlgoConfigGetAttribute(
&heuristic.algo, attr, &value, sizeof(value), &written));
return static_cast<int>(value);
};
const int algo_id = get_algo_attr_i32(CUBLASLT_ALGO_CONFIG_ID);
const int tile_id = get_algo_attr_u32(CUBLASLT_ALGO_CONFIG_TILE_ID);
const int splitk = get_algo_attr_i32(CUBLASLT_ALGO_CONFIG_SPLITK_NUM);
const int stages = get_algo_attr_u32(CUBLASLT_ALGO_CONFIG_STAGES_ID);
const int inner_shape = get_algo_attr_u16(CUBLASLT_ALGO_CONFIG_INNER_SHAPE_ID);
const int cluster_shape = get_algo_attr_u16(CUBLASLT_ALGO_CONFIG_CLUSTER_SHAPE_ID);
const float alpha = 1.0f;
const float beta = 0.0f;
auto matmul = [&]() {
CHECK_CUBLAS(cublasLtMatmul(lt, op_desc, &alpha, d_a, a_desc, d_b, b_desc,
&beta, d_d, d_desc, d_d, d_desc,
&heuristic.algo, workspace, workspace_bytes, 0));
};
for (int i = 0; i < args.warmup; ++i) {
matmul();
}
CHECK_CUDA(cudaDeviceSynchronize());
cudaEvent_t start, stop;
CHECK_CUDA(cudaEventCreate(&start));
CHECK_CUDA(cudaEventCreate(&stop));
CHECK_CUDA(cudaEventRecord(start));
for (int i = 0; i < args.iterations; ++i) {
matmul();
}
CHECK_CUDA(cudaEventRecord(stop));
CHECK_CUDA(cudaEventSynchronize(stop));
float elapsed_ms = 0.0f;
CHECK_CUDA(cudaEventElapsedTime(&elapsed_ms, start, stop));
const double flops =
2.0 * static_cast<double>(m) * static_cast<double>(n) *
static_cast<double>(k) * static_cast<double>(args.iterations);
const double tflops = flops / (static_cast<double>(elapsed_ms) / 1000.0) / 1e12;
std::printf(
" {\"index\": %d, \"fp8_tflops\": %.1f, \"algo_id\": %d, "
"\"tile_id\": %d, \"splitk\": %d, \"stages_id\": %d, "
"\"inner_shape_id\": %d, \"cluster_shape_id\": %d}%s\n",
gpu, tflops, algo_id, tile_id, splitk, stages, inner_shape, cluster_shape,
(gpu + 1 == args.first_gpu + args.gpu_count) ? "" : ",");
std::fflush(stdout);
CHECK_CUDA(cudaEventDestroy(start));
CHECK_CUDA(cudaEventDestroy(stop));
CHECK_CUBLAS(cublasLtMatmulPreferenceDestroy(preference));
CHECK_CUBLAS(cublasLtMatrixLayoutDestroy(a_desc));
CHECK_CUBLAS(cublasLtMatrixLayoutDestroy(b_desc));
CHECK_CUBLAS(cublasLtMatrixLayoutDestroy(d_desc));
CHECK_CUBLAS(cublasLtMatmulDescDestroy(op_desc));
CHECK_CUBLAS(cublasLtDestroy(lt));
CHECK_CUDA(cudaFree(d_a));
CHECK_CUDA(cudaFree(d_b));
CHECK_CUDA(cudaFree(d_d));
CHECK_CUDA(cudaFree(workspace));
CHECK_CUDA(cudaFree(d_scale_a));
CHECK_CUDA(cudaFree(d_scale_b));
CHECK_CUDA(cudaDeviceSynchronize());
return tflops;
}
int main(int argc, char **argv) {
Args args = parse_args(argc, argv);
int device_count = 0;
CHECK_CUDA(cudaGetDeviceCount(&device_count));
if (args.gpu_count < 0) {
args.gpu_count = device_count - args.first_gpu;
}
if (args.first_gpu < 0 || args.first_gpu + args.gpu_count > device_count) {
std::fprintf(stderr, "Invalid GPU range first=%d count=%d device_count=%d\n",
args.first_gpu, args.gpu_count, device_count);
return 2;
}
std::vector<double> values;
std::printf("{\n");
std::printf(" \"source\": \"cuBLASLt\",\n");
std::printf(" \"dtype\": \"fp8_e4m3_inputs_bf16_output_fp32_accum\",\n");
std::printf(" \"matrix_size\": %d,\n", args.matrix_size);
std::printf(" \"warmup\": %d,\n", args.warmup);
std::printf(" \"iterations\": %d,\n", args.iterations);
std::printf(" \"fast_accum\": %d,\n", args.fast_accum ? 1 : 0);
std::printf(" \"per_gpu\": [\n");
for (int i = 0; i < args.gpu_count; ++i) {
int gpu = args.first_gpu + i;
double tflops = run_one_gpu(gpu, args);
values.push_back(tflops);
}
double mean = std::accumulate(values.begin(), values.end(), 0.0) / values.size();
auto minmax = std::minmax_element(values.begin(), values.end());
double spread = ((*minmax.second - *minmax.first) / mean) * 100.0;
std::printf(" ],\n");
std::printf(" \"mean_tflops\": %.1f,\n", mean);
std::printf(" \"min_tflops\": %.1f,\n", *minmax.first);
std::printf(" \"max_tflops\": %.1f,\n", *minmax.second);
std::printf(" \"spread_pct\": %.2f\n", spread);
std::printf("}\n");
return mean >= 1400.0 ? 0 : 1;
}

View File

@ -1,5 +0,0 @@
/gpu_node_snapshot*.json
/gpu_node_profile*.html
/gpu_topology_3d*.html
/b300_*_live.json
/b300_*_live.html

View File

@ -1,155 +0,0 @@
# GPU Node Profile
`gpu_node_profile.py` 通过只读命令采集 GPU、NVLink、PCIe/NUMA、RoCE
Bond、RDMA 和 InfiniBand 信息,生成通用结构化 JSON,并使用 `templates/`
中的两个独立 HTML 源模板,从同一份 JSON 渲染完全离线、无 Mermaid、
无 CDN 依赖的运维画像与 3D 拓扑 CT。
页面顶部按“系统与软件 / GPU / CPU与内存”展示整机概览;中部的
GPU-RDMA 拓扑支持“总览 / NUMA 分览”即时切换,并继续展示 InfiniBand、
OS 可见的带内管理网卡和可读取的 BMC 带外信息;最底部横向展示健康检查
与验证结论。
3D CT 页面使用原生 Canvas 对结构化 JSON 动态布局,包含 NUMA 半透明地域、
CPU Socket、PCIe Root Complex、PCIe Switch 分组、GPU、NVSwitch、RoCE、
InfiniBand 和管理网卡。NUMA 与 PCIe 均来自数据,不包含 B300 专用坐标,
因此同一渲染器也可用于 A800 等其他 GPU 服务器。
页面提供递归内部镜头,可按“整机 → NUMA → PCIe 分组 → Bond → 物理网卡”
逐层钻取。进入任一层后会隐藏外层与无关的同级设备,只保留当前壳体和后代
节点;顶部面包屑说明当前位置,“上一层”按原路径返回。双击壳体继续深入,
双击普通设备时也会沿该设备的父级路径每次深入一层,例如连续双击 GPU 会
依次进入整机、NUMA 和对应 PCIe 分组;到达最深层后才执行设备聚焦。
`Shift + 拖动`可平移,`Esc`返回整机外观。
Canvas 标签使用高对比浅色底与黑色粗体。每个 RoCE Bond 会作为透明聚合
容器展示,并在内部继续拆出物理成员网卡、速率和 BDF。NVLink/NVSwitch
连线在设备盒子上层高亮绘制,GPU 端点带编号;该 Fabric 只连接 GPU,
不会连接网卡。
网络模型同时支持两种形态:存在 Bond 时渲染
`GPU → RoCE Bond → 物理成员端口`;不存在 Bond 时根据
`nvidia-smi topo -m`、RDMA 名称和 PCIe 路径渲染
`GPU → 独立 RoCE/InfiniBand 端点`。多张 GPU 与多个等距端点会进行
全局最优、确定性的唯一分配,避免顺序贪心导致所有 GPU 错误指向同一张
网卡;端点少于 GPU 时才保留可共享的最近路径语义。亲和关系只接受
`nvidia-smi topo -m` 暴露的 `PIX/PXB/PHB/NODE/SYS` 证据;若独立 IB
未出现在矩阵中,会保留设备和 PCIe/NUMA 信息并标记 `not-exposed`,但不会
推测一条 GPU 连线。
JSON 中的 `network_endpoints` 是两张页面共用的标准网络模型:
```text
network_endpoints[]
id / type(bond|standalone) / protocol(RoCE|InfiniBand)
rdma / numa / bdf / state / topology_visibility
members[] # Bond 为多个物理口,直连端点为一个物理口
gpu_links[] # GPU、PIX/PXB/...、用途及证据来源
gpus[].primary_network_id # 指向主要计算 RDMA 端点
gpus[].network_links[] # 保留主要端点和额外最近端点
```
CPU 与内存概览、NUMA 标尺和每张 GPU 卡片会同时展示 `nvidia-smi topo -m`
解析出的推荐 CPU 范围及内存 NUMA。该信息描述硬件拓扑的近端路径,不表示
训练或推理进程已经通过 `taskset`、cgroup、Slurm 或 Kubernetes 完成实际绑核。
GPU 概览和单卡画像会展示采集时刻的 `power.draw`、当前 `power.limit` 与
P-State。Power Limit 表示当前生效的 GPU 功耗上限,不等于整机额定功耗;
P0 表示最高性能状态,也不单独等价于 GPU 持续满载。
## 在 GPU 节点执行
```bash
python3 gpu_node_profile.py \
--output gpu_node_profile.html \
--topology-output gpu_topology_3d.html \
--snapshot-out gpu_node_snapshot.json \
--expected-gpus 8 \
--expected-bonds 8 \
--expected-ib 4
```
脚本不会修改网卡、驱动、Bond 或系统配置。唯一写操作是生成用户指定的
HTML/JSON 文件。`--expected-bonds` 校验的是 RDMA/RoCE Bond;普通管理
Bond 会保留在原始网卡信息中,但不会计入加速网络端点数量。
三个产物的关系:
```text
只读采集 → gpu_node_snapshot.json
├─→ gpu_node_profile.html
└─→ gpu_topology_3d.html
```
源码与演示数据的关系:
```text
gpu_node_profile.py # 只读采集、校验和渲染入口
templates/gpu_node_profile.html # 运维画像源模板
templates/gpu_topology_3d.html # 3D CT 源模板
examples/b300_demo.json # 由真机快照脱敏得到的演示数据
examples/b300_profile_demo.html # demo.json 生成的自包含示例
examples/b300_topology_3d_demo.html
```
最终生成的两张 HTML 会把 JSON 嵌入页面,因此可以直接双击、复制到其他机器
或离线归档。它们不会在打开时读取旁边的 `demo.json`;修改或替换 JSON 后,
需要重新运行 Python 渲染:
```bash
python3 gpu_node_profile.py \
--snapshot-in examples/b300_demo.json \
--output examples/b300_profile_demo.html \
--topology-output examples/b300_topology_3d_demo.html \
--expected-gpus 8 \
--expected-bonds 8 \
--expected-ib 4
```
## 使用已有快照离线渲染
```bash
python3 gpu_node_profile.py \
--snapshot-in examples/b300_demo.json \
--output b300_profile_demo.html \
--topology-output b300_topology_3d_demo.html \
--expected-gpus 8 \
--expected-bonds 8 \
--expected-ib 4
```
只采集 JSON 并写到标准输出,适合通过 SSH 在远端只读采集、在本地渲染:
```bash
python3 gpu_node_profile.py --snapshot-stdout
```
## 测试
```bash
python3 -m unittest discover -s tests -v
```
## 采集边界
- `nvcc` 表示当前 PATH 指向的 CUDA Toolkit,不代表容器或应用实际加载的
CUDA Runtime。
- LACP Bond 展示的是成员链路聚合容量,不承诺单连接可达到聚合带宽。
- 虚拟机中的 PCIe/NUMA 数据表示 Guest 可见拓扑,不能代替宿主机物理布线
和交换机侧验证。
- `pci_fabric` 保存 Guest 可见的 PCIe 祖先路径、桥和端点。3D 页面中的空间
坐标由这些逻辑关系动态生成,不代表设备在机箱中的厘米级真实位置。
- `PIX/PXB/PHB/NODE/SYS` 表示拓扑距离。`PIX` 可作为 GPUDirect RDMA 的
良好候选路径,但只有完成驱动能力和实际 RDMA 传输验证后才能标记为
`GDR Verified`。
- CPU 亲和表示 NVIDIA 拓扑给出的推荐 CPU/内存范围,不是运行中进程的
`Cpus_allowed_list`。评估实际 H2D 性能还需结合进程绑核、内存落点、
pinned memory 和带宽实测。
- GPU 功耗为报告生成时的一次瞬时采样;周期性训练负载的平均功耗和峰值
需要连续采样。整机功耗还需结合 BMC/DCMI 或机柜 PDU,不能简单等同于
所有 GPU Power Limit 之和。
- 独立BMC口通常不进入宿主OS网络命名空间。脚本会尝试从常见LAN通道只读
执行 `ipmitool lan print`;读取不到时会在画像中明确标记“OS不可见”。
- 管理网络区域只渲染物理 PCI 网卡、Bond 或默认路由接口;CNI、veth 等
容器虚拟接口仍可保留在原始 JSON 的 `nics` 中,但不会污染整机画像。

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

View File

@ -1,246 +0,0 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="color-scheme" content="dark">
<title>@@TITLE@@</title>
<style>
:root {
--bg:#07100e; --surface:#0d1916; --surface-2:#12231e; --line:#284238;
--text:#edf8f3; --muted:#8da79b; --green:#76b900; --green-2:#a5e34b;
--cyan:#39d5d0; --amber:#ffbd4a; --red:#ff6b6b; --shadow:0 24px 60px rgba(0,0,0,.28);
}
* { box-sizing:border-box; }
html { background:var(--bg); }
body {
margin:0; color:var(--text); background:
radial-gradient(circle at 18% -5%,rgba(118,185,0,.14),transparent 29rem),
radial-gradient(circle at 92% 8%,rgba(57,213,208,.08),transparent 24rem), var(--bg);
font:14px/1.5 Inter,ui-sans-serif,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;
}
body::before {
content:""; position:fixed; inset:0; pointer-events:none; opacity:.26;
background-image:linear-gradient(rgba(255,255,255,.025) 1px,transparent 1px),
linear-gradient(90deg,rgba(255,255,255,.025) 1px,transparent 1px);
background-size:32px 32px;
}
.hero,main,footer { width:min(1680px,calc(100% - 48px)); margin-inline:auto; position:relative; }
.hero { padding:44px 0 28px; display:flex; align-items:flex-end; justify-content:space-between; gap:24px; }
.kicker,.eyebrow { color:var(--green-2); font-size:11px; font-weight:800; letter-spacing:.18em; }
h1 { margin:6px 0 2px; font-size:clamp(34px,4vw,58px); letter-spacing:-.045em; line-height:1; }
.hero p,.gpu-card p,.nic-card p { color:var(--muted); margin:8px 0 0; }
.hero-actions { display:flex; align-items:center; gap:12px; }
button {
border:1px solid var(--line); background:var(--surface-2); color:var(--text);
border-radius:12px; padding:10px 14px; font-weight:750; cursor:pointer;
}
button:hover { border-color:var(--green); }
.overall { display:flex; gap:9px; align-items:center; font-weight:800; padding:10px 14px; border-radius:999px; border:1px solid var(--line); }
.overall i { width:9px; height:9px; border-radius:50%; box-shadow:0 0 18px currentColor; }
.overall.healthy { color:var(--green-2); }.overall.healthy i{background:var(--green-2)}
.overall.attention { color:var(--amber); }.overall.attention i{background:var(--amber)}
.overall.critical { color:var(--red); }.overall.critical i{background:var(--red)}
.overview-grid { display:grid; grid-template-columns:1.15fr .9fr 1fr; gap:12px; margin-bottom:14px; }
.overview-card,.panel {
background:linear-gradient(145deg,rgba(18,35,30,.94),rgba(10,21,18,.96));
border:1px solid var(--line); box-shadow:var(--shadow);
}
.overview-card { min-height:290px; border-radius:18px; padding:20px; overflow:hidden; }
.overview-head { display:flex; align-items:flex-start; justify-content:space-between; gap:14px; }
.overview-lead { color:var(--muted); margin:12px 0 0; min-height:42px; }
.overview-primary { display:flex; align-items:baseline; gap:9px; margin:8px 0 17px; }
.overview-primary b { font-size:52px; line-height:1; letter-spacing:-.055em; }
.overview-primary span { color:var(--muted); font-weight:750; }
.gpu-power-summary { margin:-5px 0 12px; }
.gpu-power-summary>div:first-child { display:flex; justify-content:space-between; align-items:baseline; gap:10px; font-size:10px; color:var(--muted); }
.gpu-power-summary>div:first-child b { color:var(--text); font-size:12px; }
.gpu-power-summary>small { display:block; margin-top:4px; color:var(--muted); font-size:8px; }
.overview-metrics { display:grid; grid-template-columns:repeat(2,1fr); gap:1px; background:var(--line); border:1px solid var(--line); border-radius:10px; overflow:hidden; }
.overview-metrics span { background:rgba(7,16,14,.72); padding:10px; min-width:0; }
.overview-metrics b,.overview-metrics small { display:block; }
.overview-metrics b { font-size:17px; }.overview-metrics small { color:var(--muted); margin-top:2px; font-size:9px; }
.overview-facts { display:grid; grid-template-columns:repeat(2,1fr); gap:1px; background:var(--line); border:1px solid var(--line); border-radius:10px; overflow:hidden; margin-top:12px; }
.overview-facts div { background:rgba(7,16,14,.72); padding:9px 10px; min-width:0; }
.overview-facts dt { font-size:9px; letter-spacing:.08em; text-transform:uppercase; }
.overview-facts dd { text-align:left; margin-top:3px; font-size:11px; font-weight:700; overflow-wrap:anywhere; }
.overview-facts small { color:var(--muted); }
.compact { padding:7px 9px; font-size:11px; }
.memory-row { display:flex; justify-content:space-between; gap:10px; color:var(--muted); font-size:10px; }
.memory-row b { color:var(--text); }
.memory-meter { height:6px; background:#172821; border-radius:10px; margin:7px 0 14px; overflow:hidden; }
.memory-meter i { display:block; height:100%; background:linear-gradient(90deg,var(--cyan),var(--green-2)); }
.cpu-metrics { grid-template-columns:repeat(3,1fr); }
.cpu-affinity-summary { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:6px; margin-top:9px; }
.cpu-affinity-summary span { border:1px solid var(--line); background:rgba(7,16,14,.55); border-radius:8px; padding:7px 8px; min-width:0; }
.cpu-affinity-summary b,.cpu-affinity-summary small { display:block; }
.cpu-affinity-summary b { color:var(--green-2); font-size:10px; }.cpu-affinity-summary small { color:var(--muted); font-size:8px; overflow-wrap:anywhere; }
.panel { border-radius:20px; padding:22px; margin-bottom:14px; overflow:hidden; }
.section-head { display:flex; justify-content:space-between; align-items:flex-start; gap:20px; margin-bottom:20px; }
h2 { margin:3px 0 0; font-size:24px; letter-spacing:-.025em; }
.legend { display:flex; flex-wrap:wrap; gap:7px; justify-content:flex-end; }
.badge { display:inline-flex; align-items:center; width:max-content; border:1px solid var(--line); color:var(--muted); border-radius:999px; padding:3px 8px; font-size:10px; font-weight:850; letter-spacing:.05em; white-space:nowrap; }
.badge.ok { color:var(--green-2); border-color:rgba(165,227,75,.34); background:rgba(118,185,0,.09); }
.badge.warn { color:var(--amber); border-color:rgba(255,189,74,.35); background:rgba(255,189,74,.08); }
.badge.bad { color:var(--red); border-color:rgba(255,107,107,.35); background:rgba(255,107,107,.08); }
.topology-viewbar { display:flex; align-items:center; justify-content:space-between; gap:14px; margin:-5px 0 14px; padding:10px 12px; border:1px solid var(--line); border-radius:12px; background:rgba(7,16,14,.48); }
.topology-viewbar b,.topology-viewbar small { display:block; }
.topology-viewbar b { font-size:11px; }.topology-viewbar small { color:var(--muted); font-size:9px; margin-top:2px; }
.numa-switcher { display:flex; align-items:center; gap:4px; padding:3px; border:1px solid var(--line); border-radius:10px; background:#07100e; }
.numa-switch { min-width:72px; padding:6px 10px; border:0; border-radius:7px; background:transparent; color:var(--muted); font-size:10px; }
.numa-switch:hover { border-color:transparent; color:var(--text); }
.numa-switch.active { color:#07100e; background:var(--green-2); box-shadow:0 0 18px rgba(165,227,75,.12); }
.numa-ruler { display:grid; grid-template-columns:1fr 1fr; margin:0 0 8px; gap:2px; }
.numa-ruler span { text-align:center; padding:5px; color:var(--muted); font-size:11px; font-weight:800; letter-spacing:.14em; border-bottom:1px solid var(--line); }
.numa-ruler small { display:block; margin-top:2px; color:var(--green-2); font-size:9px; letter-spacing:.04em; }
.affinity-note { color:var(--muted); text-align:center; font-size:9px; margin:3px 0 10px; }
.nvlink-fabric {
border:1px solid rgba(118,185,0,.5); background:linear-gradient(90deg,rgba(118,185,0,.18),rgba(57,213,208,.08),rgba(118,185,0,.18));
border-radius:13px; padding:11px 16px; display:flex; align-items:center; justify-content:center; gap:18px; color:var(--green-2); margin-bottom:16px;
}
.nvlink-fabric span { font-size:11px; letter-spacing:.12em; font-weight:900; }
.nvlink-fabric small { color:var(--muted); }
.rails { display:grid; gap:10px; }
.topology-panel { overflow-x:auto; }
.rail { position:relative; display:grid; grid-template-rows:auto 54px 1fr; }
.rail[hidden],.numa-ruler [hidden] { display:none!important; }
.rail::before { content:""; position:absolute; z-index:0; left:50%; top:150px; bottom:150px; width:2px; background:linear-gradient(var(--green),var(--cyan)); opacity:.6; }
.gpu-card,.nic-card,.ib-card { position:relative; z-index:1; border:1px solid var(--line); border-radius:14px; background:rgba(7,16,14,.92); }
.gpu-card { padding:13px; border-top:3px solid var(--green); min-height:278px; }
.card-head { display:flex; justify-content:space-between; align-items:center; gap:8px; }
.gpu-card h3,.nic-card h4,.ib-card h4 { margin:10px 0 0; line-height:1.14; }
.gpu-card h3 { min-height:33px; font-size:15px; }
.mono { font:10px/1.4 ui-monospace,SFMono-Regular,Menlo,monospace; }
.gpu-affinity { display:grid; grid-template-columns:1.4fr .6fr; gap:5px; margin-top:8px; }
.gpu-affinity span { min-width:0; background:var(--surface); border:1px solid var(--line); border-radius:7px; padding:5px 6px; }
.gpu-affinity small,.gpu-affinity b { display:block; }
.gpu-affinity small { color:var(--muted); font-size:7px; letter-spacing:.08em; }
.gpu-affinity b { color:var(--green-2); margin-top:1px; font:9px/1.25 ui-monospace,monospace; overflow-wrap:anywhere; }
.gpu-stats { display:grid; grid-template-columns:1.05fr .75fr .9fr; gap:3px; margin-top:15px; }
.gpu-stats span { min-width:0; }.gpu-stats b,.gpu-stats small { display:block; }
.gpu-stats b { font-size:12px; }.gpu-stats small { color:var(--muted); font-size:9px; }
.meter { height:4px; background:#172821; border-radius:10px; margin-top:11px; overflow:hidden; }
.meter i { display:block; height:100%; background:linear-gradient(90deg,var(--green),var(--green-2)); }
.gpu-power { margin-top:10px; padding-top:8px; border-top:1px solid var(--line); }
.gpu-power>div:first-child { display:flex; align-items:flex-start; justify-content:space-between; gap:5px; }
.gpu-power span,.gpu-power b,.gpu-power small { display:block; min-width:0; }
.gpu-power b { font-size:10px; }.gpu-power small { color:var(--muted); font-size:7px; margin-top:1px; }
.gpu-power em { flex:0 0 auto; color:var(--cyan); border:1px solid rgba(57,213,208,.3); border-radius:999px; padding:2px 5px; font:normal 800 8px/1.2 ui-monospace,monospace; }
.power-meter { height:5px; margin-top:6px; overflow:hidden; border-radius:999px; background:#172821; }
.power-meter i { display:block; height:100%; background:linear-gradient(90deg,var(--cyan),var(--green-2),var(--amber)); }
.relation { position:relative; z-index:2; align-self:center; justify-self:center; display:flex; align-items:center; flex-direction:column; gap:3px; color:var(--muted); font-size:9px; }
.nic-card { padding:13px; border-top:3px solid var(--cyan); min-height:234px; }
.nic-card h4,.ib-card h4 { font-size:18px; }.nic-card p { font:10px ui-monospace,monospace; }
.bond-capacity { display:flex; align-items:baseline; gap:6px; margin:14px 0 10px; }
.bond-capacity b { font-size:24px; }.bond-capacity span { color:var(--muted); font-size:9px; }
.ports { display:grid; gap:6px; }
.port { display:flex; align-items:center; gap:8px; background:var(--surface); border:1px solid var(--line); border-radius:8px; padding:7px; }
.port-dot { width:7px; height:7px; flex:0 0 auto; border-radius:50%; background:var(--muted); }
.port.up .port-dot { background:var(--green-2); box-shadow:0 0 9px rgba(165,227,75,.6); }
.port.down .port-dot { background:var(--red); }
.port b,.port small { display:block; }.port b { font-size:11px; }.port small { color:var(--muted); font:8px ui-monospace,monospace; margin-top:2px; }
.capacity-note { color:var(--muted); margin:16px 0 0; font-size:11px; }
.network-grid { display:grid; grid-template-columns:1fr 1fr; gap:14px; }
.ib-grid { display:grid; grid-template-columns:repeat(4,minmax(0,1fr)); gap:10px; }
.ib-card { min-width:0; padding:14px; border-top:3px solid var(--amber); overflow:hidden; }
.ib-card .card-head { align-items:flex-start; flex-wrap:wrap; row-gap:7px; }
.ib-card .card-head .eyebrow { min-width:0; overflow-wrap:anywhere; font-size:9px; letter-spacing:.13em; }
.ib-card .card-head .badge { flex:0 0 auto; margin-left:auto; max-width:100%; padding:3px 6px; font-size:8px; letter-spacing:.03em; }
.ib-rate { color:var(--amber); font-size:30px; font-weight:850; margin:13px 0; }
.ib-rate small { font-size:11px; margin-left:3px; color:var(--muted); }
dl { margin:0; }.ib-card dl { display:grid; gap:5px; }
.ib-card dl div { display:flex; justify-content:space-between; gap:8px; font-size:10px; }
.ib-card dd { min-width:0; overflow-wrap:anywhere; }
dt { color:var(--muted); } dd { margin:0; text-align:right; }
.management-grid { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:10px; }
.management-card { border:1px solid var(--line); border-top:3px solid var(--cyan); border-radius:14px; background:rgba(7,16,14,.92); padding:14px; min-height:205px; }
.management-card.bmc-card { border-top-color:#8da79b; }
.management-card.unavailable { border-top-color:var(--line); }
.management-title { display:flex; justify-content:space-between; align-items:baseline; gap:10px; margin-top:12px; }
.management-title h4 { margin:0; font-size:19px; }.management-title b { color:var(--cyan); font-size:18px; }
.management-card .address { color:var(--text); background:var(--surface); border:1px solid var(--line); border-radius:8px; padding:8px; min-height:38px; font:10px/1.5 ui-monospace,monospace; }
.management-card dl { display:grid; gap:4px; }
.management-card dl div { display:flex; justify-content:space-between; gap:8px; font-size:9px; }
.management-card .card-note { color:var(--muted); font-size:10px; margin-top:12px; }
.health-panel { margin-bottom:14px; }
.checks { list-style:none; padding:0; margin:0; display:grid; grid-template-columns:repeat(auto-fit,minmax(175px,1fr)); gap:8px; }
.check { display:flex; align-items:center; gap:11px; padding:12px; min-height:68px; border:1px solid var(--line); border-radius:10px; background:rgba(7,16,14,.55); }
.check-icon { width:25px; height:25px; display:grid; place-items:center; border-radius:50%; font-weight:900; flex:0 0 auto; }
.check.ok .check-icon { color:var(--green-2); background:rgba(118,185,0,.13); }
.check.warning .check-icon { color:var(--amber); background:rgba(255,189,74,.12); }
.check.critical .check-icon { color:var(--red); background:rgba(255,107,107,.12); }
.check b,.check small { display:block; }.check small { color:var(--muted); }
.ghost { background:transparent; }
pre { white-space:pre-wrap; word-break:break-word; background:#050b09; color:#b8cdc3; border:1px solid var(--line); border-radius:12px; padding:16px; max-height:480px; overflow:auto; font:11px/1.6 ui-monospace,monospace; }
dialog { width:min(1100px,calc(100% - 32px)); max-height:86vh; color:var(--text); background:var(--surface); border:1px solid var(--line); border-radius:18px; padding:20px; box-shadow:0 35px 100px rgba(0,0,0,.65); }
dialog::backdrop { background:rgba(0,0,0,.72); backdrop-filter:blur(5px); }
.dialog-head { display:flex; align-items:flex-start; justify-content:space-between; gap:20px; }
dialog pre { max-height:68vh; }
.empty-state { color:var(--muted); border:1px dashed var(--line); border-radius:12px; padding:30px; text-align:center; grid-column:1/-1; }
footer { padding:8px 0 30px; color:var(--muted); display:flex; justify-content:space-between; font-size:11px; }
@media (max-width:1100px) {
.overview-grid { grid-template-columns:1fr 1fr; }
.system-overview { grid-column:1/-1; }
.network-grid { grid-template-columns:1fr; }
}
@media (max-width:640px) {
.hero,main,footer { width:min(100% - 24px,1680px); }
.hero { align-items:flex-start; flex-direction:column; padding-top:28px; }
.overview-grid { grid-template-columns:1fr; }.system-overview { grid-column:auto; }
.panel { padding:15px; }.section-head { flex-direction:column; }
.ib-grid,.management-grid { grid-template-columns:1fr; }
.topology-viewbar { align-items:flex-start; flex-direction:column; }
.numa-switcher { width:100%; }.numa-switch { flex:1; min-width:0; }
.nvlink-fabric { align-items:flex-start; flex-direction:column; gap:2px; }
}
@media print {
:root { --bg:#fff; --surface:#fff; --surface-2:#f3f6f4; --line:#cbd6d0; --text:#102018; --muted:#52675d; --shadow:none; }
body { background:#fff; }.hero,main,footer { width:100%; }.hero { padding-top:10px; }
button,dialog { display:none!important; }.panel,.overview-card { break-inside:avoid; }
.topology-panel { overflow:visible; zoom:.72; }.rails { min-width:0; }
}
</style>
</head>
<body>
@@BODY@@
<script>
const rawDialog=document.getElementById('raw-dialog');
document.querySelector('[data-open-raw]')?.addEventListener('click',()=>rawDialog?.showModal());
document.querySelector('[data-close-raw]')?.addEventListener('click',()=>rawDialog?.close());
rawDialog?.addEventListener('click',event=>{if(event.target===rawDialog)rawDialog.close();});
const topologyRails=document.querySelector('.rails');
const numaRuler=document.querySelector('.numa-ruler');
const numaSwitches=[...document.querySelectorAll('[data-numa-filter]')];
function setNumaView(value){
const rails=[...document.querySelectorAll('.rail[data-numa]')];
const rulers=[...document.querySelectorAll('.numa-ruler [data-numa]')];
let visibleRails=0;
rails.forEach(rail=>{
const visible=value==='all'||rail.dataset.numa===value;
rail.hidden=!visible;
if(visible)visibleRails+=1;
});
const visibleRulers=[];
rulers.forEach(ruler=>{
const visible=value==='all'||ruler.dataset.numa===value;
ruler.hidden=!visible;
if(visible)visibleRulers.push(ruler);
});
if(topologyRails){
const count=Math.max(1,visibleRails);
topologyRails.style.gridTemplateColumns=`repeat(${count},minmax(132px,1fr))`;
topologyRails.style.minWidth=`${count*132+Math.max(0,count-1)*10}px`;
}
if(numaRuler){
numaRuler.style.gridTemplateColumns=visibleRulers.map(item=>`${item.dataset.count||1}fr`).join(' ')||'1fr';
}
numaSwitches.forEach(button=>{
const active=button.dataset.numaFilter===value;
button.classList.toggle('active',active);
button.setAttribute('aria-pressed',String(active));
});
}
numaSwitches.forEach(button=>button.addEventListener('click',()=>setNumaView(button.dataset.numaFilter)));
</script>
</body>
</html>

View File

@ -1,980 +0,0 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="color-scheme" content="dark">
<title>@@TITLE@@</title>
<style>
:root {
--bg:#050c0a; --surface:#0b1713; --surface-2:#10241d; --line:#28493d;
--text:#eef9f4; --muted:#8fa99e; --green:#a5e34b; --cyan:#51d9d2;
--amber:#ffbd4a; --purple:#b592ff; --red:#ff6b6b;
}
* { box-sizing:border-box; }
html,body { margin:0; min-height:100%; background:var(--bg); color:var(--text); }
body {
font:14px/1.45 Inter,ui-sans-serif,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;
background:
radial-gradient(circle at 16% 0,rgba(118,185,0,.15),transparent 30rem),
radial-gradient(circle at 90% 8%,rgba(81,217,210,.08),transparent 28rem),
var(--bg);
}
button { font:inherit; }
.shell { min-height:100vh; display:grid; grid-template-rows:auto 1fr auto; }
.topbar {
min-width:0; display:flex; align-items:flex-end; justify-content:space-between;
gap:24px; padding:22px 28px 16px; border-bottom:1px solid var(--line);
}
.kicker { color:var(--green); font-size:10px; font-weight:850; letter-spacing:.19em; }
h1 { margin:4px 0 0; font-size:clamp(25px,3vw,42px); letter-spacing:-.045em; line-height:1; }
.subtitle { margin:7px 0 0; color:var(--muted); }
.top-stats { display:flex; flex-wrap:wrap; justify-content:flex-end; gap:8px; }
.stat {
min-width:96px; padding:8px 11px; border:1px solid var(--line); border-radius:10px;
background:rgba(11,23,19,.76);
}
.stat b,.stat small { display:block; }.stat b { font-size:15px; }.stat small { color:var(--muted); font-size:9px; }
.workspace { min-height:0; display:grid; grid-template-columns:minmax(0,1fr) 330px; gap:0; }
.stage { min-width:0; min-height:720px; position:relative; overflow:hidden; }
#topology-canvas { display:block; width:100%; height:100%; min-height:720px; outline:none; cursor:grab; }
#topology-canvas:active { cursor:grabbing; }
#topology-canvas:focus-visible { box-shadow:inset 0 0 0 2px var(--green); }
.toolbar {
position:absolute; z-index:4; left:18px; right:18px; top:16px;
display:flex; align-items:flex-start; justify-content:space-between; gap:12px;
pointer-events:none;
}
.toolbar-group {
pointer-events:auto; display:flex; flex-wrap:wrap; align-items:center; gap:5px;
padding:5px; border:1px solid var(--line); border-radius:11px;
background:rgba(5,12,10,.86); backdrop-filter:blur(10px);
}
.toolbar-label { padding:0 7px; color:var(--muted); font-size:9px; font-weight:800; letter-spacing:.12em; }
.tool-button {
border:0; border-radius:8px; padding:7px 10px; color:var(--muted);
background:transparent; cursor:pointer; font-size:10px; font-weight:800;
}
.tool-button:hover { color:var(--text); background:rgba(255,255,255,.05); }
.tool-button[aria-pressed="true"] { color:#07100e; background:var(--green); }
.tool-button:disabled { opacity:.35; cursor:not-allowed; }
.tool-button.reset { color:var(--text); border:1px solid var(--line); }
.camera-status {
position:absolute; z-index:3; left:50%; top:72px; transform:translateX(-50%);
padding:6px 10px; border:1px solid rgba(165,227,75,.32); border-radius:999px;
color:var(--green); background:rgba(5,12,10,.76); backdrop-filter:blur(8px);
font:800 9px ui-monospace,monospace; letter-spacing:.1em; pointer-events:none;
}
.canvas-note {
position:absolute; z-index:3; left:22px; bottom:18px; max-width:min(630px,calc(100% - 44px));
padding:8px 11px; color:var(--muted); font-size:10px; border:1px solid var(--line);
border-radius:9px; background:rgba(5,12,10,.8); backdrop-filter:blur(8px);
}
.legend {
position:absolute; z-index:3; right:20px; bottom:18px; display:flex; flex-wrap:wrap;
justify-content:flex-end; gap:9px 12px; max-width:52%; padding:8px 10px;
border:1px solid var(--line); border-radius:9px; background:rgba(5,12,10,.8);
}
.legend span { display:flex; align-items:center; gap:5px; color:var(--muted); font-size:9px; }
.legend i { width:18px; height:3px; border-radius:9px; background:var(--line-color); }
.inspector {
min-width:0; padding:20px; border-left:1px solid var(--line);
background:linear-gradient(180deg,rgba(13,28,23,.97),rgba(7,16,13,.97));
overflow:auto;
}
.inspector h2 { margin:4px 0 5px; font-size:21px; }.inspector>p { color:var(--muted); margin:0 0 18px; }
.scope {
margin-bottom:16px; padding:10px; color:var(--muted); border:1px solid var(--line);
border-left:3px solid var(--amber); border-radius:9px; background:rgba(255,189,74,.04); font-size:10px;
}
.mini-grid { display:grid; grid-template-columns:1fr 1fr; gap:7px; margin-bottom:18px; }
.mini-grid div { min-width:0; padding:9px; border:1px solid var(--line); border-radius:8px; background:rgba(5,12,10,.5); }
.mini-grid b,.mini-grid small { display:block; }.mini-grid b { font-size:16px; }.mini-grid small { color:var(--muted); font-size:9px; }
.detail {
padding-top:15px; border-top:1px solid var(--line);
}
.detail-head { display:flex; justify-content:space-between; gap:12px; align-items:flex-start; }
.detail h3 { margin:3px 0 0; font-size:19px; }.detail-kind { color:var(--green); font-size:9px; font-weight:850; letter-spacing:.15em; }
.detail-status { color:var(--cyan); font:800 10px ui-monospace,monospace; }
.detail dl { display:grid; gap:1px; margin:14px 0 0; border:1px solid var(--line); border-radius:9px; overflow:hidden; }
.detail dl div { display:grid; grid-template-columns:92px minmax(0,1fr); gap:8px; padding:8px 9px; background:rgba(5,12,10,.52); border-bottom:1px solid var(--line); }
.detail dl div:last-child { border-bottom:0; }
.detail dt { color:var(--muted); font-size:9px; }.detail dd { margin:0; text-align:right; overflow-wrap:anywhere; font:10px/1.4 ui-monospace,monospace; }
.path-guide { margin-top:18px; }.path-guide h3 { margin:0 0 9px; font-size:13px; }
.path-guide ul { margin:0; padding-left:17px; color:var(--muted); font-size:10px; }
.path-guide li+li { margin-top:5px; }
footer { display:flex; justify-content:space-between; gap:16px; padding:10px 28px 16px; color:var(--muted); font-size:9px; border-top:1px solid var(--line); }
@media (max-width:980px) {
.workspace { grid-template-columns:1fr; }
.inspector { border-left:0; border-top:1px solid var(--line); }
.stage,#topology-canvas { min-height:620px; }
.mini-grid { grid-template-columns:repeat(4,minmax(0,1fr)); }
}
@media (max-width:640px) {
.topbar { align-items:flex-start; flex-direction:column; padding-inline:16px; }
.top-stats { justify-content:flex-start; }.workspace { display:block; }
.stage,#topology-canvas { min-height:560px; }
.toolbar { left:10px; right:10px; flex-direction:column; }
.toolbar-group { width:100%; }.tool-button { flex:1; }
.legend { display:none; }.canvas-note { left:10px; bottom:10px; max-width:calc(100% - 20px); }
.mini-grid { grid-template-columns:1fr 1fr; }
footer { padding-inline:16px; flex-direction:column; }
}
</style>
</head>
<body>
<div class="shell">
<header class="topbar">
<div>
<span class="kicker">MACHINE CT · GUEST-VISIBLE TOPOLOGY</span>
<h1 id="machine-title">GPU Machine 3D CT</h1>
<p class="subtitle" id="machine-subtitle"></p>
</div>
<div class="top-stats" id="top-stats"></div>
</header>
<main class="workspace">
<section class="stage" aria-label="交互式 3D 硬件拓扑">
<div class="toolbar">
<div class="toolbar-group" id="view-controls"><span class="toolbar-label">视图</span></div>
<div class="toolbar-group" id="camera-controls">
<span class="toolbar-label">镜头</span>
<button class="tool-button" type="button" data-camera="outside" aria-pressed="true">外观</button>
<button class="tool-button" type="button" data-camera="inside" aria-pressed="false">进入内部</button>
<button class="tool-button" type="button" id="camera-back" disabled>上一层</button>
<button class="tool-button" type="button" id="focus-selected" disabled>聚焦所选</button>
</div>
<div class="toolbar-group" id="layer-controls">
<span class="toolbar-label">图层</span>
<button class="tool-button" type="button" data-layer="cpu" aria-pressed="true">CPU</button>
<button class="tool-button" type="button" data-layer="pcie" aria-pressed="true">PCIe</button>
<button class="tool-button" type="button" data-layer="gpu" aria-pressed="true">GPU</button>
<button class="tool-button" type="button" data-layer="fabric" aria-pressed="true">NVSwitch</button>
<button class="tool-button" type="button" data-layer="network" aria-pressed="true">网络</button>
<button class="tool-button reset" type="button" id="reset-view">复位</button>
</div>
</div>
<canvas id="topology-canvas" tabindex="0" aria-label="可旋转的 GPU、CPU、PCIe、NVSwitch 和网卡三维拓扑"></canvas>
<div class="camera-status" id="camera-status">整机外观 · 双击壳体逐层进入</div>
<div class="canvas-note">双击壳体逐层进入 · 上一层返回 · 拖动旋转 · Shift+拖动平移 · Esc 返回整机</div>
<div class="legend" aria-label="连线图例">
<span><i style="--line-color:#a5e34b"></i>NVLink(仅连 GPU)</span>
<span><i style="--line-color:#51d9d2"></i>GPUDirect 候选</span>
<span><i style="--line-color:#ffbd4a"></i>PCIe</span>
<span><i style="--line-color:#b592ff"></i>CPU 亲和</span>
<span><i style="--line-color:#ff6b6b"></i>跨 NUMA / SYS</span>
</div>
</section>
<aside class="inspector">
<span class="kicker">TOPOLOGY INSPECTOR</span>
<h2>逻辑拓扑切片</h2>
<p>NUMA 是地域,PCIe 是道路,GPUDirect 是设备间数据路径。</p>
<div class="scope">当前视图来自 Guest 可见的 PCIe/NUMA 拓扑。直通或虚拟 PCIe 可能隐藏宿主机真实插槽几何位置。</div>
<div class="mini-grid" id="machine-metrics"></div>
<section class="detail" id="selection-detail" aria-live="polite"></section>
<section class="path-guide">
<h3>距离语义</h3>
<ul>
<li>PIX:最多经过一个 PCIe Switch,通常是最优 GPU–NIC 路径。</li>
<li>PXB/PHB:经过更多 Switch 或 Host Bridge。</li>
<li>NODE:同 NUMA 内跨 Host Bridge。</li>
<li>SYS:跨 NUMA/CPU 互联,路径最长。</li>
</ul>
</section>
</aside>
</main>
<footer><span>GPU Machine CT v@@VERSION@@ · 原生 Canvas · 无外部依赖</span><span id="generated-at"></span></footer>
</div>
<script type="application/json" id="machine-data">@@MACHINE_DATA@@</script>
<script>
(() => {
'use strict';
const data = JSON.parse(document.getElementById('machine-data').textContent);
const canvas = document.getElementById('topology-canvas');
const ctx = canvas.getContext('2d');
const system = data.system || {};
const gpus = data.gpus || [];
const networkEndpoints = data.network_endpoints || [];
const allNics = data.nics || [];
const endpointById = new Map(networkEndpoints.map(endpoint => [endpoint.id,endpoint]));
const networkByGpu = new Map(gpus.map(gpu => [Number(gpu.index),endpointById.get(gpu.primary_network_id)]).filter(([,endpoint]) => endpoint));
const numaIds = [...new Set(gpus.map(gpu => gpu.numa).filter(value => value !== null && value !== undefined))];
if (!numaIds.length) numaIds.push(0);
document.getElementById('machine-title').textContent = `${system.hostname || 'GPU Node'} · 3D Hardware CT`;
document.getElementById('machine-subtitle').textContent = `${system.manufacturer || ''} ${system.product || ''} · ${system.os || ''}`.trim();
document.getElementById('generated-at').textContent = `采集时间 ${data.generated_at || '—'} · ${data.pci_fabric?.scope || 'guest-visible'}`;
const totalPower = gpus.reduce((sum,gpu) => sum + (Number(gpu.power_w) || 0),0);
const totalLimit = gpus.reduce((sum,gpu) => sum + (Number(gpu.power_limit_w) || 0),0);
const metrics = [
['GPU', gpus.length],
['CPU', system.cpu_physical_cores || system.cpu_logical || '—'],
['NUMA', numaIds.length],
['GPU Power', totalPower ? `${(totalPower/1000).toFixed(2)} kW` : '—']
];
document.getElementById('top-stats').innerHTML = metrics.map(([label,value]) => `<span class="stat"><b>${escapeHtml(value)}</b><small>${escapeHtml(label)}</small></span>`).join('');
document.getElementById('machine-metrics').innerHTML = [
['CPU Sockets', system.cpu_sockets || '—'],
['PCI Bridges', data.pci_fabric?.bridges?.length || '—'],
['NVLinks', data.nvlink?.active_links || '—'],
['Power Limit', totalLimit ? `${(totalLimit/1000).toFixed(2)} kW` : '—']
].map(([label,value]) => `<div><b>${escapeHtml(value)}</b><small>${escapeHtml(label)}</small></div>`).join('');
const state = {
yaw:-0.64, pitch:0.52, zoom:1, focus:'all', selected:null,
cameraMode:'outside', target:{x:0,y:3.4,z:0}, scopeStack:[],
layers:new Set(['cpu','pcie','gpu','fabric','network'])
};
let scene = {boxes:[],edges:[]};
let hitTargets = [];
let pointer = null;
let cameraAnimation = 0;
function escapeHtml(value) {
return String(value ?? '').replace(/[&<>"']/g, char => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[char]));
}
function deepestCommonPath(paths) {
const valid = paths.filter(path => Array.isArray(path) && path.length);
if (valid.length < 2) return '';
const shortest = Math.min(...valid.map(path => path.length));
let common = '';
for (let index=0; index<shortest; index+=1) {
const value = valid[0][index];
if (valid.every(path => path[index] === value)) common = value;
else break;
}
return common;
}
function addBox(box) {
scene.boxes.push({
selectable:true, opacity:1, ...box,
size:box.size || {x:2,y:1,z:1}
});
return box.id;
}
function addEdge(from,to,type,label,layer,meta={}) {
scene.edges.push({from,to,type,label,layer,meta});
}
function buildScene() {
scene = {boxes:[],edges:[]};
const visibleNumas = state.focus === 'all' ? numaIds : numaIds.filter(id => String(id) === String(state.focus));
const domainGap = 18;
const totalWidth = Math.max(18,visibleNumas.length * domainGap);
addBox({
id:'chassis', kind:'chassis', layer:'pcie', selectable:true,
label:`${system.virtualization || 'HOST'} · GUEST VISIBLE`,
position:{x:0,y:3.4,z:0}, size:{x:totalWidth,y:10,z:17},
color:'#56756a', opacity:.08, wire:true
});
const gpuNodeIds = [];
visibleNumas.forEach((numa,domainIndex) => {
const centerX = (domainIndex - (visibleNumas.length-1)/2) * domainGap;
const domainGpus = gpus.filter(gpu => String(gpu.numa) === String(numa));
const cpuAffinity = domainGpus.find(gpu => gpu.cpu_affinity)?.cpu_affinity || '未解析';
const domainColor = numaIds.indexOf(numa) % 2 ? '#4fc6e8' : '#76b900';
addBox({
id:`numa-${numa}`, kind:'numa', layer:'cpu', selectable:true,
parentId:'chassis',
label:`NUMA ${numa}`, sublabel:`CPU ${cpuAffinity}`,
position:{x:centerX,y:3.2,z:0}, size:{x:16,y:8,z:15},
color:domainColor, opacity:.11, wire:true,
meta:{'CPU Affinity':cpuAffinity,'Memory NUMA':numa,'GPU Count':domainGpus.length}
});
addBox({
id:`cpu-${numa}`, kind:'cpu', layer:'cpu', label:`CPU / NUMA ${numa}`,
parentId:`numa-${numa}`,
sublabel:`Cores ${cpuAffinity}`, position:{x:centerX,y:1.4,z:4.7},
size:{x:5.8,y:1.3,z:2.3}, color:'#b592ff',
meta:{NUMA:numa,'CPU Affinity':cpuAffinity,Model:system.cpu_model || '—'}
});
addBox({
id:`root-${numa}`, kind:'pcie-root', layer:'pcie', label:'PCIe Root Complex',
parentId:`numa-${numa}`,
sublabel:`NUMA ${numa}`, position:{x:centerX,y:2.6,z:2.2},
size:{x:6.8,y:.8,z:1.6}, color:'#ffbd4a',
meta:{NUMA:numa,Scope:'Guest-visible host bridge'}
});
addEdge(`cpu-${numa}`,`root-${numa}`,'control','CPU / Host Bridge','pcie');
const columns = Math.max(1,Math.min(4,domainGpus.length));
domainGpus.forEach((gpu,index) => {
const row = Math.floor(index/columns);
const column = index%columns;
const x = centerX + (column-(columns-1)/2)*3.45;
const z = -.7-row*4.4;
const gpuId = `gpu-${gpu.index}`;
const endpoint = networkByGpu.get(Number(gpu.index));
const nicId = endpoint ? `network-${endpoint.id}` : '';
const primaryOwners = (endpoint?.gpu_links || [])
.filter(link => link.role === 'primary')
.map(link => Number(link.gpu_index))
.sort((left,right) => left-right);
const endpointOwner = primaryOwners.length > 0
&& primaryOwners[0] === Number(gpu.index)
&& (endpoint?.type === 'bond' || primaryOwners.length === 1);
const memberPaths = (endpoint?.members || []).map(member => member.pci_path);
const sharedBridge = deepestCommonPath([gpu.pci_path,...memberPaths]);
const relation = gpu.network_relation || '未解析';
addBox({
id:`pair-${gpu.index}`, kind:'pcie-group', layer:'pcie', selectable:true,
parentId:`numa-${numa}`,
label:sharedBridge ? `PCIe Switch ${sharedBridge}` : `PCIe Group · ${relation}`,
sublabel:`GPU${gpu.index} ↔ ${endpoint?.name || 'RDMA ?'}`,
position:{x,y:1.55,z:z-1.05}, size:{x:3.15,y:3.4,z:4.35},
color:relation === 'PIX' ? '#51d9d2' : '#ffbd4a', opacity:.08, wire:true,
meta:{Relation:relation,'Shared Bridge':sharedBridge || '未暴露','GPU BDF':gpu.bdf || '—'}
});
addBox({
id:gpuId, kind:'gpu', layer:'gpu', label:`GPU ${gpu.index}`,
parentId:`pair-${gpu.index}`,
sublabel:gpu.name || 'NVIDIA GPU', position:{x,y:2.35,z},
size:{x:2.55,y:1.45,z:1.7}, color:'#76b900',
meta:{
NUMA:gpu.numa,'CPU Affinity':gpu.cpu_affinity || '—',BDF:gpu.bdf || '—',
'PCI Path':(gpu.pci_path || []).join(' → ') || '未采集',
Power:gpu.power_w ? `${Number(gpu.power_w).toFixed(0)} W / ${Number(gpu.power_limit_w || 0).toFixed(0)} W` : '—',
PState:gpu.pstate || '—',Utilization:gpu.utilization_pct !== null ? `${gpu.utilization_pct}%` : '—'
}
});
gpuNodeIds.push(gpuId);
addEdge(`root-${numa}`,gpuId,'pcie',relation,'pcie',{relation});
addEdge(`cpu-${numa}`,gpuId,'affinity',`CPU ${cpuAffinity}`,'cpu');
if (endpoint?.type === 'bond' && endpointOwner) {
const slaves=endpoint.members || [];
const capacity=Number(endpoint.capacity_mbps)||0;
addBox({
id:nicId, kind:'bond', layer:'network', label:endpoint.name,
parentId:`pair-${gpu.index}`,
sublabel:`${slaves.length} ports · ${endpoint.rdma || 'RoCE / RDMA'}`,
position:{x,y:.62,z:z-2.15}, size:{x:2.85,y:1.45,z:1.65},
color:'#51d9d2',opacity:.1,wire:true,labelOffset:.85,
meta:{
NUMA:slaves[0]?.numa ?? numa,Relation:relation,
RDMA:endpoint.rdma || '—',Capacity:capacity ? `${capacity/1000} Gb/s` : '—',
Members:slaves.map(slave => `${slave.name} ${slave.bdf || ''}`).join(', ') || '—',
'Shared Bridge':sharedBridge || '未暴露'
}
});
slaves.forEach((slave,slaveIndex) => {
const portId=`nic-${endpoint.name}-${slave.name}`;
const offset=(slaveIndex-(slaves.length-1)/2)*1.18;
const speed=Number(slave.speed_mbps) ? `${Number(slave.speed_mbps)/1000}G` : '—';
addBox({
id:portId,kind:'roce-port',layer:'network',label:slave.name,
parentId:nicId,
sublabel:speed,position:{x:x+offset,y:.68,z:z-2.15},
size:{x:1.02,y:.78,z:1.08},color:'#79f4ee',
meta:{
Bond:endpoint.name,RDMA:endpoint.rdma || '—',NUMA:slave.numa ?? numa,
BDF:slave.bdf || '—',Speed:speed,State:slave.status || '—',
Driver:slave.driver || '—','PCI Path':(slave.pci_path || []).join(' → ') || '未采集'
}
});
addEdge(portId,nicId,'bond-member','Bond member','network');
addEdge(`root-${numa}`,portId,'pcie',relation,'pcie',{relation});
});
}
if (endpoint?.type === 'bond') {
(endpoint.members || []).forEach(slave => {
addEdge(
gpuId,
`nic-${endpoint.name}-${slave.name}`,
'gdr',
`${relation} · GDR candidate`,
'network',
{relation,port:slave.name}
);
});
} else if (endpoint && endpointOwner) {
const member=(endpoint.members || [])[0] || endpoint;
const speed=Number(endpoint.capacity_mbps) ? `${Number(endpoint.capacity_mbps)/1000}G` : '—';
const kind=endpoint.protocol === 'InfiniBand' ? 'infiniband' : 'roce-port';
addBox({
id:nicId,kind,layer:'network',label:endpoint.name,
parentId:`pair-${gpu.index}`,
sublabel:`${endpoint.protocol || 'RDMA'} · ${speed}`,
position:{x,y:.68,z:z-2.15},size:{x:2.35,y:1.05,z:1.35},
color:endpoint.protocol === 'InfiniBand' ? '#ff9f43' : '#79f4ee',
meta:{
Mode:'Standalone RDMA',RDMA:endpoint.rdma || '—',
NUMA:endpoint.numa ?? member.numa ?? numa,Relation:relation,
Evidence:endpoint.topology_visibility === 'nvidia-smi-topo'
? 'nvidia-smi topo -m' : 'Topology relation not exposed',
BDF:endpoint.bdf || member.bdf || '—',Speed:speed,
State:endpoint.state || member.status || '—',
'PCI Path':(endpoint.pci_path || member.pci_path || []).join(' → ') || '未采集'
}
});
addEdge(`root-${numa}`,nicId,'pcie',relation,'pcie',{relation});
}
if (endpoint?.type !== 'bond' && endpoint) {
addEdge(gpuId,nicId,'gdr',`${relation} · GDR candidate`,'network',{relation,port:endpoint.name});
}
});
const unpairedEndpoints = networkEndpoints.filter(endpoint =>
endpoint.type === 'standalone'
&& String(endpoint.numa) === String(numa)
&& (endpoint.gpu_links || []).filter(link => link.role === 'primary').length !== 1
);
unpairedEndpoints.forEach((endpoint,index) => {
const x = centerX - 5.4 + index*2.8;
const id = `network-${endpoint.id}`;
const isIb = endpoint.protocol === 'InfiniBand';
const speed = Number(endpoint.capacity_mbps)
? `${Number(endpoint.capacity_mbps)/1000} Gb/s` : '—';
addBox({
id,kind:isIb ? 'infiniband' : 'roce-port',
layer:'network',label:endpoint.name,
parentId:`numa-${numa}`,
sublabel:`${endpoint.protocol || 'RDMA'} ${speed} · GPU relation ${endpoint.topology_visibility === 'nvidia-smi-topo' ? 'exposed' : 'not exposed'}`,
position:{x,y:.75,z:6.35},
size:{x:2.35,y:1.05,z:1.4},color:isIb ? '#ff9f43' : '#79f4ee',
meta:{
NUMA:endpoint.numa,BDF:endpoint.bdf || '—',
RDMA:endpoint.rdma || '—',State:endpoint.state || '—',
'GPU Topology':endpoint.topology_visibility === 'nvidia-smi-topo'
? 'nvidia-smi topo -m' : 'Not exposed; no affinity inferred',
'PCI Path':(endpoint.pci_path || []).join(' → ') || '未采集'
}
});
addEdge(`root-${numa}`,id,'pcie',isIb ? 'Storage / IB' : 'Standalone RoCE','pcie');
(endpoint?.gpu_links || []).filter(link => link.role==='nearest').forEach(link => {
const gpuId=`gpu-${link.gpu_index}`;
if (scene.boxes.some(box => box.id===gpuId)) {
addEdge(gpuId,id,'gdr',`${link.relation} · nearest ${endpoint.protocol}`,'network',{relation:link.relation,port:endpoint.name});
}
});
});
const acceleratorMembers = new Set(networkEndpoints.flatMap(endpoint => (endpoint.members || []).map(member => member.name)));
const acceleratorGroups = new Set(networkEndpoints.filter(endpoint => endpoint.type==='bond').map(endpoint => endpoint.name));
const management = allNics.filter(nic =>
String(nic.numa) === String(numa)
&& !acceleratorMembers.has(nic.name)
&& !acceleratorGroups.has(nic.name)
&& (nic.bdf || nic.default_route || nic.is_bond)
);
management.slice(0,2).forEach((nic,index) => {
const id = `mgmt-${nic.name}`;
addBox({
id,kind:'management',layer:'network',label:nic.name,
parentId:`numa-${numa}`,
sublabel:nic.default_route ? 'In-band management' : 'OS network',
position:{x:centerX+5.4-index*2.7,y:.65,z:6.35},size:{x:2.25,y:1,z:1.35},color:'#7d9b90',
meta:{NUMA:nic.numa,BDF:nic.bdf || '—',Driver:nic.driver || '—',Address:(nic.addresses || []).join(', ') || '—'}
});
addEdge(`root-${numa}`,id,'pcie','Management','pcie');
});
});
if (gpuNodeIds.length > 1 && data.nvlink?.active_links) {
addBox({
id:'nvswitch',kind:'nvswitch',layer:'fabric',label:data.nvlink.fabric_type || 'NVLink Fabric',
parentId:'chassis',
sublabel:`${gpuNodeIds.length}/${gpus.length} GPU · GPU endpoints only`,
position:{x:0,y:8.15,z:-1.2},size:{x:Math.max(9,totalWidth-3),y:.8,z:2.1},color:'#a5e34b',
meta:{
Connection:'GPU endpoints only · not NIC',
'Active Links':data.nvlink.active_links,
'Links / GPU':data.nvlink.links_per_gpu || '—',
'Speed / Link':`${data.nvlink.speed_gbs || '—'} GB/s`
}
});
gpuNodeIds.forEach(id => {
const gpuIndex=Number(id.replace('gpu-',''));
addEdge(id,'nvswitch','nvlink',`GPU ${gpuIndex} ↔ NVSwitch`,'fabric',{gpuIndex});
});
}
renderDetail();
}
const viewControls = document.getElementById('view-controls');
function buildViewControls() {
const views = [['all','总览'],...numaIds.map(id => [String(id),`NUMA ${id}`])];
viewControls.querySelectorAll('button').forEach(button => button.remove());
views.forEach(([value,label]) => {
const button = document.createElement('button');
button.type='button'; button.className='tool-button'; button.textContent=label;
button.dataset.view=value; button.setAttribute('aria-pressed',String(state.focus === value));
button.addEventListener('click',() => {
state.focus=value; state.selected=null;
state.scopeStack=state.cameraMode==='inside' ? ['chassis'] : [];
buildViewControls(); buildScene();
setCameraMode(state.cameraMode,null,false);
});
viewControls.appendChild(button);
});
}
function cameraPreset(mode,box=null) {
if (mode === 'inside') {
const zoomByKind={chassis:1.48,numa:1.72,'pcie-group':2.5,bond:3.25};
return {
yaw:-.2,pitch:.27,zoom:box ? (zoomByKind[box.kind] || 2.15) : 1.48,
target:box ? {...box.position} : {x:0,y:2.25,z:-1.2}
};
}
return {yaw:-.64,pitch:.52,zoom:1,target:{x:0,y:3.4,z:0}};
}
function animateCamera(next,duration=380) {
const animation=++cameraAnimation;
const start={
yaw:state.yaw,pitch:state.pitch,zoom:state.zoom,target:{...state.target}
};
const started=performance.now();
function frame(now) {
if (animation!==cameraAnimation) return;
const raw=Math.min(1,(now-started)/duration);
const progress=1-Math.pow(1-raw,3);
state.yaw=start.yaw+(next.yaw-start.yaw)*progress;
state.pitch=start.pitch+(next.pitch-start.pitch)*progress;
state.zoom=start.zoom+(next.zoom-start.zoom)*progress;
state.target={
x:start.target.x+(next.target.x-start.target.x)*progress,
y:start.target.y+(next.target.y-start.target.y)*progress,
z:start.target.z+(next.target.z-start.target.z)*progress
};
draw();
if (raw<1) requestAnimationFrame(frame);
}
requestAnimationFrame(frame);
}
function updateCameraControls() {
document.querySelectorAll('[data-camera]').forEach(button => {
button.setAttribute('aria-pressed',String(button.dataset.camera===state.cameraMode));
});
const selectedBox=scene.boxes.find(box => box.id===state.selected && box.selectable);
const scopeBoxes=state.scopeStack.map(id => scene.boxes.find(box => box.id===id)).filter(Boolean);
const breadcrumb=['整机',...scopeBoxes.filter(box => box.kind!=='chassis').map(box => box.label)].join(' › ');
document.getElementById('focus-selected').disabled=!selectedBox;
document.getElementById('camera-back').disabled=state.cameraMode!=='inside';
document.getElementById('camera-status').textContent=state.cameraMode==='inside'
? `${breadcrumb} · 当前层之外已隐藏`
: '整机外观 · 双击壳体逐层进入';
}
function setCameraMode(mode,box=null,animate=true) {
state.cameraMode=mode;
if (mode==='outside') state.scopeStack=[];
const next=cameraPreset(mode,box);
updateCameraControls();
if (animate) animateCamera(next);
else {
cameraAnimation+=1;
state.yaw=next.yaw;state.pitch=next.pitch;state.zoom=next.zoom;
state.target={...next.target};draw();
}
}
const shellKinds=new Set(['chassis','numa','pcie-group','bond']);
function isShell(box) {
return Boolean(box && shellKinds.has(box.kind));
}
function activeScopeId() {
return state.scopeStack[state.scopeStack.length-1] || null;
}
function isDescendant(boxId,scopeId) {
let current=scene.boxes.find(box => box.id===boxId);
const visited=new Set();
while (current?.parentId && !visited.has(current.id)) {
if (current.parentId===scopeId) return true;
visited.add(current.id);
current=scene.boxes.find(box => box.id===current.parentId);
}
return false;
}
function scopePath(box) {
const path=[];
let current=box;
const visited=new Set();
while (current && !visited.has(current.id)) {
visited.add(current.id);
if (isShell(current)) path.unshift(current.id);
current=current.parentId ? scene.boxes.find(item => item.id===current.parentId) : null;
}
return path;
}
function nextScopeToward(box) {
const path=scopePath(box);
if (!path.length) return null;
const current=activeScopeId();
if (!current) return scene.boxes.find(item => item.id===path[0]) || null;
const currentIndex=path.indexOf(current);
const nextId=currentIndex>=0 ? path[currentIndex+1] : null;
return nextId ? scene.boxes.find(item => item.id===nextId) || null : null;
}
function enterScope(box,animate=true) {
const scope=isShell(box) ? box : scene.boxes.find(item => item.id==='chassis');
if (!scope) return;
const current=activeScopeId();
const targetPath=scopePath(scope);
if (state.cameraMode!=='inside' || !current) {
state.scopeStack=targetPath;
} else if (scope.id!==current && isDescendant(scope.id,current)) {
const currentIndex=targetPath.indexOf(current);
state.scopeStack=[
...state.scopeStack,
...targetPath.slice(Math.max(0,currentIndex+1)).filter(id => !state.scopeStack.includes(id))
];
}
state.cameraMode='inside';
state.selected=scope.kind==='chassis' ? null : scope.id;
renderDetail();setCameraMode('inside',scope,animate);
}
function leaveScope() {
if (state.cameraMode!=='inside') return;
state.scopeStack.pop();
const parentId=activeScopeId();
if (!parentId) {
state.selected=null;renderDetail();setCameraMode('outside');
return;
}
const parent=scene.boxes.find(box => box.id===parentId);
state.selected=parent?.kind==='chassis' ? null : parent?.id || null;
renderDetail();setCameraMode('inside',parent);
}
document.querySelectorAll('[data-camera]').forEach(button => button.addEventListener('click',() => {
const selectedBox=scene.boxes.find(box => box.id===state.selected && box.selectable);
const nextScope=selectedBox ? nextScopeToward(selectedBox) : null;
if (button.dataset.camera==='outside') {
state.selected=null;renderDetail();setCameraMode('outside');
} else if (nextScope) {
enterScope(nextScope);
} else if (isShell(selectedBox)) {
enterScope(selectedBox);
} else if (state.cameraMode!=='inside') {
enterScope(scene.boxes.find(box => box.id==='chassis'));
} else if (selectedBox) {
setCameraMode('inside',selectedBox);
}
}));
document.getElementById('camera-back').addEventListener('click',leaveScope);
document.getElementById('focus-selected').addEventListener('click',() => {
const selectedBox=scene.boxes.find(box => box.id===state.selected && box.selectable);
if (selectedBox) setCameraMode('inside',selectedBox);
});
document.querySelectorAll('[data-layer]').forEach(button => button.addEventListener('click',() => {
const layer = button.dataset.layer;
if (state.layers.has(layer)) state.layers.delete(layer); else state.layers.add(layer);
button.setAttribute('aria-pressed',String(state.layers.has(layer)));
if (state.selected && !visibleBox(scene.boxes.find(box => box.id === state.selected))) state.selected=null;
renderDetail(); draw();
}));
document.getElementById('reset-view').addEventListener('click',() => {
state.selected=null;setCameraMode('outside');renderDetail();
});
function visibleBox(box) {
if (!box) return false;
if (state.cameraMode==='outside') {
return box.kind==='chassis' || box.kind==='numa' || state.layers.has(box.layer);
}
const scopeId=activeScopeId() || 'chassis';
if (box.id===scopeId) return box.kind!=='chassis';
if (!isDescendant(box.id,scopeId)) return false;
return box.kind==='numa' || state.layers.has(box.layer);
}
function visibleEdge(edge) {
const from=scene.boxes.find(box => box.id===edge.from);
const to=scene.boxes.find(box => box.id===edge.to);
return state.layers.has(edge.layer) && visibleBox(from) && visibleBox(to);
}
function cameraPoint(point) {
const cy=Math.cos(state.yaw), sy=Math.sin(state.yaw);
const cp=Math.cos(state.pitch), sp=Math.sin(state.pitch);
const translated={
x:point.x-state.target.x,y:point.y-state.target.y,z:point.z-state.target.z
};
const x=translated.x*cy-translated.z*sy;
const z=translated.x*sy+translated.z*cy;
return {x,y:translated.y*cp-z*sp,z:translated.y*sp+z*cp};
}
function project(point,scale) {
const p=cameraPoint(point);
return {x:canvas.clientWidth/2+p.x*scale,y:canvas.clientHeight*.58-p.y*scale,z:p.z};
}
function corners(box) {
const {position:p,size:s}=box;
const points=[];
[-1,1].forEach(dx => [-1,1].forEach(dy => [-1,1].forEach(dz => points.push({x:p.x+dx*s.x/2,y:p.y+dy*s.y/2,z:p.z+dz*s.z/2}))));
return points;
}
const faceIndexes=[[0,1,3,2],[4,6,7,5],[0,4,5,1],[2,3,7,6],[0,2,6,4],[1,5,7,3]];
function hexToRgba(hex,alpha) {
const value=hex.replace('#','');
const number=parseInt(value.length===3?value.split('').map(x=>x+x).join(''):value,16);
return `rgba(${(number>>16)&255},${(number>>8)&255},${number&255},${alpha})`;
}
function boxCenter(id) {
const box=scene.boxes.find(item => item.id===id);
return box?.position || null;
}
function relationColor(edge) {
if (edge.meta?.relation === 'SYS') return '#ff6b6b';
return {
nvlink:'#b8ff56',gdr:'#51d9d2',pcie:'#ffbd4a',affinity:'#b592ff',
control:'#8fa99e','bond-member':'#79f4ee'
}[edge.type] || '#8fa99e';
}
function roundedRectPath(x,y,width,height,radius) {
const r=Math.min(radius,width/2,height/2);
ctx.beginPath();
ctx.moveTo(x+r,y);ctx.lineTo(x+width-r,y);ctx.quadraticCurveTo(x+width,y,x+width,y+r);
ctx.lineTo(x+width,y+height-r);ctx.quadraticCurveTo(x+width,y+height,x+width-r,y+height);
ctx.lineTo(x+r,y+height);ctx.quadraticCurveTo(x,y+height,x,y+height-r);
ctx.lineTo(x,y+r);ctx.quadraticCurveTo(x,y,x+r,y);ctx.closePath();
}
function drawLabelChip(box,point,dimmed) {
const isDomain=box.kind==='numa';
const mainFont=isDomain?'900 12px ui-sans-serif,sans-serif':'900 10px ui-sans-serif,sans-serif';
const subFont='800 8px ui-monospace,monospace';
ctx.save();ctx.globalAlpha=dimmed ? .34 : 1;ctx.textAlign='center';
ctx.font=mainFont;
const mainWidth=ctx.measureText(box.label).width;
ctx.font=subFont;
const subWidth=box.sublabel ? ctx.measureText(box.sublabel).width : 0;
const width=Math.max(mainWidth,subWidth)+14;
const height=box.sublabel ? 29 : 18;
const left=point.x-width/2,top=point.y-13;
roundedRectPath(left,top,width,height,5);
ctx.fillStyle=isDomain ? hexToRgba(box.color,.96) : 'rgba(238,249,244,.96)';
ctx.fill();
ctx.strokeStyle=hexToRgba(box.color,.9);ctx.lineWidth=1.2;ctx.stroke();
ctx.fillStyle='#06100c';ctx.font=mainFont;
ctx.fillText(box.label,point.x,top+12);
if (box.sublabel) {
ctx.fillStyle='#17251f';ctx.font=subFont;
ctx.fillText(box.sublabel,point.x,top+23);
}
ctx.restore();
}
function drawEdgeTag(text,x,y,color) {
ctx.save();ctx.font='900 8px ui-monospace,monospace';ctx.textAlign='center';
const width=ctx.measureText(text).width+9;
roundedRectPath(x-width/2,y-8,width,14,4);
ctx.fillStyle=color;ctx.fill();
ctx.fillStyle='#06100c';ctx.fillText(text,x,y+2);
ctx.restore();
}
function drawEdge(edge,scale,selected,overlay=false) {
const from=boxCenter(edge.from),to=boxCenter(edge.to);
if (!from || !to) return;
const a=project(from,scale),b=project(to,scale);
const related=!selected || edge.from===selected || edge.to===selected;
const color=relationColor(edge);
ctx.save();
ctx.strokeStyle=hexToRgba(color,related ? (overlay ? 1 : .85) : (overlay ? .24 : .13));
ctx.lineWidth=related
? (overlay ? 4.8 : edge.type==='gdr' ? 3 : edge.type==='bond-member' ? 2 : 1.3)
: (overlay ? 2 : 1);
if (overlay) {
ctx.shadowColor=color;ctx.shadowBlur=related ? 12 : 3;
ctx.lineCap='round';
}
if (edge.type==='affinity') ctx.setLineDash([5,5]);
if (edge.meta?.relation==='SYS') ctx.setLineDash([8,4]);
ctx.beginPath();ctx.moveTo(a.x,a.y);ctx.lineTo(b.x,b.y);ctx.stroke();
ctx.shadowBlur=0;
if (overlay) {
[a,b].forEach(point => {
ctx.beginPath();ctx.arc(point.x,point.y,related ? 4.5 : 3,0,Math.PI*2);
ctx.fillStyle=hexToRgba(color,related ? 1 : .3);ctx.fill();
ctx.strokeStyle='#06100c';ctx.lineWidth=1.5;ctx.stroke();
});
if (related) {
const markerX=a.x+(b.x-a.x)*.18,markerY=a.y+(b.y-a.y)*.18;
drawEdgeTag(`GPU ${edge.meta?.gpuIndex ?? '?'}`,markerX,markerY,color);
}
} else if (related && selected && Math.hypot(a.x-b.x,a.y-b.y)>70) {
drawEdgeTag(edge.label,(a.x+b.x)/2,(a.y+b.y)/2-4,color);
}
ctx.restore();
}
function drawGrid(scale) {
ctx.save(); ctx.strokeStyle='rgba(122,164,147,.08)'; ctx.lineWidth=1;
for (let value=-24; value<=24; value+=2) {
const a=project({x:value,y:0,z:-12},scale),b=project({x:value,y:0,z:12},scale);
const c=project({x:-24,y:0,z:value},scale),d=project({x:24,y:0,z:value},scale);
ctx.beginPath();ctx.moveTo(a.x,a.y);ctx.lineTo(b.x,b.y);ctx.stroke();
ctx.beginPath();ctx.moveTo(c.x,c.y);ctx.lineTo(d.x,d.y);ctx.stroke();
}
ctx.restore();
}
function draw() {
const width=canvas.clientWidth,height=canvas.clientHeight;
const dpr=Math.min(2,window.devicePixelRatio || 1);
if (canvas.width!==Math.round(width*dpr) || canvas.height!==Math.round(height*dpr)) {
canvas.width=Math.round(width*dpr);canvas.height=Math.round(height*dpr);
}
ctx.setTransform(dpr,0,0,dpr,0,0);
ctx.clearRect(0,0,width,height);
const domainCount=state.focus==='all'?Math.max(1,numaIds.length):1;
const scale=Math.max(18,Math.min(width/(domainCount*22),height/23))*state.zoom;
drawGrid(scale);
const selected=state.selected;
scene.edges.filter(edge => visibleEdge(edge) && edge.layer!=='fabric').forEach(edge => {
drawEdge(edge,scale,selected);
});
const faces=[];
const labels=[];
hitTargets=[];
scene.boxes.filter(visibleBox).forEach(box => {
const projected=corners(box).map(point => project(point,scale));
const alpha = box.opacity ?? 1;
faceIndexes.forEach((indexes,faceIndex) => {
const points=indexes.map(index => projected[index]);
faces.push({box,points,depth:points.reduce((sum,p)=>sum+p.z,0)/points.length,faceIndex,alpha});
});
const xs=projected.map(p=>p.x),ys=projected.map(p=>p.y);
hitTargets.push({box,rect:{left:Math.min(...xs),right:Math.max(...xs),top:Math.min(...ys),bottom:Math.max(...ys)},depth:cameraPoint(box.position).z});
labels.push({
box,
point:project({
x:box.position.x,
y:box.position.y+box.size.y/2+(box.labelOffset ?? .18),
z:box.position.z
},scale)
});
});
faces.sort((a,b) => a.depth-b.depth).forEach(face => {
const isSelected=face.box.id===selected;
const dimmed=selected && !isSelected && face.box.kind!=='numa' && face.box.kind!=='chassis';
const internalBoundary=state.cameraMode==='inside' && (
face.box.kind==='numa' || face.box.kind==='pcie-group' || face.box.kind==='bond'
);
const baseAlpha=internalBoundary
? face.alpha*.22
: face.box.wire ? face.alpha : Math.min(.86,face.alpha*(face.faceIndex===4?1:.72));
ctx.save();
ctx.fillStyle=hexToRgba(face.box.color,dimmed?baseAlpha*.16:baseAlpha);
ctx.strokeStyle=hexToRgba(face.box.color,isSelected ? 1 : internalBoundary ? .26 : face.box.wire ? .55 : .75);
ctx.lineWidth=isSelected ? 2.8 : face.box.wire ? 1.1 : 1;
if (face.box.wire) ctx.setLineDash([7,5]);
ctx.beginPath();face.points.forEach((point,index) => index?ctx.lineTo(point.x,point.y):ctx.moveTo(point.x,point.y));ctx.closePath();
ctx.fill();ctx.stroke();ctx.restore();
});
scene.edges.filter(edge => visibleEdge(edge) && edge.layer==='fabric').forEach(edge => {
drawEdge(edge,scale,selected,true);
});
labels.sort((a,b)=>a.point.z-b.point.z).forEach(({box,point}) => {
if (box.kind==='chassis') return;
if (state.cameraMode==='inside' && box.kind==='numa') return;
const dimmed=selected && box.id!==selected && box.kind!=='numa';
drawLabelChip(box,point,dimmed);
});
}
function renderDetail() {
const root=document.getElementById('selection-detail');
const box=scene.boxes.find(item => item.id===state.selected);
if (!box) {
root.innerHTML=`<span class="detail-kind">MACHINE OVERVIEW</span><h3>${escapeHtml(system.hostname || 'GPU Node')}</h3><dl>
<div><dt>CPU Model</dt><dd>${escapeHtml(system.cpu_model || '—')}</dd></div>
<div><dt>CPU Topology</dt><dd>${escapeHtml(`${system.cpu_sockets || '—'} sockets · ${system.cpu_logical || '—'} logical`)}</dd></div>
<div><dt>GPU</dt><dd>${escapeHtml(`${gpus.length} × ${gpus[0]?.name || '—'}`)}</dd></div>
<div><dt>Virtualization</dt><dd>${escapeHtml(system.virtualization || 'bare metal / unknown')}</dd></div>
<div><dt>PCI Scope</dt><dd>${escapeHtml(data.pci_fabric?.scope || 'guest-visible')}</dd></div>
</dl>`;
return;
}
const meta=box.meta || {};
const fields=Object.entries(meta).map(([key,value]) => `<div><dt>${escapeHtml(key)}</dt><dd>${escapeHtml(value)}</dd></div>`).join('');
root.innerHTML=`<div class="detail-head"><div><span class="detail-kind">${escapeHtml(box.kind.toUpperCase())}</span><h3>${escapeHtml(box.label)}</h3></div><span class="detail-status">${escapeHtml(box.sublabel || '')}</span></div><dl>${fields || '<div><dt>Info</dt><dd>无附加字段</dd></div>'}</dl>`;
}
function pickTarget(x,y) {
return hitTargets
.filter(hit => hit.box.selectable && x>=hit.rect.left && x<=hit.rect.right && y>=hit.rect.top && y<=hit.rect.bottom)
.sort((a,b) => {
const areaA=(a.rect.right-a.rect.left)*(a.rect.bottom-a.rect.top);
const areaB=(b.rect.right-b.rect.left)*(b.rect.bottom-b.rect.top);
return areaA-areaB || b.depth-a.depth;
})[0];
}
canvas.addEventListener('pointerdown',event => {
cameraAnimation+=1;
canvas.setPointerCapture(event.pointerId);
pointer={
id:event.pointerId,x:event.clientX,y:event.clientY,startX:event.clientX,startY:event.clientY,
yaw:state.yaw,pitch:state.pitch,target:{...state.target},pan:event.shiftKey || event.button===1
};
});
canvas.addEventListener('pointermove',event => {
if (!pointer || pointer.id!==event.pointerId) return;
if (pointer.pan) {
const factor=.022/state.zoom;
state.target.x=pointer.target.x-(event.clientX-pointer.x)*factor;
state.target.y=pointer.target.y+(event.clientY-pointer.y)*factor;
} else {
state.yaw=pointer.yaw+(event.clientX-pointer.x)*.008;
state.pitch=Math.max(-.15,Math.min(1.1,pointer.pitch+(event.clientY-pointer.y)*.006));
}
draw();
});
canvas.addEventListener('pointerup',event => {
if (!pointer || pointer.id!==event.pointerId) return;
const moved=Math.hypot(event.clientX-pointer.startX,event.clientY-pointer.startY);
if (moved<6) {
const rect=canvas.getBoundingClientRect(),x=event.clientX-rect.left,y=event.clientY-rect.top;
const target=pickTarget(x,y);
state.selected=target?.box.id || null;renderDetail();updateCameraControls();draw();
}
pointer=null;
});
canvas.addEventListener('dblclick',event => {
const rect=canvas.getBoundingClientRect(),x=event.clientX-rect.left,y=event.clientY-rect.top;
const target=pickTarget(x,y);
if (!target) return;
state.selected=target.box.id;
const nextScope=nextScopeToward(target.box);
if (nextScope) enterScope(nextScope);
else if (isShell(target.box)) enterScope(target.box);
else { renderDetail();setCameraMode('inside',target.box); }
});
canvas.addEventListener('wheel',event => {
event.preventDefault();
cameraAnimation+=1;
const limits=state.cameraMode==='inside' ? [.72,3.8] : [.55,2.3];
state.zoom=Math.max(limits[0],Math.min(limits[1],state.zoom*(event.deltaY>0 ? .92 : 1.08)));draw();
},{passive:false});
canvas.addEventListener('keydown',event => {
const key=event.key;
if (['ArrowLeft','ArrowRight','ArrowUp','ArrowDown'].includes(key)) {
event.preventDefault();
cameraAnimation+=1;
if (key==='ArrowLeft') state.yaw-=.08;if (key==='ArrowRight') state.yaw+=.08;
if (key==='ArrowUp') state.pitch=Math.min(1.1,state.pitch+.06);if (key==='ArrowDown') state.pitch=Math.max(-.15,state.pitch-.06);
draw();
}
if (key==='Escape') {
event.preventDefault();state.selected=null;renderDetail();setCameraMode('outside');
}
});
new ResizeObserver(draw).observe(canvas);
buildViewControls();buildScene();updateCameraControls();draw();
})();
</script>
</body>
</html>

View File

@ -1,133 +0,0 @@
{
"schema_version": 1,
"generated_at": "2026-07-25T09:45:54+08:00",
"collector_version": "1.0.0",
"system": {
"hostname": "root-2",
"os": "Ubuntu 24.04.1 LTS",
"kernel": "6.8.0-71-generic",
"architecture": "x86_64",
"manufacturer": "Baidu Cloud",
"product": "Baidu Cloud BCC",
"product_version": "6.10.0",
"virtualization": "kvm",
"cpu_model": "Intel(R) Xeon(R) 6787P",
"numa_nodes": 2,
"cpu_logical": 332,
"cpu_sockets": 2,
"cpu_cores_per_socket": 83,
"cpu_physical_cores": 166,
"cpu_threads_per_core": 2,
"memory_total_kib": 4227858432,
"memory_available_kib": 3549581076,
"memory_used_kib": 678277356
},
"software": {
"nvidia_driver": "595.71.05",
"cuda_toolkit": "12.8",
"cuda_note": "Toolkit selected by nvcc; application/container runtime may differ."
},
"gpus": [
{"index": 0, "name": "NVIDIA B30Z", "uuid": "GPU-0", "bdf": "63:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 34720, "temperature_c": 48, "utilization_pct": 12, "pstate": "P0", "power_w": 312, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 0},
{"index": 1, "name": "NVIDIA B30Z", "uuid": "GPU-1", "bdf": "67:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 50812, "temperature_c": 51, "utilization_pct": 18, "pstate": "P0", "power_w": 326, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 0},
{"index": 2, "name": "NVIDIA B30Z", "uuid": "GPU-2", "bdf": "6b:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 19236, "temperature_c": 47, "utilization_pct": 8, "pstate": "P0", "power_w": 296, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 0},
{"index": 3, "name": "NVIDIA B30Z", "uuid": "GPU-3", "bdf": "6f:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 62840, "temperature_c": 54, "utilization_pct": 23, "pstate": "P0", "power_w": 341, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 0},
{"index": 4, "name": "NVIDIA B30Z", "uuid": "GPU-4", "bdf": "a3:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 41560, "temperature_c": 50, "utilization_pct": 15, "pstate": "P0", "power_w": 318, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 1},
{"index": 5, "name": "NVIDIA B30Z", "uuid": "GPU-5", "bdf": "a7:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 28410, "temperature_c": 49, "utilization_pct": 10, "pstate": "P0", "power_w": 307, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 1},
{"index": 6, "name": "NVIDIA B30Z", "uuid": "GPU-6", "bdf": "ab:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 73320, "temperature_c": 56, "utilization_pct": 29, "pstate": "P0", "power_w": 359, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 1},
{"index": 7, "name": "NVIDIA B30Z", "uuid": "GPU-7", "bdf": "af:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 22350, "temperature_c": 48, "utilization_pct": 9, "pstate": "P0", "power_w": 301, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 1}
],
"topology": {
"rows": {},
"nic_legend": {
"NIC1": "mlx5_bond_0", "NIC2": "mlx5_bond_1", "NIC3": "mlx5_bond_2", "NIC4": "mlx5_bond_3",
"NIC5": "mlx5_bond_4", "NIC6": "mlx5_bond_5", "NIC7": "mlx5_bond_6", "NIC8": "mlx5_bond_7"
},
"gpu_nic": {
"GPU0": [{"slot": "NIC1", "rdma": "mlx5_bond_0", "relation": "PIX"}],
"GPU1": [{"slot": "NIC2", "rdma": "mlx5_bond_1", "relation": "PIX"}],
"GPU2": [{"slot": "NIC3", "rdma": "mlx5_bond_2", "relation": "PIX"}],
"GPU3": [{"slot": "NIC4", "rdma": "mlx5_bond_3", "relation": "PIX"}],
"GPU4": [{"slot": "NIC5", "rdma": "mlx5_bond_4", "relation": "PIX"}],
"GPU5": [{"slot": "NIC6", "rdma": "mlx5_bond_5", "relation": "PIX"}],
"GPU6": [{"slot": "NIC7", "rdma": "mlx5_bond_6", "relation": "PIX"}],
"GPU7": [{"slot": "NIC8", "rdma": "mlx5_bond_7", "relation": "PIX"}]
},
"gpu_affinity": {
"GPU0": {"cpu_affinity": "0-165", "numa_affinity": "0", "gpu_numa_id": "N/A"},
"GPU1": {"cpu_affinity": "0-165", "numa_affinity": "0", "gpu_numa_id": "N/A"},
"GPU2": {"cpu_affinity": "0-165", "numa_affinity": "0", "gpu_numa_id": "N/A"},
"GPU3": {"cpu_affinity": "0-165", "numa_affinity": "0", "gpu_numa_id": "N/A"},
"GPU4": {"cpu_affinity": "166-331", "numa_affinity": "1", "gpu_numa_id": "N/A"},
"GPU5": {"cpu_affinity": "166-331", "numa_affinity": "1", "gpu_numa_id": "N/A"},
"GPU6": {"cpu_affinity": "166-331", "numa_affinity": "1", "gpu_numa_id": "N/A"},
"GPU7": {"cpu_affinity": "166-331", "numa_affinity": "1", "gpu_numa_id": "N/A"}
}
},
"pci_fabric": {
"scope": "guest-visible",
"virtualization": "passthrough-or-virtual-pci",
"bridges": [
{"bdf": "61:00.0", "parent_bdf": "60:00.0", "numa": 0, "description": "PCI bridge", "children": ["63:00.0", "64:00.0"]},
{"bdf": "a1:00.0", "parent_bdf": "a0:00.0", "numa": 1, "description": "PCI bridge", "children": ["a3:00.0", "a4:00.0"]}
],
"endpoints": [
{"kind": "gpu", "label": "GPU0", "bdf": "63:00.0", "numa": 0, "path": ["60:00.0", "61:00.0", "63:00.0"], "description": "NVIDIA GPU"},
{"kind": "nic", "label": "eth1", "bdf": "64:00.0", "numa": 0, "path": ["60:00.0", "61:00.0", "64:00.0"], "description": "Ethernet controller"},
{"kind": "gpu", "label": "GPU4", "bdf": "a3:00.0", "numa": 1, "path": ["a0:00.0", "a1:00.0", "a3:00.0"], "description": "NVIDIA GPU"},
{"kind": "nic", "label": "eth9", "bdf": "a4:00.0", "numa": 1, "path": ["a0:00.0", "a1:00.0", "a4:00.0"], "description": "Ethernet controller"}
]
},
"nvlink": {
"fabric_type": "NVSwitch Fabric",
"per_gpu": {
"GPU0": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125],
"GPU1": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125],
"GPU2": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125],
"GPU3": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125],
"GPU4": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125],
"GPU5": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125],
"GPU6": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125],
"GPU7": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125]
},
"active_links": 144,
"links_per_gpu": 18,
"speed_gbs": 53.125
},
"nics": [
{
"name": "eth0",
"driver": "mlx5_core",
"bdf": "02:00.0",
"numa": 0,
"speed_mbps": 200000,
"state": "up",
"carrier": 1,
"mac": "fa:20:20:2e:4a:3f",
"addresses": ["192.0.2.10/24"],
"default_route": true
}
],
"bonds": [
{"name": "bond0", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_0", "slaves": [{"name": "eth1", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "64:00.0", "numa": 0}, {"name": "eth2", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "64:00.1", "numa": 0}]},
{"name": "bond1", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_1", "slaves": [{"name": "eth3", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "68:00.0", "numa": 0}, {"name": "eth4", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "68:00.1", "numa": 0}]},
{"name": "bond2", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_2", "slaves": [{"name": "eth5", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "6c:00.0", "numa": 0}, {"name": "eth6", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "6c:00.1", "numa": 0}]},
{"name": "bond3", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_3", "slaves": [{"name": "eth7", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "70:00.0", "numa": 0}, {"name": "eth8", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "70:00.1", "numa": 0}]},
{"name": "bond4", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_4", "slaves": [{"name": "eth9", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "a4:00.0", "numa": 1}, {"name": "eth10", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "a4:00.1", "numa": 1}]},
{"name": "bond5", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_5", "slaves": [{"name": "eth11", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "a8:00.0", "numa": 1}, {"name": "eth12", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "a8:00.1", "numa": 1}]},
{"name": "bond6", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_6", "slaves": [{"name": "eth13", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "ac:00.0", "numa": 1}, {"name": "eth14", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "ac:00.1", "numa": 1}]},
{"name": "bond7", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_7", "slaves": [{"name": "eth15", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "b0:00.0", "numa": 1}, {"name": "eth16", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "b0:00.1", "numa": 1}]}
],
"ib_devices": [
{"name": "mlx5_17", "ca_type": "MT4129", "firmware": "28.46.5500", "state": "Active", "physical_state": "LinkUp", "rate_gbps": 100, "lid": "1", "link_layer": "InfiniBand", "bdf": "b3:00.0", "numa": 1},
{"name": "mlx5_18", "ca_type": "MT4129", "firmware": "28.46.5500", "state": "Active", "physical_state": "LinkUp", "rate_gbps": 100, "lid": "4", "link_layer": "InfiniBand", "bdf": "b4:00.0", "numa": 1},
{"name": "mlx5_19", "ca_type": "MT4129", "firmware": "28.46.5500", "state": "Active", "physical_state": "LinkUp", "rate_gbps": 100, "lid": "5", "link_layer": "InfiniBand", "bdf": "b5:00.0", "numa": 1},
{"name": "mlx5_20", "ca_type": "MT4129", "firmware": "28.46.5500", "state": "Active", "physical_state": "LinkUp", "rate_gbps": 100, "lid": "6", "link_layer": "InfiniBand", "bdf": "b6:00.0", "numa": 1}
],
"bmc": {
"detected": false,
"note": "独立 BMC 带外口未向当前 OS/IPMI 通道暴露"
},
"rdma_map": {},
"notes": []
}

View File

@ -1,430 +0,0 @@
import io
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
import gpu_node_profile as profile
ROOT = Path(__file__).resolve().parents[1]
FIXTURE = ROOT / "tests" / "fixtures" / "b300_snapshot.json"
class ParserTests(unittest.TestCase):
def test_normalize_bdf_uses_endpoint_from_sysfs_path(self):
path = (
"/sys/devices/pci0000:a0/0000:a0:00.4/0000:b1:00.0/"
"0000:b2:00.0/0000:b3:00.0"
)
self.assertEqual(profile.normalize_bdf(path), "b3:00.0")
def test_cuda_build_version_is_concise(self):
self.assertEqual(profile.concise_version("12.8.20250303"), "12.8")
def test_parse_topology_and_mapping(self):
text = """
GPU0 GPU1 NIC0 NIC1 CPU Affinity NUMA Affinity
GPU0 X NV18 PIX SYS 0-165 0
GPU1 NV18 X SYS PIX 166-331 1
NIC Legend:
NIC0: mlx5_bond_0
NIC1: mlx5_bond_1
"""
parsed = profile.parse_topology(text)
self.assertEqual(parsed["nic_legend"]["NIC0"], "mlx5_bond_0")
self.assertEqual(parsed["gpu_nic"]["GPU0"][0]["relation"], "PIX")
self.assertEqual(
parsed["gpu_affinity"]["GPU0"]["cpu_affinity"], "0-165"
)
self.assertEqual(
parsed["gpu_affinity"]["GPU1"]["numa_affinity"], "1"
)
self.assertNotIn("CPU", parsed["rows"]["GPU0"])
def test_parse_bond(self):
text = """
Bonding Mode: IEEE 802.3ad Dynamic link aggregation
Transmit Hash Policy: layer3+4 (1)
MII Status: up
Slave Interface: eth1
MII Status: up
Speed: 400000 Mbps
Link Failure Count: 0
"""
bond = profile.parse_bond_text("bond0", text)
self.assertEqual(bond["status"], "up")
self.assertEqual(bond["slaves"][0]["speed_mbps"], 400000)
def test_nvlink_fabric_type_does_not_assume_nvswitch(self):
direct = {"rows": {"GPU0": {"GPU1": "NV4"}, "GPU1": {"GPU0": "NV4"}}}
self.assertEqual(
profile.infer_nvlink_fabric_type(direct, 2), "NVLink Fabric"
)
switched = {
"rows": {
f"GPU{source}": {
f"GPU{target}": "NV18"
for target in range(4)
if target != source
}
for source in range(4)
}
}
self.assertEqual(
profile.infer_nvlink_fabric_type(switched, 4), "NVSwitch Fabric"
)
def test_render_is_offline_native_html(self):
snapshot = json.loads(FIXTURE.read_text(encoding="utf-8"))
profile.attach_gpu_bond_mapping(snapshot)
report = profile.render_html(
snapshot, {"gpus": 8, "bonds": 8, "ib": 4}
)
self.assertIn("GPU ↔ RDMA 亲和画像", report)
self.assertIn("系统与软件", report)
self.assertIn("CPU 与内存", report)
self.assertIn("带内与带外管理网络", report)
self.assertIn("eth0", report)
self.assertIn("带外 · BMC", report)
self.assertIn("HEALTH 与验证结果", report)
self.assertIn("CPU / NUMA 亲和", report)
self.assertIn("GPU 功耗遥测", report)
self.assertIn("GPU 瞬时功耗", report)
self.assertIn("2.56 kW / 8.80 kW", report)
self.assertIn("瞬时 / 1.10 kW 上限", report)
self.assertIn(">P0<", report)
self.assertIn("GPU0–3", report)
self.assertIn("推荐 CPU 0-165", report)
self.assertIn("不代表当前训练进程已经绑核", report)
self.assertIn('data-numa-filter="all"', report)
self.assertIn('data-numa-filter="0"', report)
self.assertIn('data-numa-filter="1"', report)
self.assertIn("function setNumaView(value)", report)
self.assertIn(".rail[hidden],.numa-ruler [hidden]", report)
self.assertIn(".ib-card .card-head { align-items:flex-start; flex-wrap:wrap;", report)
self.assertIn("eth0 承载默认路由且链路 Up", report)
self.assertIn("mlx5_bond_7", report)
self.assertIn("NVLINK / NVSWITCH FABRIC", report)
self.assertLess(report.index("系统与软件"), report.index("GPU ↔ RDMA 亲和画像"))
self.assertLess(report.index("带内与带外管理网络"), report.index("HEALTH 与验证结果"))
self.assertNotIn("mermaid", report.lower())
self.assertNotIn("https://", report.lower())
self.assertNotIn("http://", report.lower())
def test_render_3d_topology_is_offline_and_json_driven(self):
snapshot = json.loads(FIXTURE.read_text(encoding="utf-8"))
profile.attach_gpu_bond_mapping(snapshot)
report = profile.render_topology_3d_html(snapshot)
self.assertIn("GPU Machine 3D CT", report)
self.assertIn('id="machine-data"', report)
self.assertIn('id="topology-canvas"', report)
self.assertIn("function buildScene()", report)
self.assertIn("function deepestCommonPath(paths)", report)
self.assertIn('data-camera="inside"', report)
self.assertIn("function setCameraMode(mode,box=null,animate=true)", report)
self.assertIn("双击壳体逐层进入", report)
self.assertIn("state.cameraMode==='inside'", report)
self.assertIn("kind:'bond'", report)
self.assertIn("kind:'roce-port'", report)
self.assertIn("GPU endpoints only · not NIC", report)
self.assertIn("function drawEdge(edge,scale,selected,overlay=false)", report)
self.assertIn("function scopePath(box)", report)
self.assertIn("function nextScopeToward(box)", report)
self.assertIn("function enterScope(box,animate=true)", report)
self.assertIn('id="camera-back"', report)
self.assertIn("parentId:`pair-${gpu.index}`", report)
self.assertIn("PCIe Root Complex", report)
self.assertIn("GDR candidate", report)
self.assertIn('"scope": "guest-visible"', report)
self.assertNotIn("https://", report.lower())
self.assertNotIn("http://", report.lower())
def test_html_templates_are_separate_source_files(self):
operations = profile.load_html_template("gpu_node_profile.html")
topology = profile.load_html_template("gpu_topology_3d.html")
self.assertIn("@@BODY@@", operations)
self.assertIn("@@MACHINE_DATA@@", topology)
self.assertNotIn("https://", operations.lower())
self.assertNotIn("https://", topology.lower())
def test_build_pci_fabric_preserves_bridge_hierarchy(self):
gpus = [
{
"index": 0,
"bdf": "63:00.0",
"numa": 0,
"pci_path": ["60:00.0", "61:00.0", "63:00.0"],
}
]
nics = [
{
"name": "eth1",
"bdf": "64:00.0",
"numa": 0,
"pci_path": ["60:00.0", "61:00.0", "64:00.0"],
}
]
with patch.object(profile, "pci_numa", return_value=0):
fabric = profile.build_pci_fabric(gpus, nics, [], {})
self.assertEqual(fabric["scope"], "guest-visible")
bridge = next(item for item in fabric["bridges"] if item["bdf"] == "61:00.0")
self.assertEqual(bridge["children"], ["63:00.0", "64:00.0"])
def test_standalone_roce_endpoints_map_uniquely_to_gpus(self):
nics = [
{
"name": "eth0",
"bdf": "0b:00.0",
"numa": 0,
"pci_path": ["08:00.0", "0b:00.0"],
"speed_mbps": 100000,
"state": "up",
"carrier": 1,
"default_route": True,
"is_bond": False,
},
{
"name": "eth1",
"bdf": "5a:00.0",
"numa": 0,
"pci_path": ["4c:00.0", "54:00.0", "5a:00.0"],
"speed_mbps": 100000,
"state": "up",
"carrier": 1,
"default_route": False,
"is_bond": False,
},
{
"name": "eth2",
"bdf": "5a:00.1",
"numa": 0,
"pci_path": ["4c:00.0", "54:00.0", "5a:00.1"],
"speed_mbps": 100000,
"state": "up",
"carrier": 1,
"default_route": False,
"is_bond": False,
},
]
rdma_map = {"mlx5_0": "eth0", "mlx5_1": "eth1", "mlx5_2": "eth2"}
endpoints = profile.build_network_endpoints(
nics,
[],
[],
rdma_map,
{
"mlx5_1": {"state": "ACTIVE"},
"mlx5_2": {"state": "ACTIVE"},
},
)
self.assertEqual([item["name"] for item in endpoints], ["eth1", "eth2"])
self.assertTrue(all(item["type"] == "standalone" for item in endpoints))
snapshot = {
"gpus": [
{"index": 0, "pci_path": ["4c:00.0", "53:00.0"], "numa": 0},
{"index": 1, "pci_path": ["4c:00.0", "58:00.0"], "numa": 0},
],
"nics": nics,
"bonds": [],
"ib_devices": [],
"rdma_map": rdma_map,
"network_endpoints": endpoints,
"topology": {
"gpu_affinity": {},
"gpu_nic": {
"GPU0": [
{"rdma": "mlx5_1", "relation": "PXB"},
{"rdma": "mlx5_2", "relation": "PXB"},
],
"GPU1": [
{"rdma": "mlx5_1", "relation": "PXB"},
{"rdma": "mlx5_2", "relation": "PXB"},
],
},
},
}
profile.attach_gpu_network_mapping(snapshot)
primary_ids = {
gpu.get("primary_network_id") for gpu in snapshot["gpus"]
}
self.assertEqual(primary_ids, {"rdma:mlx5_1", "rdma:mlx5_2"})
self.assertTrue(
all(gpu.get("network_relation") == "PXB" for gpu in snapshot["gpus"])
)
health = profile.evaluate(
snapshot, {"gpus": 2, "bonds": None, "ib": None}
)
affinity = next(
item for item in health["checks"] if item["title"] == "GPU ↔ RDMA 亲和"
)
self.assertEqual(affinity["level"], "ok")
self.assertIn("PXB 2", affinity["detail"])
operations = profile.render_html(
snapshot, {"gpus": 2, "bonds": None, "ib": None}
)
topology = profile.render_topology_3d_html(snapshot)
self.assertEqual(operations.count("ROCE · DIRECT"), 2)
self.assertNotIn("未发现对应网络路径", operations)
self.assertIn("Mode:'Standalone RDMA'", topology)
self.assertIn("endpoint.topology_visibility", topology)
def test_unique_mapping_uses_global_minimum_not_greedy_first_match(self):
endpoints = [
{
"id": "rdma:mlx5_1",
"type": "standalone",
"protocol": "RoCE",
"name": "eth1",
"rdma": "mlx5_1",
"role": "compute",
"members": [{"name": "eth1", "pci_path": []}],
},
{
"id": "rdma:mlx5_2",
"type": "standalone",
"protocol": "RoCE",
"name": "eth2",
"rdma": "mlx5_2",
"role": "compute",
"members": [{"name": "eth2", "pci_path": []}],
},
]
snapshot = {
"gpus": [
{"index": 0, "pci_path": [], "numa": 0},
{"index": 1, "pci_path": [], "numa": 0},
],
"bonds": [],
"network_endpoints": endpoints,
"topology": {
"gpu_affinity": {},
"gpu_nic": {
"GPU0": [
{"rdma": "mlx5_1", "relation": "PIX"},
{"rdma": "mlx5_2", "relation": "PIX"},
],
"GPU1": [
{"rdma": "mlx5_1", "relation": "PIX"},
{"rdma": "mlx5_2", "relation": "PHB"},
],
},
},
}
profile.attach_gpu_network_mapping(snapshot)
self.assertEqual(snapshot["gpus"][0]["network"], "eth2")
self.assertEqual(snapshot["gpus"][1]["network"], "eth1")
self.assertTrue(
all(gpu["network_relation"] == "PIX" for gpu in snapshot["gpus"])
)
def test_non_rdma_bond_is_not_an_accelerator_endpoint(self):
management_bond = {
"name": "bond-mgmt",
"rdma": "",
"status": "up",
"slaves": [{"name": "eno1", "status": "up"}],
}
endpoints = profile.build_network_endpoints(
[],
[management_bond],
[],
{},
)
self.assertEqual(endpoints, [])
health = profile.evaluate(
{
"gpus": [],
"bonds": [management_bond],
"ib_devices": [],
"network_endpoints": [],
"topology": {},
},
{"gpus": None, "bonds": 0, "ib": None},
)
self.assertFalse(
any(item["title"] == "Bond 数量不符合预期" for item in health["checks"])
)
def test_power_telemetry_health_does_not_flag_p0_as_an_error(self):
snapshot = json.loads(FIXTURE.read_text(encoding="utf-8"))
profile.attach_gpu_bond_mapping(snapshot)
health = profile.evaluate(snapshot, {"gpus": 8, "bonds": 8, "ib": 4})
power_check = next(
item for item in health["checks"] if item["title"] == "GPU 功耗遥测"
)
self.assertEqual(power_check["level"], "ok")
self.assertIn("P0 8/8", power_check["detail"])
def test_collect_gpus_keeps_power_when_optional_range_is_unsupported(self):
core = (
"0, NVIDIA Test, GPU-0, 00000000:63:00.0, 595.71.05, "
"275040, 1024, 42, 75"
)
power = "0, P0, 875.5, 1100"
with (
patch.object(profile, "run", side_effect=[core, power, ""]),
patch.object(profile, "pci_numa", return_value=0),
):
gpus = profile.collect_gpus()
self.assertEqual(len(gpus), 1)
self.assertEqual(gpus[0]["pstate"], "P0")
self.assertEqual(gpus[0]["power_w"], 875.5)
self.assertEqual(gpus[0]["power_limit_w"], 1100)
self.assertNotIn("power_max_limit_w", gpus[0])
def test_incomplete_power_telemetry_is_a_warning(self):
snapshot = json.loads(FIXTURE.read_text(encoding="utf-8"))
snapshot["gpus"][0].pop("power_limit_w")
profile.attach_gpu_bond_mapping(snapshot)
health = profile.evaluate(snapshot, {"gpus": 8, "bonds": 8, "ib": 4})
power_check = next(
item for item in health["checks"] if item["title"] == "GPU功耗信息不完整"
)
self.assertEqual(power_check["level"], "warning")
report = profile.render_html(snapshot, {"gpus": 8, "bonds": 8, "ib": 4})
self.assertIn("遥测不完整", report)
def test_cli_renders_snapshot(self):
with tempfile.TemporaryDirectory() as directory:
output = Path(directory) / "profile.html"
topology_output = Path(directory) / "topology-3d.html"
result = profile.main(
[
"--snapshot-in",
str(FIXTURE),
"--output",
str(output),
"--topology-output",
str(topology_output),
"--expected-gpus",
"8",
"--expected-bonds",
"8",
"--expected-ib",
"4",
]
)
self.assertEqual(result, 0)
self.assertTrue(output.exists())
self.assertGreater(output.stat().st_size, 30_000)
self.assertTrue(topology_output.exists())
self.assertIn(
'id="topology-canvas"',
topology_output.read_text(encoding="utf-8"),
)
def test_cli_can_emit_structured_snapshot_only(self):
buffer = io.StringIO()
with patch.object(profile.sys, "stdout", buffer):
result = profile.main(["--snapshot-in", str(FIXTURE), "--snapshot-stdout"])
self.assertEqual(result, 0)
snapshot = json.loads(buffer.getvalue())
self.assertEqual(snapshot["schema_version"], 1)
self.assertEqual(snapshot["pci_fabric"]["scope"], "guest-visible")
if __name__ == "__main__":
unittest.main()

View File

@ -1,34 +0,0 @@
10.184.0.20 slots=8
10.184.0.21 slots=8
10.184.0.22 slots=8
10.184.0.23 slots=8
10.184.0.24 slots=8
10.184.0.25 slots=8
10.184.0.26 slots=8
10.184.0.27 slots=8
10.184.0.28 slots=8
10.184.0.29 slots=8
10.184.0.30 slots=8
10.184.0.31 slots=8
10.184.0.32 slots=8
10.184.0.33 slots=8
10.184.0.34 slots=8
10.184.0.35 slots=8
10.184.0.36 slots=8
10.184.0.37 slots=8
10.184.0.38 slots=8
10.184.0.39 slots=8
10.184.0.40 slots=8
10.184.0.41 slots=8
10.184.0.42 slots=8
10.184.0.43 slots=8
10.184.0.44 slots=8
10.184.0.45 slots=8
10.184.0.46 slots=8
10.184.0.47 slots=8
10.184.0.48 slots=8
10.184.0.49 slots=8
10.184.0.50 slots=8
10.184.0.51 slots=8
10.184.0.52 slots=8
10.184.0.55 slots=8

View File

@ -1,28 +0,0 @@
#!/bin/bash
# ===== 多机 NCCL 测试环境 — H20-3e 集群 (34台, 8xNDR400 per node) =====
# 主节点: gpu-20 (10.184.0.20), 在此节点 source 后执行 mpirun
# 验证日期: 2026-06-06
# OpenMPI 不在默认 loader 路径,必须显式加
export PATH=/usr/mpi/gcc/openmpi-4.1.7rc1/bin:$PATH
export LD_LIBRARY_PATH=/usr/mpi/gcc/openmpi-4.1.7rc1/lib:$LD_LIBRARY_PATH
# NCCL: 使用 8 张 GPU 直连 IB 网卡 (PIX 级别直连)
# 排除 mlx5_6/7: 虽为 IB Active, 但无 GPU PIX 直连, 为存储 NIC
# 排除 mlx5_bond_0: 25G 以太管理口
# GPU↔NIC 对应: GPU0↔mlx5_3, GPU1↔mlx5_2, GPU2↔mlx5_1, GPU3↔mlx5_0
# GPU4↔mlx5_5, GPU5↔mlx5_4, GPU6↔mlx5_9, GPU7↔mlx5_8
export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9
export NCCL_IB_DISABLE=0
# NCCL bootstrap 网卡: 节点上有 k8s 虚拟网卡 (lvscare/kube-ipvs0),
# 不指定会选错导致 connection refused, 内网 IP 在 bond0 上
# 注意: 不能用 ^bond (会把 bond0 也排掉), 必须明确写 bond0
export NCCL_SOCKET_IFNAME=bond0
export NCCL_IB_DISABLE=0
export NCCL_DEBUG=${NCCL_DEBUG:-WARN}
# IB 为 InfiniBand link_layer (非 RoCE), GID 一般不用设
# 如报 GID 问题再放开:
# export NCCL_IB_GID_INDEX=3

View File

@ -1,76 +0,0 @@
#!/bin/bash
# ===== 跨交换机多机 NCCL 通信测试 =====
# 适用场景: 验证不同交换机的节点加入集群后跨交换机通信是否正常
# 用法: bash run_cross_switch_test.sh
# 主节点运行: gpu-20 (需与目标节点已打通免密 SSH)
#
# 默认分三个阶段:
# old2node — 旧集群内 2 台基准 (gpu-20/21)
# new2node — 新机 2 台同交换机 (gpu-51/55)
# cross4node — 跨交换机 4 台联合
#
# 新机接入前置步骤:
# 1. 四台节点互相打通 authorized_keys
# 2. 新机上重编 nccl-tests: cd /opt/nccl-tests && make clean &&
# make MPI=1 MPI_HOME=/usr/mpi/gcc/openmpi-4.1.7rc1 CUDA_HOME=/usr/local/cuda -j
set -euo pipefail
source "$(dirname "$0")/nccl_env.sh"
NT=/opt/nccl-tests/build
OUT=~/nccl_kit/results_cross_$(date +%Y%m%d_%H%M%S)
mkdir -p "$OUT"
SUMMARY="$OUT/summary.csv"
echo "test,nodes,op,size,busbw_GBps" > "$SUMMARY"
MPI=(mpirun --allow-run-as-root
--mca plm_rsh_args "-oStrictHostKeyChecking=no"
-x PATH -x LD_LIBRARY_PATH
-x NCCL_IB_HCA -x NCCL_IB_DISABLE
-x NCCL_SOCKET_IFNAME -x NCCL_DEBUG)
run_test() {
local tag="$1" hostfile="$2"
local np=$(( $(wc -l < "$hostfile") * 8 ))
echo ""
echo "======== $tag ($np GPUs) ========"
for op in all_reduce alltoall all_gather reduce_scatter; do
log="$OUT/${tag}_${op}.log"
"${MPI[@]}" -np "$np" -N 8 --hostfile "$hostfile" \
"$NT/${op}_perf" -b 8 -e 4G -f 2 -g 1 -w 5 -n 30 > "$log" 2>&1
row=$(awk '$1==4294967296{print}' "$log" | tail -1)
busbw=$(echo "$row" | awk '{print $(NF-1)}')
[ -z "$busbw" ] && busbw="ERR(见 $log)"
printf " %-14s 4G busbw=%s GB/s\n" "$op" "$busbw"
echo "$tag,$(wc -l < "$hostfile"),$op,4G,$busbw" >> "$SUMMARY"
done
}
# === 阶段1: 旧集群内 2 台 (基准) ===
cat > "$OUT/hosts_old2" << HEOF
10.184.0.20 slots=8
10.184.0.21 slots=8
HEOF
run_test "old2node" "$OUT/hosts_old2"
# === 阶段2: 新机 2 台同交换机 ===
cat > "$OUT/hosts_new2" << HEOF
10.184.0.51 slots=8
10.184.0.55 slots=8
HEOF
run_test "new2node" "$OUT/hosts_new2"
# === 阶段3: 跨交换机 4 台 ===
cat > "$OUT/hosts_cross4" << HEOF
10.184.0.20 slots=8
10.184.0.21 slots=8
10.184.0.51 slots=8
10.184.0.55 slots=8
HEOF
run_test "cross4node" "$OUT/hosts_cross4"
echo ""
echo "==== 汇总 ===="
column -t -s, "$SUMMARY"
echo "结果目录: $OUT"

View File

@ -1,63 +0,0 @@
#!/bin/bash
# ===== 多机 NCCL Scaling 测试 =====
# 用法: bash run_scaling.sh [节点级别...]
# 默认: "2 4 8 16 32"
# 示例: bash run_scaling.sh 2 4 8
# 前置: source nccl_env.sh, hosts_all 文件与本脚本同目录或指定 HOSTS_ALL
# 主节点运行: gpu-20
set -euo pipefail
source "$(dirname "$0")/nccl_env.sh"
NT=/opt/nccl-tests/build
HOSTS_ALL="${HOSTS_ALL:-$(dirname "$0")/hosts_all}"
LEVELS="${*:-2 4 8 16 32}"
OUT=~/nccl_kit/results_$(date +%Y%m%d_%H%M%S)
mkdir -p "$OUT"
SUMMARY="$OUT/summary.csv"
echo "nodes,gpus,op,size,busbw_GBps" > "$SUMMARY"
MPI=(mpirun --allow-run-as-root
--mca plm_rsh_args "-oStrictHostKeyChecking=no"
-x PATH -x LD_LIBRARY_PATH
-x NCCL_IB_HCA -x NCCL_IB_DISABLE
-x NCCL_SOCKET_IFNAME -x NCCL_DEBUG)
echo "Scaling levels: $LEVELS"
echo "结果目录: $OUT"
for N in $LEVELS; do
hf="$OUT/hosts_$N"
head -"$N" "$HOSTS_ALL" > "$hf"
avail=$(wc -l < "$hf")
if [ "$avail" -lt "$N" ]; then
echo "!! hosts_all 只有 $avail 台, 跳过 $N 节点"
continue
fi
np=$((N * 8))
echo ""
echo "================ $N nodes / $np GPUs ================"
for op in all_reduce alltoall all_gather reduce_scatter; do
log="$OUT/${op}_${N}n.log"
"${MPI[@]}" -np "$np" -N 8 --hostfile "$hf" \
"$NT/${op}_perf" -b 8 -e 8G -f 2 -g 1 -w 5 -n 50 > "$log" 2>&1
# AllToAll 在大消息 (>2G) + 多节点会 hang, 优先取 1G 行
if [[ "$op" == "alltoall" ]]; then
row=$(awk '$1==1073741824{print}' "$log" | tail -1)
sz="1G"
else
row=$(awk '$1==8589934592{print}' "$log" | tail -1)
sz="8G"
fi
busbw=$(echo "$row" | awk '{print $(NF-1)}')
[ -z "$busbw" ] && busbw="ERR(见 $log)"
printf " %-14s %s busbw=%s GB/s\n" "$op" "$sz" "$busbw"
echo "$N,$np,$op,$sz,$busbw" >> "$SUMMARY"
done
done
echo ""
echo "==== 汇总 (AllReduce 应随节点数基本持平 ~317 GB/s) ===="
column -t -s, "$SUMMARY"
echo "结果目录: $OUT"

View File

@ -0,0 +1,425 @@
#!/usr/bin/env bash
set -euo pipefail
# Deep-diagnose multi-node NCCL behavior from the coordinator node.
# Default values match the current 2-node H100 cross-leaf investigation.
MODE="${1:-all}"
MPI_BIN="${MPI_BIN:-/usr/mpi/gcc/openmpi-4.1.9a1/bin/mpirun}"
NCCL_TESTS_DIR="${NCCL_TESTS_DIR:-/data/nccl-tests-latest/build}"
HOSTS="${HOSTS:-172.72.8.12:8,172.72.8.16:8}"
PEER_HOST="${PEER_HOST:-172.72.8.16}"
SSH_USER="${SSH_USER:-root}"
HCAS="${HCAS:-mlx5_0 mlx5_1 mlx5_6 mlx5_7}"
HCA_CSV="${HCA_CSV:-mlx5_0,mlx5_1,mlx5_6,mlx5_7}"
OUT_DIR="${OUT_DIR:-/tmp/nccl_deep_diagnose_$(date +%Y%m%d_%H%M%S)}"
BEGIN_SIZE="${BEGIN_SIZE:-16G}"
END_SIZE="${END_SIZE:-16G}"
WARMUP_ITERS="${WARMUP_ITERS:-10}"
ITERS="${ITERS:-10}"
GRAPH_WARMUP_ITERS="${GRAPH_WARMUP_ITERS:-1}"
GRAPH_ITERS="${GRAPH_ITERS:-1}"
SWEEP_WARMUP_ITERS="${SWEEP_WARMUP_ITERS:-3}"
SWEEP_ITERS="${SWEEP_ITERS:-5}"
NCCL_LD_LIBRARY_PATH="${NCCL_LD_LIBRARY_PATH:-/usr/mpi/gcc/openmpi-4.1.9a1/lib:/tmp/nccl-2.27.7-cuda12.4/usr/lib/x86_64-linux-gnu:/usr/local/cuda-12.4/targets/x86_64-linux/lib}"
DEFAULT_NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
COUNTERS="${COUNTERS:-port_xmit_data port_rcv_data port_xmit_packets port_rcv_packets port_xmit_wait port_xmit_discards port_rcv_errors port_rcv_remote_physical_errors port_rcv_switch_relay_errors port_xmit_constraint_errors port_rcv_constraint_errors symbol_error link_error_recovery link_downed local_link_integrity_errors excessive_buffer_overrun_errors VL15_dropped}"
HW_COUNTERS="${HW_COUNTERS:-roce_adp_retrans roce_adp_retrans_to roce_slow_restart roce_slow_restart_cnps roce_slow_restart_trans packet_seq_err out_of_sequence out_of_buffer duplicate_request implied_nak_seq_err local_ack_timeout_err req_transport_retries_exceeded rnr_nak_retry_err rx_write_requests rx_read_requests}"
mkdir -p "$OUT_DIR"
mpi_base=(
"$MPI_BIN"
--allow-run-as-root
--mca btl_openib_warn_no_device_params_found 0
--mca btl_tcp_if_include bond0
--mca oob_tcp_if_include bond0
--mca plm_rsh_args "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=10"
-H "$HOSTS"
--map-by ppr:8:node
-np 16
)
base_exports=(
LD_LIBRARY_PATH
NCCL_IB_GID_INDEX NCCL_IB_SL NCCL_IB_TC NCCL_SOCKET_IFNAME
NCCL_DEBUG NCCL_DEBUG_SUBSYS NCCL_IB_TIMEOUT NCCL_IB_HCA
NCCL_NET_PLUGIN NCCL_NVLS_ENABLE NCCL_NET_GDR_LEVEL NCCL_NET_GDR_READ
NCCL_DMABUF_ENABLE NCCL_PXN_DISABLE NCCL_IB_QPS_PER_CONNECTION
NCCL_IB_SPLIT_DATA_ON_QPS NCCL_MIN_NCHANNELS NCCL_MAX_NCHANNELS
NCCL_BUFFSIZE NCCL_P2P_NET_CHUNKSIZE NCCL_NCHANNELS_PER_NET_PEER
NCCL_IB_AR_THRESHOLD
)
set_common_env() {
unset NCCL_DEBUG_SUBSYS NCCL_PXN_DISABLE NCCL_IB_QPS_PER_CONNECTION
unset NCCL_IB_SPLIT_DATA_ON_QPS NCCL_MIN_NCHANNELS NCCL_MAX_NCHANNELS
unset NCCL_BUFFSIZE NCCL_P2P_NET_CHUNKSIZE NCCL_NCHANNELS_PER_NET_PEER
unset NCCL_IB_AR_THRESHOLD
export LD_LIBRARY_PATH="$NCCL_LD_LIBRARY_PATH"
export NCCL_IB_GID_INDEX="${NCCL_IB_GID_INDEX:-3}"
export NCCL_IB_SL="${NCCL_IB_SL:-5}"
export NCCL_IB_TC="${NCCL_IB_TC:-136}"
export NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-bond0}"
export NCCL_DEBUG="$DEFAULT_NCCL_DEBUG"
export NCCL_IB_TIMEOUT="${NCCL_IB_TIMEOUT:-22}"
export NCCL_IB_HCA="$HCA_CSV"
export NCCL_NET_PLUGIN="${NCCL_NET_PLUGIN:-none}"
export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-1}"
export NCCL_NET_GDR_LEVEL="${NCCL_NET_GDR_LEVEL:-5}"
export NCCL_NET_GDR_READ="${NCCL_NET_GDR_READ:-1}"
export NCCL_DMABUF_ENABLE="${NCCL_DMABUF_ENABLE:-0}"
}
mpi_xargs() {
for name in "${base_exports[@]}"; do
if [[ -n "${!name+x}" ]]; then
printf -- '-x\n%s\n' "$name"
fi
done
}
run_nccl() {
local op="$1"
local bin="$2"
local log="$3"
local warmup="$4"
local iters="$5"
mapfile -t xargs < <(mpi_xargs)
"${mpi_base[@]}" "${xargs[@]}" \
"$bin" -b "$BEGIN_SIZE" -e "$END_SIZE" -g 1 -f 2 -w "$warmup" -n "$iters" \
>"$log" 2>&1
awk -v op="$op" '/Avg bus bandwidth/ {print op, $0}' "$log"
}
read_one_snapshot() {
local host_label="$1"
local out="$2"
: >"$out"
for hca in $HCAS; do
for c in $COUNTERS; do
local f="/sys/class/infiniband/$hca/ports/1/counters/$c"
if [[ -r "$f" ]]; then
printf '%s %s counters %s %s\n' "$host_label" "$hca" "$c" "$(cat "$f" 2>/dev/null || echo 0)" >>"$out"
fi
done
for c in $HW_COUNTERS; do
local f="/sys/class/infiniband/$hca/ports/1/hw_counters/$c"
if [[ -r "$f" ]]; then
printf '%s %s hw_counters %s %s\n' "$host_label" "$hca" "$c" "$(cat "$f" 2>/dev/null || echo 0)" >>"$out"
fi
done
done
}
read_remote_snapshot() {
local out="$1"
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-o BatchMode=yes -o ConnectTimeout=5 "${SSH_USER}@${PEER_HOST}" \
"HCAS='$HCAS' COUNTERS='$COUNTERS' HW_COUNTERS='$HW_COUNTERS' bash -s" <<'EOS' >"$out"
for hca in $HCAS; do
for c in $COUNTERS; do
f="/sys/class/infiniband/$hca/ports/1/counters/$c"
if [ -r "$f" ]; then
printf '%s %s counters %s %s\n' "$HOSTNAME" "$hca" "$c" "$(cat "$f" 2>/dev/null || echo 0)"
fi
done
for c in $HW_COUNTERS; do
f="/sys/class/infiniband/$hca/ports/1/hw_counters/$c"
if [ -r "$f" ]; then
printf '%s %s hw_counters %s %s\n' "$HOSTNAME" "$hca" "$c" "$(cat "$f" 2>/dev/null || echo 0)"
fi
done
done
EOS
}
summarize_counter_delta() {
local before_a="$1"
local before_b="$2"
local after_a="$3"
local after_b="$4"
local out="$5"
python3 - "$before_a" "$before_b" "$after_a" "$after_b" >"$out" <<'PY'
import pathlib
import sys
interesting = {
"port_xmit_wait", "port_xmit_discards", "port_rcv_errors",
"port_rcv_remote_physical_errors", "port_rcv_switch_relay_errors",
"port_xmit_constraint_errors", "port_rcv_constraint_errors",
"symbol_error", "link_error_recovery", "link_downed",
"local_link_integrity_errors", "excessive_buffer_overrun_errors",
"VL15_dropped", "roce_adp_retrans", "roce_adp_retrans_to",
"roce_slow_restart", "roce_slow_restart_cnps", "roce_slow_restart_trans",
"packet_seq_err", "out_of_sequence", "out_of_buffer",
"duplicate_request", "implied_nak_seq_err", "local_ack_timeout_err",
"req_transport_retries_exceeded", "rnr_nak_retry_err",
}
def load(path):
data = {}
for line in pathlib.Path(path).read_text().splitlines():
parts = line.split()
if len(parts) != 5:
continue
host, hca, kind, counter, value = parts
try:
data[(host, hca, kind, counter)] = int(value)
except ValueError:
pass
return data
before = {}
after = {}
before.update(load(sys.argv[1]))
before.update(load(sys.argv[2]))
after.update(load(sys.argv[3]))
after.update(load(sys.argv[4]))
print("NONZERO_DELTAS")
for key in sorted(set(before) | set(after)):
delta = after.get(key, 0) - before.get(key, 0)
if not delta:
continue
host, hca, kind, counter = key
if counter in {"port_xmit_data", "port_rcv_data"}:
gib = delta * 4 / (1024 ** 3)
print(f"{host} {hca} {kind} {counter} {delta} words4B {gib:.2f} GiB")
else:
print(f"{host} {hca} {kind} {counter} {delta}")
print("ERROR_OR_CONGESTION_DELTAS")
seen = False
for key in sorted(set(before) | set(after)):
delta = after.get(key, 0) - before.get(key, 0)
if delta and key[3] in interesting:
seen = True
print(*key, delta)
if not seen:
print("none")
PY
}
run_counter_case() {
local op="$1"
local bin="$2"
local extra="${3:-}"
set_common_env
if [[ -n "$extra" ]]; then
eval "export $extra"
fi
local dir="$OUT_DIR/${op}_counter"
mkdir -p "$dir"
read_one_snapshot "$(hostname)" "$dir/before.local"
read_remote_snapshot "$dir/before.remote"
run_nccl "$op" "$bin" "$dir/${op}.log" "$WARMUP_ITERS" "$ITERS"
read_one_snapshot "$(hostname)" "$dir/after.local"
read_remote_snapshot "$dir/after.remote"
summarize_counter_delta "$dir/before.local" "$dir/before.remote" "$dir/after.local" "$dir/after.remote" "$dir/counter_delta.txt"
echo "$dir"
}
summarize_graph_log() {
local log="$1"
local out="$2"
python3 - "$log" >"$out" <<'PY'
from pathlib import Path
import collections
import re
import sys
text = Path(sys.argv[1]).read_text(errors="ignore")
print("avg_busbw", (re.findall(r"Avg bus bandwidth\s*:\s*([0-9.]+)", text) or ["NA"])[-1])
print("nccl_version", sorted(set(re.findall(r"NCCL version ([^\s]+)", text))))
print("plugin_missing", len(re.findall(r"Could not find: none libnccl-net-none\.so", text)))
print("gdr_enabled_lines", len(re.findall(r"GPU Direct RDMA Enabled", text)))
print("using_hca")
for value, count in collections.Counter(re.findall(r"NET/IB : Using \[(.*?)\]; OOB", text)).most_common(4):
print(f" {count} {value}")
print("pattern_counts")
patterns = re.findall(
r"Pattern (\d+), crossNic (\d+), nChannels (\d+), bw ([0-9.]+)/([0-9.]+), type ([^,]+), sameChannels (\d+)",
text,
)
for key, count in collections.Counter(patterns).most_common():
print(f" {count} {key}")
print("channel_summary")
for value, count in collections.Counter(
re.findall(r"(\d+ coll channels, \d+ collnet channels, \d+ nvls channels, \d+ p2p channels, \d+ p2p channels per peer)", text)
).most_common():
print(f" {count} {value}")
print("p2p_chunks", collections.Counter(re.findall(r"P2P Chunksize set to (\d+)", text)))
print("check_p2p", collections.Counter(re.findall(r"Check P2P Type ([^\n]+)", text)))
for token in ["NET/IB/0/GDRDMA", "NET/IB/1/GDRDMA", "NET/IB/2/GDRDMA", "NET/IB/3/GDRDMA", "P2P/CUMEM", "P2P/IPC", "SHM"]:
print(token, text.count(token))
print("channel_edge_lines", len([line for line in text.splitlines() if "Channel " in line and ("via NET/IB" in line or "via P2P" in line)]))
PY
}
run_graph_case() {
local op="$1"
local bin="$2"
local extra="${3:-}"
set_common_env
export NCCL_DEBUG=INFO
export NCCL_DEBUG_SUBSYS=INIT,NET,GRAPH,TUNING,COLL
if [[ -n "$extra" ]]; then
eval "export $extra"
fi
local dir="$OUT_DIR/graph"
mkdir -p "$dir"
local log="$dir/${op}.log"
run_nccl "$op" "$bin" "$log" "$GRAPH_WARMUP_ITERS" "$GRAPH_ITERS"
summarize_graph_log "$log" "$dir/${op}_summary.txt"
echo "$dir/${op}_summary.txt"
}
run_pxn_sweep() {
local dir="$OUT_DIR/pxn_sweep"
mkdir -p "$dir"
local cases=(
"baseline|"
"nvls_off|NCCL_NVLS_ENABLE=0"
"qps4_split1|NCCL_IB_QPS_PER_CONNECTION=4 NCCL_IB_SPLIT_DATA_ON_QPS=1"
"qps8_split1|NCCL_IB_QPS_PER_CONNECTION=8 NCCL_IB_SPLIT_DATA_ON_QPS=1"
"qps4_split0|NCCL_IB_QPS_PER_CONNECTION=4 NCCL_IB_SPLIT_DATA_ON_QPS=0"
"channels16|NCCL_MIN_NCHANNELS=16 NCCL_MAX_NCHANNELS=16"
"buff8m|NCCL_BUFFSIZE=8388608"
"p2pchunk4m|NCCL_P2P_NET_CHUNKSIZE=4194304"
"netpeer8|NCCL_NCHANNELS_PER_NET_PEER=8"
"ar0|NCCL_IB_AR_THRESHOLD=0"
)
: >"$dir/summary.txt"
for item in "${cases[@]}"; do
local name="${item%%|*}"
local extra="${item#*|}"
set_common_env
export NCCL_PXN_DISABLE=1
if [[ -n "$extra" ]]; then
eval "export $extra"
fi
local log="$dir/${name}.log"
{
echo "===== CASE $name ====="
echo "extra: ${extra:-none}"
run_nccl "alltoall" "$NCCL_TESTS_DIR/alltoall_perf" "$log" "$SWEEP_WARMUP_ITERS" "$SWEEP_ITERS"
awk '/Avg bus bandwidth/ {print}' "$log" | tail -1
} | tee -a "$dir/summary.txt"
done
echo "$dir/summary.txt"
}
run_preflight() {
set_common_env
local out="$OUT_DIR/preflight.txt"
{
echo "===== LOCAL ====="
echo "hostname: $(hostname)"
echo "mpirun: $MPI_BIN"
if [[ -x "$MPI_BIN" ]]; then
"$MPI_BIN" --version 2>&1 | sed -n '1p'
else
echo "MISSING executable: $MPI_BIN"
fi
for bin in "$NCCL_TESTS_DIR/all_reduce_perf" "$NCCL_TESTS_DIR/alltoall_perf"; do
if [[ -x "$bin" ]]; then
echo "OK executable: $bin"
else
echo "MISSING executable: $bin"
fi
done
for hca in $HCAS; do
local state="/sys/class/infiniband/$hca/ports/1/state"
local rate="/sys/class/infiniband/$hca/ports/1/rate"
if [[ -r "$state" ]]; then
echo "OK HCA: $hca state=$(cat "$state") rate=$(cat "$rate" 2>/dev/null || echo unknown)"
else
echo "MISSING HCA path: $hca"
fi
done
echo "===== REMOTE ====="
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-o BatchMode=yes -o ConnectTimeout=5 "${SSH_USER}@${PEER_HOST}" \
"MPI_BIN='$MPI_BIN' NCCL_TESTS_DIR='$NCCL_TESTS_DIR' HCAS='$HCAS' bash -s" <<'EOS'
echo "hostname: $(hostname)"
echo "mpirun: $MPI_BIN"
if [ -x "$MPI_BIN" ]; then
"$MPI_BIN" --version 2>&1 | sed -n '1p'
else
echo "MISSING executable: $MPI_BIN"
fi
for bin in "$NCCL_TESTS_DIR/all_reduce_perf" "$NCCL_TESTS_DIR/alltoall_perf"; do
if [ -x "$bin" ]; then
echo "OK executable: $bin"
else
echo "MISSING executable: $bin"
fi
done
for hca in $HCAS; do
state="/sys/class/infiniband/$hca/ports/1/state"
rate="/sys/class/infiniband/$hca/ports/1/rate"
if [ -r "$state" ]; then
echo "OK HCA: $hca state=$(cat "$state") rate=$(cat "$rate" 2>/dev/null || echo unknown)"
else
echo "MISSING HCA path: $hca"
fi
done
EOS
} | tee "$out"
echo "$out"
}
usage() {
cat <<EOF
Usage: $0 [preflight|all|allreduce-counter|alltoall-counter|graph|pxn-sweep]
Outputs are written to: $OUT_DIR
Common overrides:
HOSTS, PEER_HOST, HCAS, HCA_CSV, MPI_BIN, NCCL_TESTS_DIR,
NCCL_LD_LIBRARY_PATH, BEGIN_SIZE, END_SIZE, WARMUP_ITERS, ITERS
EOF
}
case "$MODE" in
preflight)
run_preflight
;;
all)
run_preflight
run_counter_case allreduce "$NCCL_TESTS_DIR/all_reduce_perf" ""
run_counter_case alltoall_pxn "$NCCL_TESTS_DIR/alltoall_perf" "NCCL_PXN_DISABLE=1"
run_graph_case allreduce "$NCCL_TESTS_DIR/all_reduce_perf" ""
run_graph_case alltoall_pxn "$NCCL_TESTS_DIR/alltoall_perf" "NCCL_PXN_DISABLE=1"
run_pxn_sweep
;;
allreduce-counter)
run_counter_case allreduce "$NCCL_TESTS_DIR/all_reduce_perf" ""
;;
alltoall-counter)
run_counter_case alltoall_pxn "$NCCL_TESTS_DIR/alltoall_perf" "NCCL_PXN_DISABLE=1"
;;
graph)
run_graph_case allreduce "$NCCL_TESTS_DIR/all_reduce_perf" ""
run_graph_case alltoall_pxn "$NCCL_TESTS_DIR/alltoall_perf" "NCCL_PXN_DISABLE=1"
;;
pxn-sweep)
run_pxn_sweep
;;
-h|--help|help)
usage
;;
*)
usage
exit 2
;;
esac
echo "OUT_DIR=$OUT_DIR"

View File

@ -0,0 +1,169 @@
#!/usr/bin/env bash
set -euo pipefail
# Collect a lightweight NCCL/RDMA environment snapshot on one node.
# This script does not run NCCL workloads and is safe to use before deeper tests.
HOST="$(hostname 2>/dev/null || echo unknown)"
TS="$(date +%Y%m%d_%H%M%S)"
OUT_FILE="${1:-${OUT_FILE:-/tmp/nccl_environment_snapshot_${HOST}_${TS}.md}}"
PDF_ALLREDUCE_BUSBW="${PDF_ALLREDUCE_BUSBW:-491.84}"
PDF_ALLTOALL_BUSBW="${PDF_ALLTOALL_BUSBW:-76.54}"
PLUGIN_SEARCH_ROOTS="${PLUGIN_SEARCH_ROOTS:-/usr /opt /tmp /root}"
mkdir -p "$(dirname "$OUT_FILE")"
shopt -s nullglob
have_cmd() {
command -v "$1" >/dev/null 2>&1
}
emit_cmd() {
local title="$1"
shift
{
echo
echo "### $title"
echo
echo '```text'
"$@" 2>&1 || true
echo '```'
} >>"$OUT_FILE"
}
active_400g_hcas=()
non_400g_rows=()
{
echo "# NCCL/RDMA 环境快照"
echo
echo "- Host: \`$HOST\`"
echo "- Time: \`$(date -Is 2>/dev/null || date)\`"
echo "- Kernel: \`$(uname -r 2>/dev/null || echo unknown)\`"
echo
echo "## HCA / Port 状态"
echo
echo "| HCA | Port | State | Phys State | Rate | Link Layer | 400G IB Rail |"
echo "|---|---:|---|---|---:|---|---|"
} >"$OUT_FILE"
hca_paths=(/sys/class/infiniband/mlx5_*)
if ((${#hca_paths[@]})); then
for hca_path in "${hca_paths[@]}"; do
hca="$(basename "$hca_path")"
for port_path in "$hca_path"/ports/*; do
[[ -d "$port_path" ]] || continue
port="$(basename "$port_path")"
state="$(cat "$port_path/state" 2>/dev/null || echo NA)"
phys_state="$(cat "$port_path/phys_state" 2>/dev/null || echo NA)"
rate="$(cat "$port_path/rate" 2>/dev/null || echo NA)"
layer="$(cat "$port_path/link_layer" 2>/dev/null || echo NA)"
is_400g="NO"
if [[ "$state" == *"ACTIVE"* && "$rate" == 400\ Gb/sec* && "$layer" == "InfiniBand" ]]; then
is_400g="YES"
active_400g_hcas+=("$hca")
else
non_400g_rows+=("$hca port=$port state=$state rate=$rate layer=$layer")
fi
printf '| `%s` | `%s` | `%s` | `%s` | `%s` | `%s` | `%s` |\n' \
"$hca" "$port" "$state" "$phys_state" "$rate" "$layer" "$is_400g" >>"$OUT_FILE"
done
done
else
printf '| N/A | N/A | `%s` | N/A | N/A | N/A | NO |\n' "/sys/class/infiniband/mlx5_* not found" >>"$OUT_FILE"
fi
{
echo
echo "## Rail 摘要"
echo
if ((${#active_400g_hcas[@]})); then
hca_csv="$(IFS=,; echo "${active_400g_hcas[*]}")"
echo "- Active 400G IB rail count: \`${#active_400g_hcas[@]}\`"
echo "- Candidate \`NCCL_IB_HCA\`: \`$hca_csv\`"
echo "- Theoretical one-way raw bandwidth: \`${#active_400g_hcas[@]} * 400Gb/s / 8 = $((${#active_400g_hcas[@]} * 50)) GB/s\`"
else
echo "- Active 400G IB rail count: \`0\`"
echo "- Candidate \`NCCL_IB_HCA\`: \`N/A\`"
fi
echo
echo "Non-400G / non-IB / down ports:"
echo
if ((${#non_400g_rows[@]})); then
for row in "${non_400g_rows[@]}"; do
echo "- \`$row\`"
done
else
echo "- none"
fi
echo
echo "## PDF 目标换算"
echo
echo "- PDF allreduce busbw target: \`${PDF_ALLREDUCE_BUSBW} GB/s\`"
echo "- PDF alltoall busbw target: \`${PDF_ALLTOALL_BUSBW} GB/s\`"
} >>"$OUT_FILE"
python3 - "$PDF_ALLREDUCE_BUSBW" "${#active_400g_hcas[@]}" >>"$OUT_FILE" <<'PY' || true
import sys
busbw = float(sys.argv[1])
rail_count = int(sys.argv[2])
algbw = busbw / 1.875
raw = rail_count * 50.0
print(f"- 16-rank allreduce implied algbw: `{algbw:.2f} GB/s`")
if rail_count:
pct = algbw / raw * 100
print(f"- Implied algbw / current raw 400G rail bandwidth: `{pct:.1f}%`")
if algbw > raw:
print("- Interpretation: PDF allreduce target is above current 400G rail one-way raw bandwidth.")
else:
print("- Interpretation: PDF allreduce target is within current 400G rail one-way raw bandwidth.")
else:
print("- Interpretation: no active 400G IB rail was detected.")
PY
{
echo
echo "## NCCL Net Plugin / SHARP 文件"
echo
echo '```text'
} >>"$OUT_FILE"
read -r -a plugin_roots <<<"$PLUGIN_SEARCH_ROOTS"
find "${plugin_roots[@]}" \( -name 'libnccl-net*.so*' -o -name 'libsharp*.so*' \) \
2>/dev/null | sort >>"$OUT_FILE" || true
if ! grep -q 'libnccl-net\|libsharp' "$OUT_FILE"; then
echo "none found under $PLUGIN_SEARCH_ROOTS" >>"$OUT_FILE"
fi
echo '```' >>"$OUT_FILE"
if have_cmd dpkg; then
emit_cmd "Relevant Debian packages" bash -lc "dpkg -l | egrep -i 'nccl|sharp|hcoll|ucx|ofed|mlnx' | sed -n '1,160p'"
else
emit_cmd "Relevant packages" bash -lc "echo 'dpkg not found'"
fi
if have_cmd nvidia-smi; then
emit_cmd "nvidia-smi topo -m" nvidia-smi topo -m
else
emit_cmd "nvidia-smi topo -m" bash -lc "echo 'nvidia-smi not found'"
fi
if have_cmd ibstat; then
emit_cmd "ibstat" ibstat
fi
{
echo
echo "## 建议判断"
echo
echo "1. 如果 Active 400G IB rail 少于 PDF 参考环境,不能直接按 PDF 阈值判断等价。"
echo "2. 如果没有 \`libnccl-net*.so*\` / \`libsharp*.so*\`,NCCL 可能只能走 internal IB plugin。"
echo "3. 若要追 PDF 2x8 目标,请先确认 rail 数量、SHARP/NCCL net plugin、跨 Leaf 交换策略是否与 PDF 环境一致。"
echo
echo "Snapshot written to: \`$OUT_FILE\`"
} >>"$OUT_FILE"
echo "$OUT_FILE"

277
scripts/pytorch_fp8_path_bench.py Executable file
View File

@ -0,0 +1,277 @@
#!/usr/bin/env python3
"""Compare FP8 GEMM paths used for H100/H200 acceptance debugging.
Paths:
A. torch._scaled_mm eager, default accumulation
B. torch._scaled_mm eager, use_fast_accum=True
C. CUDA Graph replay of torch._scaled_mm(out=..., use_fast_accum=True)
D. Transformer Engine Linear under fp8_autocast, when installed
"""
from __future__ import annotations
import argparse
import json
import statistics
import sys
import time
from typing import Any, Callable
import torch
def tflops_from_ms(matrix_size: int, iterations: int, elapsed_ms: float) -> float:
flops = 2.0 * matrix_size * matrix_size * matrix_size * iterations
return flops / (elapsed_ms / 1000.0) / 1e12
def cuda_event_bench(
name: str,
matrix_size: int,
iterations: int,
warmup: int,
func: Callable[[int], Any],
) -> dict[str, Any]:
for i in range(warmup):
func(i)
torch.cuda.synchronize()
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
wall_start = time.perf_counter()
start.record()
for i in range(iterations):
func(i)
end.record()
torch.cuda.synchronize()
wall_elapsed = time.perf_counter() - wall_start
elapsed_ms = start.elapsed_time(end)
return {
"name": name,
"status": "ok",
"matrix_size": matrix_size,
"iterations": iterations,
"warmup": warmup,
"event_ms_total": round(elapsed_ms, 3),
"event_us_per_iter": round(elapsed_ms * 1000.0 / iterations, 3),
"wall_ms_total": round(wall_elapsed * 1000.0, 3),
"tflops": round(tflops_from_ms(matrix_size, iterations, elapsed_ms), 1),
}
def make_fp8_inputs(matrix_size: int, pools: int, device: str) -> tuple[list[torch.Tensor], list[torch.Tensor]]:
a = [
torch.randn(matrix_size, matrix_size, device=device, dtype=torch.float32).to(torch.float8_e4m3fn)
for _ in range(pools)
]
b = [
torch.randn(matrix_size, matrix_size, device=device, dtype=torch.float32).to(torch.float8_e4m3fn)
for _ in range(pools)
]
torch.cuda.synchronize()
return a, b
def bench_scaled_mm(args: argparse.Namespace) -> list[dict[str, Any]]:
device = f"cuda:{args.gpu_index}"
torch.cuda.set_device(args.gpu_index)
scale_a = torch.tensor(1.0, device=device)
scale_b = torch.tensor(1.0, device=device)
pools_a, pools_b = make_fp8_inputs(args.matrix_size, args.pools, device)
results: list[dict[str, Any]] = []
def eager_default(i: int) -> torch.Tensor:
idx = i % args.pools
return torch._scaled_mm(
pools_a[idx],
pools_b[idx].T,
scale_a=scale_a,
scale_b=scale_b,
out_dtype=torch.bfloat16,
)
def eager_fast(i: int) -> torch.Tensor:
idx = i % args.pools
return torch._scaled_mm(
pools_a[idx],
pools_b[idx].T,
scale_a=scale_a,
scale_b=scale_b,
out_dtype=torch.bfloat16,
use_fast_accum=True,
)
results.append(
cuda_event_bench(
"A_eager_scaled_mm_default",
args.matrix_size,
args.iterations,
args.warmup,
eager_default,
)
)
results.append(
cuda_event_bench(
"B_eager_scaled_mm_fast_accum",
args.matrix_size,
args.iterations,
args.warmup,
eager_fast,
)
)
graph_out = torch.empty(
(args.matrix_size, args.matrix_size),
device=device,
dtype=torch.bfloat16,
)
static_a = pools_a[0]
static_b_t = pools_b[0].T
try:
side_stream = torch.cuda.Stream()
side_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(side_stream):
for _ in range(max(3, args.warmup // 2)):
torch._scaled_mm(
static_a,
static_b_t,
scale_a=scale_a,
scale_b=scale_b,
out_dtype=torch.bfloat16,
use_fast_accum=True,
out=graph_out,
)
torch.cuda.current_stream().wait_stream(side_stream)
torch.cuda.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
torch._scaled_mm(
static_a,
static_b_t,
scale_a=scale_a,
scale_b=scale_b,
out_dtype=torch.bfloat16,
use_fast_accum=True,
out=graph_out,
)
def graph_replay(_: int) -> None:
graph.replay()
results.append(
cuda_event_bench(
"C_cuda_graph_scaled_mm_fast_accum",
args.matrix_size,
args.iterations,
3,
graph_replay,
)
)
except Exception as exc: # noqa: BLE001
results.append(
{
"name": "C_cuda_graph_scaled_mm_fast_accum",
"status": "unavailable",
"reason": f"{type(exc).__name__}: {exc}",
}
)
return results
def bench_transformer_engine(args: argparse.Namespace) -> dict[str, Any]:
try:
import transformer_engine.pytorch as te # type: ignore[import-not-found]
from transformer_engine.common.recipe import DelayedScaling, Format # type: ignore[import-not-found]
except Exception as exc: # noqa: BLE001
return {
"name": "D_transformer_engine_fp8_linear",
"status": "unavailable",
"reason": f"{type(exc).__name__}: {exc}",
}
device = f"cuda:{args.gpu_index}"
x = torch.randn(args.matrix_size, args.matrix_size, device=device, dtype=torch.bfloat16)
layer = te.Linear(
args.matrix_size,
args.matrix_size,
bias=False,
params_dtype=torch.bfloat16,
device=device,
)
recipe = DelayedScaling(fp8_format=Format.HYBRID)
def run(_: int) -> torch.Tensor:
with te.fp8_autocast(enabled=True, fp8_recipe=recipe):
return layer(x)
try:
result = cuda_event_bench(
"D_transformer_engine_fp8_linear",
args.matrix_size,
args.iterations,
args.warmup,
run,
)
except Exception as exc: # noqa: BLE001
return {
"name": "D_transformer_engine_fp8_linear",
"status": "error",
"reason": f"{type(exc).__name__}: {exc}",
}
result["note"] = "Transformer Engine Linear forward under fp8_autocast; includes TE module/cast overhead."
return result
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--matrix-size", type=int, default=8192)
parser.add_argument("--warmup", type=int, default=20)
parser.add_argument("--iterations", type=int, default=100)
parser.add_argument("--gpu-index", type=int, default=0)
parser.add_argument("--pools", type=int, default=4)
args = parser.parse_args()
if not torch.cuda.is_available():
print(json.dumps({"error": "cuda unavailable"}, indent=2))
return 1
if not hasattr(torch, "_scaled_mm") or not hasattr(torch, "float8_e4m3fn"):
print(json.dumps({"error": "torch FP8 _scaled_mm unavailable"}, indent=2))
return 1
torch.cuda.set_device(args.gpu_index)
props = torch.cuda.get_device_properties(args.gpu_index)
payload = {
"source": "pytorch_fp8_path_bench",
"torch": torch.__version__,
"cuda": torch.version.cuda,
"gpu_index": args.gpu_index,
"gpu_name": props.name,
"matrix_size": args.matrix_size,
"warmup": args.warmup,
"iterations": args.iterations,
"results": [],
}
try:
payload["results"].extend(bench_scaled_mm(args))
payload["results"].append(bench_transformer_engine(args))
except torch.cuda.OutOfMemoryError as exc:
payload["error"] = f"CUDA OOM: {exc}"
print(json.dumps(payload, indent=2))
return 1
ok_values = [r["tflops"] for r in payload["results"] if r.get("status") == "ok"]
if ok_values:
payload["summary"] = {
"max_tflops": round(max(ok_values), 1),
"min_tflops": round(min(ok_values), 1),
"mean_tflops": round(statistics.mean(ok_values), 1),
}
print(json.dumps(payload, indent=2))
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@ -0,0 +1,45 @@
#!/usr/bin/env bash
set -uo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
PROJECT_DIR="$(cd -- "$SCRIPT_DIR/.." >/dev/null 2>&1 && pwd)"
CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}"
NVCC="${NVCC:-$CUDA_HOME/bin/nvcc}"
OUT_DIR="${OUT_DIR:-$PROJECT_DIR/reports}"
MATRIX_SIZE="${MATRIX_SIZE:-8192}"
WARMUP="${WARMUP:-20}"
ITERATIONS="${ITERATIONS:-200}"
GPU_COUNT="${GPU_COUNT:-8}"
FIRST_GPU="${FIRST_GPU:-0}"
WORKSPACE_MB="${WORKSPACE_MB:-256}"
if [[ ! -x "$NVCC" ]]; then
echo "nvcc not found: $NVCC" >&2
exit 1
fi
mkdir -p "$OUT_DIR" "$PROJECT_DIR/build"
HOST="$(hostname 2>/dev/null || echo unknown)"
TS="$(date +%Y%m%d_%H%M%S)"
BIN="$PROJECT_DIR/build/cublaslt_fp8_gemm_bench"
REPORT="$OUT_DIR/cublaslt_fp8_gemm_${HOST}_${TS}.json"
"$NVCC" -O3 -std=c++17 -arch=sm_90 \
"$PROJECT_DIR/scripts/cublaslt_fp8_gemm_bench.cu" \
-lcublasLt -lcublas -o "$BIN"
set +e
"$BIN" \
--matrix-size "$MATRIX_SIZE" \
--warmup "$WARMUP" \
--iterations "$ITERATIONS" \
--first-gpu "$FIRST_GPU" \
--gpu-count "$GPU_COUNT" \
--workspace-mb "$WORKSPACE_MB" \
| tee "$REPORT"
status=${PIPESTATUS[0]}
set -e
echo "Report written to: $REPORT"
exit "$status"

View File

@ -0,0 +1,93 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
PROJECT_DIR="$(cd -- "$SCRIPT_DIR/.." >/dev/null 2>&1 && pwd)"
PYTHON="${PYTHON:-/root/gpu-test-venv/bin/python}"
CUDA_HOME="${CUDA_HOME:-/usr/local/cuda-12.4}"
NVCC="${NVCC:-$CUDA_HOME/bin/nvcc}"
OUT_DIR="${OUT_DIR:-$PROJECT_DIR/reports}"
MATRIX_SIZE="${MATRIX_SIZE:-8192}"
WARMUP="${WARMUP:-20}"
ITERATIONS="${ITERATIONS:-100}"
GPU_INDEX="${GPU_INDEX:-0}"
WORKSPACE_MB="${WORKSPACE_MB:-256}"
VENV_SITE_PACKAGES="$("$PYTHON" - <<'PY'
import site
print(site.getsitepackages()[0])
PY
)"
export LD_LIBRARY_PATH="$VENV_SITE_PACKAGES/nvidia/cudnn/lib:$VENV_SITE_PACKAGES/nvidia/nccl/lib:${LD_LIBRARY_PATH:-}"
mkdir -p "$PROJECT_DIR/build" "$OUT_DIR"
HOST="$(hostname 2>/dev/null || echo unknown)"
TS="$(date +%Y%m%d_%H%M%S)"
PY_REPORT="$OUT_DIR/fp8_paths_pytorch_${HOST}_${TS}.json"
CUBLAS_REPORT="$OUT_DIR/fp8_paths_cublaslt_${HOST}_${TS}.json"
COMBINED_REPORT="$OUT_DIR/fp8_paths_combined_${HOST}_${TS}.json"
"$PYTHON" "$PROJECT_DIR/scripts/pytorch_fp8_path_bench.py" \
--matrix-size "$MATRIX_SIZE" \
--warmup "$WARMUP" \
--iterations "$ITERATIONS" \
--gpu-index "$GPU_INDEX" | tee "$PY_REPORT"
"$NVCC" -O3 -std=c++17 -arch=sm_90 \
"$PROJECT_DIR/scripts/cublaslt_fp8_gemm_bench.cu" \
-lcublasLt -lcublas -o "$PROJECT_DIR/build/cublaslt_fp8_gemm_bench"
"$PROJECT_DIR/build/cublaslt_fp8_gemm_bench" \
--matrix-size "$MATRIX_SIZE" \
--warmup "$WARMUP" \
--iterations "$ITERATIONS" \
--first-gpu "$GPU_INDEX" \
--gpu-count 1 \
--workspace-mb "$WORKSPACE_MB" \
--fast-accum 1 | tee "$CUBLAS_REPORT"
"$PYTHON" - "$PY_REPORT" "$CUBLAS_REPORT" "$COMBINED_REPORT" <<'PY'
import json
import pathlib
import sys
py_report = pathlib.Path(sys.argv[1])
cublas_report = pathlib.Path(sys.argv[2])
combined_report = pathlib.Path(sys.argv[3])
with py_report.open() as f:
py_payload = json.load(f)
with cublas_report.open() as f:
cublas_payload = json.load(f)
combined = {
"source": "fp8_path_comparison",
"host": cublas_payload.get("host"),
"matrix_size": py_payload.get("matrix_size"),
"gpu_index": py_payload.get("gpu_index"),
"pytorch": py_payload,
"cublaslt": cublas_payload,
"results": [],
}
combined["results"].extend(py_payload.get("results", []))
per_gpu = cublas_payload.get("per_gpu", [])
if per_gpu:
row = dict(per_gpu[0])
row.update({
"name": "E_direct_cublaslt_fast_accum",
"status": "ok",
"tflops": row.pop("fp8_tflops"),
"matrix_size": cublas_payload.get("matrix_size"),
"iterations": cublas_payload.get("iterations"),
"warmup": cublas_payload.get("warmup"),
"fast_accum": cublas_payload.get("fast_accum"),
"note": "Direct cuBLASLt FP8 GEMM, bypasses PyTorch eager.",
})
combined["results"].append(row)
combined_report.write_text(json.dumps(combined, indent=2), encoding="utf-8")
print(f"Combined report written to: {combined_report}")
PY
echo "$COMBINED_REPORT"

View File

@ -0,0 +1,134 @@
#!/usr/bin/env bash
set -uo pipefail
# Run the single-node H100 acceptance suite and keep the raw report paths stable.
# The suite itself still lives in gpu_tester.py; this wrapper only standardizes
# snapshot/report naming for repeated machine-level runs.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
PROJECT_DIR="$(cd -- "$SCRIPT_DIR/.." >/dev/null 2>&1 && pwd)"
PYTHON_BIN="${PYTHON_BIN:-/root/gpu-test-venv/bin/python}"
CONFIG_FILE="${CONFIG_FILE:-$PROJECT_DIR/configs/default.yaml}"
OUT_DIR="${OUT_DIR:-$PROJECT_DIR/reports}"
FORMAT="${FORMAT:-md}"
DRY_RUN=0
SNAPSHOT=1
usage() {
cat <<'EOF'
Usage: run_h100_single_node_all.sh [options]
Options:
--python PATH Python executable (default: /root/gpu-test-venv/bin/python)
--config PATH gpu_tester config file (default: configs/default.yaml)
--out-dir PATH Report output directory (default: reports)
--format FORMAT Report format: md, json, or html (default: md)
--no-snapshot Do not run nccl_environment_snapshot.sh first
--dry-run Print commands without running them
-h, --help Show this help
EOF
}
while (($#)); do
case "$1" in
--python)
PYTHON_BIN="$2"
shift 2
;;
--config)
CONFIG_FILE="$2"
shift 2
;;
--out-dir)
OUT_DIR="$2"
shift 2
;;
--format)
FORMAT="$2"
shift 2
;;
--no-snapshot)
SNAPSHOT=0
shift
;;
--dry-run)
DRY_RUN=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ "$FORMAT" != "md" && "$FORMAT" != "json" && "$FORMAT" != "html" ]]; then
echo "Unsupported format: $FORMAT" >&2
exit 2
fi
if [[ ! -x "$PYTHON_BIN" ]]; then
PYTHON_BIN="$(command -v python3 || true)"
fi
if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then
echo "Python executable not found. Set --python or PYTHON_BIN." >&2
exit 1
fi
HOST="$(hostname 2>/dev/null || echo unknown)"
TS="$(date +%Y%m%d_%H%M%S)"
mkdir -p "$OUT_DIR"
SNAPSHOT_FILE="$OUT_DIR/nccl_environment_snapshot_${HOST}_${TS}.md"
REPORT_FILE="$OUT_DIR/h100_single_node_all_${HOST}_${TS}.${FORMAT}"
snapshot_cmd=(bash "$PROJECT_DIR/scripts/nccl_environment_snapshot.sh" "$SNAPSHOT_FILE")
test_cmd=(
"$PYTHON_BIN" "$PROJECT_DIR/gpu_tester.py"
--config "$CONFIG_FILE"
--test all
--report
--format "$FORMAT"
--output "$REPORT_FILE"
)
echo "Project: $PROJECT_DIR"
echo "Host: $HOST"
echo "Config: $CONFIG_FILE"
echo "Report: $REPORT_FILE"
if ((SNAPSHOT)); then
echo "Snapshot: $SNAPSHOT_FILE"
fi
if ((DRY_RUN)); then
if ((SNAPSHOT)); then
printf 'DRY RUN snapshot:'
printf ' %q' "${snapshot_cmd[@]}"
printf '\n'
fi
printf 'DRY RUN test:'
printf ' %q' "${test_cmd[@]}"
printf '\n'
exit 0
fi
if ((SNAPSHOT)); then
"${snapshot_cmd[@]}"
fi
"${test_cmd[@]}"
status=$?
echo "Report written to: $REPORT_FILE"
if ((SNAPSHOT)); then
echo "Snapshot written to: $SNAPSHOT_FILE"
fi
exit "$status"

View File

@ -0,0 +1,147 @@
#!/usr/bin/env bash
set -uo pipefail
# Run a two-node, eight-GPU-per-node NCCL evidence pass across the six
# collectives used by the single-node H100 acceptance flow.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
PROJECT_DIR="$(cd -- "$SCRIPT_DIR/.." >/dev/null 2>&1 && pwd)"
PYTHON_BIN="${PYTHON_BIN:-/root/gpu-test-venv/bin/python}"
CONFIG_FILE="${CONFIG_FILE:-$PROJECT_DIR/configs/multinode_nccl_nccl227_all_collectives_2x8.yaml}"
OUT_DIR="${OUT_DIR:-$PROJECT_DIR/reports}"
FORMAT="${FORMAT:-md}"
DRY_RUN=0
RUN_PREFLIGHT=1
PREFLIGHT_ONLY=0
usage() {
cat <<'EOF'
Usage: run_multinode_nccl_all_collectives.sh [options]
Options:
--python PATH Python executable (default: /root/gpu-test-venv/bin/python)
--config PATH Config file (default: configs/multinode_nccl_nccl227_all_collectives_2x8.yaml)
--out-dir PATH Report output directory (default: reports)
--format FORMAT Report format: md, json, or html (default: md)
--no-preflight Skip scripts/multinode_nccl_deep_diagnose.sh preflight
--preflight-only Run only the preflight check, not the workload
--dry-run Print commands without running them
-h, --help Show this help
EOF
}
while (($#)); do
case "$1" in
--python)
PYTHON_BIN="$2"
shift 2
;;
--config)
CONFIG_FILE="$2"
shift 2
;;
--out-dir)
OUT_DIR="$2"
shift 2
;;
--format)
FORMAT="$2"
shift 2
;;
--no-preflight)
RUN_PREFLIGHT=0
shift
;;
--preflight-only)
PREFLIGHT_ONLY=1
shift
;;
--dry-run)
DRY_RUN=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ "$FORMAT" != "md" && "$FORMAT" != "json" && "$FORMAT" != "html" ]]; then
echo "Unsupported format: $FORMAT" >&2
exit 2
fi
if [[ ! -x "$PYTHON_BIN" ]]; then
PYTHON_BIN="$(command -v python3 || true)"
fi
if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then
echo "Python executable not found. Set --python or PYTHON_BIN." >&2
exit 1
fi
TS="$(date +%Y%m%d_%H%M%S)"
mkdir -p "$OUT_DIR"
REPORT_FILE="$OUT_DIR/multinode_nccl_all_collectives_${TS}.${FORMAT}"
ARTIFACT_DIR="$OUT_DIR/multinode_nccl_all_collectives_${TS}_artifacts"
PREFLIGHT_CMD=(bash "$PROJECT_DIR/scripts/multinode_nccl_deep_diagnose.sh" preflight)
RUN_CMD=(
"$PYTHON_BIN" "$PROJECT_DIR/gpu_tester.py"
--config "$CONFIG_FILE"
--test multinode-nccl
--report
--format "$FORMAT"
--output "$REPORT_FILE"
)
echo "Project: $PROJECT_DIR"
echo "Config: $CONFIG_FILE"
echo "Report: $REPORT_FILE"
echo "Artifacts: $ARTIFACT_DIR"
echo "Collectives: allreduce, alltoall, broadcast, reducescatter, allgather, sendrecv"
echo "Topology: 2 nodes x 8 GPUs per node; 16G"
if ((DRY_RUN)); then
if ((RUN_PREFLIGHT)); then
printf 'DRY RUN preflight:'
printf ' %q' "${PREFLIGHT_CMD[@]}"
printf '\n'
fi
if ((PREFLIGHT_ONLY)); then
exit 0
fi
printf 'DRY RUN workload:'
printf ' MULTINODE_NCCL_ARTIFACT_DIR=%q' "$ARTIFACT_DIR"
printf ' %q' "${RUN_CMD[@]}"
printf '\n'
exit 0
fi
if ((RUN_PREFLIGHT)); then
"${PREFLIGHT_CMD[@]}"
preflight_status=$?
if ((preflight_status != 0)); then
echo "Preflight failed with exit code $preflight_status" >&2
exit "$preflight_status"
fi
fi
if ((PREFLIGHT_ONLY)); then
exit 0
fi
mkdir -p "$ARTIFACT_DIR"
MULTINODE_NCCL_ARTIFACT_DIR="$ARTIFACT_DIR" "${RUN_CMD[@]}"
status=$?
echo "Report written to: $REPORT_FILE"
echo "Artifacts written to: $ARTIFACT_DIR"
exit "$status"

View File

@ -0,0 +1,147 @@
#!/usr/bin/env bash
set -uo pipefail
# Run the formal cross-node NCCL PDF matrix for the current two-node H100 pair.
# This wrapper standardizes the command, output naming, and preflight hook; the
# actual benchmark implementation remains in gpu_tester.py / MultiNodeNCCLTest.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
PROJECT_DIR="$(cd -- "$SCRIPT_DIR/.." >/dev/null 2>&1 && pwd)"
PYTHON_BIN="${PYTHON_BIN:-/root/gpu-test-venv/bin/python}"
CONFIG_FILE="${CONFIG_FILE:-$PROJECT_DIR/configs/multinode_nccl_nccl227_pdf_matrix.yaml}"
OUT_DIR="${OUT_DIR:-$PROJECT_DIR/reports}"
FORMAT="${FORMAT:-md}"
DRY_RUN=0
RUN_PREFLIGHT=1
PREFLIGHT_ONLY=0
usage() {
cat <<'EOF'
Usage: run_multinode_nccl_pdf_matrix.sh [options]
Options:
--python PATH Python executable (default: /root/gpu-test-venv/bin/python)
--config PATH Matrix config file (default: configs/multinode_nccl_nccl227_pdf_matrix.yaml)
--out-dir PATH Report output directory (default: reports)
--format FORMAT Report format: md, json, or html (default: md)
--no-preflight Skip scripts/multinode_nccl_deep_diagnose.sh preflight
--preflight-only Run only the preflight check, not the matrix workload
--dry-run Print commands without running them
-h, --help Show this help
EOF
}
while (($#)); do
case "$1" in
--python)
PYTHON_BIN="$2"
shift 2
;;
--config)
CONFIG_FILE="$2"
shift 2
;;
--out-dir)
OUT_DIR="$2"
shift 2
;;
--format)
FORMAT="$2"
shift 2
;;
--no-preflight)
RUN_PREFLIGHT=0
shift
;;
--preflight-only)
PREFLIGHT_ONLY=1
shift
;;
--dry-run)
DRY_RUN=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ "$FORMAT" != "md" && "$FORMAT" != "json" && "$FORMAT" != "html" ]]; then
echo "Unsupported format: $FORMAT" >&2
exit 2
fi
if [[ ! -x "$PYTHON_BIN" ]]; then
PYTHON_BIN="$(command -v python3 || true)"
fi
if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then
echo "Python executable not found. Set --python or PYTHON_BIN." >&2
exit 1
fi
TS="$(date +%Y%m%d_%H%M%S)"
mkdir -p "$OUT_DIR"
REPORT_FILE="$OUT_DIR/multinode_nccl_pdf_matrix_${TS}.${FORMAT}"
ARTIFACT_DIR="$OUT_DIR/multinode_nccl_pdf_matrix_${TS}_artifacts"
PREFLIGHT_CMD=(bash "$PROJECT_DIR/scripts/multinode_nccl_deep_diagnose.sh" preflight)
MATRIX_CMD=(
"$PYTHON_BIN" "$PROJECT_DIR/gpu_tester.py"
--config "$CONFIG_FILE"
--test multinode-nccl
--report
--format "$FORMAT"
--output "$REPORT_FILE"
)
echo "Project: $PROJECT_DIR"
echo "Config: $CONFIG_FILE"
echo "Report: $REPORT_FILE"
echo "Artifacts: $ARTIFACT_DIR"
echo "Matrix: 2 nodes x {1,2,4,8} GPUs per node; all_reduce_perf + alltoall_perf; 16G"
if ((DRY_RUN)); then
if ((RUN_PREFLIGHT)); then
printf 'DRY RUN preflight:'
printf ' %q' "${PREFLIGHT_CMD[@]}"
printf '\n'
fi
if ((PREFLIGHT_ONLY)); then
exit 0
fi
printf 'DRY RUN matrix:'
printf ' MULTINODE_NCCL_ARTIFACT_DIR=%q' "$ARTIFACT_DIR"
printf ' %q' "${MATRIX_CMD[@]}"
printf '\n'
exit 0
fi
if ((RUN_PREFLIGHT)); then
"${PREFLIGHT_CMD[@]}"
preflight_status=$?
if ((preflight_status != 0)); then
echo "Preflight failed with exit code $preflight_status" >&2
exit "$preflight_status"
fi
fi
if ((PREFLIGHT_ONLY)); then
exit 0
fi
mkdir -p "$ARTIFACT_DIR"
MULTINODE_NCCL_ARTIFACT_DIR="$ARTIFACT_DIR" "${MATRIX_CMD[@]}"
status=$?
echo "Report written to: $REPORT_FILE"
echo "Artifacts written to: $ARTIFACT_DIR"
exit "$status"

View File

@ -1,90 +0,0 @@
import os
import tempfile
import unittest
from unittest.mock import patch
from modules.benchmark import Benchmark
from modules.gpu_specs import get_gpu_specs
from modules.nccl_test import NCCLTest
from modules.rdma_test import RDMATest
from modules.report import ReportGenerator
from modules.training_sim import TrainingSim
class B300JudgementTests(unittest.TestCase):
def test_b300_training_excludes_cudnn_sdpa(self):
self.assertTrue(TrainingSim._requires_b300_sdpa_workaround("b300", None, 91900))
self.assertTrue(TrainingSim._requires_b300_sdpa_workaround("auto", (10, 3), 91900))
self.assertFalse(TrainingSim._requires_b300_sdpa_workaround("b300", (10, 3), 92000))
self.assertFalse(TrainingSim._requires_b300_sdpa_workaround("h200", (9, 0), 91900))
def test_b300_uses_dense_peaks_and_calibrated_floors(self):
specs = get_gpu_specs("b300")
self.assertEqual(specs["fp8_tflops"], 4500)
self.assertEqual(specs["bf16_tflops"], 2250)
self.assertEqual(specs["compute_pass_thresholds_tflops"]["fp8"], 2800)
self.assertEqual(specs["nccl_pass_thresholds_gbps"]["allreduce"], 750)
def test_compute_verdict_uses_slowest_gpu_not_average(self):
result = {
"compute": {
"per_dtype_tflops": {"fp8": 3000.0},
"worst_per_dtype_tflops": {"fp8": 2500.0},
"pass_thresholds_tflops": {"fp8": 2800.0},
"efficiency_pct": {"fp8": 66.7},
}
}
judged = Benchmark.judge_compute(result)
self.assertEqual(judged["verdict"], "FAIL")
self.assertEqual(judged["rows"][0][1], 2500.0)
def test_rdma_without_peer_is_skip_not_false_failure(self):
rdma = RDMATest({"rdma": {}})
with (
patch.object(rdma, "_get_ib_devices", return_value=["mlx5_0"]),
patch.object(rdma, "_get_ib_ports", return_value=["1"]),
patch.object(rdma, "_read_sys", side_effect=["InfiniBand", "4: ACTIVE"]),
patch.object(rdma, "_collect_device_info", return_value=[]),
):
result = rdma.run()
self.assertEqual(result["status"], "SKIP")
self.assertTrue(result["skipped"])
self.assertIn("peer not configured", result["reason"])
def test_nccl_runtime_env_preserves_and_prepends_venv_libraries(self):
with (
patch("modules.nccl_test.sysconfig.get_paths", return_value={"purelib": "/venv/site"}),
patch("modules.nccl_test.os.path.isdir", return_value=True),
patch.dict(os.environ, {"LD_LIBRARY_PATH": "/system/lib"}),
):
env = NCCLTest._runtime_env()
self.assertEqual(
env["LD_LIBRARY_PATH"],
"/venv/site/nvidia/nccl/lib:/venv/site/torch/lib:/system/lib",
)
self.assertEqual(env["NCCL_DEBUG"], "WARN")
def test_markdown_summary_keeps_warn_and_measured_semantics(self):
results = {
"nccl": {"status": "WARN", "passed": True, "tests": {}},
"training": {
"throughput_tokens_per_sec": 1234,
"gpu_scope": "single_gpu:0",
},
}
generator = ReportGenerator({"report": {"output_dir": "."}})
summary = dict(generator._build_summary(results))
self.assertEqual(summary["NCCL"], "WARN")
self.assertTrue(summary["Training"].startswith("MEASURED"))
with tempfile.TemporaryDirectory() as tmpdir:
path = os.path.join(tmpdir, "report.md")
generator.generate(results, fmt="md", output=path)
with open(path, encoding="utf-8") as report_file:
report = report_file.read()
self.assertIn("| NCCL | WARN |", report)
self.assertIn("synthetic workload; not an 8-GPU scaling acceptance test", report)
if __name__ == "__main__":
unittest.main()