diff --git a/README.md b/README.md index 9530f68..8dfb408 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ - [配置说明](#配置说明) - [测试 SOP(标准操作流程)](#测试-sop标准操作流程) - [报告输出](#报告输出) +- [GPU 节点画像工具](#gpu-节点画像工具) - [故障排除](#故障排除) --- @@ -42,6 +43,11 @@ servertest/ │ ├── training_sim.py # 训练模拟 │ └── report.py # 报告生成 ├── scripts/ +│ ├── gpu_node_profile/ # JSON 驱动的运维画像与 3D CT +│ │ ├── gpu_node_profile.py # 只读采集与渲染入口 +│ │ ├── templates/ # 两个独立 HTML 源模板 +│ │ ├── examples/ # 脱敏真机 JSON 与自包含示例页面 +│ │ └── tests/ # 解析与渲染回归测试 │ └── multinode_nccl/ # 独立 shell 脚本版 (历史备用) │ ├── nccl_env.sh # 环境变量 (MPI路径/IB网卡/bond0) │ ├── hosts_all # 全量 34 台节点列表 @@ -677,6 +683,27 @@ python3 gpu_tester.py --test all --format html --output report.html --- +## GPU 节点画像工具 + +`scripts/gpu_node_profile/` 提供独立、只读的 GPU 节点画像工具。采集器可在 +一次执行中生成结构化 JSON,并由同一份 JSON 分别渲染运维画像和可逐层钻取 +的 3D CT 页面。两个 HTML 源模板位于 `templates/`;`examples/` 包含脱敏 +真机 JSON 和由其生成的两张自包含示例页面。支持 NUMA、PCIe、GPU、 +NVLink/NVSwitch、RoCE Bond、无 Bond 的独立 RoCE/InfiniBand 端口、 +物理网卡、CPU 亲和性和功耗信息。GPU 到 RDMA 端点的亲和关系来自 +`nvidia-smi topo -m`,页面不会为矩阵未暴露的设备推测连线;详细命令与 +采集边界见该目录的 `README.md`。 + +```bash +cd scripts/gpu_node_profile +python3 gpu_node_profile.py \ + --snapshot-out gpu_node_snapshot.json \ + --output gpu_node_profile.html \ + --topology-output gpu_topology_3d.html +``` + +--- + ## 故障排除 | 问题 | 原因 | 解决方案 | diff --git a/scripts/gpu_node_profile/.gitignore b/scripts/gpu_node_profile/.gitignore new file mode 100644 index 0000000..2e4f324 --- /dev/null +++ b/scripts/gpu_node_profile/.gitignore @@ -0,0 +1,5 @@ +/gpu_node_snapshot*.json +/gpu_node_profile*.html +/gpu_topology_3d*.html +/b300_*_live.json +/b300_*_live.html diff --git a/scripts/gpu_node_profile/README.md b/scripts/gpu_node_profile/README.md new file mode 100644 index 0000000..5044873 --- /dev/null +++ b/scripts/gpu_node_profile/README.md @@ -0,0 +1,155 @@ +# GPU Node Profile + +`gpu_node_profile.py` 通过只读命令采集 GPU、NVLink、PCIe/NUMA、RoCE +Bond、RDMA 和 InfiniBand 信息,生成通用结构化 JSON,并使用 `templates/` +中的两个独立 HTML 源模板,从同一份 JSON 渲染完全离线、无 Mermaid、 +无 CDN 依赖的运维画像与 3D 拓扑 CT。 + +页面顶部按“系统与软件 / GPU / CPU与内存”展示整机概览;中部的 +GPU-RDMA 拓扑支持“总览 / NUMA 分览”即时切换,并继续展示 InfiniBand、 +OS 可见的带内管理网卡和可读取的 BMC 带外信息;最底部横向展示健康检查 +与验证结论。 + +3D CT 页面使用原生 Canvas 对结构化 JSON 动态布局,包含 NUMA 半透明地域、 +CPU Socket、PCIe Root Complex、PCIe Switch 分组、GPU、NVSwitch、RoCE、 +InfiniBand 和管理网卡。NUMA 与 PCIe 均来自数据,不包含 B300 专用坐标, +因此同一渲染器也可用于 A800 等其他 GPU 服务器。 + +页面提供递归内部镜头,可按“整机 → NUMA → PCIe 分组 → Bond → 物理网卡” +逐层钻取。进入任一层后会隐藏外层与无关的同级设备,只保留当前壳体和后代 +节点;顶部面包屑说明当前位置,“上一层”按原路径返回。双击壳体继续深入, +双击普通设备时也会沿该设备的父级路径每次深入一层,例如连续双击 GPU 会 +依次进入整机、NUMA 和对应 PCIe 分组;到达最深层后才执行设备聚焦。 +`Shift + 拖动`可平移,`Esc`返回整机外观。 + +Canvas 标签使用高对比浅色底与黑色粗体。每个 RoCE Bond 会作为透明聚合 +容器展示,并在内部继续拆出物理成员网卡、速率和 BDF。NVLink/NVSwitch +连线在设备盒子上层高亮绘制,GPU 端点带编号;该 Fabric 只连接 GPU, +不会连接网卡。 + +网络模型同时支持两种形态:存在 Bond 时渲染 +`GPU → RoCE Bond → 物理成员端口`;不存在 Bond 时根据 +`nvidia-smi topo -m`、RDMA 名称和 PCIe 路径渲染 +`GPU → 独立 RoCE/InfiniBand 端点`。多张 GPU 与多个等距端点会进行 +全局最优、确定性的唯一分配,避免顺序贪心导致所有 GPU 错误指向同一张 +网卡;端点少于 GPU 时才保留可共享的最近路径语义。亲和关系只接受 +`nvidia-smi topo -m` 暴露的 `PIX/PXB/PHB/NODE/SYS` 证据;若独立 IB +未出现在矩阵中,会保留设备和 PCIe/NUMA 信息并标记 `not-exposed`,但不会 +推测一条 GPU 连线。 + +JSON 中的 `network_endpoints` 是两张页面共用的标准网络模型: + +```text +network_endpoints[] + id / type(bond|standalone) / protocol(RoCE|InfiniBand) + rdma / numa / bdf / state / topology_visibility + members[] # Bond 为多个物理口,直连端点为一个物理口 + gpu_links[] # GPU、PIX/PXB/...、用途及证据来源 + +gpus[].primary_network_id # 指向主要计算 RDMA 端点 +gpus[].network_links[] # 保留主要端点和额外最近端点 +``` + +CPU 与内存概览、NUMA 标尺和每张 GPU 卡片会同时展示 `nvidia-smi topo -m` +解析出的推荐 CPU 范围及内存 NUMA。该信息描述硬件拓扑的近端路径,不表示 +训练或推理进程已经通过 `taskset`、cgroup、Slurm 或 Kubernetes 完成实际绑核。 + +GPU 概览和单卡画像会展示采集时刻的 `power.draw`、当前 `power.limit` 与 +P-State。Power Limit 表示当前生效的 GPU 功耗上限,不等于整机额定功耗; +P0 表示最高性能状态,也不单独等价于 GPU 持续满载。 + +## 在 GPU 节点执行 + +```bash +python3 gpu_node_profile.py \ + --output gpu_node_profile.html \ + --topology-output gpu_topology_3d.html \ + --snapshot-out gpu_node_snapshot.json \ + --expected-gpus 8 \ + --expected-bonds 8 \ + --expected-ib 4 +``` + +脚本不会修改网卡、驱动、Bond 或系统配置。唯一写操作是生成用户指定的 +HTML/JSON 文件。`--expected-bonds` 校验的是 RDMA/RoCE Bond;普通管理 +Bond 会保留在原始网卡信息中,但不会计入加速网络端点数量。 + +三个产物的关系: + +```text +只读采集 → gpu_node_snapshot.json + ├─→ gpu_node_profile.html + └─→ gpu_topology_3d.html +``` + +源码与演示数据的关系: + +```text +gpu_node_profile.py # 只读采集、校验和渲染入口 +templates/gpu_node_profile.html # 运维画像源模板 +templates/gpu_topology_3d.html # 3D CT 源模板 +examples/b300_demo.json # 由真机快照脱敏得到的演示数据 +examples/b300_profile_demo.html # demo.json 生成的自包含示例 +examples/b300_topology_3d_demo.html +``` + +最终生成的两张 HTML 会把 JSON 嵌入页面,因此可以直接双击、复制到其他机器 +或离线归档。它们不会在打开时读取旁边的 `demo.json`;修改或替换 JSON 后, +需要重新运行 Python 渲染: + +```bash +python3 gpu_node_profile.py \ + --snapshot-in examples/b300_demo.json \ + --output examples/b300_profile_demo.html \ + --topology-output examples/b300_topology_3d_demo.html \ + --expected-gpus 8 \ + --expected-bonds 8 \ + --expected-ib 4 +``` + +## 使用已有快照离线渲染 + +```bash +python3 gpu_node_profile.py \ + --snapshot-in examples/b300_demo.json \ + --output b300_profile_demo.html \ + --topology-output b300_topology_3d_demo.html \ + --expected-gpus 8 \ + --expected-bonds 8 \ + --expected-ib 4 +``` + +只采集 JSON 并写到标准输出,适合通过 SSH 在远端只读采集、在本地渲染: + +```bash +python3 gpu_node_profile.py --snapshot-stdout +``` + +## 测试 + +```bash +python3 -m unittest discover -s tests -v +``` + +## 采集边界 + +- `nvcc` 表示当前 PATH 指向的 CUDA Toolkit,不代表容器或应用实际加载的 + CUDA Runtime。 +- LACP Bond 展示的是成员链路聚合容量,不承诺单连接可达到聚合带宽。 +- 虚拟机中的 PCIe/NUMA 数据表示 Guest 可见拓扑,不能代替宿主机物理布线 + 和交换机侧验证。 +- `pci_fabric` 保存 Guest 可见的 PCIe 祖先路径、桥和端点。3D 页面中的空间 + 坐标由这些逻辑关系动态生成,不代表设备在机箱中的厘米级真实位置。 +- `PIX/PXB/PHB/NODE/SYS` 表示拓扑距离。`PIX` 可作为 GPUDirect RDMA 的 + 良好候选路径,但只有完成驱动能力和实际 RDMA 传输验证后才能标记为 + `GDR Verified`。 +- CPU 亲和表示 NVIDIA 拓扑给出的推荐 CPU/内存范围,不是运行中进程的 + `Cpus_allowed_list`。评估实际 H2D 性能还需结合进程绑核、内存落点、 + pinned memory 和带宽实测。 +- GPU 功耗为报告生成时的一次瞬时采样;周期性训练负载的平均功耗和峰值 + 需要连续采样。整机功耗还需结合 BMC/DCMI 或机柜 PDU,不能简单等同于 + 所有 GPU Power Limit 之和。 +- 独立BMC口通常不进入宿主OS网络命名空间。脚本会尝试从常见LAN通道只读 + 执行 `ipmitool lan print`;读取不到时会在画像中明确标记“OS不可见”。 +- 管理网络区域只渲染物理 PCI 网卡、Bond 或默认路由接口;CNI、veth 等 + 容器虚拟接口仍可保留在原始 JSON 的 `nics` 中,但不会污染整机画像。 diff --git a/scripts/gpu_node_profile/examples/b300_demo.json b/scripts/gpu_node_profile/examples/b300_demo.json new file mode 100644 index 0000000..25393c6 --- /dev/null +++ b/scripts/gpu_node_profile/examples/b300_demo.json @@ -0,0 +1,3458 @@ +{ + "schema_version": 1, + "generated_at": "2026-07-25T11:49:43+08:00", + "collector_version": "2.0.0", + "system": { + "hostname": "b300-demo", + "os": "Ubuntu 24.04.1 LTS", + "kernel": "6.8.0-71-generic", + "architecture": "x86_64", + "manufacturer": "Baidu Cloud", + "product": "Baidu Cloud BCC", + "product_version": "6.10.0", + "virtualization": "kvm", + "cpu_model": "Intel(R) Xeon(R) 6787P", + "numa_nodes": 2, + "cpu_logical": 332, + "cpu_sockets": 2, + "cpu_cores_per_socket": 83, + "cpu_physical_cores": 166, + "cpu_threads_per_core": 2, + "memory_total_kib": 3032272904, + "memory_available_kib": 2538918872, + "memory_used_kib": 493354032 + }, + "software": { + "nvidia_driver": "595.71.05", + "cuda_toolkit": "12.8", + "cuda_note": "Toolkit selected by nvcc; application/container runtime may differ." + }, + "gpus": [ + { + "index": 0, + "name": "NVIDIA B30Z", + "uuid": "GPU-DEMO-00", + "bdf": "63:00.0", + "driver": "595.71.05", + "memory_total_mib": 275040.0, + "memory_used_mib": 207582.0, + "temperature_c": 67.0, + "utilization_pct": 100.0, + "numa": 0, + "pci_path": [ + "60:00.0", + "61:00.0", + "62:00.0", + "63:00.0" + ], + "pstate": "P0", + "power_w": 1089.75, + "power_limit_w": 1100.0, + "power_default_limit_w": 1100.0, + "power_min_limit_w": 200.0, + "power_max_limit_w": 1100.0, + "cpu_affinity": "0-165", + "numa_affinity": "0", + "gpu_numa_id": "N/A", + "network_links": [ + { + "endpoint_id": "bond:bond0", + "name": "bond0", + "rdma": "mlx5_bond_0", + "protocol": "RoCE", + "type": "bond", + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "primary_network_id": "bond:bond0", + "network_relation": "PIX", + "rdma": "mlx5_bond_0", + "network": "bond0", + "bond": "bond0" + }, + { + "index": 1, + "name": "NVIDIA B30Z", + "uuid": "GPU-DEMO-01", + "bdf": "67:00.0", + "driver": "595.71.05", + "memory_total_mib": 275040.0, + "memory_used_mib": 207582.0, + "temperature_c": 56.0, + "utilization_pct": 100.0, + "numa": 0, + "pci_path": [ + "60:00.1", + "65:00.0", + "66:00.0", + "67:00.0" + ], + "pstate": "P0", + "power_w": 1066.45, + "power_limit_w": 1100.0, + "power_default_limit_w": 1100.0, + "power_min_limit_w": 200.0, + "power_max_limit_w": 1100.0, + "cpu_affinity": "0-165", + "numa_affinity": "0", + "gpu_numa_id": "N/A", + "network_links": [ + { + "endpoint_id": "bond:bond1", + "name": "bond1", + "rdma": "mlx5_bond_1", + "protocol": "RoCE", + "type": "bond", + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "primary_network_id": "bond:bond1", + "network_relation": "PIX", + "rdma": "mlx5_bond_1", + "network": "bond1", + "bond": "bond1" + }, + { + "index": 2, + "name": "NVIDIA B30Z", + "uuid": "GPU-DEMO-02", + "bdf": "6b:00.0", + "driver": "595.71.05", + "memory_total_mib": 275040.0, + "memory_used_mib": 207582.0, + "temperature_c": 63.0, + "utilization_pct": 100.0, + "numa": 0, + "pci_path": [ + "60:00.2", + "69:00.0", + "6a:00.0", + "6b:00.0" + ], + "pstate": "P0", + "power_w": 1049.08, + "power_limit_w": 1100.0, + "power_default_limit_w": 1100.0, + "power_min_limit_w": 200.0, + "power_max_limit_w": 1100.0, + "cpu_affinity": "0-165", + "numa_affinity": "0", + "gpu_numa_id": "N/A", + "network_links": [ + { + "endpoint_id": "bond:bond2", + "name": "bond2", + "rdma": "mlx5_bond_2", + "protocol": "RoCE", + "type": "bond", + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "primary_network_id": "bond:bond2", + "network_relation": "PIX", + "rdma": "mlx5_bond_2", + "network": "bond2", + "bond": "bond2" + }, + { + "index": 3, + "name": "NVIDIA B30Z", + "uuid": "GPU-DEMO-03", + "bdf": "6f:00.0", + "driver": "595.71.05", + "memory_total_mib": 275040.0, + "memory_used_mib": 207582.0, + "temperature_c": 56.0, + "utilization_pct": 100.0, + "numa": 0, + "pci_path": [ + "60:00.3", + "6d:00.0", + "6e:00.0", + "6f:00.0" + ], + "pstate": "P0", + "power_w": 1079.78, + "power_limit_w": 1100.0, + "power_default_limit_w": 1100.0, + "power_min_limit_w": 200.0, + "power_max_limit_w": 1100.0, + "cpu_affinity": "0-165", + "numa_affinity": "0", + "gpu_numa_id": "N/A", + "network_links": [ + { + "endpoint_id": "bond:bond3", + "name": "bond3", + "rdma": "mlx5_bond_3", + "protocol": "RoCE", + "type": "bond", + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "primary_network_id": "bond:bond3", + "network_relation": "PIX", + "rdma": "mlx5_bond_3", + "network": "bond3", + "bond": "bond3" + }, + { + "index": 4, + "name": "NVIDIA B30Z", + "uuid": "GPU-DEMO-04", + "bdf": "a3:00.0", + "driver": "595.71.05", + "memory_total_mib": 275040.0, + "memory_used_mib": 207582.0, + "temperature_c": 55.0, + "utilization_pct": 100.0, + "numa": 1, + "pci_path": [ + "a0:00.0", + "a1:00.0", + "a2:00.0", + "a3:00.0" + ], + "pstate": "P0", + "power_w": 1067.78, + "power_limit_w": 1100.0, + "power_default_limit_w": 1100.0, + "power_min_limit_w": 200.0, + "power_max_limit_w": 1100.0, + "cpu_affinity": "166-331", + "numa_affinity": "1", + "gpu_numa_id": "N/A", + "network_links": [ + { + "endpoint_id": "bond:bond4", + "name": "bond4", + "rdma": "mlx5_bond_4", + "protocol": "RoCE", + "type": "bond", + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "primary_network_id": "bond:bond4", + "network_relation": "PIX", + "rdma": "mlx5_bond_4", + "network": "bond4", + "bond": "bond4" + }, + { + "index": 5, + "name": "NVIDIA B30Z", + "uuid": "GPU-DEMO-05", + "bdf": "a7:00.0", + "driver": "595.71.05", + "memory_total_mib": 275040.0, + "memory_used_mib": 207582.0, + "temperature_c": 66.0, + "utilization_pct": 100.0, + "numa": 1, + "pci_path": [ + "a0:00.1", + "a5:00.0", + "a6:00.0", + "a7:00.0" + ], + "pstate": "P0", + "power_w": 1033.6, + "power_limit_w": 1100.0, + "power_default_limit_w": 1100.0, + "power_min_limit_w": 200.0, + "power_max_limit_w": 1100.0, + "cpu_affinity": "166-331", + "numa_affinity": "1", + "gpu_numa_id": "N/A", + "network_links": [ + { + "endpoint_id": "bond:bond5", + "name": "bond5", + "rdma": "mlx5_bond_5", + "protocol": "RoCE", + "type": "bond", + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "primary_network_id": "bond:bond5", + "network_relation": "PIX", + "rdma": "mlx5_bond_5", + "network": "bond5", + "bond": "bond5" + }, + { + "index": 6, + "name": "NVIDIA B30Z", + "uuid": "GPU-DEMO-06", + "bdf": "ab:00.0", + "driver": "595.71.05", + "memory_total_mib": 275040.0, + "memory_used_mib": 207582.0, + "temperature_c": 55.0, + "utilization_pct": 100.0, + "numa": 1, + "pci_path": [ + "a0:00.2", + "a9:00.0", + "aa:00.0", + "ab:00.0" + ], + "pstate": "P0", + "power_w": 1088.35, + "power_limit_w": 1100.0, + "power_default_limit_w": 1100.0, + "power_min_limit_w": 200.0, + "power_max_limit_w": 1100.0, + "cpu_affinity": "166-331", + "numa_affinity": "1", + "gpu_numa_id": "N/A", + "network_links": [ + { + "endpoint_id": "bond:bond6", + "name": "bond6", + "rdma": "mlx5_bond_6", + "protocol": "RoCE", + "type": "bond", + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "primary_network_id": "bond:bond6", + "network_relation": "PIX", + "rdma": "mlx5_bond_6", + "network": "bond6", + "bond": "bond6" + }, + { + "index": 7, + "name": "NVIDIA B30Z", + "uuid": "GPU-DEMO-07", + "bdf": "af:00.0", + "driver": "595.71.05", + "memory_total_mib": 275040.0, + "memory_used_mib": 207582.0, + "temperature_c": 64.0, + "utilization_pct": 100.0, + "numa": 1, + "pci_path": [ + "a0:00.3", + "ad:00.0", + "ae:00.0", + "af:00.0" + ], + "pstate": "P0", + "power_w": 1063.74, + "power_limit_w": 1100.0, + "power_default_limit_w": 1100.0, + "power_min_limit_w": 200.0, + "power_max_limit_w": 1100.0, + "cpu_affinity": "166-331", + "numa_affinity": "1", + "gpu_numa_id": "N/A", + "network_links": [ + { + "endpoint_id": "bond:bond7", + "name": "bond7", + "rdma": "mlx5_bond_7", + "protocol": "RoCE", + "type": "bond", + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "primary_network_id": "bond:bond7", + "network_relation": "PIX", + "rdma": "mlx5_bond_7", + "network": "bond7", + "bond": "bond7" + } + ], + "topology": { + "rows": { + "GPU0": { + "GPU0": "X", + "GPU1": "NV18", + "GPU2": "NV18", + "GPU3": "NV18", + "GPU4": "NV18", + "GPU5": "NV18", + "GPU6": "NV18", + "GPU7": "NV18", + "NIC0": "SYS", + "NIC1": "PIX", + "NIC2": "PHB", + "NIC3": "PHB", + "NIC4": "PHB", + "NIC5": "SYS", + "NIC6": "SYS", + "NIC7": "SYS", + "NIC8": "SYS" + }, + "GPU1": { + "GPU0": "NV18", + "GPU1": "X", + "GPU2": "NV18", + "GPU3": "NV18", + "GPU4": "NV18", + "GPU5": "NV18", + "GPU6": "NV18", + "GPU7": "NV18", + "NIC0": "SYS", + "NIC1": "PHB", + "NIC2": "PIX", + "NIC3": "PHB", + "NIC4": "PHB", + "NIC5": "SYS", + "NIC6": "SYS", + "NIC7": "SYS", + "NIC8": "SYS" + }, + "GPU2": { + "GPU0": "NV18", + "GPU1": "NV18", + "GPU2": "X", + "GPU3": "NV18", + "GPU4": "NV18", + "GPU5": "NV18", + "GPU6": "NV18", + "GPU7": "NV18", + "NIC0": "SYS", + "NIC1": "PHB", + "NIC2": "PHB", + "NIC3": "PIX", + "NIC4": "PHB", + "NIC5": "SYS", + "NIC6": "SYS", + "NIC7": "SYS", + "NIC8": "SYS" + }, + "GPU3": { + "GPU0": "NV18", + "GPU1": "NV18", + "GPU2": "NV18", + "GPU3": "X", + "GPU4": "NV18", + "GPU5": "NV18", + "GPU6": "NV18", + "GPU7": "NV18", + "NIC0": "SYS", + "NIC1": "PHB", + "NIC2": "PHB", + "NIC3": "PHB", + "NIC4": "PIX", + "NIC5": "SYS", + "NIC6": "SYS", + "NIC7": "SYS", + "NIC8": "SYS" + }, + "GPU4": { + "GPU0": "NV18", + "GPU1": "NV18", + "GPU2": "NV18", + "GPU3": "NV18", + "GPU4": "X", + "GPU5": "NV18", + "GPU6": "NV18", + "GPU7": "NV18", + "NIC0": "SYS", + "NIC1": "SYS", + "NIC2": "SYS", + "NIC3": "SYS", + "NIC4": "SYS", + "NIC5": "PIX", + "NIC6": "PHB", + "NIC7": "PHB", + "NIC8": "PHB" + }, + "GPU5": { + "GPU0": "NV18", + "GPU1": "NV18", + "GPU2": "NV18", + "GPU3": "NV18", + "GPU4": "NV18", + "GPU5": "X", + "GPU6": "NV18", + "GPU7": "NV18", + "NIC0": "SYS", + "NIC1": "SYS", + "NIC2": "SYS", + "NIC3": "SYS", + "NIC4": "SYS", + "NIC5": "PHB", + "NIC6": "PIX", + "NIC7": "PHB", + "NIC8": "PHB" + }, + "GPU6": { + "GPU0": "NV18", + "GPU1": "NV18", + "GPU2": "NV18", + "GPU3": "NV18", + "GPU4": "NV18", + "GPU5": "NV18", + "GPU6": "X", + "GPU7": "NV18", + "NIC0": "SYS", + "NIC1": "SYS", + "NIC2": "SYS", + "NIC3": "SYS", + "NIC4": "SYS", + "NIC5": "PHB", + "NIC6": "PHB", + "NIC7": "PIX", + "NIC8": "PHB" + }, + "GPU7": { + "GPU0": "NV18", + "GPU1": "NV18", + "GPU2": "NV18", + "GPU3": "NV18", + "GPU4": "NV18", + "GPU5": "NV18", + "GPU6": "NV18", + "GPU7": "X", + "NIC0": "SYS", + "NIC1": "SYS", + "NIC2": "SYS", + "NIC3": "SYS", + "NIC4": "SYS", + "NIC5": "PHB", + "NIC6": "PHB", + "NIC7": "PHB", + "NIC8": "PIX" + } + }, + "nic_legend": { + "NIC0": "mlx5_0", + "NIC1": "mlx5_bond_0", + "NIC2": "mlx5_bond_1", + "NIC3": "mlx5_bond_2", + "NIC4": "mlx5_bond_3", + "NIC5": "mlx5_bond_4", + "NIC6": "mlx5_bond_5", + "NIC7": "mlx5_bond_6", + "NIC8": "mlx5_bond_7" + }, + "gpu_nic": { + "GPU0": [ + { + "slot": "NIC0", + "rdma": "mlx5_0", + "relation": "SYS" + }, + { + "slot": "NIC1", + "rdma": "mlx5_bond_0", + "relation": "PIX" + }, + { + "slot": "NIC2", + "rdma": "mlx5_bond_1", + "relation": "PHB" + }, + { + "slot": "NIC3", + "rdma": "mlx5_bond_2", + "relation": "PHB" + }, + { + "slot": "NIC4", + "rdma": "mlx5_bond_3", + "relation": "PHB" + }, + { + "slot": "NIC5", + "rdma": "mlx5_bond_4", + "relation": "SYS" + }, + { + "slot": "NIC6", + "rdma": "mlx5_bond_5", + "relation": "SYS" + }, + { + "slot": "NIC7", + "rdma": "mlx5_bond_6", + "relation": "SYS" + }, + { + "slot": "NIC8", + "rdma": "mlx5_bond_7", + "relation": "SYS" + } + ], + "GPU1": [ + { + "slot": "NIC0", + "rdma": "mlx5_0", + "relation": "SYS" + }, + { + "slot": "NIC1", + "rdma": "mlx5_bond_0", + "relation": "PHB" + }, + { + "slot": "NIC2", + "rdma": "mlx5_bond_1", + "relation": "PIX" + }, + { + "slot": "NIC3", + "rdma": "mlx5_bond_2", + "relation": "PHB" + }, + { + "slot": "NIC4", + "rdma": "mlx5_bond_3", + "relation": "PHB" + }, + { + "slot": "NIC5", + "rdma": "mlx5_bond_4", + "relation": "SYS" + }, + { + "slot": "NIC6", + "rdma": "mlx5_bond_5", + "relation": "SYS" + }, + { + "slot": "NIC7", + "rdma": "mlx5_bond_6", + "relation": "SYS" + }, + { + "slot": "NIC8", + "rdma": "mlx5_bond_7", + "relation": "SYS" + } + ], + "GPU2": [ + { + "slot": "NIC0", + "rdma": "mlx5_0", + "relation": "SYS" + }, + { + "slot": "NIC1", + "rdma": "mlx5_bond_0", + "relation": "PHB" + }, + { + "slot": "NIC2", + "rdma": "mlx5_bond_1", + "relation": "PHB" + }, + { + "slot": "NIC3", + "rdma": "mlx5_bond_2", + "relation": "PIX" + }, + { + "slot": "NIC4", + "rdma": "mlx5_bond_3", + "relation": "PHB" + }, + { + "slot": "NIC5", + "rdma": "mlx5_bond_4", + "relation": "SYS" + }, + { + "slot": "NIC6", + "rdma": "mlx5_bond_5", + "relation": "SYS" + }, + { + "slot": "NIC7", + "rdma": "mlx5_bond_6", + "relation": "SYS" + }, + { + "slot": "NIC8", + "rdma": "mlx5_bond_7", + "relation": "SYS" + } + ], + "GPU3": [ + { + "slot": "NIC0", + "rdma": "mlx5_0", + "relation": "SYS" + }, + { + "slot": "NIC1", + "rdma": "mlx5_bond_0", + "relation": "PHB" + }, + { + "slot": "NIC2", + "rdma": "mlx5_bond_1", + "relation": "PHB" + }, + { + "slot": "NIC3", + "rdma": "mlx5_bond_2", + "relation": "PHB" + }, + { + "slot": "NIC4", + "rdma": "mlx5_bond_3", + "relation": "PIX" + }, + { + "slot": "NIC5", + "rdma": "mlx5_bond_4", + "relation": "SYS" + }, + { + "slot": "NIC6", + "rdma": "mlx5_bond_5", + "relation": "SYS" + }, + { + "slot": "NIC7", + "rdma": "mlx5_bond_6", + "relation": "SYS" + }, + { + "slot": "NIC8", + "rdma": "mlx5_bond_7", + "relation": "SYS" + } + ], + "GPU4": [ + { + "slot": "NIC0", + "rdma": "mlx5_0", + "relation": "SYS" + }, + { + "slot": "NIC1", + "rdma": "mlx5_bond_0", + "relation": "SYS" + }, + { + "slot": "NIC2", + "rdma": "mlx5_bond_1", + "relation": "SYS" + }, + { + "slot": "NIC3", + "rdma": "mlx5_bond_2", + "relation": "SYS" + }, + { + "slot": "NIC4", + "rdma": "mlx5_bond_3", + "relation": "SYS" + }, + { + "slot": "NIC5", + "rdma": "mlx5_bond_4", + "relation": "PIX" + }, + { + "slot": "NIC6", + "rdma": "mlx5_bond_5", + "relation": "PHB" + }, + { + "slot": "NIC7", + "rdma": "mlx5_bond_6", + "relation": "PHB" + }, + { + "slot": "NIC8", + "rdma": "mlx5_bond_7", + "relation": "PHB" + } + ], + "GPU5": [ + { + "slot": "NIC0", + "rdma": "mlx5_0", + "relation": "SYS" + }, + { + "slot": "NIC1", + "rdma": "mlx5_bond_0", + "relation": "SYS" + }, + { + "slot": "NIC2", + "rdma": "mlx5_bond_1", + "relation": "SYS" + }, + { + "slot": "NIC3", + "rdma": "mlx5_bond_2", + "relation": "SYS" + }, + { + "slot": "NIC4", + "rdma": "mlx5_bond_3", + "relation": "SYS" + }, + { + "slot": "NIC5", + "rdma": "mlx5_bond_4", + "relation": "PHB" + }, + { + "slot": "NIC6", + "rdma": "mlx5_bond_5", + "relation": "PIX" + }, + { + "slot": "NIC7", + "rdma": "mlx5_bond_6", + "relation": "PHB" + }, + { + "slot": "NIC8", + "rdma": "mlx5_bond_7", + "relation": "PHB" + } + ], + "GPU6": [ + { + "slot": "NIC0", + "rdma": "mlx5_0", + "relation": "SYS" + }, + { + "slot": "NIC1", + "rdma": "mlx5_bond_0", + "relation": "SYS" + }, + { + "slot": "NIC2", + "rdma": "mlx5_bond_1", + "relation": "SYS" + }, + { + "slot": "NIC3", + "rdma": "mlx5_bond_2", + "relation": "SYS" + }, + { + "slot": "NIC4", + "rdma": "mlx5_bond_3", + "relation": "SYS" + }, + { + "slot": "NIC5", + "rdma": "mlx5_bond_4", + "relation": "PHB" + }, + { + "slot": "NIC6", + "rdma": "mlx5_bond_5", + "relation": "PHB" + }, + { + "slot": "NIC7", + "rdma": "mlx5_bond_6", + "relation": "PIX" + }, + { + "slot": "NIC8", + "rdma": "mlx5_bond_7", + "relation": "PHB" + } + ], + "GPU7": [ + { + "slot": "NIC0", + "rdma": "mlx5_0", + "relation": "SYS" + }, + { + "slot": "NIC1", + "rdma": "mlx5_bond_0", + "relation": "SYS" + }, + { + "slot": "NIC2", + "rdma": "mlx5_bond_1", + "relation": "SYS" + }, + { + "slot": "NIC3", + "rdma": "mlx5_bond_2", + "relation": "SYS" + }, + { + "slot": "NIC4", + "rdma": "mlx5_bond_3", + "relation": "SYS" + }, + { + "slot": "NIC5", + "rdma": "mlx5_bond_4", + "relation": "PHB" + }, + { + "slot": "NIC6", + "rdma": "mlx5_bond_5", + "relation": "PHB" + }, + { + "slot": "NIC7", + "rdma": "mlx5_bond_6", + "relation": "PHB" + }, + { + "slot": "NIC8", + "rdma": "mlx5_bond_7", + "relation": "PIX" + } + ] + }, + "gpu_affinity": { + "GPU0": { + "cpu_affinity": "0-165", + "numa_affinity": "0", + "gpu_numa_id": "N/A" + }, + "GPU1": { + "cpu_affinity": "0-165", + "numa_affinity": "0", + "gpu_numa_id": "N/A" + }, + "GPU2": { + "cpu_affinity": "0-165", + "numa_affinity": "0", + "gpu_numa_id": "N/A" + }, + "GPU3": { + "cpu_affinity": "0-165", + "numa_affinity": "0", + "gpu_numa_id": "N/A" + }, + "GPU4": { + "cpu_affinity": "166-331", + "numa_affinity": "1", + "gpu_numa_id": "N/A" + }, + "GPU5": { + "cpu_affinity": "166-331", + "numa_affinity": "1", + "gpu_numa_id": "N/A" + }, + "GPU6": { + "cpu_affinity": "166-331", + "numa_affinity": "1", + "gpu_numa_id": "N/A" + }, + "GPU7": { + "cpu_affinity": "166-331", + "numa_affinity": "1", + "gpu_numa_id": "N/A" + } + } + }, + "pci_fabric": { + "scope": "guest-visible", + "virtualization": "passthrough-or-virtual-pci", + "bridges": [ + { + "bdf": "00:02.1", + "parent_bdf": "", + "numa": -1, + "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]", + "children": [ + "02:00.0" + ] + }, + { + "bdf": "6a:00.0", + "parent_bdf": "69:00.0", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "6b:00.0" + ] + }, + { + "bdf": "6a:01.0", + "parent_bdf": "69:00.0", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "6c:00.0", + "6c:00.1" + ] + }, + { + "bdf": "6d:00.0", + "parent_bdf": "60:00.3", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)", + "children": [ + "6e:00.0", + "6e:01.0" + ] + }, + { + "bdf": "6e:00.0", + "parent_bdf": "6d:00.0", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "6f:00.0" + ] + }, + { + "bdf": "6e:01.0", + "parent_bdf": "6d:00.0", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "70:00.0", + "70:00.1" + ] + }, + { + "bdf": "60:00.0", + "parent_bdf": "", + "numa": 0, + "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]", + "children": [ + "61:00.0" + ] + }, + { + "bdf": "60:00.1", + "parent_bdf": "", + "numa": 0, + "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]", + "children": [ + "65:00.0" + ] + }, + { + "bdf": "60:00.2", + "parent_bdf": "", + "numa": 0, + "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]", + "children": [ + "69:00.0" + ] + }, + { + "bdf": "60:00.3", + "parent_bdf": "", + "numa": 0, + "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]", + "children": [ + "6d:00.0" + ] + }, + { + "bdf": "61:00.0", + "parent_bdf": "60:00.0", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)", + "children": [ + "62:00.0", + "62:01.0" + ] + }, + { + "bdf": "62:00.0", + "parent_bdf": "61:00.0", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "63:00.0" + ] + }, + { + "bdf": "62:01.0", + "parent_bdf": "61:00.0", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "64:00.0", + "64:00.1" + ] + }, + { + "bdf": "65:00.0", + "parent_bdf": "60:00.1", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)", + "children": [ + "66:00.0", + "66:01.0" + ] + }, + { + "bdf": "66:00.0", + "parent_bdf": "65:00.0", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "67:00.0" + ] + }, + { + "bdf": "66:01.0", + "parent_bdf": "65:00.0", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "68:00.0", + "68:00.1" + ] + }, + { + "bdf": "69:00.0", + "parent_bdf": "60:00.2", + "numa": 0, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)", + "children": [ + "6a:00.0", + "6a:01.0" + ] + }, + { + "bdf": "a0:00.0", + "parent_bdf": "", + "numa": 1, + "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]", + "children": [ + "a1:00.0" + ] + }, + { + "bdf": "a0:00.1", + "parent_bdf": "", + "numa": 1, + "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]", + "children": [ + "a5:00.0" + ] + }, + { + "bdf": "a0:00.2", + "parent_bdf": "", + "numa": 1, + "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]", + "children": [ + "a9:00.0" + ] + }, + { + "bdf": "a0:00.3", + "parent_bdf": "", + "numa": 1, + "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]", + "children": [ + "ad:00.0" + ] + }, + { + "bdf": "a0:00.4", + "parent_bdf": "", + "numa": 1, + "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]", + "children": [ + "b1:00.0" + ] + }, + { + "bdf": "a1:00.0", + "parent_bdf": "a0:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)", + "children": [ + "a2:00.0", + "a2:01.0" + ] + }, + { + "bdf": "a2:00.0", + "parent_bdf": "a1:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "a3:00.0" + ] + }, + { + "bdf": "a2:01.0", + "parent_bdf": "a1:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "a4:00.0", + "a4:00.1" + ] + }, + { + "bdf": "a5:00.0", + "parent_bdf": "a0:00.1", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)", + "children": [ + "a6:00.0", + "a6:01.0" + ] + }, + { + "bdf": "a6:00.0", + "parent_bdf": "a5:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "a7:00.0" + ] + }, + { + "bdf": "a6:01.0", + "parent_bdf": "a5:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "a8:00.0", + "a8:00.1" + ] + }, + { + "bdf": "a9:00.0", + "parent_bdf": "a0:00.2", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)", + "children": [ + "aa:00.0", + "aa:01.0" + ] + }, + { + "bdf": "aa:00.0", + "parent_bdf": "a9:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "ab:00.0" + ] + }, + { + "bdf": "aa:01.0", + "parent_bdf": "a9:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "ac:00.0", + "ac:00.1" + ] + }, + { + "bdf": "ad:00.0", + "parent_bdf": "a0:00.3", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)", + "children": [ + "ae:00.0", + "ae:01.0" + ] + }, + { + "bdf": "ae:00.0", + "parent_bdf": "ad:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "af:00.0" + ] + }, + { + "bdf": "ae:01.0", + "parent_bdf": "ad:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "b0:00.0", + "b0:00.1" + ] + }, + { + "bdf": "b1:00.0", + "parent_bdf": "a0:00.4", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)", + "children": [ + "b2:00.0", + "b2:01.0", + "b2:02.0", + "b2:03.0" + ] + }, + { + "bdf": "b2:00.0", + "parent_bdf": "b1:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "b3:00.0" + ] + }, + { + "bdf": "b2:01.0", + "parent_bdf": "b1:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "b4:00.0" + ] + }, + { + "bdf": "b2:02.0", + "parent_bdf": "b1:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "b5:00.0" + ] + }, + { + "bdf": "b2:03.0", + "parent_bdf": "b1:00.0", + "numa": 1, + "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)", + "children": [ + "b6:00.0" + ] + } + ], + "endpoints": [ + { + "kind": "gpu", + "label": "GPU0", + "bdf": "63:00.0", + "numa": 0, + "path": [ + "60:00.0", + "61:00.0", + "62:00.0", + "63:00.0" + ], + "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)" + }, + { + "kind": "gpu", + "label": "GPU1", + "bdf": "67:00.0", + "numa": 0, + "path": [ + "60:00.1", + "65:00.0", + "66:00.0", + "67:00.0" + ], + "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)" + }, + { + "kind": "gpu", + "label": "GPU2", + "bdf": "6b:00.0", + "numa": 0, + "path": [ + "60:00.2", + "69:00.0", + "6a:00.0", + "6b:00.0" + ], + "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)" + }, + { + "kind": "gpu", + "label": "GPU3", + "bdf": "6f:00.0", + "numa": 0, + "path": [ + "60:00.3", + "6d:00.0", + "6e:00.0", + "6f:00.0" + ], + "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)" + }, + { + "kind": "nic", + "label": "eth0", + "bdf": "02:00.0", + "numa": 0, + "path": [ + "00:02.1", + "02:00.0" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function [15b3:101e]" + }, + { + "kind": "nic", + "label": "eth1", + "bdf": "64:00.0", + "numa": 0, + "path": [ + "60:00.0", + "61:00.0", + "62:01.0", + "64:00.0" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth2", + "bdf": "64:00.1", + "numa": 0, + "path": [ + "60:00.0", + "61:00.0", + "62:01.0", + "64:00.1" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth3", + "bdf": "68:00.0", + "numa": 0, + "path": [ + "60:00.1", + "65:00.0", + "66:01.0", + "68:00.0" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth4", + "bdf": "68:00.1", + "numa": 0, + "path": [ + "60:00.1", + "65:00.0", + "66:01.0", + "68:00.1" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth5", + "bdf": "6c:00.0", + "numa": 0, + "path": [ + "60:00.2", + "69:00.0", + "6a:01.0", + "6c:00.0" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth6", + "bdf": "6c:00.1", + "numa": 0, + "path": [ + "60:00.2", + "69:00.0", + "6a:01.0", + "6c:00.1" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth7", + "bdf": "70:00.0", + "numa": 0, + "path": [ + "60:00.3", + "6d:00.0", + "6e:01.0", + "70:00.0" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth8", + "bdf": "70:00.1", + "numa": 0, + "path": [ + "60:00.3", + "6d:00.0", + "6e:01.0", + "70:00.1" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "gpu", + "label": "GPU4", + "bdf": "a3:00.0", + "numa": 1, + "path": [ + "a0:00.0", + "a1:00.0", + "a2:00.0", + "a3:00.0" + ], + "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)" + }, + { + "kind": "gpu", + "label": "GPU5", + "bdf": "a7:00.0", + "numa": 1, + "path": [ + "a0:00.1", + "a5:00.0", + "a6:00.0", + "a7:00.0" + ], + "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)" + }, + { + "kind": "gpu", + "label": "GPU6", + "bdf": "ab:00.0", + "numa": 1, + "path": [ + "a0:00.2", + "a9:00.0", + "aa:00.0", + "ab:00.0" + ], + "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)" + }, + { + "kind": "gpu", + "label": "GPU7", + "bdf": "af:00.0", + "numa": 1, + "path": [ + "a0:00.3", + "ad:00.0", + "ae:00.0", + "af:00.0" + ], + "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)" + }, + { + "kind": "infiniband", + "label": "mlx5_17", + "bdf": "b3:00.0", + "numa": 1, + "path": [ + "a0:00.4", + "b1:00.0", + "b2:00.0", + "b3:00.0" + ], + "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]" + }, + { + "kind": "infiniband", + "label": "mlx5_18", + "bdf": "b4:00.0", + "numa": 1, + "path": [ + "a0:00.4", + "b1:00.0", + "b2:01.0", + "b4:00.0" + ], + "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]" + }, + { + "kind": "infiniband", + "label": "mlx5_19", + "bdf": "b5:00.0", + "numa": 1, + "path": [ + "a0:00.4", + "b1:00.0", + "b2:02.0", + "b5:00.0" + ], + "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]" + }, + { + "kind": "infiniband", + "label": "mlx5_20", + "bdf": "b6:00.0", + "numa": 1, + "path": [ + "a0:00.4", + "b1:00.0", + "b2:03.0", + "b6:00.0" + ], + "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]" + }, + { + "kind": "nic", + "label": "eth9", + "bdf": "a4:00.0", + "numa": 1, + "path": [ + "a0:00.0", + "a1:00.0", + "a2:01.0", + "a4:00.0" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth10", + "bdf": "a4:00.1", + "numa": 1, + "path": [ + "a0:00.0", + "a1:00.0", + "a2:01.0", + "a4:00.1" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth11", + "bdf": "a8:00.0", + "numa": 1, + "path": [ + "a0:00.1", + "a5:00.0", + "a6:01.0", + "a8:00.0" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth12", + "bdf": "a8:00.1", + "numa": 1, + "path": [ + "a0:00.1", + "a5:00.0", + "a6:01.0", + "a8:00.1" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth13", + "bdf": "ac:00.0", + "numa": 1, + "path": [ + "a0:00.2", + "a9:00.0", + "aa:01.0", + "ac:00.0" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth14", + "bdf": "ac:00.1", + "numa": 1, + "path": [ + "a0:00.2", + "a9:00.0", + "aa:01.0", + "ac:00.1" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth15", + "bdf": "b0:00.0", + "numa": 1, + "path": [ + "a0:00.3", + "ad:00.0", + "ae:01.0", + "b0:00.0" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + }, + { + "kind": "nic", + "label": "eth16", + "bdf": "b0:00.1", + "numa": 1, + "path": [ + "a0:00.3", + "ad:00.0", + "ae:01.0", + "b0:00.1" + ], + "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]" + } + ] + }, + "nvlink": { + "per_gpu": { + "GPU0": [ + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125 + ], + "GPU1": [ + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125 + ], + "GPU2": [ + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125 + ], + "GPU3": [ + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125 + ], + "GPU4": [ + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125 + ], + "GPU5": [ + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125 + ], + "GPU6": [ + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125 + ], + "GPU7": [ + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125, + 53.125 + ] + }, + "active_links": 144, + "links_per_gpu": 18, + "speed_gbs": 53.125, + "fabric_type": "NVSwitch Fabric" + }, + "nics": [ + { + "name": "bond0", + "driver": "bonding", + "bdf": "", + "numa": null, + "pci_path": [], + "speed_mbps": 800000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:01", + "is_bond": true, + "addresses": [ + "198.18.0.2/24" + ], + "default_route": false + }, + { + "name": "bond1", + "driver": "bonding", + "bdf": "", + "numa": null, + "pci_path": [], + "speed_mbps": 800000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:02", + "is_bond": true, + "addresses": [ + "198.18.1.2/24" + ], + "default_route": false + }, + { + "name": "bond2", + "driver": "bonding", + "bdf": "", + "numa": null, + "pci_path": [], + "speed_mbps": 800000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:03", + "is_bond": true, + "addresses": [ + "198.18.2.2/24" + ], + "default_route": false + }, + { + "name": "bond3", + "driver": "bonding", + "bdf": "", + "numa": null, + "pci_path": [], + "speed_mbps": 800000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:04", + "is_bond": true, + "addresses": [ + "198.18.3.2/24" + ], + "default_route": false + }, + { + "name": "bond4", + "driver": "bonding", + "bdf": "", + "numa": null, + "pci_path": [], + "speed_mbps": 800000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:05", + "is_bond": true, + "addresses": [ + "198.18.4.2/24" + ], + "default_route": false + }, + { + "name": "bond5", + "driver": "bonding", + "bdf": "", + "numa": null, + "pci_path": [], + "speed_mbps": 800000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:06", + "is_bond": true, + "addresses": [ + "198.18.5.2/24" + ], + "default_route": false + }, + { + "name": "bond6", + "driver": "bonding", + "bdf": "", + "numa": null, + "pci_path": [], + "speed_mbps": 800000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:07", + "is_bond": true, + "addresses": [ + "198.18.6.2/24" + ], + "default_route": false + }, + { + "name": "bond7", + "driver": "bonding", + "bdf": "", + "numa": null, + "pci_path": [], + "speed_mbps": 800000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:08", + "is_bond": true, + "addresses": [ + "198.18.7.2/24" + ], + "default_route": false + }, + { + "name": "eth0", + "driver": "mlx5_core", + "bdf": "02:00.0", + "numa": 0, + "pci_path": [ + "00:02.1", + "02:00.0" + ], + "speed_mbps": 200000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:09", + "is_bond": false, + "addresses": [ + "192.0.2.10/24" + ], + "default_route": true + }, + { + "name": "eth1", + "driver": "mlx5_core", + "bdf": "64:00.0", + "numa": 0, + "pci_path": [ + "60:00.0", + "61:00.0", + "62:01.0", + "64:00.0" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:01", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth2", + "driver": "mlx5_core", + "bdf": "64:00.1", + "numa": 0, + "pci_path": [ + "60:00.0", + "61:00.0", + "62:01.0", + "64:00.1" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:01", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth3", + "driver": "mlx5_core", + "bdf": "68:00.0", + "numa": 0, + "pci_path": [ + "60:00.1", + "65:00.0", + "66:01.0", + "68:00.0" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:02", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth4", + "driver": "mlx5_core", + "bdf": "68:00.1", + "numa": 0, + "pci_path": [ + "60:00.1", + "65:00.0", + "66:01.0", + "68:00.1" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:02", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth5", + "driver": "mlx5_core", + "bdf": "6c:00.0", + "numa": 0, + "pci_path": [ + "60:00.2", + "69:00.0", + "6a:01.0", + "6c:00.0" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:03", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth6", + "driver": "mlx5_core", + "bdf": "6c:00.1", + "numa": 0, + "pci_path": [ + "60:00.2", + "69:00.0", + "6a:01.0", + "6c:00.1" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:03", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth7", + "driver": "mlx5_core", + "bdf": "70:00.0", + "numa": 0, + "pci_path": [ + "60:00.3", + "6d:00.0", + "6e:01.0", + "70:00.0" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:04", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth8", + "driver": "mlx5_core", + "bdf": "70:00.1", + "numa": 0, + "pci_path": [ + "60:00.3", + "6d:00.0", + "6e:01.0", + "70:00.1" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:04", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth9", + "driver": "mlx5_core", + "bdf": "a4:00.0", + "numa": 1, + "pci_path": [ + "a0:00.0", + "a1:00.0", + "a2:01.0", + "a4:00.0" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:05", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth10", + "driver": "mlx5_core", + "bdf": "a4:00.1", + "numa": 1, + "pci_path": [ + "a0:00.0", + "a1:00.0", + "a2:01.0", + "a4:00.1" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:05", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth11", + "driver": "mlx5_core", + "bdf": "a8:00.0", + "numa": 1, + "pci_path": [ + "a0:00.1", + "a5:00.0", + "a6:01.0", + "a8:00.0" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:06", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth12", + "driver": "mlx5_core", + "bdf": "a8:00.1", + "numa": 1, + "pci_path": [ + "a0:00.1", + "a5:00.0", + "a6:01.0", + "a8:00.1" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:06", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth13", + "driver": "mlx5_core", + "bdf": "ac:00.0", + "numa": 1, + "pci_path": [ + "a0:00.2", + "a9:00.0", + "aa:01.0", + "ac:00.0" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:07", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth14", + "driver": "mlx5_core", + "bdf": "ac:00.1", + "numa": 1, + "pci_path": [ + "a0:00.2", + "a9:00.0", + "aa:01.0", + "ac:00.1" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:07", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth15", + "driver": "mlx5_core", + "bdf": "b0:00.0", + "numa": 1, + "pci_path": [ + "a0:00.3", + "ad:00.0", + "ae:01.0", + "b0:00.0" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:08", + "is_bond": false, + "addresses": [], + "default_route": false + }, + { + "name": "eth16", + "driver": "mlx5_core", + "bdf": "b0:00.1", + "numa": 1, + "pci_path": [ + "a0:00.3", + "ad:00.0", + "ae:01.0", + "b0:00.1" + ], + "speed_mbps": 400000, + "state": "up", + "carrier": 1, + "mac": "02:00:00:00:00:08", + "is_bond": false, + "addresses": [], + "default_route": false + } + ], + "bonds": [ + { + "name": "bond0", + "mode": "IEEE 802.3ad Dynamic link aggregation", + "hash_policy": "layer3+4 (1)", + "status": "up", + "slaves": [ + { + "name": "eth1", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:01", + "driver": "mlx5_core", + "bdf": "64:00.0", + "numa": 0, + "pci_path": [ + "60:00.0", + "61:00.0", + "62:01.0", + "64:00.0" + ], + "carrier": 1 + }, + { + "name": "eth2", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0a", + "driver": "mlx5_core", + "bdf": "64:00.1", + "numa": 0, + "pci_path": [ + "60:00.0", + "61:00.0", + "62:01.0", + "64:00.1" + ], + "carrier": 1 + } + ], + "rdma": "mlx5_bond_0", + "gpu_index": 0, + "relation": "PIX" + }, + { + "name": "bond1", + "mode": "IEEE 802.3ad Dynamic link aggregation", + "hash_policy": "layer3+4 (1)", + "status": "up", + "slaves": [ + { + "name": "eth3", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:02", + "driver": "mlx5_core", + "bdf": "68:00.0", + "numa": 0, + "pci_path": [ + "60:00.1", + "65:00.0", + "66:01.0", + "68:00.0" + ], + "carrier": 1 + }, + { + "name": "eth4", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0b", + "driver": "mlx5_core", + "bdf": "68:00.1", + "numa": 0, + "pci_path": [ + "60:00.1", + "65:00.0", + "66:01.0", + "68:00.1" + ], + "carrier": 1 + } + ], + "rdma": "mlx5_bond_1", + "gpu_index": 1, + "relation": "PIX" + }, + { + "name": "bond2", + "mode": "IEEE 802.3ad Dynamic link aggregation", + "hash_policy": "layer3+4 (1)", + "status": "up", + "slaves": [ + { + "name": "eth5", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:03", + "driver": "mlx5_core", + "bdf": "6c:00.0", + "numa": 0, + "pci_path": [ + "60:00.2", + "69:00.0", + "6a:01.0", + "6c:00.0" + ], + "carrier": 1 + }, + { + "name": "eth6", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0c", + "driver": "mlx5_core", + "bdf": "6c:00.1", + "numa": 0, + "pci_path": [ + "60:00.2", + "69:00.0", + "6a:01.0", + "6c:00.1" + ], + "carrier": 1 + } + ], + "rdma": "mlx5_bond_2", + "gpu_index": 2, + "relation": "PIX" + }, + { + "name": "bond3", + "mode": "IEEE 802.3ad Dynamic link aggregation", + "hash_policy": "layer3+4 (1)", + "status": "up", + "slaves": [ + { + "name": "eth7", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:04", + "driver": "mlx5_core", + "bdf": "70:00.0", + "numa": 0, + "pci_path": [ + "60:00.3", + "6d:00.0", + "6e:01.0", + "70:00.0" + ], + "carrier": 1 + }, + { + "name": "eth8", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0d", + "driver": "mlx5_core", + "bdf": "70:00.1", + "numa": 0, + "pci_path": [ + "60:00.3", + "6d:00.0", + "6e:01.0", + "70:00.1" + ], + "carrier": 1 + } + ], + "rdma": "mlx5_bond_3", + "gpu_index": 3, + "relation": "PIX" + }, + { + "name": "bond4", + "mode": "IEEE 802.3ad Dynamic link aggregation", + "hash_policy": "layer3+4 (1)", + "status": "up", + "slaves": [ + { + "name": "eth9", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:05", + "driver": "mlx5_core", + "bdf": "a4:00.0", + "numa": 1, + "pci_path": [ + "a0:00.0", + "a1:00.0", + "a2:01.0", + "a4:00.0" + ], + "carrier": 1 + }, + { + "name": "eth10", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0e", + "driver": "mlx5_core", + "bdf": "a4:00.1", + "numa": 1, + "pci_path": [ + "a0:00.0", + "a1:00.0", + "a2:01.0", + "a4:00.1" + ], + "carrier": 1 + } + ], + "rdma": "mlx5_bond_4", + "gpu_index": 4, + "relation": "PIX" + }, + { + "name": "bond5", + "mode": "IEEE 802.3ad Dynamic link aggregation", + "hash_policy": "layer3+4 (1)", + "status": "up", + "slaves": [ + { + "name": "eth11", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:06", + "driver": "mlx5_core", + "bdf": "a8:00.0", + "numa": 1, + "pci_path": [ + "a0:00.1", + "a5:00.0", + "a6:01.0", + "a8:00.0" + ], + "carrier": 1 + }, + { + "name": "eth12", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0f", + "driver": "mlx5_core", + "bdf": "a8:00.1", + "numa": 1, + "pci_path": [ + "a0:00.1", + "a5:00.0", + "a6:01.0", + "a8:00.1" + ], + "carrier": 1 + } + ], + "rdma": "mlx5_bond_5", + "gpu_index": 5, + "relation": "PIX" + }, + { + "name": "bond6", + "mode": "IEEE 802.3ad Dynamic link aggregation", + "hash_policy": "layer3+4 (1)", + "status": "up", + "slaves": [ + { + "name": "eth13", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:07", + "driver": "mlx5_core", + "bdf": "ac:00.0", + "numa": 1, + "pci_path": [ + "a0:00.2", + "a9:00.0", + "aa:01.0", + "ac:00.0" + ], + "carrier": 1 + }, + { + "name": "eth14", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:10", + "driver": "mlx5_core", + "bdf": "ac:00.1", + "numa": 1, + "pci_path": [ + "a0:00.2", + "a9:00.0", + "aa:01.0", + "ac:00.1" + ], + "carrier": 1 + } + ], + "rdma": "mlx5_bond_6", + "gpu_index": 6, + "relation": "PIX" + }, + { + "name": "bond7", + "mode": "IEEE 802.3ad Dynamic link aggregation", + "hash_policy": "layer3+4 (1)", + "status": "up", + "slaves": [ + { + "name": "eth15", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:08", + "driver": "mlx5_core", + "bdf": "b0:00.0", + "numa": 1, + "pci_path": [ + "a0:00.3", + "ad:00.0", + "ae:01.0", + "b0:00.0" + ], + "carrier": 1 + }, + { + "name": "eth16", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:11", + "driver": "mlx5_core", + "bdf": "b0:00.1", + "numa": 1, + "pci_path": [ + "a0:00.3", + "ad:00.0", + "ae:01.0", + "b0:00.1" + ], + "carrier": 1 + } + ], + "rdma": "mlx5_bond_7", + "gpu_index": 7, + "relation": "PIX" + } + ], + "ib_devices": [ + { + "name": "mlx5_17", + "ca_type": "MT4129", + "firmware": "28.46.5500", + "state": "Active", + "physical_state": "LinkUp", + "rate_gbps": 100.0, + "lid": "1", + "link_layer": "InfiniBand", + "bdf": "b3:00.0", + "numa": 1, + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:00.0", + "b3:00.0" + ], + "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]", + "rdma_state": { + "port": 1, + "state": "ACTIVE", + "physical_state": "LINK_UP", + "netdev": "" + } + }, + { + "name": "mlx5_18", + "ca_type": "MT4129", + "firmware": "28.46.5500", + "state": "Active", + "physical_state": "LinkUp", + "rate_gbps": 100.0, + "lid": "4", + "link_layer": "InfiniBand", + "bdf": "b4:00.0", + "numa": 1, + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:01.0", + "b4:00.0" + ], + "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]", + "rdma_state": { + "port": 1, + "state": "ACTIVE", + "physical_state": "LINK_UP", + "netdev": "" + } + }, + { + "name": "mlx5_19", + "ca_type": "MT4129", + "firmware": "28.46.5500", + "state": "Active", + "physical_state": "LinkUp", + "rate_gbps": 100.0, + "lid": "5", + "link_layer": "InfiniBand", + "bdf": "b5:00.0", + "numa": 1, + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:02.0", + "b5:00.0" + ], + "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]", + "rdma_state": { + "port": 1, + "state": "ACTIVE", + "physical_state": "LINK_UP", + "netdev": "" + } + }, + { + "name": "mlx5_20", + "ca_type": "MT4129", + "firmware": "28.46.5500", + "state": "Active", + "physical_state": "LinkUp", + "rate_gbps": 100.0, + "lid": "6", + "link_layer": "InfiniBand", + "bdf": "b6:00.0", + "numa": 1, + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:03.0", + "b6:00.0" + ], + "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]", + "rdma_state": { + "port": 1, + "state": "ACTIVE", + "physical_state": "LINK_UP", + "netdev": "" + } + } + ], + "bmc": { + "detected": false, + "note": "独立 BMC 带外口未向当前 OS/IPMI 通道暴露" + }, + "rdma_map": { + "mlx5_0": "eth0", + "mlx5_bond_0": "bond0", + "mlx5_bond_1": "bond1", + "mlx5_bond_2": "bond2", + "mlx5_bond_3": "bond3", + "mlx5_bond_4": "bond4", + "mlx5_bond_5": "bond5", + "mlx5_bond_6": "bond6", + "mlx5_bond_7": "bond7" + }, + "notes": [ + "Demo snapshot derived from a live B300 node; hostname, GPU UUIDs, MAC and IP addresses are sanitized." + ], + "network_endpoints": [ + { + "id": "bond:bond0", + "type": "bond", + "protocol": "RoCE", + "name": "bond0", + "rdma": "mlx5_bond_0", + "role": "compute", + "state": "UP", + "numa": 0, + "capacity_mbps": 800000.0, + "members": [ + { + "name": "eth1", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:01", + "driver": "mlx5_core", + "bdf": "64:00.0", + "numa": 0, + "pci_path": [ + "60:00.0", + "61:00.0", + "62:01.0", + "64:00.0" + ], + "carrier": 1 + }, + { + "name": "eth2", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0a", + "driver": "mlx5_core", + "bdf": "64:00.1", + "numa": 0, + "pci_path": [ + "60:00.0", + "61:00.0", + "62:01.0", + "64:00.1" + ], + "carrier": 1 + } + ], + "gpu_links": [ + { + "gpu_index": 0, + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "topology_visibility": "nvidia-smi-topo" + }, + { + "id": "bond:bond1", + "type": "bond", + "protocol": "RoCE", + "name": "bond1", + "rdma": "mlx5_bond_1", + "role": "compute", + "state": "UP", + "numa": 0, + "capacity_mbps": 800000.0, + "members": [ + { + "name": "eth3", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:02", + "driver": "mlx5_core", + "bdf": "68:00.0", + "numa": 0, + "pci_path": [ + "60:00.1", + "65:00.0", + "66:01.0", + "68:00.0" + ], + "carrier": 1 + }, + { + "name": "eth4", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0b", + "driver": "mlx5_core", + "bdf": "68:00.1", + "numa": 0, + "pci_path": [ + "60:00.1", + "65:00.0", + "66:01.0", + "68:00.1" + ], + "carrier": 1 + } + ], + "gpu_links": [ + { + "gpu_index": 1, + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "topology_visibility": "nvidia-smi-topo" + }, + { + "id": "bond:bond2", + "type": "bond", + "protocol": "RoCE", + "name": "bond2", + "rdma": "mlx5_bond_2", + "role": "compute", + "state": "UP", + "numa": 0, + "capacity_mbps": 800000.0, + "members": [ + { + "name": "eth5", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:03", + "driver": "mlx5_core", + "bdf": "6c:00.0", + "numa": 0, + "pci_path": [ + "60:00.2", + "69:00.0", + "6a:01.0", + "6c:00.0" + ], + "carrier": 1 + }, + { + "name": "eth6", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0c", + "driver": "mlx5_core", + "bdf": "6c:00.1", + "numa": 0, + "pci_path": [ + "60:00.2", + "69:00.0", + "6a:01.0", + "6c:00.1" + ], + "carrier": 1 + } + ], + "gpu_links": [ + { + "gpu_index": 2, + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "topology_visibility": "nvidia-smi-topo" + }, + { + "id": "bond:bond3", + "type": "bond", + "protocol": "RoCE", + "name": "bond3", + "rdma": "mlx5_bond_3", + "role": "compute", + "state": "UP", + "numa": 0, + "capacity_mbps": 800000.0, + "members": [ + { + "name": "eth7", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:04", + "driver": "mlx5_core", + "bdf": "70:00.0", + "numa": 0, + "pci_path": [ + "60:00.3", + "6d:00.0", + "6e:01.0", + "70:00.0" + ], + "carrier": 1 + }, + { + "name": "eth8", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0d", + "driver": "mlx5_core", + "bdf": "70:00.1", + "numa": 0, + "pci_path": [ + "60:00.3", + "6d:00.0", + "6e:01.0", + "70:00.1" + ], + "carrier": 1 + } + ], + "gpu_links": [ + { + "gpu_index": 3, + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "topology_visibility": "nvidia-smi-topo" + }, + { + "id": "bond:bond4", + "type": "bond", + "protocol": "RoCE", + "name": "bond4", + "rdma": "mlx5_bond_4", + "role": "compute", + "state": "UP", + "numa": 1, + "capacity_mbps": 800000.0, + "members": [ + { + "name": "eth9", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:05", + "driver": "mlx5_core", + "bdf": "a4:00.0", + "numa": 1, + "pci_path": [ + "a0:00.0", + "a1:00.0", + "a2:01.0", + "a4:00.0" + ], + "carrier": 1 + }, + { + "name": "eth10", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0e", + "driver": "mlx5_core", + "bdf": "a4:00.1", + "numa": 1, + "pci_path": [ + "a0:00.0", + "a1:00.0", + "a2:01.0", + "a4:00.1" + ], + "carrier": 1 + } + ], + "gpu_links": [ + { + "gpu_index": 4, + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "topology_visibility": "nvidia-smi-topo" + }, + { + "id": "bond:bond5", + "type": "bond", + "protocol": "RoCE", + "name": "bond5", + "rdma": "mlx5_bond_5", + "role": "compute", + "state": "UP", + "numa": 1, + "capacity_mbps": 800000.0, + "members": [ + { + "name": "eth11", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:06", + "driver": "mlx5_core", + "bdf": "a8:00.0", + "numa": 1, + "pci_path": [ + "a0:00.1", + "a5:00.0", + "a6:01.0", + "a8:00.0" + ], + "carrier": 1 + }, + { + "name": "eth12", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:0f", + "driver": "mlx5_core", + "bdf": "a8:00.1", + "numa": 1, + "pci_path": [ + "a0:00.1", + "a5:00.0", + "a6:01.0", + "a8:00.1" + ], + "carrier": 1 + } + ], + "gpu_links": [ + { + "gpu_index": 5, + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "topology_visibility": "nvidia-smi-topo" + }, + { + "id": "bond:bond6", + "type": "bond", + "protocol": "RoCE", + "name": "bond6", + "rdma": "mlx5_bond_6", + "role": "compute", + "state": "UP", + "numa": 1, + "capacity_mbps": 800000.0, + "members": [ + { + "name": "eth13", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:07", + "driver": "mlx5_core", + "bdf": "ac:00.0", + "numa": 1, + "pci_path": [ + "a0:00.2", + "a9:00.0", + "aa:01.0", + "ac:00.0" + ], + "carrier": 1 + }, + { + "name": "eth14", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:10", + "driver": "mlx5_core", + "bdf": "ac:00.1", + "numa": 1, + "pci_path": [ + "a0:00.2", + "a9:00.0", + "aa:01.0", + "ac:00.1" + ], + "carrier": 1 + } + ], + "gpu_links": [ + { + "gpu_index": 6, + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "topology_visibility": "nvidia-smi-topo" + }, + { + "id": "bond:bond7", + "type": "bond", + "protocol": "RoCE", + "name": "bond7", + "rdma": "mlx5_bond_7", + "role": "compute", + "state": "UP", + "numa": 1, + "capacity_mbps": 800000.0, + "members": [ + { + "name": "eth15", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:08", + "driver": "mlx5_core", + "bdf": "b0:00.0", + "numa": 1, + "pci_path": [ + "a0:00.3", + "ad:00.0", + "ae:01.0", + "b0:00.0" + ], + "carrier": 1 + }, + { + "name": "eth16", + "status": "up", + "speed_mbps": 400000, + "duplex": "full", + "failures": 0, + "mac": "02:00:00:00:00:11", + "driver": "mlx5_core", + "bdf": "b0:00.1", + "numa": 1, + "pci_path": [ + "a0:00.3", + "ad:00.0", + "ae:01.0", + "b0:00.1" + ], + "carrier": 1 + } + ], + "gpu_links": [ + { + "gpu_index": 7, + "relation": "PIX", + "role": "primary", + "evidence": "nvidia-smi topo -m" + } + ], + "topology_visibility": "nvidia-smi-topo" + }, + { + "id": "ib:mlx5_17", + "type": "standalone", + "protocol": "InfiniBand", + "name": "mlx5_17", + "rdma": "mlx5_17", + "role": "storage", + "state": "ACTIVE", + "physical_state": "LinkUp", + "numa": 1, + "bdf": "b3:00.0", + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:00.0", + "b3:00.0" + ], + "capacity_mbps": 100000.0, + "members": [ + { + "name": "mlx5_17", + "rdma": "mlx5_17", + "bdf": "b3:00.0", + "numa": 1, + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:00.0", + "b3:00.0" + ], + "speed_mbps": 100000.0, + "status": "active" + } + ], + "gpu_links": [], + "topology_visibility": "not-exposed" + }, + { + "id": "ib:mlx5_18", + "type": "standalone", + "protocol": "InfiniBand", + "name": "mlx5_18", + "rdma": "mlx5_18", + "role": "storage", + "state": "ACTIVE", + "physical_state": "LinkUp", + "numa": 1, + "bdf": "b4:00.0", + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:01.0", + "b4:00.0" + ], + "capacity_mbps": 100000.0, + "members": [ + { + "name": "mlx5_18", + "rdma": "mlx5_18", + "bdf": "b4:00.0", + "numa": 1, + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:01.0", + "b4:00.0" + ], + "speed_mbps": 100000.0, + "status": "active" + } + ], + "gpu_links": [], + "topology_visibility": "not-exposed" + }, + { + "id": "ib:mlx5_19", + "type": "standalone", + "protocol": "InfiniBand", + "name": "mlx5_19", + "rdma": "mlx5_19", + "role": "storage", + "state": "ACTIVE", + "physical_state": "LinkUp", + "numa": 1, + "bdf": "b5:00.0", + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:02.0", + "b5:00.0" + ], + "capacity_mbps": 100000.0, + "members": [ + { + "name": "mlx5_19", + "rdma": "mlx5_19", + "bdf": "b5:00.0", + "numa": 1, + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:02.0", + "b5:00.0" + ], + "speed_mbps": 100000.0, + "status": "active" + } + ], + "gpu_links": [], + "topology_visibility": "not-exposed" + }, + { + "id": "ib:mlx5_20", + "type": "standalone", + "protocol": "InfiniBand", + "name": "mlx5_20", + "rdma": "mlx5_20", + "role": "storage", + "state": "ACTIVE", + "physical_state": "LinkUp", + "numa": 1, + "bdf": "b6:00.0", + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:03.0", + "b6:00.0" + ], + "capacity_mbps": 100000.0, + "members": [ + { + "name": "mlx5_20", + "rdma": "mlx5_20", + "bdf": "b6:00.0", + "numa": 1, + "pci_path": [ + "a0:00.4", + "b1:00.0", + "b2:03.0", + "b6:00.0" + ], + "speed_mbps": 100000.0, + "status": "active" + } + ], + "gpu_links": [], + "topology_visibility": "not-exposed" + } + ], + "normalized_by_version": "2.1.0" +} diff --git a/scripts/gpu_node_profile/examples/b300_profile_demo.html b/scripts/gpu_node_profile/examples/b300_profile_demo.html new file mode 100644 index 0000000..10d3109 --- /dev/null +++ b/scripts/gpu_node_profile/examples/b300_profile_demo.html @@ -0,0 +1,4370 @@ + + + + + + +b300-demo · GPU Hardware Portrait + + + + +
+
+ GPU NODE · HARDWARE PORTRAIT +

b300-demo

+

Baidu Cloud Baidu Cloud BCC · Ubuntu 24.04.1 LTS

+
+
+ 健康 + +
+
+ +
+
+
+
+
SYSTEM / SOFTWARE

系统与软件

+ +
+

Baidu Cloud Baidu Cloud BCC

+
+
操作系统
Ubuntu 24.04.1 LTS
+
内核
6.8.0-71-generic
+
Driver
595.71.05
+
CUDA Toolkit
12.8 nvcc
+
虚拟化
kvm
+
生成时间
2026-07-25T11:49:43+08:00
+
+
+ +
+
ACCELERATOR

GPU

NVLINK ACTIVE
+

NVIDIA B30Z

+
8GPU
+
+
GPU 瞬时功耗8.54 kW / 8.80 kW
+
+ Power Limit 上限 · 8/8 张处于 P0 · 单次瞬时采样 +
+
+ 2149 GiBGPU 显存 + 6400GRoCE 物理容量 + 18NVLink / GPU + 400GInfiniBand +
+
+ +
+
HOST COMPUTE

CPU 与内存

2 NUMA
+

Intel(R) Xeon(R) 6787P

+
166物理核心
+
系统内存470.5 GiB / 2891.8 GiB
+
+
+ 2CPU Sockets + 332逻辑 CPU + 16.3%内存使用率 +
+
+ GPU0–3CPU 0-165 · MEM NUMA 0GPU4–7CPU 166-331 · MEM NUMA 1 +
+
+
+ +
+
+
TOPOLOGY

GPU ↔ RDMA 亲和画像

+
+ PIX 最佳PXB / PHBSYS 跨 NUMA +
+
+
+
+ NUMA 视图 + 总览显示完整拓扑,分览聚焦单个 CPU / 内存域 +
+
+ + +
+
+
+ NUMA 0 推荐 CPU 0-165NUMA 1 推荐 CPU 166-331 +
+

CPU范围来自 NVIDIA 拓扑亲和,表示更近的CPU/内存路径;不代表当前训练进程已经绑核。

+ +
+
+
+
+ GPU 0 + NUMA 0 +
+

NVIDIA B30Z

+

63:00.0

+
+ CPU0-165 + MEMN0 +
+
+ 269 GiB显存 + 67°温度 + 100%利用率 +
+
+
+
+ 1.09 kW瞬时 / 1.10 kW 上限 + P0 +
+
+
+
+ +
PIXPCIe 亲和
+
+
+ ROCE LAG + UP +
+

bond0

+

mlx5_bond_0

+
800G聚合容量
+
+
+ + eth1400G · 64:00.0 +
+ +
+ + eth2400G · 64:00.1 +
+
+
+ +
+ +
+
+
+ GPU 1 + NUMA 0 +
+

NVIDIA B30Z

+

67:00.0

+
+ CPU0-165 + MEMN0 +
+
+ 269 GiB显存 + 56°温度 + 100%利用率 +
+
+
+
+ 1.07 kW瞬时 / 1.10 kW 上限 + P0 +
+
+
+
+ +
PIXPCIe 亲和
+
+
+ ROCE LAG + UP +
+

bond1

+

mlx5_bond_1

+
800G聚合容量
+
+
+ + eth3400G · 68:00.0 +
+ +
+ + eth4400G · 68:00.1 +
+
+
+ +
+ +
+
+
+ GPU 2 + NUMA 0 +
+

NVIDIA B30Z

+

6b:00.0

+
+ CPU0-165 + MEMN0 +
+
+ 269 GiB显存 + 63°温度 + 100%利用率 +
+
+
+
+ 1.05 kW瞬时 / 1.10 kW 上限 + P0 +
+
+
+
+ +
PIXPCIe 亲和
+
+
+ ROCE LAG + UP +
+

bond2

+

mlx5_bond_2

+
800G聚合容量
+
+
+ + eth5400G · 6c:00.0 +
+ +
+ + eth6400G · 6c:00.1 +
+
+
+ +
+ +
+
+
+ GPU 3 + NUMA 0 +
+

NVIDIA B30Z

+

6f:00.0

+
+ CPU0-165 + MEMN0 +
+
+ 269 GiB显存 + 56°温度 + 100%利用率 +
+
+
+
+ 1.08 kW瞬时 / 1.10 kW 上限 + P0 +
+
+
+
+ +
PIXPCIe 亲和
+
+
+ ROCE LAG + UP +
+

bond3

+

mlx5_bond_3

+
800G聚合容量
+
+
+ + eth7400G · 70:00.0 +
+ +
+ + eth8400G · 70:00.1 +
+
+
+ +
+ +
+
+
+ GPU 4 + NUMA 1 +
+

NVIDIA B30Z

+

a3:00.0

+
+ CPU166-331 + MEMN1 +
+
+ 269 GiB显存 + 55°温度 + 100%利用率 +
+
+
+
+ 1.07 kW瞬时 / 1.10 kW 上限 + P0 +
+
+
+
+ +
PIXPCIe 亲和
+
+
+ ROCE LAG + UP +
+

bond4

+

mlx5_bond_4

+
800G聚合容量
+
+
+ + eth9400G · a4:00.0 +
+ +
+ + eth10400G · a4:00.1 +
+
+
+ +
+ +
+
+
+ GPU 5 + NUMA 1 +
+

NVIDIA B30Z

+

a7:00.0

+
+ CPU166-331 + MEMN1 +
+
+ 269 GiB显存 + 66°温度 + 100%利用率 +
+
+
+
+ 1.03 kW瞬时 / 1.10 kW 上限 + P0 +
+
+
+
+ +
PIXPCIe 亲和
+
+
+ ROCE LAG + UP +
+

bond5

+

mlx5_bond_5

+
800G聚合容量
+
+
+ + eth11400G · a8:00.0 +
+ +
+ + eth12400G · a8:00.1 +
+
+
+ +
+ +
+
+
+ GPU 6 + NUMA 1 +
+

NVIDIA B30Z

+

ab:00.0

+
+ CPU166-331 + MEMN1 +
+
+ 269 GiB显存 + 55°温度 + 100%利用率 +
+
+
+
+ 1.09 kW瞬时 / 1.10 kW 上限 + P0 +
+
+
+
+ +
PIXPCIe 亲和
+
+
+ ROCE LAG + UP +
+

bond6

+

mlx5_bond_6

+
800G聚合容量
+
+
+ + eth13400G · ac:00.0 +
+ +
+ + eth14400G · ac:00.1 +
+
+
+ +
+ +
+
+
+ GPU 7 + NUMA 1 +
+

NVIDIA B30Z

+

af:00.0

+
+ CPU166-331 + MEMN1 +
+
+ 269 GiB显存 + 64°温度 + 100%利用率 +
+
+
+
+ 1.06 kW瞬时 / 1.10 kW 上限 + P0 +
+
+
+
+ +
PIXPCIe 亲和
+
+
+ ROCE LAG + UP +
+

bond7

+

mlx5_bond_7

+
800G聚合容量
+
+
+ + eth15400G · b0:00.0 +
+ +
+ + eth16400G · b0:00.1 +
+
+
+ +
+
+

有 Bond 时展示聚合容器与物理成员;无 Bond 时展示独立 RoCE/IB 端点。LACP 容量不代表单连接带宽。

+
+ +
+
+
+
STORAGE / FABRIC

InfiniBand

+
+
+
+
+ INFINIBAND + ACTIVE +
+

mlx5_17

+
100Gb/s
+
+
PCI
b3:00.0
+
NUMA
1
+
CA
MT4129
+
+
+ +
+
+ INFINIBAND + ACTIVE +
+

mlx5_18

+
100Gb/s
+
+
PCI
b4:00.0
+
NUMA
1
+
CA
MT4129
+
+
+ +
+
+ INFINIBAND + ACTIVE +
+

mlx5_19

+
100Gb/s
+
+
PCI
b5:00.0
+
NUMA
1
+
CA
MT4129
+
+
+ +
+
+ INFINIBAND + ACTIVE +
+

mlx5_20

+
100Gb/s
+
+
PCI
b6:00.0
+
NUMA
1
+
CA
MT4129
+
+
+
+
+
+
+
MANAGEMENT NETWORK

带内与带外管理网络

+
+
+
+
+ 带内 · 默认路由 + UP +
+

eth0

200G
+

192.0.2.10/24

+
+
Driver
mlx5_core
+
PCI
02:00.0
+
NUMA
0
+
MAC
02:00:00:00:00:09
+
+
+ +
+
+ 带外 · BMCOS 不可见 +
+

独立管理口

—
+

独立 BMC 带外口未向当前 OS/IPMI 通道暴露

+

独立带外口通常不进入宿主 OS 网络命名空间;需要在 BMC/IPMI 或云厂商控制台侧验证。

+
+
+
+
+ +
+
+
HEALTH / VALIDATION

HEALTH 与验证结果

+ 健康 +
+ +
+
+ + +
RAW DATA

JSON Snapshot

+
{
+  "schema_version": 1,
+  "generated_at": "2026-07-25T11:49:43+08:00",
+  "collector_version": "2.0.0",
+  "system": {
+    "hostname": "b300-demo",
+    "os": "Ubuntu 24.04.1 LTS",
+    "kernel": "6.8.0-71-generic",
+    "architecture": "x86_64",
+    "manufacturer": "Baidu Cloud",
+    "product": "Baidu Cloud BCC",
+    "product_version": "6.10.0",
+    "virtualization": "kvm",
+    "cpu_model": "Intel(R) Xeon(R) 6787P",
+    "numa_nodes": 2,
+    "cpu_logical": 332,
+    "cpu_sockets": 2,
+    "cpu_cores_per_socket": 83,
+    "cpu_physical_cores": 166,
+    "cpu_threads_per_core": 2,
+    "memory_total_kib": 3032272904,
+    "memory_available_kib": 2538918872,
+    "memory_used_kib": 493354032
+  },
+  "software": {
+    "nvidia_driver": "595.71.05",
+    "cuda_toolkit": "12.8",
+    "cuda_note": "Toolkit selected by nvcc; application/container runtime may differ."
+  },
+  "gpus": [
+    {
+      "index": 0,
+      "name": "NVIDIA B30Z",
+      "uuid": "GPU-DEMO-00",
+      "bdf": "63:00.0",
+      "driver": "595.71.05",
+      "memory_total_mib": 275040.0,
+      "memory_used_mib": 207582.0,
+      "temperature_c": 67.0,
+      "utilization_pct": 100.0,
+      "numa": 0,
+      "pci_path": [
+        "60:00.0",
+        "61:00.0",
+        "62:00.0",
+        "63:00.0"
+      ],
+      "pstate": "P0",
+      "power_w": 1089.75,
+      "power_limit_w": 1100.0,
+      "power_default_limit_w": 1100.0,
+      "power_min_limit_w": 200.0,
+      "power_max_limit_w": 1100.0,
+      "cpu_affinity": "0-165",
+      "numa_affinity": "0",
+      "gpu_numa_id": "N/A",
+      "network_links": [
+        {
+          "endpoint_id": "bond:bond0",
+          "name": "bond0",
+          "rdma": "mlx5_bond_0",
+          "protocol": "RoCE",
+          "type": "bond",
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "primary_network_id": "bond:bond0",
+      "network_relation": "PIX",
+      "rdma": "mlx5_bond_0",
+      "network": "bond0",
+      "bond": "bond0"
+    },
+    {
+      "index": 1,
+      "name": "NVIDIA B30Z",
+      "uuid": "GPU-DEMO-01",
+      "bdf": "67:00.0",
+      "driver": "595.71.05",
+      "memory_total_mib": 275040.0,
+      "memory_used_mib": 207582.0,
+      "temperature_c": 56.0,
+      "utilization_pct": 100.0,
+      "numa": 0,
+      "pci_path": [
+        "60:00.1",
+        "65:00.0",
+        "66:00.0",
+        "67:00.0"
+      ],
+      "pstate": "P0",
+      "power_w": 1066.45,
+      "power_limit_w": 1100.0,
+      "power_default_limit_w": 1100.0,
+      "power_min_limit_w": 200.0,
+      "power_max_limit_w": 1100.0,
+      "cpu_affinity": "0-165",
+      "numa_affinity": "0",
+      "gpu_numa_id": "N/A",
+      "network_links": [
+        {
+          "endpoint_id": "bond:bond1",
+          "name": "bond1",
+          "rdma": "mlx5_bond_1",
+          "protocol": "RoCE",
+          "type": "bond",
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "primary_network_id": "bond:bond1",
+      "network_relation": "PIX",
+      "rdma": "mlx5_bond_1",
+      "network": "bond1",
+      "bond": "bond1"
+    },
+    {
+      "index": 2,
+      "name": "NVIDIA B30Z",
+      "uuid": "GPU-DEMO-02",
+      "bdf": "6b:00.0",
+      "driver": "595.71.05",
+      "memory_total_mib": 275040.0,
+      "memory_used_mib": 207582.0,
+      "temperature_c": 63.0,
+      "utilization_pct": 100.0,
+      "numa": 0,
+      "pci_path": [
+        "60:00.2",
+        "69:00.0",
+        "6a:00.0",
+        "6b:00.0"
+      ],
+      "pstate": "P0",
+      "power_w": 1049.08,
+      "power_limit_w": 1100.0,
+      "power_default_limit_w": 1100.0,
+      "power_min_limit_w": 200.0,
+      "power_max_limit_w": 1100.0,
+      "cpu_affinity": "0-165",
+      "numa_affinity": "0",
+      "gpu_numa_id": "N/A",
+      "network_links": [
+        {
+          "endpoint_id": "bond:bond2",
+          "name": "bond2",
+          "rdma": "mlx5_bond_2",
+          "protocol": "RoCE",
+          "type": "bond",
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "primary_network_id": "bond:bond2",
+      "network_relation": "PIX",
+      "rdma": "mlx5_bond_2",
+      "network": "bond2",
+      "bond": "bond2"
+    },
+    {
+      "index": 3,
+      "name": "NVIDIA B30Z",
+      "uuid": "GPU-DEMO-03",
+      "bdf": "6f:00.0",
+      "driver": "595.71.05",
+      "memory_total_mib": 275040.0,
+      "memory_used_mib": 207582.0,
+      "temperature_c": 56.0,
+      "utilization_pct": 100.0,
+      "numa": 0,
+      "pci_path": [
+        "60:00.3",
+        "6d:00.0",
+        "6e:00.0",
+        "6f:00.0"
+      ],
+      "pstate": "P0",
+      "power_w": 1079.78,
+      "power_limit_w": 1100.0,
+      "power_default_limit_w": 1100.0,
+      "power_min_limit_w": 200.0,
+      "power_max_limit_w": 1100.0,
+      "cpu_affinity": "0-165",
+      "numa_affinity": "0",
+      "gpu_numa_id": "N/A",
+      "network_links": [
+        {
+          "endpoint_id": "bond:bond3",
+          "name": "bond3",
+          "rdma": "mlx5_bond_3",
+          "protocol": "RoCE",
+          "type": "bond",
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "primary_network_id": "bond:bond3",
+      "network_relation": "PIX",
+      "rdma": "mlx5_bond_3",
+      "network": "bond3",
+      "bond": "bond3"
+    },
+    {
+      "index": 4,
+      "name": "NVIDIA B30Z",
+      "uuid": "GPU-DEMO-04",
+      "bdf": "a3:00.0",
+      "driver": "595.71.05",
+      "memory_total_mib": 275040.0,
+      "memory_used_mib": 207582.0,
+      "temperature_c": 55.0,
+      "utilization_pct": 100.0,
+      "numa": 1,
+      "pci_path": [
+        "a0:00.0",
+        "a1:00.0",
+        "a2:00.0",
+        "a3:00.0"
+      ],
+      "pstate": "P0",
+      "power_w": 1067.78,
+      "power_limit_w": 1100.0,
+      "power_default_limit_w": 1100.0,
+      "power_min_limit_w": 200.0,
+      "power_max_limit_w": 1100.0,
+      "cpu_affinity": "166-331",
+      "numa_affinity": "1",
+      "gpu_numa_id": "N/A",
+      "network_links": [
+        {
+          "endpoint_id": "bond:bond4",
+          "name": "bond4",
+          "rdma": "mlx5_bond_4",
+          "protocol": "RoCE",
+          "type": "bond",
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "primary_network_id": "bond:bond4",
+      "network_relation": "PIX",
+      "rdma": "mlx5_bond_4",
+      "network": "bond4",
+      "bond": "bond4"
+    },
+    {
+      "index": 5,
+      "name": "NVIDIA B30Z",
+      "uuid": "GPU-DEMO-05",
+      "bdf": "a7:00.0",
+      "driver": "595.71.05",
+      "memory_total_mib": 275040.0,
+      "memory_used_mib": 207582.0,
+      "temperature_c": 66.0,
+      "utilization_pct": 100.0,
+      "numa": 1,
+      "pci_path": [
+        "a0:00.1",
+        "a5:00.0",
+        "a6:00.0",
+        "a7:00.0"
+      ],
+      "pstate": "P0",
+      "power_w": 1033.6,
+      "power_limit_w": 1100.0,
+      "power_default_limit_w": 1100.0,
+      "power_min_limit_w": 200.0,
+      "power_max_limit_w": 1100.0,
+      "cpu_affinity": "166-331",
+      "numa_affinity": "1",
+      "gpu_numa_id": "N/A",
+      "network_links": [
+        {
+          "endpoint_id": "bond:bond5",
+          "name": "bond5",
+          "rdma": "mlx5_bond_5",
+          "protocol": "RoCE",
+          "type": "bond",
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "primary_network_id": "bond:bond5",
+      "network_relation": "PIX",
+      "rdma": "mlx5_bond_5",
+      "network": "bond5",
+      "bond": "bond5"
+    },
+    {
+      "index": 6,
+      "name": "NVIDIA B30Z",
+      "uuid": "GPU-DEMO-06",
+      "bdf": "ab:00.0",
+      "driver": "595.71.05",
+      "memory_total_mib": 275040.0,
+      "memory_used_mib": 207582.0,
+      "temperature_c": 55.0,
+      "utilization_pct": 100.0,
+      "numa": 1,
+      "pci_path": [
+        "a0:00.2",
+        "a9:00.0",
+        "aa:00.0",
+        "ab:00.0"
+      ],
+      "pstate": "P0",
+      "power_w": 1088.35,
+      "power_limit_w": 1100.0,
+      "power_default_limit_w": 1100.0,
+      "power_min_limit_w": 200.0,
+      "power_max_limit_w": 1100.0,
+      "cpu_affinity": "166-331",
+      "numa_affinity": "1",
+      "gpu_numa_id": "N/A",
+      "network_links": [
+        {
+          "endpoint_id": "bond:bond6",
+          "name": "bond6",
+          "rdma": "mlx5_bond_6",
+          "protocol": "RoCE",
+          "type": "bond",
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "primary_network_id": "bond:bond6",
+      "network_relation": "PIX",
+      "rdma": "mlx5_bond_6",
+      "network": "bond6",
+      "bond": "bond6"
+    },
+    {
+      "index": 7,
+      "name": "NVIDIA B30Z",
+      "uuid": "GPU-DEMO-07",
+      "bdf": "af:00.0",
+      "driver": "595.71.05",
+      "memory_total_mib": 275040.0,
+      "memory_used_mib": 207582.0,
+      "temperature_c": 64.0,
+      "utilization_pct": 100.0,
+      "numa": 1,
+      "pci_path": [
+        "a0:00.3",
+        "ad:00.0",
+        "ae:00.0",
+        "af:00.0"
+      ],
+      "pstate": "P0",
+      "power_w": 1063.74,
+      "power_limit_w": 1100.0,
+      "power_default_limit_w": 1100.0,
+      "power_min_limit_w": 200.0,
+      "power_max_limit_w": 1100.0,
+      "cpu_affinity": "166-331",
+      "numa_affinity": "1",
+      "gpu_numa_id": "N/A",
+      "network_links": [
+        {
+          "endpoint_id": "bond:bond7",
+          "name": "bond7",
+          "rdma": "mlx5_bond_7",
+          "protocol": "RoCE",
+          "type": "bond",
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "primary_network_id": "bond:bond7",
+      "network_relation": "PIX",
+      "rdma": "mlx5_bond_7",
+      "network": "bond7",
+      "bond": "bond7"
+    }
+  ],
+  "topology": {
+    "rows": {
+      "GPU0": {
+        "GPU0": "X",
+        "GPU1": "NV18",
+        "GPU2": "NV18",
+        "GPU3": "NV18",
+        "GPU4": "NV18",
+        "GPU5": "NV18",
+        "GPU6": "NV18",
+        "GPU7": "NV18",
+        "NIC0": "SYS",
+        "NIC1": "PIX",
+        "NIC2": "PHB",
+        "NIC3": "PHB",
+        "NIC4": "PHB",
+        "NIC5": "SYS",
+        "NIC6": "SYS",
+        "NIC7": "SYS",
+        "NIC8": "SYS"
+      },
+      "GPU1": {
+        "GPU0": "NV18",
+        "GPU1": "X",
+        "GPU2": "NV18",
+        "GPU3": "NV18",
+        "GPU4": "NV18",
+        "GPU5": "NV18",
+        "GPU6": "NV18",
+        "GPU7": "NV18",
+        "NIC0": "SYS",
+        "NIC1": "PHB",
+        "NIC2": "PIX",
+        "NIC3": "PHB",
+        "NIC4": "PHB",
+        "NIC5": "SYS",
+        "NIC6": "SYS",
+        "NIC7": "SYS",
+        "NIC8": "SYS"
+      },
+      "GPU2": {
+        "GPU0": "NV18",
+        "GPU1": "NV18",
+        "GPU2": "X",
+        "GPU3": "NV18",
+        "GPU4": "NV18",
+        "GPU5": "NV18",
+        "GPU6": "NV18",
+        "GPU7": "NV18",
+        "NIC0": "SYS",
+        "NIC1": "PHB",
+        "NIC2": "PHB",
+        "NIC3": "PIX",
+        "NIC4": "PHB",
+        "NIC5": "SYS",
+        "NIC6": "SYS",
+        "NIC7": "SYS",
+        "NIC8": "SYS"
+      },
+      "GPU3": {
+        "GPU0": "NV18",
+        "GPU1": "NV18",
+        "GPU2": "NV18",
+        "GPU3": "X",
+        "GPU4": "NV18",
+        "GPU5": "NV18",
+        "GPU6": "NV18",
+        "GPU7": "NV18",
+        "NIC0": "SYS",
+        "NIC1": "PHB",
+        "NIC2": "PHB",
+        "NIC3": "PHB",
+        "NIC4": "PIX",
+        "NIC5": "SYS",
+        "NIC6": "SYS",
+        "NIC7": "SYS",
+        "NIC8": "SYS"
+      },
+      "GPU4": {
+        "GPU0": "NV18",
+        "GPU1": "NV18",
+        "GPU2": "NV18",
+        "GPU3": "NV18",
+        "GPU4": "X",
+        "GPU5": "NV18",
+        "GPU6": "NV18",
+        "GPU7": "NV18",
+        "NIC0": "SYS",
+        "NIC1": "SYS",
+        "NIC2": "SYS",
+        "NIC3": "SYS",
+        "NIC4": "SYS",
+        "NIC5": "PIX",
+        "NIC6": "PHB",
+        "NIC7": "PHB",
+        "NIC8": "PHB"
+      },
+      "GPU5": {
+        "GPU0": "NV18",
+        "GPU1": "NV18",
+        "GPU2": "NV18",
+        "GPU3": "NV18",
+        "GPU4": "NV18",
+        "GPU5": "X",
+        "GPU6": "NV18",
+        "GPU7": "NV18",
+        "NIC0": "SYS",
+        "NIC1": "SYS",
+        "NIC2": "SYS",
+        "NIC3": "SYS",
+        "NIC4": "SYS",
+        "NIC5": "PHB",
+        "NIC6": "PIX",
+        "NIC7": "PHB",
+        "NIC8": "PHB"
+      },
+      "GPU6": {
+        "GPU0": "NV18",
+        "GPU1": "NV18",
+        "GPU2": "NV18",
+        "GPU3": "NV18",
+        "GPU4": "NV18",
+        "GPU5": "NV18",
+        "GPU6": "X",
+        "GPU7": "NV18",
+        "NIC0": "SYS",
+        "NIC1": "SYS",
+        "NIC2": "SYS",
+        "NIC3": "SYS",
+        "NIC4": "SYS",
+        "NIC5": "PHB",
+        "NIC6": "PHB",
+        "NIC7": "PIX",
+        "NIC8": "PHB"
+      },
+      "GPU7": {
+        "GPU0": "NV18",
+        "GPU1": "NV18",
+        "GPU2": "NV18",
+        "GPU3": "NV18",
+        "GPU4": "NV18",
+        "GPU5": "NV18",
+        "GPU6": "NV18",
+        "GPU7": "X",
+        "NIC0": "SYS",
+        "NIC1": "SYS",
+        "NIC2": "SYS",
+        "NIC3": "SYS",
+        "NIC4": "SYS",
+        "NIC5": "PHB",
+        "NIC6": "PHB",
+        "NIC7": "PHB",
+        "NIC8": "PIX"
+      }
+    },
+    "nic_legend": {
+      "NIC0": "mlx5_0",
+      "NIC1": "mlx5_bond_0",
+      "NIC2": "mlx5_bond_1",
+      "NIC3": "mlx5_bond_2",
+      "NIC4": "mlx5_bond_3",
+      "NIC5": "mlx5_bond_4",
+      "NIC6": "mlx5_bond_5",
+      "NIC7": "mlx5_bond_6",
+      "NIC8": "mlx5_bond_7"
+    },
+    "gpu_nic": {
+      "GPU0": [
+        {
+          "slot": "NIC0",
+          "rdma": "mlx5_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC1",
+          "rdma": "mlx5_bond_0",
+          "relation": "PIX"
+        },
+        {
+          "slot": "NIC2",
+          "rdma": "mlx5_bond_1",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC3",
+          "rdma": "mlx5_bond_2",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC4",
+          "rdma": "mlx5_bond_3",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC5",
+          "rdma": "mlx5_bond_4",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC6",
+          "rdma": "mlx5_bond_5",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC7",
+          "rdma": "mlx5_bond_6",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC8",
+          "rdma": "mlx5_bond_7",
+          "relation": "SYS"
+        }
+      ],
+      "GPU1": [
+        {
+          "slot": "NIC0",
+          "rdma": "mlx5_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC1",
+          "rdma": "mlx5_bond_0",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC2",
+          "rdma": "mlx5_bond_1",
+          "relation": "PIX"
+        },
+        {
+          "slot": "NIC3",
+          "rdma": "mlx5_bond_2",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC4",
+          "rdma": "mlx5_bond_3",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC5",
+          "rdma": "mlx5_bond_4",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC6",
+          "rdma": "mlx5_bond_5",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC7",
+          "rdma": "mlx5_bond_6",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC8",
+          "rdma": "mlx5_bond_7",
+          "relation": "SYS"
+        }
+      ],
+      "GPU2": [
+        {
+          "slot": "NIC0",
+          "rdma": "mlx5_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC1",
+          "rdma": "mlx5_bond_0",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC2",
+          "rdma": "mlx5_bond_1",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC3",
+          "rdma": "mlx5_bond_2",
+          "relation": "PIX"
+        },
+        {
+          "slot": "NIC4",
+          "rdma": "mlx5_bond_3",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC5",
+          "rdma": "mlx5_bond_4",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC6",
+          "rdma": "mlx5_bond_5",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC7",
+          "rdma": "mlx5_bond_6",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC8",
+          "rdma": "mlx5_bond_7",
+          "relation": "SYS"
+        }
+      ],
+      "GPU3": [
+        {
+          "slot": "NIC0",
+          "rdma": "mlx5_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC1",
+          "rdma": "mlx5_bond_0",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC2",
+          "rdma": "mlx5_bond_1",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC3",
+          "rdma": "mlx5_bond_2",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC4",
+          "rdma": "mlx5_bond_3",
+          "relation": "PIX"
+        },
+        {
+          "slot": "NIC5",
+          "rdma": "mlx5_bond_4",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC6",
+          "rdma": "mlx5_bond_5",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC7",
+          "rdma": "mlx5_bond_6",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC8",
+          "rdma": "mlx5_bond_7",
+          "relation": "SYS"
+        }
+      ],
+      "GPU4": [
+        {
+          "slot": "NIC0",
+          "rdma": "mlx5_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC1",
+          "rdma": "mlx5_bond_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC2",
+          "rdma": "mlx5_bond_1",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC3",
+          "rdma": "mlx5_bond_2",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC4",
+          "rdma": "mlx5_bond_3",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC5",
+          "rdma": "mlx5_bond_4",
+          "relation": "PIX"
+        },
+        {
+          "slot": "NIC6",
+          "rdma": "mlx5_bond_5",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC7",
+          "rdma": "mlx5_bond_6",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC8",
+          "rdma": "mlx5_bond_7",
+          "relation": "PHB"
+        }
+      ],
+      "GPU5": [
+        {
+          "slot": "NIC0",
+          "rdma": "mlx5_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC1",
+          "rdma": "mlx5_bond_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC2",
+          "rdma": "mlx5_bond_1",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC3",
+          "rdma": "mlx5_bond_2",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC4",
+          "rdma": "mlx5_bond_3",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC5",
+          "rdma": "mlx5_bond_4",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC6",
+          "rdma": "mlx5_bond_5",
+          "relation": "PIX"
+        },
+        {
+          "slot": "NIC7",
+          "rdma": "mlx5_bond_6",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC8",
+          "rdma": "mlx5_bond_7",
+          "relation": "PHB"
+        }
+      ],
+      "GPU6": [
+        {
+          "slot": "NIC0",
+          "rdma": "mlx5_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC1",
+          "rdma": "mlx5_bond_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC2",
+          "rdma": "mlx5_bond_1",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC3",
+          "rdma": "mlx5_bond_2",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC4",
+          "rdma": "mlx5_bond_3",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC5",
+          "rdma": "mlx5_bond_4",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC6",
+          "rdma": "mlx5_bond_5",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC7",
+          "rdma": "mlx5_bond_6",
+          "relation": "PIX"
+        },
+        {
+          "slot": "NIC8",
+          "rdma": "mlx5_bond_7",
+          "relation": "PHB"
+        }
+      ],
+      "GPU7": [
+        {
+          "slot": "NIC0",
+          "rdma": "mlx5_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC1",
+          "rdma": "mlx5_bond_0",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC2",
+          "rdma": "mlx5_bond_1",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC3",
+          "rdma": "mlx5_bond_2",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC4",
+          "rdma": "mlx5_bond_3",
+          "relation": "SYS"
+        },
+        {
+          "slot": "NIC5",
+          "rdma": "mlx5_bond_4",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC6",
+          "rdma": "mlx5_bond_5",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC7",
+          "rdma": "mlx5_bond_6",
+          "relation": "PHB"
+        },
+        {
+          "slot": "NIC8",
+          "rdma": "mlx5_bond_7",
+          "relation": "PIX"
+        }
+      ]
+    },
+    "gpu_affinity": {
+      "GPU0": {
+        "cpu_affinity": "0-165",
+        "numa_affinity": "0",
+        "gpu_numa_id": "N/A"
+      },
+      "GPU1": {
+        "cpu_affinity": "0-165",
+        "numa_affinity": "0",
+        "gpu_numa_id": "N/A"
+      },
+      "GPU2": {
+        "cpu_affinity": "0-165",
+        "numa_affinity": "0",
+        "gpu_numa_id": "N/A"
+      },
+      "GPU3": {
+        "cpu_affinity": "0-165",
+        "numa_affinity": "0",
+        "gpu_numa_id": "N/A"
+      },
+      "GPU4": {
+        "cpu_affinity": "166-331",
+        "numa_affinity": "1",
+        "gpu_numa_id": "N/A"
+      },
+      "GPU5": {
+        "cpu_affinity": "166-331",
+        "numa_affinity": "1",
+        "gpu_numa_id": "N/A"
+      },
+      "GPU6": {
+        "cpu_affinity": "166-331",
+        "numa_affinity": "1",
+        "gpu_numa_id": "N/A"
+      },
+      "GPU7": {
+        "cpu_affinity": "166-331",
+        "numa_affinity": "1",
+        "gpu_numa_id": "N/A"
+      }
+    }
+  },
+  "pci_fabric": {
+    "scope": "guest-visible",
+    "virtualization": "passthrough-or-virtual-pci",
+    "bridges": [
+      {
+        "bdf": "00:02.1",
+        "parent_bdf": "",
+        "numa": -1,
+        "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]",
+        "children": [
+          "02:00.0"
+        ]
+      },
+      {
+        "bdf": "6a:00.0",
+        "parent_bdf": "69:00.0",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "6b:00.0"
+        ]
+      },
+      {
+        "bdf": "6a:01.0",
+        "parent_bdf": "69:00.0",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "6c:00.0",
+          "6c:00.1"
+        ]
+      },
+      {
+        "bdf": "6d:00.0",
+        "parent_bdf": "60:00.3",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)",
+        "children": [
+          "6e:00.0",
+          "6e:01.0"
+        ]
+      },
+      {
+        "bdf": "6e:00.0",
+        "parent_bdf": "6d:00.0",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "6f:00.0"
+        ]
+      },
+      {
+        "bdf": "6e:01.0",
+        "parent_bdf": "6d:00.0",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "70:00.0",
+          "70:00.1"
+        ]
+      },
+      {
+        "bdf": "60:00.0",
+        "parent_bdf": "",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]",
+        "children": [
+          "61:00.0"
+        ]
+      },
+      {
+        "bdf": "60:00.1",
+        "parent_bdf": "",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]",
+        "children": [
+          "65:00.0"
+        ]
+      },
+      {
+        "bdf": "60:00.2",
+        "parent_bdf": "",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]",
+        "children": [
+          "69:00.0"
+        ]
+      },
+      {
+        "bdf": "60:00.3",
+        "parent_bdf": "",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]",
+        "children": [
+          "6d:00.0"
+        ]
+      },
+      {
+        "bdf": "61:00.0",
+        "parent_bdf": "60:00.0",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)",
+        "children": [
+          "62:00.0",
+          "62:01.0"
+        ]
+      },
+      {
+        "bdf": "62:00.0",
+        "parent_bdf": "61:00.0",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "63:00.0"
+        ]
+      },
+      {
+        "bdf": "62:01.0",
+        "parent_bdf": "61:00.0",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "64:00.0",
+          "64:00.1"
+        ]
+      },
+      {
+        "bdf": "65:00.0",
+        "parent_bdf": "60:00.1",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)",
+        "children": [
+          "66:00.0",
+          "66:01.0"
+        ]
+      },
+      {
+        "bdf": "66:00.0",
+        "parent_bdf": "65:00.0",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "67:00.0"
+        ]
+      },
+      {
+        "bdf": "66:01.0",
+        "parent_bdf": "65:00.0",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "68:00.0",
+          "68:00.1"
+        ]
+      },
+      {
+        "bdf": "69:00.0",
+        "parent_bdf": "60:00.2",
+        "numa": 0,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)",
+        "children": [
+          "6a:00.0",
+          "6a:01.0"
+        ]
+      },
+      {
+        "bdf": "a0:00.0",
+        "parent_bdf": "",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]",
+        "children": [
+          "a1:00.0"
+        ]
+      },
+      {
+        "bdf": "a0:00.1",
+        "parent_bdf": "",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]",
+        "children": [
+          "a5:00.0"
+        ]
+      },
+      {
+        "bdf": "a0:00.2",
+        "parent_bdf": "",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]",
+        "children": [
+          "a9:00.0"
+        ]
+      },
+      {
+        "bdf": "a0:00.3",
+        "parent_bdf": "",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]",
+        "children": [
+          "ad:00.0"
+        ]
+      },
+      {
+        "bdf": "a0:00.4",
+        "parent_bdf": "",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Red Hat, Inc. QEMU PCIe Root port [1b36:000c]",
+        "children": [
+          "b1:00.0"
+        ]
+      },
+      {
+        "bdf": "a1:00.0",
+        "parent_bdf": "a0:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)",
+        "children": [
+          "a2:00.0",
+          "a2:01.0"
+        ]
+      },
+      {
+        "bdf": "a2:00.0",
+        "parent_bdf": "a1:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "a3:00.0"
+        ]
+      },
+      {
+        "bdf": "a2:01.0",
+        "parent_bdf": "a1:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "a4:00.0",
+          "a4:00.1"
+        ]
+      },
+      {
+        "bdf": "a5:00.0",
+        "parent_bdf": "a0:00.1",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)",
+        "children": [
+          "a6:00.0",
+          "a6:01.0"
+        ]
+      },
+      {
+        "bdf": "a6:00.0",
+        "parent_bdf": "a5:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "a7:00.0"
+        ]
+      },
+      {
+        "bdf": "a6:01.0",
+        "parent_bdf": "a5:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "a8:00.0",
+          "a8:00.1"
+        ]
+      },
+      {
+        "bdf": "a9:00.0",
+        "parent_bdf": "a0:00.2",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)",
+        "children": [
+          "aa:00.0",
+          "aa:01.0"
+        ]
+      },
+      {
+        "bdf": "aa:00.0",
+        "parent_bdf": "a9:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "ab:00.0"
+        ]
+      },
+      {
+        "bdf": "aa:01.0",
+        "parent_bdf": "a9:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "ac:00.0",
+          "ac:00.1"
+        ]
+      },
+      {
+        "bdf": "ad:00.0",
+        "parent_bdf": "a0:00.3",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)",
+        "children": [
+          "ae:00.0",
+          "ae:01.0"
+        ]
+      },
+      {
+        "bdf": "ae:00.0",
+        "parent_bdf": "ad:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "af:00.0"
+        ]
+      },
+      {
+        "bdf": "ae:01.0",
+        "parent_bdf": "ad:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "b0:00.0",
+          "b0:00.1"
+        ]
+      },
+      {
+        "bdf": "b1:00.0",
+        "parent_bdf": "a0:00.4",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Upstream) [104c:8232] (rev 02)",
+        "children": [
+          "b2:00.0",
+          "b2:01.0",
+          "b2:02.0",
+          "b2:03.0"
+        ]
+      },
+      {
+        "bdf": "b2:00.0",
+        "parent_bdf": "b1:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "b3:00.0"
+        ]
+      },
+      {
+        "bdf": "b2:01.0",
+        "parent_bdf": "b1:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "b4:00.0"
+        ]
+      },
+      {
+        "bdf": "b2:02.0",
+        "parent_bdf": "b1:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "b5:00.0"
+        ]
+      },
+      {
+        "bdf": "b2:03.0",
+        "parent_bdf": "b1:00.0",
+        "numa": 1,
+        "description": "PCI bridge [0604]: Texas Instruments XIO3130 PCI Express Switch (Downstream) [104c:8233] (rev 01)",
+        "children": [
+          "b6:00.0"
+        ]
+      }
+    ],
+    "endpoints": [
+      {
+        "kind": "gpu",
+        "label": "GPU0",
+        "bdf": "63:00.0",
+        "numa": 0,
+        "path": [
+          "60:00.0",
+          "61:00.0",
+          "62:00.0",
+          "63:00.0"
+        ],
+        "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)"
+      },
+      {
+        "kind": "gpu",
+        "label": "GPU1",
+        "bdf": "67:00.0",
+        "numa": 0,
+        "path": [
+          "60:00.1",
+          "65:00.0",
+          "66:00.0",
+          "67:00.0"
+        ],
+        "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)"
+      },
+      {
+        "kind": "gpu",
+        "label": "GPU2",
+        "bdf": "6b:00.0",
+        "numa": 0,
+        "path": [
+          "60:00.2",
+          "69:00.0",
+          "6a:00.0",
+          "6b:00.0"
+        ],
+        "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)"
+      },
+      {
+        "kind": "gpu",
+        "label": "GPU3",
+        "bdf": "6f:00.0",
+        "numa": 0,
+        "path": [
+          "60:00.3",
+          "6d:00.0",
+          "6e:00.0",
+          "6f:00.0"
+        ],
+        "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)"
+      },
+      {
+        "kind": "nic",
+        "label": "eth0",
+        "bdf": "02:00.0",
+        "numa": 0,
+        "path": [
+          "00:02.1",
+          "02:00.0"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function [15b3:101e]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth1",
+        "bdf": "64:00.0",
+        "numa": 0,
+        "path": [
+          "60:00.0",
+          "61:00.0",
+          "62:01.0",
+          "64:00.0"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth2",
+        "bdf": "64:00.1",
+        "numa": 0,
+        "path": [
+          "60:00.0",
+          "61:00.0",
+          "62:01.0",
+          "64:00.1"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth3",
+        "bdf": "68:00.0",
+        "numa": 0,
+        "path": [
+          "60:00.1",
+          "65:00.0",
+          "66:01.0",
+          "68:00.0"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth4",
+        "bdf": "68:00.1",
+        "numa": 0,
+        "path": [
+          "60:00.1",
+          "65:00.0",
+          "66:01.0",
+          "68:00.1"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth5",
+        "bdf": "6c:00.0",
+        "numa": 0,
+        "path": [
+          "60:00.2",
+          "69:00.0",
+          "6a:01.0",
+          "6c:00.0"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth6",
+        "bdf": "6c:00.1",
+        "numa": 0,
+        "path": [
+          "60:00.2",
+          "69:00.0",
+          "6a:01.0",
+          "6c:00.1"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth7",
+        "bdf": "70:00.0",
+        "numa": 0,
+        "path": [
+          "60:00.3",
+          "6d:00.0",
+          "6e:01.0",
+          "70:00.0"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth8",
+        "bdf": "70:00.1",
+        "numa": 0,
+        "path": [
+          "60:00.3",
+          "6d:00.0",
+          "6e:01.0",
+          "70:00.1"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "gpu",
+        "label": "GPU4",
+        "bdf": "a3:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.0",
+          "a1:00.0",
+          "a2:00.0",
+          "a3:00.0"
+        ],
+        "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)"
+      },
+      {
+        "kind": "gpu",
+        "label": "GPU5",
+        "bdf": "a7:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.1",
+          "a5:00.0",
+          "a6:00.0",
+          "a7:00.0"
+        ],
+        "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)"
+      },
+      {
+        "kind": "gpu",
+        "label": "GPU6",
+        "bdf": "ab:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.2",
+          "a9:00.0",
+          "aa:00.0",
+          "ab:00.0"
+        ],
+        "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)"
+      },
+      {
+        "kind": "gpu",
+        "label": "GPU7",
+        "bdf": "af:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.3",
+          "ad:00.0",
+          "ae:00.0",
+          "af:00.0"
+        ],
+        "description": "3D controller [0302]: NVIDIA Corporation Device [10de:2bb9] (rev a1)"
+      },
+      {
+        "kind": "infiniband",
+        "label": "mlx5_17",
+        "bdf": "b3:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.4",
+          "b1:00.0",
+          "b2:00.0",
+          "b3:00.0"
+        ],
+        "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]"
+      },
+      {
+        "kind": "infiniband",
+        "label": "mlx5_18",
+        "bdf": "b4:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.4",
+          "b1:00.0",
+          "b2:01.0",
+          "b4:00.0"
+        ],
+        "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]"
+      },
+      {
+        "kind": "infiniband",
+        "label": "mlx5_19",
+        "bdf": "b5:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.4",
+          "b1:00.0",
+          "b2:02.0",
+          "b5:00.0"
+        ],
+        "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]"
+      },
+      {
+        "kind": "infiniband",
+        "label": "mlx5_20",
+        "bdf": "b6:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.4",
+          "b1:00.0",
+          "b2:03.0",
+          "b6:00.0"
+        ],
+        "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth9",
+        "bdf": "a4:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.0",
+          "a1:00.0",
+          "a2:01.0",
+          "a4:00.0"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth10",
+        "bdf": "a4:00.1",
+        "numa": 1,
+        "path": [
+          "a0:00.0",
+          "a1:00.0",
+          "a2:01.0",
+          "a4:00.1"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth11",
+        "bdf": "a8:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.1",
+          "a5:00.0",
+          "a6:01.0",
+          "a8:00.0"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth12",
+        "bdf": "a8:00.1",
+        "numa": 1,
+        "path": [
+          "a0:00.1",
+          "a5:00.0",
+          "a6:01.0",
+          "a8:00.1"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth13",
+        "bdf": "ac:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.2",
+          "a9:00.0",
+          "aa:01.0",
+          "ac:00.0"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth14",
+        "bdf": "ac:00.1",
+        "numa": 1,
+        "path": [
+          "a0:00.2",
+          "a9:00.0",
+          "aa:01.0",
+          "ac:00.1"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth15",
+        "bdf": "b0:00.0",
+        "numa": 1,
+        "path": [
+          "a0:00.3",
+          "ad:00.0",
+          "ae:01.0",
+          "b0:00.0"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      },
+      {
+        "kind": "nic",
+        "label": "eth16",
+        "bdf": "b0:00.1",
+        "numa": 1,
+        "path": [
+          "a0:00.3",
+          "ad:00.0",
+          "ae:01.0",
+          "b0:00.1"
+        ],
+        "description": "Ethernet controller [0200]: Mellanox Technologies CX8 Family [ConnectX-8] [15b3:1023]"
+      }
+    ]
+  },
+  "nvlink": {
+    "per_gpu": {
+      "GPU0": [
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125
+      ],
+      "GPU1": [
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125
+      ],
+      "GPU2": [
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125
+      ],
+      "GPU3": [
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125
+      ],
+      "GPU4": [
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125
+      ],
+      "GPU5": [
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125
+      ],
+      "GPU6": [
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125
+      ],
+      "GPU7": [
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125,
+        53.125
+      ]
+    },
+    "active_links": 144,
+    "links_per_gpu": 18,
+    "speed_gbs": 53.125,
+    "fabric_type": "NVSwitch Fabric"
+  },
+  "nics": [
+    {
+      "name": "bond0",
+      "driver": "bonding",
+      "bdf": "",
+      "numa": null,
+      "pci_path": [],
+      "speed_mbps": 800000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:01",
+      "is_bond": true,
+      "addresses": [
+        "198.18.0.2/24"
+      ],
+      "default_route": false
+    },
+    {
+      "name": "bond1",
+      "driver": "bonding",
+      "bdf": "",
+      "numa": null,
+      "pci_path": [],
+      "speed_mbps": 800000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:02",
+      "is_bond": true,
+      "addresses": [
+        "198.18.1.2/24"
+      ],
+      "default_route": false
+    },
+    {
+      "name": "bond2",
+      "driver": "bonding",
+      "bdf": "",
+      "numa": null,
+      "pci_path": [],
+      "speed_mbps": 800000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:03",
+      "is_bond": true,
+      "addresses": [
+        "198.18.2.2/24"
+      ],
+      "default_route": false
+    },
+    {
+      "name": "bond3",
+      "driver": "bonding",
+      "bdf": "",
+      "numa": null,
+      "pci_path": [],
+      "speed_mbps": 800000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:04",
+      "is_bond": true,
+      "addresses": [
+        "198.18.3.2/24"
+      ],
+      "default_route": false
+    },
+    {
+      "name": "bond4",
+      "driver": "bonding",
+      "bdf": "",
+      "numa": null,
+      "pci_path": [],
+      "speed_mbps": 800000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:05",
+      "is_bond": true,
+      "addresses": [
+        "198.18.4.2/24"
+      ],
+      "default_route": false
+    },
+    {
+      "name": "bond5",
+      "driver": "bonding",
+      "bdf": "",
+      "numa": null,
+      "pci_path": [],
+      "speed_mbps": 800000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:06",
+      "is_bond": true,
+      "addresses": [
+        "198.18.5.2/24"
+      ],
+      "default_route": false
+    },
+    {
+      "name": "bond6",
+      "driver": "bonding",
+      "bdf": "",
+      "numa": null,
+      "pci_path": [],
+      "speed_mbps": 800000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:07",
+      "is_bond": true,
+      "addresses": [
+        "198.18.6.2/24"
+      ],
+      "default_route": false
+    },
+    {
+      "name": "bond7",
+      "driver": "bonding",
+      "bdf": "",
+      "numa": null,
+      "pci_path": [],
+      "speed_mbps": 800000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:08",
+      "is_bond": true,
+      "addresses": [
+        "198.18.7.2/24"
+      ],
+      "default_route": false
+    },
+    {
+      "name": "eth0",
+      "driver": "mlx5_core",
+      "bdf": "02:00.0",
+      "numa": 0,
+      "pci_path": [
+        "00:02.1",
+        "02:00.0"
+      ],
+      "speed_mbps": 200000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:09",
+      "is_bond": false,
+      "addresses": [
+        "192.0.2.10/24"
+      ],
+      "default_route": true
+    },
+    {
+      "name": "eth1",
+      "driver": "mlx5_core",
+      "bdf": "64:00.0",
+      "numa": 0,
+      "pci_path": [
+        "60:00.0",
+        "61:00.0",
+        "62:01.0",
+        "64:00.0"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:01",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth2",
+      "driver": "mlx5_core",
+      "bdf": "64:00.1",
+      "numa": 0,
+      "pci_path": [
+        "60:00.0",
+        "61:00.0",
+        "62:01.0",
+        "64:00.1"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:01",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth3",
+      "driver": "mlx5_core",
+      "bdf": "68:00.0",
+      "numa": 0,
+      "pci_path": [
+        "60:00.1",
+        "65:00.0",
+        "66:01.0",
+        "68:00.0"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:02",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth4",
+      "driver": "mlx5_core",
+      "bdf": "68:00.1",
+      "numa": 0,
+      "pci_path": [
+        "60:00.1",
+        "65:00.0",
+        "66:01.0",
+        "68:00.1"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:02",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth5",
+      "driver": "mlx5_core",
+      "bdf": "6c:00.0",
+      "numa": 0,
+      "pci_path": [
+        "60:00.2",
+        "69:00.0",
+        "6a:01.0",
+        "6c:00.0"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:03",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth6",
+      "driver": "mlx5_core",
+      "bdf": "6c:00.1",
+      "numa": 0,
+      "pci_path": [
+        "60:00.2",
+        "69:00.0",
+        "6a:01.0",
+        "6c:00.1"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:03",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth7",
+      "driver": "mlx5_core",
+      "bdf": "70:00.0",
+      "numa": 0,
+      "pci_path": [
+        "60:00.3",
+        "6d:00.0",
+        "6e:01.0",
+        "70:00.0"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:04",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth8",
+      "driver": "mlx5_core",
+      "bdf": "70:00.1",
+      "numa": 0,
+      "pci_path": [
+        "60:00.3",
+        "6d:00.0",
+        "6e:01.0",
+        "70:00.1"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:04",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth9",
+      "driver": "mlx5_core",
+      "bdf": "a4:00.0",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.0",
+        "a1:00.0",
+        "a2:01.0",
+        "a4:00.0"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:05",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth10",
+      "driver": "mlx5_core",
+      "bdf": "a4:00.1",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.0",
+        "a1:00.0",
+        "a2:01.0",
+        "a4:00.1"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:05",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth11",
+      "driver": "mlx5_core",
+      "bdf": "a8:00.0",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.1",
+        "a5:00.0",
+        "a6:01.0",
+        "a8:00.0"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:06",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth12",
+      "driver": "mlx5_core",
+      "bdf": "a8:00.1",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.1",
+        "a5:00.0",
+        "a6:01.0",
+        "a8:00.1"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:06",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth13",
+      "driver": "mlx5_core",
+      "bdf": "ac:00.0",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.2",
+        "a9:00.0",
+        "aa:01.0",
+        "ac:00.0"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:07",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth14",
+      "driver": "mlx5_core",
+      "bdf": "ac:00.1",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.2",
+        "a9:00.0",
+        "aa:01.0",
+        "ac:00.1"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:07",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth15",
+      "driver": "mlx5_core",
+      "bdf": "b0:00.0",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.3",
+        "ad:00.0",
+        "ae:01.0",
+        "b0:00.0"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:08",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    },
+    {
+      "name": "eth16",
+      "driver": "mlx5_core",
+      "bdf": "b0:00.1",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.3",
+        "ad:00.0",
+        "ae:01.0",
+        "b0:00.1"
+      ],
+      "speed_mbps": 400000,
+      "state": "up",
+      "carrier": 1,
+      "mac": "02:00:00:00:00:08",
+      "is_bond": false,
+      "addresses": [],
+      "default_route": false
+    }
+  ],
+  "bonds": [
+    {
+      "name": "bond0",
+      "mode": "IEEE 802.3ad Dynamic link aggregation",
+      "hash_policy": "layer3+4 (1)",
+      "status": "up",
+      "slaves": [
+        {
+          "name": "eth1",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:01",
+          "driver": "mlx5_core",
+          "bdf": "64:00.0",
+          "numa": 0,
+          "pci_path": [
+            "60:00.0",
+            "61:00.0",
+            "62:01.0",
+            "64:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth2",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0a",
+          "driver": "mlx5_core",
+          "bdf": "64:00.1",
+          "numa": 0,
+          "pci_path": [
+            "60:00.0",
+            "61:00.0",
+            "62:01.0",
+            "64:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "rdma": "mlx5_bond_0",
+      "gpu_index": 0,
+      "relation": "PIX"
+    },
+    {
+      "name": "bond1",
+      "mode": "IEEE 802.3ad Dynamic link aggregation",
+      "hash_policy": "layer3+4 (1)",
+      "status": "up",
+      "slaves": [
+        {
+          "name": "eth3",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:02",
+          "driver": "mlx5_core",
+          "bdf": "68:00.0",
+          "numa": 0,
+          "pci_path": [
+            "60:00.1",
+            "65:00.0",
+            "66:01.0",
+            "68:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth4",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0b",
+          "driver": "mlx5_core",
+          "bdf": "68:00.1",
+          "numa": 0,
+          "pci_path": [
+            "60:00.1",
+            "65:00.0",
+            "66:01.0",
+            "68:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "rdma": "mlx5_bond_1",
+      "gpu_index": 1,
+      "relation": "PIX"
+    },
+    {
+      "name": "bond2",
+      "mode": "IEEE 802.3ad Dynamic link aggregation",
+      "hash_policy": "layer3+4 (1)",
+      "status": "up",
+      "slaves": [
+        {
+          "name": "eth5",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:03",
+          "driver": "mlx5_core",
+          "bdf": "6c:00.0",
+          "numa": 0,
+          "pci_path": [
+            "60:00.2",
+            "69:00.0",
+            "6a:01.0",
+            "6c:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth6",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0c",
+          "driver": "mlx5_core",
+          "bdf": "6c:00.1",
+          "numa": 0,
+          "pci_path": [
+            "60:00.2",
+            "69:00.0",
+            "6a:01.0",
+            "6c:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "rdma": "mlx5_bond_2",
+      "gpu_index": 2,
+      "relation": "PIX"
+    },
+    {
+      "name": "bond3",
+      "mode": "IEEE 802.3ad Dynamic link aggregation",
+      "hash_policy": "layer3+4 (1)",
+      "status": "up",
+      "slaves": [
+        {
+          "name": "eth7",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:04",
+          "driver": "mlx5_core",
+          "bdf": "70:00.0",
+          "numa": 0,
+          "pci_path": [
+            "60:00.3",
+            "6d:00.0",
+            "6e:01.0",
+            "70:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth8",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0d",
+          "driver": "mlx5_core",
+          "bdf": "70:00.1",
+          "numa": 0,
+          "pci_path": [
+            "60:00.3",
+            "6d:00.0",
+            "6e:01.0",
+            "70:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "rdma": "mlx5_bond_3",
+      "gpu_index": 3,
+      "relation": "PIX"
+    },
+    {
+      "name": "bond4",
+      "mode": "IEEE 802.3ad Dynamic link aggregation",
+      "hash_policy": "layer3+4 (1)",
+      "status": "up",
+      "slaves": [
+        {
+          "name": "eth9",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:05",
+          "driver": "mlx5_core",
+          "bdf": "a4:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.0",
+            "a1:00.0",
+            "a2:01.0",
+            "a4:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth10",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0e",
+          "driver": "mlx5_core",
+          "bdf": "a4:00.1",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.0",
+            "a1:00.0",
+            "a2:01.0",
+            "a4:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "rdma": "mlx5_bond_4",
+      "gpu_index": 4,
+      "relation": "PIX"
+    },
+    {
+      "name": "bond5",
+      "mode": "IEEE 802.3ad Dynamic link aggregation",
+      "hash_policy": "layer3+4 (1)",
+      "status": "up",
+      "slaves": [
+        {
+          "name": "eth11",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:06",
+          "driver": "mlx5_core",
+          "bdf": "a8:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.1",
+            "a5:00.0",
+            "a6:01.0",
+            "a8:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth12",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0f",
+          "driver": "mlx5_core",
+          "bdf": "a8:00.1",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.1",
+            "a5:00.0",
+            "a6:01.0",
+            "a8:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "rdma": "mlx5_bond_5",
+      "gpu_index": 5,
+      "relation": "PIX"
+    },
+    {
+      "name": "bond6",
+      "mode": "IEEE 802.3ad Dynamic link aggregation",
+      "hash_policy": "layer3+4 (1)",
+      "status": "up",
+      "slaves": [
+        {
+          "name": "eth13",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:07",
+          "driver": "mlx5_core",
+          "bdf": "ac:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.2",
+            "a9:00.0",
+            "aa:01.0",
+            "ac:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth14",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:10",
+          "driver": "mlx5_core",
+          "bdf": "ac:00.1",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.2",
+            "a9:00.0",
+            "aa:01.0",
+            "ac:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "rdma": "mlx5_bond_6",
+      "gpu_index": 6,
+      "relation": "PIX"
+    },
+    {
+      "name": "bond7",
+      "mode": "IEEE 802.3ad Dynamic link aggregation",
+      "hash_policy": "layer3+4 (1)",
+      "status": "up",
+      "slaves": [
+        {
+          "name": "eth15",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:08",
+          "driver": "mlx5_core",
+          "bdf": "b0:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.3",
+            "ad:00.0",
+            "ae:01.0",
+            "b0:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth16",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:11",
+          "driver": "mlx5_core",
+          "bdf": "b0:00.1",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.3",
+            "ad:00.0",
+            "ae:01.0",
+            "b0:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "rdma": "mlx5_bond_7",
+      "gpu_index": 7,
+      "relation": "PIX"
+    }
+  ],
+  "ib_devices": [
+    {
+      "name": "mlx5_17",
+      "ca_type": "MT4129",
+      "firmware": "28.46.5500",
+      "state": "Active",
+      "physical_state": "LinkUp",
+      "rate_gbps": 100.0,
+      "lid": "1",
+      "link_layer": "InfiniBand",
+      "bdf": "b3:00.0",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.4",
+        "b1:00.0",
+        "b2:00.0",
+        "b3:00.0"
+      ],
+      "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]",
+      "rdma_state": {
+        "port": 1,
+        "state": "ACTIVE",
+        "physical_state": "LINK_UP",
+        "netdev": ""
+      }
+    },
+    {
+      "name": "mlx5_18",
+      "ca_type": "MT4129",
+      "firmware": "28.46.5500",
+      "state": "Active",
+      "physical_state": "LinkUp",
+      "rate_gbps": 100.0,
+      "lid": "4",
+      "link_layer": "InfiniBand",
+      "bdf": "b4:00.0",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.4",
+        "b1:00.0",
+        "b2:01.0",
+        "b4:00.0"
+      ],
+      "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]",
+      "rdma_state": {
+        "port": 1,
+        "state": "ACTIVE",
+        "physical_state": "LINK_UP",
+        "netdev": ""
+      }
+    },
+    {
+      "name": "mlx5_19",
+      "ca_type": "MT4129",
+      "firmware": "28.46.5500",
+      "state": "Active",
+      "physical_state": "LinkUp",
+      "rate_gbps": 100.0,
+      "lid": "5",
+      "link_layer": "InfiniBand",
+      "bdf": "b5:00.0",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.4",
+        "b1:00.0",
+        "b2:02.0",
+        "b5:00.0"
+      ],
+      "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]",
+      "rdma_state": {
+        "port": 1,
+        "state": "ACTIVE",
+        "physical_state": "LINK_UP",
+        "netdev": ""
+      }
+    },
+    {
+      "name": "mlx5_20",
+      "ca_type": "MT4129",
+      "firmware": "28.46.5500",
+      "state": "Active",
+      "physical_state": "LinkUp",
+      "rate_gbps": 100.0,
+      "lid": "6",
+      "link_layer": "InfiniBand",
+      "bdf": "b6:00.0",
+      "numa": 1,
+      "pci_path": [
+        "a0:00.4",
+        "b1:00.0",
+        "b2:03.0",
+        "b6:00.0"
+      ],
+      "description": "Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]",
+      "rdma_state": {
+        "port": 1,
+        "state": "ACTIVE",
+        "physical_state": "LINK_UP",
+        "netdev": ""
+      }
+    }
+  ],
+  "bmc": {
+    "detected": false,
+    "note": "独立 BMC 带外口未向当前 OS/IPMI 通道暴露"
+  },
+  "rdma_map": {
+    "mlx5_0": "eth0",
+    "mlx5_bond_0": "bond0",
+    "mlx5_bond_1": "bond1",
+    "mlx5_bond_2": "bond2",
+    "mlx5_bond_3": "bond3",
+    "mlx5_bond_4": "bond4",
+    "mlx5_bond_5": "bond5",
+    "mlx5_bond_6": "bond6",
+    "mlx5_bond_7": "bond7"
+  },
+  "notes": [
+    "Demo snapshot derived from a live B300 node; hostname, GPU UUIDs, MAC and IP addresses are sanitized."
+  ],
+  "network_endpoints": [
+    {
+      "id": "bond:bond0",
+      "type": "bond",
+      "protocol": "RoCE",
+      "name": "bond0",
+      "rdma": "mlx5_bond_0",
+      "role": "compute",
+      "state": "UP",
+      "numa": 0,
+      "capacity_mbps": 800000.0,
+      "members": [
+        {
+          "name": "eth1",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:01",
+          "driver": "mlx5_core",
+          "bdf": "64:00.0",
+          "numa": 0,
+          "pci_path": [
+            "60:00.0",
+            "61:00.0",
+            "62:01.0",
+            "64:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth2",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0a",
+          "driver": "mlx5_core",
+          "bdf": "64:00.1",
+          "numa": 0,
+          "pci_path": [
+            "60:00.0",
+            "61:00.0",
+            "62:01.0",
+            "64:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "gpu_links": [
+        {
+          "gpu_index": 0,
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "topology_visibility": "nvidia-smi-topo"
+    },
+    {
+      "id": "bond:bond1",
+      "type": "bond",
+      "protocol": "RoCE",
+      "name": "bond1",
+      "rdma": "mlx5_bond_1",
+      "role": "compute",
+      "state": "UP",
+      "numa": 0,
+      "capacity_mbps": 800000.0,
+      "members": [
+        {
+          "name": "eth3",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:02",
+          "driver": "mlx5_core",
+          "bdf": "68:00.0",
+          "numa": 0,
+          "pci_path": [
+            "60:00.1",
+            "65:00.0",
+            "66:01.0",
+            "68:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth4",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0b",
+          "driver": "mlx5_core",
+          "bdf": "68:00.1",
+          "numa": 0,
+          "pci_path": [
+            "60:00.1",
+            "65:00.0",
+            "66:01.0",
+            "68:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "gpu_links": [
+        {
+          "gpu_index": 1,
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "topology_visibility": "nvidia-smi-topo"
+    },
+    {
+      "id": "bond:bond2",
+      "type": "bond",
+      "protocol": "RoCE",
+      "name": "bond2",
+      "rdma": "mlx5_bond_2",
+      "role": "compute",
+      "state": "UP",
+      "numa": 0,
+      "capacity_mbps": 800000.0,
+      "members": [
+        {
+          "name": "eth5",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:03",
+          "driver": "mlx5_core",
+          "bdf": "6c:00.0",
+          "numa": 0,
+          "pci_path": [
+            "60:00.2",
+            "69:00.0",
+            "6a:01.0",
+            "6c:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth6",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0c",
+          "driver": "mlx5_core",
+          "bdf": "6c:00.1",
+          "numa": 0,
+          "pci_path": [
+            "60:00.2",
+            "69:00.0",
+            "6a:01.0",
+            "6c:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "gpu_links": [
+        {
+          "gpu_index": 2,
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "topology_visibility": "nvidia-smi-topo"
+    },
+    {
+      "id": "bond:bond3",
+      "type": "bond",
+      "protocol": "RoCE",
+      "name": "bond3",
+      "rdma": "mlx5_bond_3",
+      "role": "compute",
+      "state": "UP",
+      "numa": 0,
+      "capacity_mbps": 800000.0,
+      "members": [
+        {
+          "name": "eth7",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:04",
+          "driver": "mlx5_core",
+          "bdf": "70:00.0",
+          "numa": 0,
+          "pci_path": [
+            "60:00.3",
+            "6d:00.0",
+            "6e:01.0",
+            "70:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth8",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0d",
+          "driver": "mlx5_core",
+          "bdf": "70:00.1",
+          "numa": 0,
+          "pci_path": [
+            "60:00.3",
+            "6d:00.0",
+            "6e:01.0",
+            "70:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "gpu_links": [
+        {
+          "gpu_index": 3,
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "topology_visibility": "nvidia-smi-topo"
+    },
+    {
+      "id": "bond:bond4",
+      "type": "bond",
+      "protocol": "RoCE",
+      "name": "bond4",
+      "rdma": "mlx5_bond_4",
+      "role": "compute",
+      "state": "UP",
+      "numa": 1,
+      "capacity_mbps": 800000.0,
+      "members": [
+        {
+          "name": "eth9",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:05",
+          "driver": "mlx5_core",
+          "bdf": "a4:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.0",
+            "a1:00.0",
+            "a2:01.0",
+            "a4:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth10",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0e",
+          "driver": "mlx5_core",
+          "bdf": "a4:00.1",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.0",
+            "a1:00.0",
+            "a2:01.0",
+            "a4:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "gpu_links": [
+        {
+          "gpu_index": 4,
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "topology_visibility": "nvidia-smi-topo"
+    },
+    {
+      "id": "bond:bond5",
+      "type": "bond",
+      "protocol": "RoCE",
+      "name": "bond5",
+      "rdma": "mlx5_bond_5",
+      "role": "compute",
+      "state": "UP",
+      "numa": 1,
+      "capacity_mbps": 800000.0,
+      "members": [
+        {
+          "name": "eth11",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:06",
+          "driver": "mlx5_core",
+          "bdf": "a8:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.1",
+            "a5:00.0",
+            "a6:01.0",
+            "a8:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth12",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:0f",
+          "driver": "mlx5_core",
+          "bdf": "a8:00.1",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.1",
+            "a5:00.0",
+            "a6:01.0",
+            "a8:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "gpu_links": [
+        {
+          "gpu_index": 5,
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "topology_visibility": "nvidia-smi-topo"
+    },
+    {
+      "id": "bond:bond6",
+      "type": "bond",
+      "protocol": "RoCE",
+      "name": "bond6",
+      "rdma": "mlx5_bond_6",
+      "role": "compute",
+      "state": "UP",
+      "numa": 1,
+      "capacity_mbps": 800000.0,
+      "members": [
+        {
+          "name": "eth13",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:07",
+          "driver": "mlx5_core",
+          "bdf": "ac:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.2",
+            "a9:00.0",
+            "aa:01.0",
+            "ac:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth14",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:10",
+          "driver": "mlx5_core",
+          "bdf": "ac:00.1",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.2",
+            "a9:00.0",
+            "aa:01.0",
+            "ac:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "gpu_links": [
+        {
+          "gpu_index": 6,
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "topology_visibility": "nvidia-smi-topo"
+    },
+    {
+      "id": "bond:bond7",
+      "type": "bond",
+      "protocol": "RoCE",
+      "name": "bond7",
+      "rdma": "mlx5_bond_7",
+      "role": "compute",
+      "state": "UP",
+      "numa": 1,
+      "capacity_mbps": 800000.0,
+      "members": [
+        {
+          "name": "eth15",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:08",
+          "driver": "mlx5_core",
+          "bdf": "b0:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.3",
+            "ad:00.0",
+            "ae:01.0",
+            "b0:00.0"
+          ],
+          "carrier": 1
+        },
+        {
+          "name": "eth16",
+          "status": "up",
+          "speed_mbps": 400000,
+          "duplex": "full",
+          "failures": 0,
+          "mac": "02:00:00:00:00:11",
+          "driver": "mlx5_core",
+          "bdf": "b0:00.1",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.3",
+            "ad:00.0",
+            "ae:01.0",
+            "b0:00.1"
+          ],
+          "carrier": 1
+        }
+      ],
+      "gpu_links": [
+        {
+          "gpu_index": 7,
+          "relation": "PIX",
+          "role": "primary",
+          "evidence": "nvidia-smi topo -m"
+        }
+      ],
+      "topology_visibility": "nvidia-smi-topo"
+    },
+    {
+      "id": "ib:mlx5_17",
+      "type": "standalone",
+      "protocol": "InfiniBand",
+      "name": "mlx5_17",
+      "rdma": "mlx5_17",
+      "role": "storage",
+      "state": "ACTIVE",
+      "physical_state": "LinkUp",
+      "numa": 1,
+      "bdf": "b3:00.0",
+      "pci_path": [
+        "a0:00.4",
+        "b1:00.0",
+        "b2:00.0",
+        "b3:00.0"
+      ],
+      "capacity_mbps": 100000.0,
+      "members": [
+        {
+          "name": "mlx5_17",
+          "rdma": "mlx5_17",
+          "bdf": "b3:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.4",
+            "b1:00.0",
+            "b2:00.0",
+            "b3:00.0"
+          ],
+          "speed_mbps": 100000.0,
+          "status": "active"
+        }
+      ],
+      "gpu_links": [],
+      "topology_visibility": "not-exposed"
+    },
+    {
+      "id": "ib:mlx5_18",
+      "type": "standalone",
+      "protocol": "InfiniBand",
+      "name": "mlx5_18",
+      "rdma": "mlx5_18",
+      "role": "storage",
+      "state": "ACTIVE",
+      "physical_state": "LinkUp",
+      "numa": 1,
+      "bdf": "b4:00.0",
+      "pci_path": [
+        "a0:00.4",
+        "b1:00.0",
+        "b2:01.0",
+        "b4:00.0"
+      ],
+      "capacity_mbps": 100000.0,
+      "members": [
+        {
+          "name": "mlx5_18",
+          "rdma": "mlx5_18",
+          "bdf": "b4:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.4",
+            "b1:00.0",
+            "b2:01.0",
+            "b4:00.0"
+          ],
+          "speed_mbps": 100000.0,
+          "status": "active"
+        }
+      ],
+      "gpu_links": [],
+      "topology_visibility": "not-exposed"
+    },
+    {
+      "id": "ib:mlx5_19",
+      "type": "standalone",
+      "protocol": "InfiniBand",
+      "name": "mlx5_19",
+      "rdma": "mlx5_19",
+      "role": "storage",
+      "state": "ACTIVE",
+      "physical_state": "LinkUp",
+      "numa": 1,
+      "bdf": "b5:00.0",
+      "pci_path": [
+        "a0:00.4",
+        "b1:00.0",
+        "b2:02.0",
+        "b5:00.0"
+      ],
+      "capacity_mbps": 100000.0,
+      "members": [
+        {
+          "name": "mlx5_19",
+          "rdma": "mlx5_19",
+          "bdf": "b5:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.4",
+            "b1:00.0",
+            "b2:02.0",
+            "b5:00.0"
+          ],
+          "speed_mbps": 100000.0,
+          "status": "active"
+        }
+      ],
+      "gpu_links": [],
+      "topology_visibility": "not-exposed"
+    },
+    {
+      "id": "ib:mlx5_20",
+      "type": "standalone",
+      "protocol": "InfiniBand",
+      "name": "mlx5_20",
+      "rdma": "mlx5_20",
+      "role": "storage",
+      "state": "ACTIVE",
+      "physical_state": "LinkUp",
+      "numa": 1,
+      "bdf": "b6:00.0",
+      "pci_path": [
+        "a0:00.4",
+        "b1:00.0",
+        "b2:03.0",
+        "b6:00.0"
+      ],
+      "capacity_mbps": 100000.0,
+      "members": [
+        {
+          "name": "mlx5_20",
+          "rdma": "mlx5_20",
+          "bdf": "b6:00.0",
+          "numa": 1,
+          "pci_path": [
+            "a0:00.4",
+            "b1:00.0",
+            "b2:03.0",
+            "b6:00.0"
+          ],
+          "speed_mbps": 100000.0,
+          "status": "active"
+        }
+      ],
+      "gpu_links": [],
+      "topology_visibility": "not-exposed"
+    }
+  ],
+  "normalized_by_version": "2.1.0"
+}
+
+ + + + + + + diff --git a/scripts/gpu_node_profile/examples/b300_topology_3d_demo.html b/scripts/gpu_node_profile/examples/b300_topology_3d_demo.html new file mode 100644 index 0000000..72d1eb1 --- /dev/null +++ b/scripts/gpu_node_profile/examples/b300_topology_3d_demo.html @@ -0,0 +1,980 @@ + + + + + + +b300-demo · 3D Topology CT + + + +
+
+
+ MACHINE CT · GUEST-VISIBLE TOPOLOGY +

GPU Machine 3D CT

+

+
+
+
+
+
+
+
视图
+
+ 镜头 + + + + +
+
+ 图层 + + + + + + +
+
+ +
整机外观 · 双击壳体逐层进入
+
双击壳体逐层进入 · 上一层返回 · 拖动旋转 · Shift+拖动平移 · Esc 返回整机
+
+ NVLink(仅连 GPU) + GPUDirect 候选 + PCIe + CPU 亲和 + 跨 NUMA / SYS +
+
+ +
+ +
+ + + + diff --git a/scripts/gpu_node_profile/gpu_node_profile.py b/scripts/gpu_node_profile/gpu_node_profile.py new file mode 100755 index 0000000..5906931 --- /dev/null +++ b/scripts/gpu_node_profile/gpu_node_profile.py @@ -0,0 +1,2155 @@ +#!/usr/bin/env python3 +"""Generate a self-contained, native HTML profile for a GPU node. + +The collector only executes read-only commands. The generated report has no +Mermaid, JavaScript framework, external font, or CDN dependency. +""" + +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import html +import json +import os +import platform +import re +import shutil +import socket +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any, Iterable + + +VERSION = "2.1.0" +ANSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") +PCI_RE = re.compile(r"(?:00000000:|0000:)?([0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7])") + + +def run(command: list[str], timeout: int = 12) -> str: + """Run a read-only command and return stdout without raising.""" + if not command or shutil.which(command[0]) is None: + return "" + try: + result = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + timeout=timeout, + env={**os.environ, "LC_ALL": "C"}, + ) + except (OSError, subprocess.SubprocessError): + return "" + return ANSI_RE.sub("", result.stdout).strip() + + +def read_text(path: str | Path) -> str: + try: + return Path(path).read_text(encoding="utf-8", errors="replace").strip() + except OSError: + return "" + + +def read_int(path: str | Path) -> int | None: + value = read_text(path) + try: + return int(value) + except (TypeError, ValueError): + return None + + +def normalize_bdf(value: str) -> str: + matches = PCI_RE.findall(value or "") + return matches[-1].lower() if matches else "" + + +def pci_numa(bdf: str) -> int | None: + bdf = normalize_bdf(bdf) + return read_int(f"/sys/bus/pci/devices/0000:{bdf}/numa_node") if bdf else None + + +def pci_path(bdf: str) -> list[str]: + """Return the guest-visible PCI ancestor path ending at the endpoint.""" + bdf = normalize_bdf(bdf) + if not bdf: + return [] + try: + resolved = str(Path(f"/sys/bus/pci/devices/0000:{bdf}").resolve()) + except OSError: + return [bdf] + path: list[str] = [] + for match in PCI_RE.findall(resolved): + normalized = normalize_bdf(match) + if normalized and normalized not in path: + path.append(normalized) + if bdf not in path: + path.append(bdf) + return path + + +def dmi_value(name: str) -> str: + return read_text(f"/sys/class/dmi/id/{name}") + + +def parse_csv_rows(text: str) -> list[list[str]]: + if not text: + return [] + return [[field.strip() for field in row] for row in csv.reader(text.splitlines())] + + +def collect_gpus() -> list[dict[str, Any]]: + core_fields = [ + "index", + "name", + "uuid", + "pci.bus_id", + "driver_version", + "memory.total", + "memory.used", + "temperature.gpu", + "utilization.gpu", + ] + output = run( + [ + "nvidia-smi", + f"--query-gpu={','.join(core_fields)}", + "--format=csv,noheader,nounits", + ] + ) + power_fields = [ + "index", + "pstate", + "power.draw", + "power.limit", + ] + power_output = run( + [ + "nvidia-smi", + f"--query-gpu={','.join(power_fields)}", + "--format=csv,noheader,nounits", + ] + ) + power_by_index: dict[int, dict[str, Any]] = {} + for row in parse_csv_rows(power_output): + if len(row) < len(power_fields) or not row[0].isdigit(): + continue + power_by_index[int(row[0])] = { + "pstate": row[1] if row[1] not in {"", "N/A", "[N/A]"} else "", + "power_w": number(row[2]), + "power_limit_w": number(row[3]), + } + range_fields = [ + "index", + "power.default_limit", + "power.min_limit", + "power.max_limit", + ] + range_output = run( + [ + "nvidia-smi", + f"--query-gpu={','.join(range_fields)}", + "--format=csv,noheader,nounits", + ] + ) + for row in parse_csv_rows(range_output): + if len(row) < len(range_fields) or not row[0].isdigit(): + continue + power_by_index.setdefault(int(row[0]), {}).update( + { + "power_default_limit_w": number(row[1]), + "power_min_limit_w": number(row[2]), + "power_max_limit_w": number(row[3]), + } + ) + gpus: list[dict[str, Any]] = [] + for row in parse_csv_rows(output): + if len(row) < len(core_fields): + continue + bdf = normalize_bdf(row[3]) + index = int(row[0]) if row[0].isdigit() else len(gpus) + gpus.append( + { + "index": index, + "name": row[1], + "uuid": row[2], + "bdf": bdf, + "driver": row[4], + "memory_total_mib": number(row[5]), + "memory_used_mib": number(row[6]), + "temperature_c": number(row[7]), + "utilization_pct": number(row[8]), + "numa": pci_numa(bdf), + "pci_path": pci_path(bdf), + **power_by_index.get(index, {}), + } + ) + return sorted(gpus, key=lambda item: item["index"]) + + +def number(value: Any) -> float | None: + try: + return float(str(value).strip()) + except (TypeError, ValueError): + return None + + +def parse_topology(text: str) -> dict[str, Any]: + """Parse nvidia-smi topo -m into GPU relationships and NIC legend.""" + text = ANSI_RE.sub("", text or "") + lines = [line.rstrip() for line in text.splitlines()] + header: list[str] = [] + rows: dict[str, dict[str, str]] = {} + nic_legend: dict[str, str] = {} + gpu_affinity: dict[str, dict[str, str]] = {} + + for line in lines: + stripped = line.strip() + if not stripped: + continue + legend_match = re.match(r"(NIC\d+):\s+(\S+)", stripped) + if legend_match: + nic_legend[legend_match.group(1)] = legend_match.group(2) + continue + fields = re.split(r"\s+", stripped) + if not header and fields and fields[0] == "GPU0": + # Only retain the square GPU/NIC matrix. Labels such as + # "CPU Affinity" contain spaces while row values do not, so keeping + # them in the same whitespace-split header would shift columns. + matrix_end = fields.index("CPU") if "CPU" in fields else len(fields) + header = fields[:matrix_end] + continue + if header and fields and re.fullmatch(r"GPU\d+", fields[0]): + matrix_end = 1 + len(header) + values = fields[1:matrix_end] + gpu_name = fields[0] + rows[gpu_name] = dict(zip(header, values)) + suffix = fields[matrix_end:] + gpu_affinity[gpu_name] = { + "cpu_affinity": suffix[0] if len(suffix) > 0 else "", + "numa_affinity": suffix[1] if len(suffix) > 1 else "", + "gpu_numa_id": suffix[2] if len(suffix) > 2 else "", + } + + gpu_nic: dict[str, list[dict[str, str]]] = {} + for gpu_name, relations in rows.items(): + links: list[dict[str, str]] = [] + for column, relation in relations.items(): + if column.startswith("NIC"): + links.append( + { + "slot": column, + "rdma": nic_legend.get(column, column), + "relation": relation, + } + ) + gpu_nic[gpu_name] = links + return { + "rows": rows, + "nic_legend": nic_legend, + "gpu_nic": gpu_nic, + "gpu_affinity": gpu_affinity, + } + + +def parse_nvlink_status(text: str) -> dict[str, Any]: + current = "" + links: dict[str, list[float]] = {} + for raw_line in (text or "").splitlines(): + line = raw_line.strip() + gpu_match = re.match(r"GPU\s+(\d+):", line) + if gpu_match: + current = f"GPU{gpu_match.group(1)}" + links[current] = [] + continue + link_match = re.match(r"Link\s+\d+:\s+([\d.]+)\s+GB/s", line) + if current and link_match: + links[current].append(float(link_match.group(1))) + counts = [len(values) for values in links.values()] + speeds = [speed for values in links.values() for speed in values] + return { + "per_gpu": links, + "active_links": sum(counts), + "links_per_gpu": min(counts) if counts else 0, + "speed_gbs": min(speeds) if speeds else None, + } + + +def infer_nvlink_fabric_type(topology: dict[str, Any], gpu_count: int) -> str: + """Avoid calling every NVLink configuration an NVSwitch fabric.""" + if gpu_count < 2: + return "" + nv_pairs = 0 + for row_name, relations in topology.get("rows", {}).items(): + if not str(row_name).startswith("GPU"): + continue + nv_pairs += sum( + str(peer).startswith("GPU") and str(relation).startswith("NV") + for peer, relation in relations.items() + ) + if gpu_count > 2 and nv_pairs >= gpu_count * (gpu_count - 1): + return "NVSwitch Fabric" + return "NVLink Fabric" + + +def parse_bond_text(name: str, text: str) -> dict[str, Any]: + bond: dict[str, Any] = { + "name": name, + "mode": "", + "hash_policy": "", + "status": "", + "slaves": [], + } + current: dict[str, Any] | None = None + for raw_line in (text or "").splitlines(): + line = raw_line.strip() + if line.startswith("Bonding Mode:"): + bond["mode"] = line.split(":", 1)[1].strip() + elif line.startswith("Transmit Hash Policy:"): + bond["hash_policy"] = line.split(":", 1)[1].strip() + elif line.startswith("MII Status:") and current is None: + bond["status"] = line.split(":", 1)[1].strip().lower() + elif line.startswith("Slave Interface:"): + current = {"name": line.split(":", 1)[1].strip()} + bond["slaves"].append(current) + elif current is not None and ":" in line: + key, value = [part.strip() for part in line.split(":", 1)] + if key == "MII Status": + current["status"] = value.lower() + elif key == "Speed": + current["speed_mbps"] = int(value.split()[0]) if value.split()[0].isdigit() else None + elif key == "Duplex": + current["duplex"] = value + elif key == "Link Failure Count": + current["failures"] = int(value) if value.isdigit() else None + elif key == "Permanent HW addr": + current["mac"] = value + return bond + + +def parse_ibdev2netdev(text: str) -> tuple[dict[str, str], dict[str, str]]: + rdma_to_net: dict[str, str] = {} + net_to_rdma: dict[str, str] = {} + pattern = re.compile(r"(\S+)\s+port\s+\d+\s+==>\s+(\S+)\s+\(([^)]+)\)") + for line in (text or "").splitlines(): + match = pattern.search(line) + if match: + rdma, netdev = match.group(1), match.group(2) + rdma_to_net[rdma] = netdev + net_to_rdma[netdev] = rdma + return rdma_to_net, net_to_rdma + + +def parse_rdma_links(text: str) -> dict[str, dict[str, Any]]: + links: dict[str, dict[str, Any]] = {} + for line in (text or "").splitlines(): + match = re.match(r"link\s+([^/]+)/(\d+)\s+(.*)", line.strip()) + if not match: + continue + name, port, tail = match.groups() + links[name] = { + "port": int(port), + "state": capture_token(tail, "state"), + "physical_state": capture_token(tail, "physical_state"), + "netdev": capture_token(tail, "netdev"), + } + return links + + +def capture_token(text: str, key: str) -> str: + match = re.search(rf"(?:^|\s){re.escape(key)}\s+(\S+)", text) + return match.group(1) if match else "" + + +def parse_ibstat(text: str) -> list[dict[str, Any]]: + devices: list[dict[str, Any]] = [] + current: dict[str, Any] | None = None + in_port = False + for raw_line in (text or "").splitlines(): + line = raw_line.strip() + ca_match = re.match(r"CA '([^']+)'", line) + if ca_match: + current = {"name": ca_match.group(1)} + devices.append(current) + in_port = False + continue + if current is None: + continue + if line.startswith("CA type:"): + current["ca_type"] = line.split(":", 1)[1].strip() + elif line.startswith("Firmware version:"): + current["firmware"] = line.split(":", 1)[1].strip() + elif line.startswith("Port 1:"): + in_port = True + elif in_port and line.startswith("State:"): + current["state"] = line.split(":", 1)[1].strip() + elif in_port and line.startswith("Physical state:"): + current["physical_state"] = line.split(":", 1)[1].strip() + elif in_port and line.startswith("Rate:"): + current["rate_gbps"] = number(line.split(":", 1)[1].strip()) + elif in_port and line.startswith("Base lid:"): + current["lid"] = line.split(":", 1)[1].strip() + elif in_port and line.startswith("Link layer:"): + current["link_layer"] = line.split(":", 1)[1].strip() + return devices + + +def collect_nics() -> list[dict[str, Any]]: + base = Path("/sys/class/net") + if not base.exists(): + return [] + nics: list[dict[str, Any]] = [] + for entry in sorted(base.iterdir(), key=lambda item: natural_key(item.name)): + name = entry.name + if name == "lo": + continue + is_bond = (entry / "bonding").exists() + driver_info = run(["ethtool", "-i", name]) + driver = field_from_lines(driver_info, "driver") or ("bonding" if is_bond else "") + bus = normalize_bdf(field_from_lines(driver_info, "bus-info")) + if not driver and not bus: + continue + speed_text = field_from_lines(run(["ethtool", name]), "Speed") + speed_match = re.search(r"(\d+)", speed_text) + nics.append( + { + "name": name, + "driver": driver, + "bdf": bus, + "numa": pci_numa(bus), + "pci_path": pci_path(bus), + "speed_mbps": int(speed_match.group(1)) if speed_match else None, + "state": read_text(entry / "operstate").lower(), + "carrier": read_int(entry / "carrier"), + "mac": read_text(entry / "address"), + "is_bond": is_bond, + } + ) + attach_network_context(nics) + return nics + + +def attach_network_context(nics: list[dict[str, Any]]) -> None: + """Attach addresses and default-route roles to OS-visible interfaces.""" + addresses: dict[str, list[str]] = {} + defaults: set[str] = set() + try: + address_rows = json.loads(run(["ip", "-j", "address", "show"]) or "[]") + for row in address_rows: + values = [] + for item in row.get("addr_info", []): + local = item.get("local") + prefix = item.get("prefixlen") + if local and item.get("scope") != "link": + values.append(f"{local}/{prefix}" if prefix is not None else str(local)) + addresses[str(row.get("ifname", ""))] = values + route_rows = json.loads(run(["ip", "-j", "route", "show", "default"]) or "[]") + defaults = {str(row.get("dev")) for row in route_rows if row.get("dev")} + except (TypeError, ValueError, json.JSONDecodeError): + pass + for nic in nics: + name = nic["name"] + nic["addresses"] = addresses.get(name, []) + nic["default_route"] = name in defaults + + +def collect_bmc() -> dict[str, Any]: + """Read common BMC LAN channels when IPMI is locally available.""" + if shutil.which("ipmitool") is None: + return { + "detected": False, + "note": "独立 BMC 带外口未向当前 OS/IPMI 通道暴露", + } + last_values: dict[str, Any] = {} + for channel in (1, 8): + output = run(["ipmitool", "lan", "print", str(channel)], timeout=8) + values: dict[str, Any] = {"channel": channel} + wanted = { + "IP Address Source": "source", + "IP Address": "ip", + "Subnet Mask": "subnet", + "MAC Address": "mac", + "Default Gateway IP": "gateway", + "802.1q VLAN ID": "vlan", + } + for line in output.splitlines(): + if ":" not in line: + continue + key, value = [part.strip() for part in line.split(":", 1)] + if key in wanted: + values[wanted[key]] = value + last_values = values + if values.get("ip") and values["ip"] not in {"0.0.0.0", "0.0.0.0/0"}: + return {"detected": True, **values, "note": ""} + return { + "detected": False, + **last_values, + "note": "IPMI 可访问,但常用 LAN 通道未读取到有效地址", + } + + +def field_from_lines(text: str, field: str) -> str: + prefix = f"{field}:" + for line in (text or "").splitlines(): + if line.strip().startswith(prefix): + return line.split(":", 1)[1].strip() + return "" + + +def natural_key(value: str) -> list[Any]: + return [int(part) if part.isdigit() else part for part in re.split(r"(\d+)", value)] + + +def collect_bonds(net_to_rdma: dict[str, str], nics: list[dict[str, Any]]) -> list[dict[str, Any]]: + nic_map = {nic["name"]: nic for nic in nics} + bond_dir = Path("/proc/net/bonding") + if not bond_dir.exists(): + return [] + bonds: list[dict[str, Any]] = [] + for path in sorted(bond_dir.iterdir(), key=lambda item: natural_key(item.name)): + bond = parse_bond_text(path.name, read_text(path)) + bond["rdma"] = net_to_rdma.get(path.name, "") + for slave in bond["slaves"]: + slave.update( + { + key: value + for key, value in nic_map.get(slave["name"], {}).items() + if key in {"bdf", "numa", "driver", "carrier", "pci_path"} + } + ) + bonds.append(bond) + return bonds + + +def build_network_endpoints( + nics: list[dict[str, Any]], + bonds: list[dict[str, Any]], + ib_devices: list[dict[str, Any]], + rdma_to_net: dict[str, str], + rdma_links: dict[str, dict[str, Any]] | None = None, +) -> list[dict[str, Any]]: + """Normalize bonded RoCE, standalone RoCE and native IB as RDMA endpoints.""" + rdma_links = rdma_links or {} + nic_by_name = {str(nic.get("name")): nic for nic in nics} + consumed_rdma: set[str] = set() + consumed_netdevs: set[str] = set() + endpoints: list[dict[str, Any]] = [] + + for bond in bonds: + rdma = str(bond.get("rdma") or "") + # A host may also use ordinary management/storage bonds. Only a bond + # exposed by the RDMA stack belongs in the accelerator topology. + if not rdma: + continue + members = [dict(member) for member in bond.get("slaves", [])] + consumed_rdma.add(rdma) + consumed_netdevs.update(str(member.get("name")) for member in members) + endpoint = { + "id": f"bond:{bond.get('name')}", + "type": "bond", + "protocol": "RoCE", + "name": bond.get("name"), + "rdma": rdma, + "role": "compute", + "state": str(bond.get("status") or "").upper(), + "numa": next( + ( + member.get("numa") + for member in members + if member.get("numa") is not None + ), + None, + ), + "capacity_mbps": sum( + number(member.get("speed_mbps")) or 0 for member in members + ), + "members": members, + "gpu_links": [], + } + endpoints.append(endpoint) + + for device in ib_devices: + rdma = str(device.get("name") or "") + consumed_rdma.add(rdma) + endpoints.append( + { + "id": f"ib:{rdma}", + "type": "standalone", + "protocol": "InfiniBand", + "name": rdma, + "rdma": rdma, + "role": "storage", + "state": str(device.get("state") or "").upper(), + "physical_state": str(device.get("physical_state") or ""), + "numa": device.get("numa"), + "bdf": device.get("bdf") or "", + "pci_path": device.get("pci_path") or [], + "capacity_mbps": ( + (number(device.get("rate_gbps")) or 0) * 1000 + ), + "members": [ + { + "name": rdma, + "rdma": rdma, + "bdf": device.get("bdf") or "", + "numa": device.get("numa"), + "pci_path": device.get("pci_path") or [], + "speed_mbps": ( + (number(device.get("rate_gbps")) or 0) * 1000 + ), + "status": str(device.get("state") or "").lower(), + } + ], + "gpu_links": [], + } + ) + + for rdma, netdev in sorted( + rdma_to_net.items(), key=lambda item: natural_key(item[0]) + ): + if rdma in consumed_rdma or netdev in consumed_netdevs: + continue + nic = nic_by_name.get(netdev) + if ( + not nic + or not nic.get("bdf") + or nic.get("default_route") + or nic.get("is_bond") + ): + continue + rdma_state = rdma_links.get(rdma, {}) + state = str(rdma_state.get("state") or nic.get("state") or "").upper() + member = { + **nic, + "rdma": rdma, + "status": str(nic.get("state") or "").lower(), + } + endpoints.append( + { + "id": f"rdma:{rdma}", + "type": "standalone", + "protocol": "RoCE", + "name": netdev, + "rdma": rdma, + "role": "compute", + "state": state, + "physical_state": str( + rdma_state.get("physical_state") or "" + ), + "numa": nic.get("numa"), + "bdf": nic.get("bdf") or "", + "pci_path": nic.get("pci_path") or [], + "capacity_mbps": number(nic.get("speed_mbps")) or 0, + "members": [member], + "gpu_links": [], + } + ) + + return sorted( + endpoints, + key=lambda endpoint: ( + 0 if endpoint.get("role") == "compute" else 1, + str(endpoint.get("numa")), + natural_key(str(endpoint.get("name") or "")), + ), + ) + + +def parse_lspci_devices(text: str) -> dict[str, dict[str, str]]: + devices: dict[str, dict[str, str]] = {} + pattern = re.compile(r"^[0-9a-fA-F]{4}:([0-9a-fA-F:.]+)\s+(.+)$") + for line in (text or "").splitlines(): + match = pattern.match(line.strip()) + if not match: + continue + bdf, description = match.groups() + devices[normalize_bdf(bdf)] = { + "bdf": normalize_bdf(bdf), + "description": description, + } + return devices + + +def parse_lspci_network(text: str) -> dict[str, dict[str, str]]: + return { + bdf: device + for bdf, device in parse_lspci_devices(text).items() + if re.search( + r"Ethernet|Infiniband|Network controller", + device.get("description", ""), + re.I, + ) + } + + +def build_pci_fabric( + gpus: list[dict[str, Any]], + nics: list[dict[str, Any]], + ib_devices: list[dict[str, Any]], + pci_devices: dict[str, dict[str, str]], +) -> dict[str, Any]: + """Build a device-neutral PCI graph for both 2D and 3D renderers.""" + endpoints: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + + def add_endpoint( + kind: str, + label: str, + bdf: str, + numa: Any, + path: list[str] | None, + ) -> None: + bdf = normalize_bdf(bdf) + key = (kind, bdf) + if not bdf or key in seen: + return + seen.add(key) + endpoints.append( + { + "kind": kind, + "label": label, + "bdf": bdf, + "numa": numa, + "path": path or pci_path(bdf), + "description": pci_devices.get(bdf, {}).get("description", ""), + } + ) + + for gpu in gpus: + add_endpoint( + "gpu", + f"GPU{gpu.get('index')}", + str(gpu.get("bdf") or ""), + gpu.get("numa"), + gpu.get("pci_path"), + ) + for nic in nics: + add_endpoint( + "nic", + str(nic.get("name") or "NIC"), + str(nic.get("bdf") or ""), + nic.get("numa"), + nic.get("pci_path"), + ) + for device in ib_devices: + add_endpoint( + "infiniband", + str(device.get("name") or "IB"), + str(device.get("bdf") or ""), + device.get("numa"), + device.get("pci_path"), + ) + + bridges: dict[str, dict[str, Any]] = {} + for endpoint in endpoints: + path = endpoint.get("path") or [] + for index, bdf in enumerate(path[:-1]): + bridge = bridges.setdefault( + bdf, + { + "bdf": bdf, + "parent_bdf": path[index - 1] if index else "", + "numa": pci_numa(bdf), + "description": pci_devices.get(bdf, {}).get( + "description", "PCI bridge" + ), + "children": [], + }, + ) + child = path[index + 1] + if child not in bridge["children"]: + bridge["children"].append(child) + + return { + "scope": "guest-visible", + "virtualization": "passthrough-or-virtual-pci", + "bridges": sorted(bridges.values(), key=lambda item: natural_key(item["bdf"])), + "endpoints": sorted( + endpoints, + key=lambda item: (str(item.get("numa")), item["kind"], natural_key(item["label"])), + ), + } + + +def collect_cuda_toolkit() -> str: + for executable in ("nvcc", "/usr/local/cuda/bin/nvcc"): + output = run([executable, "--version"]) + match = re.search(r"release\s+([\d.]+)", output) + if match: + return concise_version(match.group(1)) + for path in ("/usr/local/cuda/version.json", "/usr/local/cuda/version.txt"): + text = read_text(path) + match = re.search(r"(?:CUDA Version|\"version\"\s*:)\s*\"?([\d.]+)", text) + if match: + return concise_version(match.group(1)) + return "" + + +def concise_version(value: str) -> str: + match = re.match(r"(\d+\.\d+)", value or "") + return match.group(1) if match else value + + +def collect_snapshot() -> dict[str, Any]: + topo_text = run(["nvidia-smi", "topo", "-m"]) + topology = parse_topology(topo_text) + rdma_to_net, net_to_rdma = parse_ibdev2netdev(run(["ibdev2netdev"])) + rdma_links = parse_rdma_links(run(["rdma", "link", "show"])) + nics = collect_nics() + bonds = collect_bonds(net_to_rdma, nics) + lspci_text = run(["lspci", "-Dnn"]) + pci_devices = parse_lspci_devices(lspci_text) + pci_network = parse_lspci_network(lspci_text) + + ib_devices = [] + bonded_rdma = {bond.get("rdma") for bond in bonds} + for device in parse_ibstat(run(["ibstat"])): + if ( + device["name"] in bonded_rdma + or str(device.get("link_layer", "")).lower() != "infiniband" + ): + continue + bdf = "" + device_path = Path(f"/sys/class/infiniband/{device['name']}/device") + try: + bdf = normalize_bdf(str(device_path.resolve())) + except OSError: + pass + device.update( + { + "bdf": bdf, + "numa": pci_numa(bdf), + "pci_path": pci_path(bdf), + "description": pci_network.get(bdf, {}).get("description", ""), + "rdma_state": rdma_links.get(device["name"], {}), + } + ) + ib_devices.append(device) + network_endpoints = build_network_endpoints( + nics, + bonds, + ib_devices, + rdma_to_net, + rdma_links, + ) + + system = { + "hostname": socket.gethostname(), + "os": os_release(), + "kernel": platform.release(), + "architecture": platform.machine(), + "manufacturer": dmi_value("sys_vendor"), + "product": dmi_value("product_name"), + "product_version": dmi_value("product_version"), + "virtualization": run(["systemd-detect-virt"]), + "cpu_model": cpu_model(), + "numa_nodes": numa_count(), + **collect_cpu_topology(), + **collect_memory(), + } + gpus = collect_gpus() + pci_fabric = build_pci_fabric(gpus, nics, ib_devices, pci_devices) + driver = next((gpu.get("driver") for gpu in gpus if gpu.get("driver")), "") + nvlink = parse_nvlink_status(run(["nvidia-smi", "nvlink", "--status"])) + nvlink["fabric_type"] = infer_nvlink_fabric_type(topology, len(gpus)) + snapshot = { + "schema_version": 1, + "generated_at": dt.datetime.now().astimezone().isoformat(timespec="seconds"), + "collector_version": VERSION, + "system": system, + "software": { + "nvidia_driver": driver, + "cuda_toolkit": collect_cuda_toolkit(), + "cuda_note": "Toolkit selected by nvcc; application/container runtime may differ.", + }, + "gpus": gpus, + "topology": topology, + "pci_fabric": pci_fabric, + "nvlink": nvlink, + "nics": nics, + "bonds": bonds, + "ib_devices": ib_devices, + "rdma_links": rdma_links, + "network_endpoints": network_endpoints, + "bmc": collect_bmc(), + "rdma_map": rdma_to_net, + "notes": [], + } + attach_gpu_network_mapping(snapshot) + return snapshot + + +def os_release() -> str: + for line in read_text("/etc/os-release").splitlines(): + if line.startswith("PRETTY_NAME="): + return line.split("=", 1)[1].strip().strip('"') + return platform.platform() + + +def cpu_model() -> str: + for line in read_text("/proc/cpuinfo").splitlines(): + if line.lower().startswith("model name"): + return line.split(":", 1)[1].strip() + return "" + + +def numa_count() -> int: + nodes = list(Path("/sys/devices/system/node").glob("node[0-9]*")) + return len(nodes) + + +def collect_cpu_topology() -> dict[str, Any]: + values: dict[str, str] = {} + output = run(["lscpu", "-J"]) + try: + for item in json.loads(output or "{}").get("lscpu", []): + key = str(item.get("field", "")).rstrip(":") + values[key] = str(item.get("data", "")).strip() + except (TypeError, ValueError, json.JSONDecodeError): + pass + + def integer(key: str) -> int | None: + match = re.search(r"\d+", values.get(key, "")) + return int(match.group(0)) if match else None + + sockets = integer("Socket(s)") + cores_per_socket = integer("Core(s) per socket") + return { + "cpu_logical": integer("CPU(s)") or os.cpu_count(), + "cpu_sockets": sockets, + "cpu_cores_per_socket": cores_per_socket, + "cpu_physical_cores": ( + sockets * cores_per_socket + if sockets is not None and cores_per_socket is not None + else None + ), + "cpu_threads_per_core": integer("Thread(s) per core"), + } + + +def collect_memory() -> dict[str, Any]: + values: dict[str, int] = {} + for line in read_text("/proc/meminfo").splitlines(): + if ":" not in line: + continue + key, tail = line.split(":", 1) + match = re.search(r"\d+", tail) + if match: + values[key] = int(match.group(0)) + total = values.get("MemTotal") + available = values.get("MemAvailable") + return { + "memory_total_kib": total, + "memory_available_kib": available, + "memory_used_kib": ( + total - available + if total is not None and available is not None + else None + ), + } + + +def common_path_depth(left: list[str], right: list[str]) -> int: + depth = 0 + for left_item, right_item in zip(left or [], right or []): + if left_item != right_item: + break + depth += 1 + return depth + + +def hungarian_assignment(costs: list[list[int]]) -> list[tuple[int, int]]: + """Return minimum-cost unique row/column pairs for a rectangular matrix.""" + if not costs: + return [] + row_count = len(costs) + column_count = len(costs[0]) + if row_count > column_count: + raise ValueError("Hungarian assignment requires rows <= columns") + potentials_rows = [0] * (row_count + 1) + potentials_columns = [0] * (column_count + 1) + column_match = [0] * (column_count + 1) + path = [0] * (column_count + 1) + for row in range(1, row_count + 1): + column_match[0] = row + minimums = [10**18] * (column_count + 1) + used = [False] * (column_count + 1) + column = 0 + while True: + used[column] = True + current_row = column_match[column] + delta = 10**18 + next_column = 0 + for candidate in range(1, column_count + 1): + if used[candidate]: + continue + reduced = ( + costs[current_row - 1][candidate - 1] + - potentials_rows[current_row] + - potentials_columns[candidate] + ) + if reduced < minimums[candidate]: + minimums[candidate] = reduced + path[candidate] = column + if minimums[candidate] < delta: + delta = minimums[candidate] + next_column = candidate + for candidate in range(column_count + 1): + if used[candidate]: + potentials_rows[column_match[candidate]] += delta + potentials_columns[candidate] -= delta + else: + minimums[candidate] -= delta + column = next_column + if column_match[column] == 0: + break + while True: + previous = path[column] + column_match[column] = column_match[previous] + column = previous + if column == 0: + break + return sorted( + (row - 1, column - 1) + for column, row in enumerate(column_match[1:], start=1) + if row + ) + + +def attach_gpu_network_mapping(snapshot: dict[str, Any]) -> None: + """Map GPUs to bonded or standalone RDMA endpoints using topology distance.""" + endpoints = snapshot.get("network_endpoints") + if not isinstance(endpoints, list) or ( + not endpoints + and ( + snapshot.get("bonds") + or snapshot.get("ib_devices") + or snapshot.get("rdma_map") + ) + ): + endpoints = build_network_endpoints( + snapshot.get("nics", []), + snapshot.get("bonds", []), + snapshot.get("ib_devices", []), + snapshot.get("rdma_map", {}), + snapshot.get("rdma_links", {}), + ) + snapshot["network_endpoints"] = endpoints + snapshot["normalized_by_version"] = VERSION + + endpoint_by_id = { + str(endpoint.get("id")): endpoint + for endpoint in endpoints + if endpoint.get("id") + } + bond_by_name = { + str(bond.get("name")): bond for bond in snapshot.get("bonds", []) + } + for endpoint in endpoints: + endpoint["gpu_links"] = [] + endpoint["topology_visibility"] = "not-exposed" + for bond in snapshot.get("bonds", []): + bond.pop("gpu_index", None) + bond.pop("relation", None) + + topology = snapshot.get("topology", {}).get("gpu_nic", {}) + affinity = snapshot.get("topology", {}).get("gpu_affinity", {}) + gpus = snapshot.get("gpus", []) + relations: dict[tuple[int, str], str] = {} + + for gpu in gpus: + for key in ( + "network_relation", + "rdma", + "bond", + "network", + "primary_network_id", + ): + gpu.pop(key, None) + gpu["network_links"] = [] + gpu_key = f"GPU{gpu['index']}" + gpu.update( + { + key: value + for key, value in affinity.get(gpu_key, {}).items() + if value + } + ) + by_rdma = { + str(item.get("rdma")): str(item.get("relation") or "") + for item in topology.get(gpu_key, []) + } + for endpoint in endpoints: + relation = by_rdma.get(str(endpoint.get("rdma") or ""), "") + if relation: + relations[(int(gpu["index"]), str(endpoint["id"]))] = relation + endpoint["topology_visibility"] = "nvidia-smi-topo" + + primary_pool = [ + endpoint for endpoint in endpoints if endpoint.get("role") == "compute" + ] + if not primary_pool: + primary_pool = list(endpoints) + + sorted_gpus = sorted(gpus, key=lambda item: int(item["index"])) + sorted_endpoints = sorted( + primary_pool, key=lambda item: natural_key(str(item.get("name") or "")) + ) + + def pair_cost(gpu: dict[str, Any], endpoint: dict[str, Any]) -> int: + gpu_index = int(gpu["index"]) + endpoint_id = str(endpoint["id"]) + rank = relation_rank(relations.get((gpu_index, endpoint_id), "")) + if rank >= 9: + return 10**9 + member_paths = [ + member.get("pci_path") or [] + for member in endpoint.get("members", []) + ] or [endpoint.get("pci_path") or []] + proximity = max( + ( + common_path_depth(gpu.get("pci_path") or [], path) + for path in member_paths + ), + default=0, + ) + # Topology relation is authoritative. PCI path only breaks equal-distance + # ties, followed by stable natural ordering of GPUs and endpoints. + return rank * 1_000_000 + (100 - min(100, proximity)) * 1_000 + + primary_assignments: dict[int, tuple[dict[str, Any], str]] = {} + if sorted_gpus and sorted_endpoints: + if len(sorted_gpus) <= len(sorted_endpoints): + costs = [ + [ + pair_cost(gpu, endpoint) + endpoint_index + for endpoint_index, endpoint in enumerate(sorted_endpoints) + ] + for gpu in sorted_gpus + ] + unique_pairs = [ + (sorted_gpus[gpu_index], sorted_endpoints[endpoint_index]) + for gpu_index, endpoint_index in hungarian_assignment(costs) + if costs[gpu_index][endpoint_index] < 10**9 + ] + else: + costs = [ + [ + pair_cost(gpu, endpoint) + gpu_index + for gpu_index, gpu in enumerate(sorted_gpus) + ] + for endpoint in sorted_endpoints + ] + unique_pairs = [ + (sorted_gpus[gpu_index], sorted_endpoints[endpoint_index]) + for endpoint_index, gpu_index in hungarian_assignment(costs) + if costs[endpoint_index][gpu_index] < 10**9 + ] + for gpu, endpoint in unique_pairs: + gpu_index = int(gpu["index"]) + endpoint_id = str(endpoint["id"]) + primary_assignments[gpu_index] = ( + endpoint, + relations[(gpu_index, endpoint_id)], + ) + + for gpu in gpus: + gpu_index = int(gpu["index"]) + if gpu_index in primary_assignments: + continue + candidates = [ + ( + relation_rank(relations.get((gpu_index, str(endpoint["id"])), "")), + str(endpoint.get("name") or ""), + endpoint, + ) + for endpoint in primary_pool + if relation_rank( + relations.get((gpu_index, str(endpoint["id"])), "") + ) + < 9 + ] + if candidates: + _, _, endpoint = min(candidates) + primary_assignments[gpu_index] = ( + endpoint, + relations[(gpu_index, str(endpoint["id"]))], + ) + + def attach_link( + gpu: dict[str, Any], + endpoint: dict[str, Any], + relation: str, + role: str, + ) -> None: + link = { + "endpoint_id": endpoint["id"], + "name": endpoint.get("name"), + "rdma": endpoint.get("rdma"), + "protocol": endpoint.get("protocol"), + "type": endpoint.get("type"), + "relation": relation, + "role": role, + "evidence": "nvidia-smi topo -m", + } + gpu["network_links"].append(link) + endpoint["gpu_links"].append( + { + "gpu_index": gpu["index"], + "relation": relation, + "role": role, + "evidence": "nvidia-smi topo -m", + } + ) + + for gpu in gpus: + gpu_index = int(gpu["index"]) + primary = primary_assignments.get(gpu_index) + if primary: + endpoint, relation = primary + gpu["primary_network_id"] = endpoint["id"] + gpu["network_relation"] = relation + gpu["rdma"] = endpoint.get("rdma") + gpu["network"] = endpoint.get("name") + if endpoint.get("type") == "bond": + gpu["bond"] = endpoint.get("name") + bond = bond_by_name.get(str(endpoint.get("name"))) + if bond is not None: + bond["gpu_index"] = gpu["index"] + bond["relation"] = relation + attach_link(gpu, endpoint, relation, "primary") + + ib_candidates = [ + ( + relation_rank(relations.get((gpu_index, str(endpoint["id"])), "")), + str(endpoint.get("name") or ""), + endpoint, + ) + for endpoint in endpoints + if endpoint.get("protocol") == "InfiniBand" + and relation_rank( + relations.get((gpu_index, str(endpoint["id"])), "") + ) + < 9 + and (not primary or endpoint["id"] != primary[0]["id"]) + ] + if ib_candidates: + _, _, endpoint = min(ib_candidates) + attach_link( + gpu, + endpoint, + relations[(gpu_index, str(endpoint["id"]))], + "nearest", + ) + + +def attach_gpu_bond_mapping(snapshot: dict[str, Any]) -> None: + """Backward-compatible alias for callers using the former function name.""" + attach_gpu_network_mapping(snapshot) + + +def relation_rank(relation: str) -> int: + return {"PIX": 0, "PXB": 1, "PHB": 2, "NODE": 3, "SYS": 4}.get(relation, 9) + + +def evaluate(snapshot: dict[str, Any], expected: dict[str, int | None]) -> dict[str, Any]: + checks: list[dict[str, str]] = [] + + def add(level: str, title: str, detail: str) -> None: + checks.append({"level": level, "title": title, "detail": detail}) + + gpus = snapshot.get("gpus", []) + bonds = snapshot.get("bonds", []) + rdma_bonds = [bond for bond in bonds if bond.get("rdma")] + ib_devices = snapshot.get("ib_devices", []) + network_endpoints = snapshot.get("network_endpoints", []) + if not gpus: + add("critical", "未发现 NVIDIA GPU", "nvidia-smi 无可解析输出") + elif expected.get("gpus") is not None and len(gpus) != expected["gpus"]: + add("warning", "GPU 数量不符合预期", f"发现 {len(gpus)},预期 {expected['gpus']}") + else: + add("ok", "GPU 数量", f"发现 {len(gpus)} 张 GPU") + + unhealthy_bonds = [ + bond + for bond in rdma_bonds + if bond.get("status") != "up" + or any(slave.get("status") != "up" for slave in bond["slaves"]) + ] + if unhealthy_bonds: + add("critical", "Bond 链路异常", "、".join(bond["name"] for bond in unhealthy_bonds)) + elif rdma_bonds: + add( + "ok", + "Bond 链路", + f"{len(rdma_bonds)} 个 RDMA Bond 及全部成员均为 Up", + ) + if ( + expected.get("bonds") is not None + and len(rdma_bonds) != expected["bonds"] + ): + add( + "warning", + "Bond 数量不符合预期", + f"发现 {len(rdma_bonds)} 个 RDMA Bond,预期 {expected['bonds']}", + ) + + failures = sum( + int(slave.get("failures") or 0) + for bond in rdma_bonds + for slave in bond.get("slaves", []) + ) + if failures: + add("warning", "存在历史链路失败", f"Bond 成员累计 Link Failure Count = {failures}") + elif rdma_bonds: + add("ok", "链路失败计数", "全部 Bond 成员为 0") + + def endpoint_active(endpoint: dict[str, Any]) -> bool: + state_ok = str(endpoint.get("state") or "").upper() in {"UP", "ACTIVE"} + physical = ( + str(endpoint.get("physical_state") or "") + .upper() + .replace("_", "") + ) + physical_ok = not physical or physical in {"LINKUP", "ACTIVE", "UP"} + members_ok = all( + not member.get("status") + or str(member.get("status")).lower() in {"up", "active"} + for member in endpoint.get("members", []) + ) + return state_ok and physical_ok and members_ok + + inactive_endpoints = [ + endpoint for endpoint in network_endpoints if not endpoint_active(endpoint) + ] + if inactive_endpoints: + add( + "warning", + "RDMA 端点未全部激活", + "、".join(str(endpoint.get("name")) for endpoint in inactive_endpoints), + ) + elif network_endpoints: + roce_count = sum( + endpoint.get("protocol") == "RoCE" + for endpoint in network_endpoints + ) + ib_endpoint_count = sum( + endpoint.get("protocol") == "InfiniBand" + for endpoint in network_endpoints + ) + add( + "ok", + "RDMA 网络端点", + f"RoCE {roce_count} · InfiniBand {ib_endpoint_count},均为 Active/Up", + ) + elif snapshot.get("topology", {}).get("nic_legend"): + add( + "warning", + "RDMA 端点建模不完整", + "拓扑矩阵包含 NIC,但未构建 Bond、独立 RoCE 或 IB 端点", + ) + + mapped_gpus = [ + gpu for gpu in gpus if gpu.get("primary_network_id") + ] + relation_counts: dict[str, int] = {} + for gpu in mapped_gpus: + relation = str(gpu.get("network_relation") or "N/A") + relation_counts[relation] = relation_counts.get(relation, 0) + 1 + relation_detail = " / ".join( + f"{name} {relation_counts[name]}" + for name in ("PIX", "PXB", "PHB", "NODE", "SYS") + if relation_counts.get(name) + ) + if gpus and len(mapped_gpus) != len(gpus): + add( + "warning", + "GPU ↔ RDMA 映射不完整", + f"仅 {len(mapped_gpus)}/{len(gpus)} 张 GPU 有最近网络端点", + ) + elif mapped_gpus and relation_counts.get("SYS"): + add( + "warning", + "GPU ↔ RDMA 存在跨 NUMA 路径", + relation_detail, + ) + elif mapped_gpus: + add( + "ok", + "GPU ↔ RDMA 亲和", + f"{len(mapped_gpus)}/{len(gpus)} 已映射 · {relation_detail}", + ) + + affinity_gpus = [ + gpu + for gpu in gpus + if gpu.get("cpu_affinity") and gpu.get("cpu_affinity") != "N/A" + ] + affinity_mismatches = [] + for gpu in affinity_gpus: + gpu_numa = gpu.get("numa") + affinity_numa = number(gpu.get("numa_affinity")) + if ( + isinstance(gpu_numa, int) + and gpu_numa >= 0 + and affinity_numa is not None + and gpu_numa != int(affinity_numa) + ): + affinity_mismatches.append(f"GPU{gpu.get('index')}") + if affinity_mismatches: + add( + "warning", + "CPU / NUMA 亲和冲突", + "、".join(affinity_mismatches) + " 的PCI NUMA与内存亲和不一致", + ) + elif gpus and len(affinity_gpus) == len(gpus): + add("ok", "CPU / NUMA 亲和", f"{len(gpus)}/{len(gpus)} 张GPU均有推荐CPU范围") + elif gpus: + add( + "warning", + "CPU亲和信息不完整", + f"仅 {len(affinity_gpus)}/{len(gpus)} 张GPU可解析", + ) + + power_gpus = [ + gpu + for gpu in gpus + if number(gpu.get("power_w")) is not None + and number(gpu.get("power_limit_w")) is not None + and bool(gpu.get("pstate")) + ] + if gpus and len(power_gpus) == len(gpus): + total_power = sum(number(gpu.get("power_w")) or 0 for gpu in power_gpus) + total_limit = sum(number(gpu.get("power_limit_w")) or 0 for gpu in power_gpus) + p0_count = sum(str(gpu.get("pstate", "")).upper() == "P0" for gpu in gpus) + add( + "ok", + "GPU 功耗遥测", + f"瞬时 {fmt_power(total_power)} / 上限 {fmt_power(total_limit)} · P0 {p0_count}/{len(gpus)}", + ) + elif gpus: + add( + "warning", + "GPU功耗信息不完整", + f"仅 {len(power_gpus)}/{len(gpus)} 张GPU可读取当前功耗、上限和P-State", + ) + + active_ib = sum(str(device.get("state", "")).lower() == "active" for device in ib_devices) + if ib_devices and active_ib != len(ib_devices): + add("warning", "IB 端口未全部激活", f"{active_ib}/{len(ib_devices)} Active") + elif ib_devices: + add("ok", "IB 端口", f"{active_ib}/{len(ib_devices)} Active") + if expected.get("ib") is not None and len(ib_devices) != expected["ib"]: + add("warning", "IB 数量不符合预期", f"发现 {len(ib_devices)},预期 {expected['ib']}") + + default_route_nics = [ + nic for nic in snapshot.get("nics", []) if nic.get("default_route") + ] + unhealthy_management = [ + nic + for nic in default_route_nics + if str(nic.get("state", "")).lower() != "up" or nic.get("carrier") == 0 + ] + if unhealthy_management: + add( + "warning", + "带内管理网异常", + "、".join(str(nic.get("name")) for nic in unhealthy_management), + ) + elif default_route_nics: + add( + "ok", + "带内管理网", + "、".join(str(nic.get("name")) for nic in default_route_nics) + " 承载默认路由且链路 Up", + ) + + nvlink = snapshot.get("nvlink", {}) + if len(gpus) > 1 and not nvlink.get("active_links"): + add("warning", "未解析到 NVLink", "无法确认机内 NVLink 状态") + elif nvlink.get("active_links"): + add( + "ok", + "NVLink", + f"共 {nvlink['active_links']} 条 Active,单 GPU 至少 {nvlink.get('links_per_gpu', 0)} 条", + ) + + severity = {"ok": 0, "warning": 1, "critical": 2} + worst = max((severity[item["level"]] for item in checks), default=1) + return { + "status": ("healthy", "attention", "critical")[worst], + "checks": checks, + } + + +def esc(value: Any) -> str: + return html.escape("" if value is None else str(value), quote=True) + + +def fmt_rate(mbps: Any) -> str: + value = number(mbps) + if value is None: + return "未知" + if value >= 1000: + gbps = value / 1000 + return f"{gbps:g}G" + return f"{value:g}M" + + +def fmt_mib(mib: Any) -> str: + value = number(mib) + if value is None: + return "未知" + return f"{value / 1024:.0f} GiB" + + +def fmt_kib(kib: Any) -> str: + value = number(kib) + if value is None: + return "未知" + return f"{value / 1024 / 1024:.1f} GiB" + + +def fmt_power(watts: Any) -> str: + value = number(watts) + if value is None: + return "未知" + if value >= 1000: + return f"{value / 1000:.2f} kW" + return f"{value:.0f} W" + + +def fmt_number(value: Any, fallback: str = "—") -> str: + parsed = number(value) + return f"{parsed:g}" if parsed is not None else fallback + + +def compact_gpu_indices(indices: list[int]) -> str: + if not indices: + return "GPU?" + ordered = sorted(indices) + if ordered == list(range(ordered[0], ordered[-1] + 1)): + return f"GPU{ordered[0]}" if len(ordered) == 1 else f"GPU{ordered[0]}–{ordered[-1]}" + return ",".join(f"GPU{index}" for index in ordered) + + +def status_label(value: str) -> str: + return {"healthy": "健康", "attention": "需关注", "critical": "异常"}.get(value, "未知") + + +def badge(text: str, tone: str = "neutral") -> str: + return f'{esc(text)}' + + +def render_gpu_lane( + gpu: dict[str, Any], endpoint: dict[str, Any] | None +) -> str: + utilization = gpu.get("utilization_pct") + memory_total = gpu.get("memory_total_mib") + memory_used = gpu.get("memory_used_mib") + memory_pct = ( + 100 * memory_used / memory_total + if number(memory_used) is not None and number(memory_total) + else 0 + ) + relation = gpu.get("network_relation", "未映射") + relation_tone = "ok" if relation == "PIX" else "warn" + memory_numa = gpu.get("numa_affinity") + if memory_numa in {None, "", "N/A"}: + memory_numa = gpu.get("numa") + power_w = number(gpu.get("power_w")) + power_limit_w = number(gpu.get("power_limit_w")) + power_pct = ( + 100 * power_w / power_limit_w + if power_w is not None and power_limit_w + else 0 + ) + pstate = str(gpu.get("pstate") or "—").upper() + if endpoint and endpoint.get("type") == "bond": + members = endpoint.get("members", []) + slaves = "".join( + f""" +
+ + {esc(slave.get('name'))}{esc(fmt_rate(slave.get('speed_mbps')))} · {esc(slave.get('bdf'))} +
+ """ + for slave in members + ) + aggregate = number(endpoint.get("capacity_mbps")) or 0 + endpoint_up = str(endpoint.get("state") or "").upper() in { + "UP", + "ACTIVE", + } + network_html = f""" +
{badge(relation, relation_tone)}PCIe 亲和
+
+
+ ROCE LAG + {badge("UP" if endpoint_up else "DOWN", "ok" if endpoint_up else "bad")} +
+

{esc(endpoint.get("name"))}

+

{esc(endpoint.get("rdma") or "无 RDMA 映射")}

+
{esc(fmt_rate(aggregate))}聚合容量
+
{slaves}
+
+ """ + elif endpoint: + member = (endpoint.get("members") or [{}])[0] + endpoint_up = str(endpoint.get("state") or "").upper() in { + "UP", + "ACTIVE", + } + protocol = str(endpoint.get("protocol") or "RDMA") + network_html = f""" +
{badge(relation, relation_tone)}PCIe 亲和
+
+
+ {esc(protocol.upper())} · DIRECT + {badge("ACTIVE" if endpoint_up else (endpoint.get("state") or "UNKNOWN"), "ok" if endpoint_up else "bad")} +
+

{esc(endpoint.get("name"))}

+

{esc(endpoint.get("rdma") or "无 RDMA 映射")}

+
{esc(fmt_rate(endpoint.get("capacity_mbps")))}物理端口
+
+
+ + {esc(member.get('name') or endpoint.get('name'))}{esc(member.get('bdf') or endpoint.get('bdf') or '—')} · NUMA {esc(member.get('numa') if member.get('numa') is not None else '—')} +
+
+
+ """ + else: + network_html = """ +
未发现对应网络路径
+

没有可映射的 Bond、独立 RoCE 或 IB 端点

+ """ + return f""" +
+
+
+ GPU {esc(gpu.get('index'))} + {badge(f"NUMA {gpu.get('numa')}" if gpu.get('numa') is not None else "NUMA ?", "neutral")} +
+

{esc(gpu.get('name') or 'Unknown GPU')}

+

{esc(gpu.get('bdf'))}

+
+ CPU{esc(gpu.get('cpu_affinity') or '—')} + MEMN{esc(memory_numa if memory_numa is not None else '—')} +
+
+ {esc(fmt_mib(memory_total))}显存 + {esc(fmt_number(gpu.get('temperature_c')))}°温度 + {esc(fmt_number(utilization))}%利用率 +
+
+
+
+ {esc(fmt_power(power_w))}瞬时 / {esc(fmt_power(power_limit_w))} 上限 + {esc(pstate)} +
+
+
+
+ {network_html} +
+ """ + + +def render_ib_card(device: dict[str, Any]) -> str: + active = str(device.get("state", "")).lower() == "active" + return f""" +
+
+ INFINIBAND + {badge("ACTIVE" if active else (device.get("state") or "UNKNOWN"), "ok" if active else "warn")} +
+

{esc(device.get('name'))}

+
{esc(fmt_number(device.get('rate_gbps')))}Gb/s
+
+
PCI
{esc(device.get('bdf') or '—')}
+
NUMA
{esc(device.get('numa') if device.get('numa') is not None else '—')}
+
CA
{esc(device.get('ca_type') or '—')}
+
+
+ """ + + +def render_management_card(nic: dict[str, Any]) -> str: + state = str(nic.get("state", "")).lower() + up = state == "up" and nic.get("carrier") != 0 + addresses = nic.get("addresses") or [] + if nic.get("is_bond"): + role = "带内 · 管理 Bond" + elif nic.get("default_route"): + role = "带内 · 默认路由" + elif addresses: + role = "带内 · OS 网络" + else: + role = "其他 OS 网卡" + address_html = "
".join(esc(value) for value in addresses) or "未配置三层地址" + return f""" +
+
+ {esc(role)} + {badge("UP" if up else (state.upper() or "UNKNOWN"), "ok" if up else "warn")} +
+

{esc(nic.get('name'))}

{esc(fmt_rate(nic.get('speed_mbps')))}
+

{address_html}

+
+
Driver
{esc(nic.get('driver') or '—')}
+
PCI
{esc(nic.get('bdf') or '—')}
+
NUMA
{esc(nic.get('numa') if nic.get('numa') is not None else '—')}
+
MAC
{esc(nic.get('mac') or '—')}
+
+
+ """ + + +def render_bmc_card(bmc: dict[str, Any]) -> str: + if bmc.get("detected"): + return f""" +
+
+ 带外 · BMC{badge("DETECTED", "ok")} +
+

独立管理口

{esc(bmc.get('source') or 'IPMI')}
+

{esc(bmc.get('ip') or '—')}

+
+
Subnet
{esc(bmc.get('subnet') or '—')}
+
Gateway
{esc(bmc.get('gateway') or '—')}
+
Channel / VLAN
{esc(bmc.get('channel') or '—')} / {esc(bmc.get('vlan') or '—')}
+
MAC
{esc(bmc.get('mac') or '—')}
+
+
+ """ + return f""" +
+
+ 带外 · BMC{badge("OS 不可见", "neutral")} +
+

独立管理口

—
+

{esc(bmc.get('note') or '未发现可读取的 BMC LAN 信息')}

+

独立带外口通常不进入宿主 OS 网络命名空间;需要在 BMC/IPMI 或云厂商控制台侧验证。

+
+ """ + + +def render_check(item: dict[str, str]) -> str: + icon = {"ok": "✓", "warning": "!", "critical": "×"}.get(item["level"], "·") + return f""" +
  • + {icon} + {esc(item['title'])}{esc(item['detail'])} +
  • + """ + + +def render_html(snapshot: dict[str, Any], expected: dict[str, int | None]) -> str: + health = evaluate(snapshot, expected) + system = snapshot.get("system", {}) + gpus = snapshot.get("gpus", []) + bonds = snapshot.get("bonds", []) + ib_devices = snapshot.get("ib_devices", []) + network_endpoints = snapshot.get("network_endpoints", []) + endpoint_by_id = { + endpoint.get("id"): endpoint + for endpoint in network_endpoints + if endpoint.get("id") + } + lanes = "".join( + render_gpu_lane( + gpu, endpoint_by_id.get(gpu.get("primary_network_id")) + ) + for gpu in gpus + ) + numa_groups: list[tuple[Any, int, str]] = [] + for gpu in gpus: + numa = gpu.get("numa") + cpu_range = str(gpu.get("cpu_affinity") or "—") + if ( + not numa_groups + or numa_groups[-1][0] != numa + or numa_groups[-1][2] != cpu_range + ): + numa_groups.append((numa, 1, cpu_range)) + else: + previous_numa, previous_count, previous_cpu_range = numa_groups[-1] + numa_groups[-1] = ( + previous_numa, + previous_count + 1, + previous_cpu_range, + ) + numa_columns = " ".join(f"{count}fr" for _, count, _ in numa_groups) or "1fr" + numa_ruler = "".join( + f'' + f"{esc(f'NUMA {numa}' if numa is not None else 'NUMA ?')} " + f"推荐 CPU {esc(cpu_range)}" + for numa, count, cpu_range in numa_groups + ) or "NUMA 未知" + numa_values = list( + dict.fromkeys( + str(gpu.get("numa")) + for gpu in gpus + if gpu.get("numa") is not None + ) + ) + numa_switches = "".join( + f'' + for numa in numa_values + ) + rail_count = max(1, len(gpus)) + rail_min_width = max(0, rail_count * 132 + max(0, rail_count - 1) * 10) + affinity_groups: dict[tuple[Any, str], list[int]] = {} + for gpu in gpus: + affinity_numa = gpu.get("numa_affinity") + key = ( + affinity_numa if affinity_numa not in (None, "") else gpu.get("numa"), + str(gpu.get("cpu_affinity") or "—"), + ) + affinity_groups.setdefault(key, []).append(int(gpu.get("index", 0))) + affinity_summary = "".join( + f"{esc(compact_gpu_indices(indices))}CPU {esc(cpu_range)} · MEM NUMA {esc(numa)}" + for (numa, cpu_range), indices in affinity_groups.items() + ) + ib_cards = "".join(render_ib_card(device) for device in ib_devices) + checks = "".join(render_check(item) for item in health["checks"]) + accelerator_nic_names = { + member.get("name") + for endpoint in network_endpoints + for member in endpoint.get("members", []) + if member.get("name") + } + accelerator_nic_names.update( + endpoint.get("name") + for endpoint in network_endpoints + if endpoint.get("type") == "bond" + ) + management_nics = [ + nic + for nic in snapshot.get("nics", []) + if nic.get("name") not in accelerator_nic_names + and ( + nic.get("bdf") + or nic.get("default_route") + or nic.get("is_bond") + ) + ] + management_cards = "".join(render_management_card(nic) for nic in management_nics) + bmc = snapshot.get("bmc") or { + "detected": False, + "note": "快照中没有 BMC/IPMI 信息", + } + management_cards += render_bmc_card(bmc) + + total_memory = sum(number(gpu.get("memory_total_mib")) or 0 for gpu in gpus) + roce_mbps = sum( + number(endpoint.get("capacity_mbps")) or 0 + for endpoint in network_endpoints + if endpoint.get("protocol") == "RoCE" + and str(endpoint.get("state") or "").upper() in {"UP", "ACTIVE"} + ) + ib_gbps = sum( + number(device.get("rate_gbps")) or 0 + for device in ib_devices + if str(device.get("state", "")).lower() == "active" + ) + power_data_complete = bool(gpus) and all( + number(gpu.get("power_w")) is not None + and number(gpu.get("power_limit_w")) is not None + and bool(gpu.get("pstate")) + for gpu in gpus + ) + total_power_w = ( + sum(number(gpu.get("power_w")) or 0 for gpu in gpus) + if power_data_complete + else 0 + ) + total_power_limit_w = ( + sum(number(gpu.get("power_limit_w")) or 0 for gpu in gpus) + if power_data_complete + else 0 + ) + power_pct = ( + 100 * total_power_w / total_power_limit_w + if total_power_limit_w + else 0 + ) + p0_count = sum(str(gpu.get("pstate", "")).upper() == "P0" for gpu in gpus) + nvlink = snapshot.get("nvlink", {}) + software = snapshot.get("software", {}) + generated = snapshot.get("generated_at", "") + memory_total_kib = number(system.get("memory_total_kib")) + memory_used_kib = number(system.get("memory_used_kib")) + memory_pct = ( + 100 * memory_used_kib / memory_total_kib + if memory_used_kib is not None and memory_total_kib + else 0 + ) + cpu_count = system.get("cpu_physical_cores") or system.get("cpu_logical") or "—" + cpu_count_label = "物理核心" if system.get("cpu_physical_cores") else "逻辑 CPU" + + payload = json.dumps(snapshot, ensure_ascii=False).replace(" +
    + GPU NODE · HARDWARE PORTRAIT +

    {esc(system.get('hostname') or 'GPU Node')}

    +

    {esc(system.get('manufacturer'))} {esc(system.get('product'))} · {esc(system.get('os'))}

    +
    +
    + {esc(status_label(health['status']))} + +
    + + +
    +
    +
    +
    +
    SYSTEM / SOFTWARE

    系统与软件

    + +
    +

    {esc(system.get('manufacturer'))} {esc(system.get('product'))}

    +
    +
    操作系统
    {esc(system.get('os') or '—')}
    +
    内核
    {esc(system.get('kernel') or '—')}
    +
    Driver
    {esc(software.get('nvidia_driver') or '—')}
    +
    CUDA Toolkit
    {esc(software.get('cuda_toolkit') or '—')} nvcc
    +
    虚拟化
    {esc(system.get('virtualization') or 'Bare metal / 未识别')}
    +
    生成时间
    {esc(generated)}
    +
    +
    + +
    +
    ACCELERATOR

    GPU

    {badge("NVLINK ACTIVE" if nvlink.get('active_links') else "NVLINK 未知", "ok" if nvlink.get('active_links') else "warn")}
    +

    {esc(gpus[0].get('name') if gpus else '未发现 NVIDIA GPU')}

    +
    {len(gpus)}GPU
    +
    +
    GPU 瞬时功耗{esc(fmt_power(total_power_w) if power_data_complete else '遥测不完整')} / {esc(fmt_power(total_power_limit_w) if power_data_complete else '—')}
    +
    + Power Limit 上限 · {p0_count}/{len(gpus)} 张处于 P0 · 单次瞬时采样 +
    +
    + {esc(fmt_mib(total_memory))}GPU 显存 + {esc(fmt_rate(roce_mbps))}RoCE 物理容量 + {esc(nvlink.get('links_per_gpu') or '—')}NVLink / GPU + {ib_gbps:g}GInfiniBand +
    +
    + +
    +
    HOST COMPUTE

    CPU 与内存

    {badge(f"{system.get('numa_nodes') or '—'} NUMA", "neutral")}
    +

    {esc(system.get('cpu_model') or '未知 CPU')}

    +
    {esc(cpu_count)}{esc(cpu_count_label)}
    +
    系统内存{esc(fmt_kib(memory_used_kib))} / {esc(fmt_kib(memory_total_kib))}
    +
    +
    + {esc(system.get('cpu_sockets') or '—')}CPU Sockets + {esc(system.get('cpu_logical') or '—')}逻辑 CPU + {memory_pct:.1f}%内存使用率 +
    +
    + {affinity_summary or '未解析CPU / NUMA 推荐亲和'} +
    +
    +
    + +
    +
    +
    TOPOLOGY

    GPU ↔ RDMA 亲和画像

    +
    + {badge("PIX 最佳", "ok")}{badge("PXB / PHB", "warn")}{badge("SYS 跨 NUMA", "bad")} +
    +
    +
    +
    + NUMA 视图 + 总览显示完整拓扑,分览聚焦单个 CPU / 内存域 +
    +
    + + {numa_switches} +
    +
    +
    + {numa_ruler} +
    +

    CPU范围来自 NVIDIA 拓扑亲和,表示更近的CPU/内存路径;不代表当前训练进程已经绑核。

    + +
    {lanes or '
    未采集到 GPU 数据
    '}
    +

    有 Bond 时展示聚合容器与物理成员;无 Bond 时展示独立 RoCE/IB 端点。LACP 容量不代表单连接带宽。

    +
    + +
    +
    +
    +
    STORAGE / FABRIC

    InfiniBand

    +
    +
    {ib_cards or '
    未发现独立 InfiniBand 设备
    '}
    +
    +
    +
    +
    MANAGEMENT NETWORK

    带内与带外管理网络

    +
    +
    {management_cards}
    +
    +
    + +
    +
    +
    HEALTH / VALIDATION

    HEALTH 与验证结果

    + {esc(status_label(health['status']))} +
    + +
    +
    + + +
    RAW DATA

    JSON Snapshot

    +
    {esc(json.dumps(snapshot, ensure_ascii=False, indent=2))}
    +
    + + + + """ + title = f"{system.get('hostname') or 'GPU Node'} · GPU Hardware Portrait" + template = load_html_template("gpu_node_profile.html") + return clean_generated_text( + template.replace("@@TITLE@@", esc(title)).replace("@@BODY@@", body) + ) + + +def clean_generated_text(content: str) -> str: + """Remove template indentation residue without changing line structure.""" + cleaned = "\n".join(line.rstrip() for line in content.splitlines()) + return cleaned + ("\n" if content.endswith("\n") else "") + + +def load_html_template(filename: str) -> str: + """Load a renderer template beside the collector script.""" + template_path = Path(__file__).resolve().parent / "templates" / filename + try: + return template_path.read_text(encoding="utf-8") + except OSError as exc: + raise OSError( + f"HTML template not found: {template_path}. " + "Keep gpu_node_profile.py and templates/ together." + ) from exc + + +def render_topology_3d_html(snapshot: dict[str, Any]) -> str: + payload = json.dumps(snapshot, ensure_ascii=False).replace(" None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: + handle.write(content) + os.replace(temp_name, path) + except Exception: + try: + os.unlink(temp_name) + except OSError: + pass + raise + + +def load_snapshot(path: Path) -> dict[str, Any]: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict) or data.get("schema_version") != 1: + raise ValueError("snapshot schema_version must be 1") + nvlink = data.setdefault("nvlink", {}) + nvlink.setdefault( + "fabric_type", + infer_nvlink_fabric_type(data.get("topology", {}), len(data.get("gpus", []))), + ) + attach_gpu_bond_mapping(data) + return data + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate a native, self-contained HTML portrait for a GPU node." + ) + parser.add_argument("-o", "--output", type=Path, default=Path("gpu_node_profile.html")) + parser.add_argument( + "--topology-output", + type=Path, + help="Generate the standalone 3D topology CT HTML from the same snapshot", + ) + parser.add_argument("--snapshot-in", type=Path, help="Render an existing JSON snapshot") + parser.add_argument("--snapshot-out", type=Path, help="Save collected JSON snapshot") + parser.add_argument("--expected-gpus", type=int) + parser.add_argument("--expected-bonds", type=int) + parser.add_argument("--expected-ib", type=int) + output_group = parser.add_mutually_exclusive_group() + output_group.add_argument("--stdout", action="store_true", help="Write operations HTML to stdout") + output_group.add_argument( + "--snapshot-stdout", + action="store_true", + help="Write collected structured JSON to stdout and skip HTML rendering", + ) + parser.add_argument("--version", action="version", version=f"%(prog)s {VERSION}") + return parser + + +def main(argv: Iterable[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + snapshot = load_snapshot(args.snapshot_in) if args.snapshot_in else collect_snapshot() + if args.snapshot_stdout: + sys.stdout.write(json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n") + return 0 + if args.snapshot_out: + atomic_write( + args.snapshot_out.expanduser().resolve(), + json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n", + ) + expected = { + "gpus": args.expected_gpus, + "bonds": args.expected_bonds, + "ib": args.expected_ib, + } + report = render_html(snapshot, expected) + if args.stdout: + sys.stdout.write(report) + else: + output = args.output.expanduser().resolve() + atomic_write(output, report) + print(f"GPU node profile generated: {output}") + if args.topology_output: + topology_output = args.topology_output.expanduser().resolve() + atomic_write(topology_output, render_topology_3d_html(snapshot)) + if not args.stdout: + print(f"GPU 3D topology generated: {topology_output}") + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gpu_node_profile/templates/gpu_node_profile.html b/scripts/gpu_node_profile/templates/gpu_node_profile.html new file mode 100644 index 0000000..c3a4f8e --- /dev/null +++ b/scripts/gpu_node_profile/templates/gpu_node_profile.html @@ -0,0 +1,246 @@ + + + + + + +@@TITLE@@ + + + +@@BODY@@ + + + diff --git a/scripts/gpu_node_profile/templates/gpu_topology_3d.html b/scripts/gpu_node_profile/templates/gpu_topology_3d.html new file mode 100644 index 0000000..5d23719 --- /dev/null +++ b/scripts/gpu_node_profile/templates/gpu_topology_3d.html @@ -0,0 +1,980 @@ + + + + + + +@@TITLE@@ + + + +
    +
    +
    + MACHINE CT · GUEST-VISIBLE TOPOLOGY +

    GPU Machine 3D CT

    +

    +
    +
    +
    +
    +
    +
    +
    视图
    +
    + 镜头 + + + + +
    +
    + 图层 + + + + + + +
    +
    + +
    整机外观 · 双击壳体逐层进入
    +
    双击壳体逐层进入 · 上一层返回 · 拖动旋转 · Shift+拖动平移 · Esc 返回整机
    +
    + NVLink(仅连 GPU) + GPUDirect 候选 + PCIe + CPU 亲和 + 跨 NUMA / SYS +
    +
    + +
    + +
    + + + + diff --git a/scripts/gpu_node_profile/tests/fixtures/b300_snapshot.json b/scripts/gpu_node_profile/tests/fixtures/b300_snapshot.json new file mode 100644 index 0000000..0414c4b --- /dev/null +++ b/scripts/gpu_node_profile/tests/fixtures/b300_snapshot.json @@ -0,0 +1,133 @@ +{ + "schema_version": 1, + "generated_at": "2026-07-25T09:45:54+08:00", + "collector_version": "1.0.0", + "system": { + "hostname": "root-2", + "os": "Ubuntu 24.04.1 LTS", + "kernel": "6.8.0-71-generic", + "architecture": "x86_64", + "manufacturer": "Baidu Cloud", + "product": "Baidu Cloud BCC", + "product_version": "6.10.0", + "virtualization": "kvm", + "cpu_model": "Intel(R) Xeon(R) 6787P", + "numa_nodes": 2, + "cpu_logical": 332, + "cpu_sockets": 2, + "cpu_cores_per_socket": 83, + "cpu_physical_cores": 166, + "cpu_threads_per_core": 2, + "memory_total_kib": 4227858432, + "memory_available_kib": 3549581076, + "memory_used_kib": 678277356 + }, + "software": { + "nvidia_driver": "595.71.05", + "cuda_toolkit": "12.8", + "cuda_note": "Toolkit selected by nvcc; application/container runtime may differ." + }, + "gpus": [ + {"index": 0, "name": "NVIDIA B30Z", "uuid": "GPU-0", "bdf": "63:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 34720, "temperature_c": 48, "utilization_pct": 12, "pstate": "P0", "power_w": 312, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 0}, + {"index": 1, "name": "NVIDIA B30Z", "uuid": "GPU-1", "bdf": "67:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 50812, "temperature_c": 51, "utilization_pct": 18, "pstate": "P0", "power_w": 326, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 0}, + {"index": 2, "name": "NVIDIA B30Z", "uuid": "GPU-2", "bdf": "6b:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 19236, "temperature_c": 47, "utilization_pct": 8, "pstate": "P0", "power_w": 296, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 0}, + {"index": 3, "name": "NVIDIA B30Z", "uuid": "GPU-3", "bdf": "6f:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 62840, "temperature_c": 54, "utilization_pct": 23, "pstate": "P0", "power_w": 341, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 0}, + {"index": 4, "name": "NVIDIA B30Z", "uuid": "GPU-4", "bdf": "a3:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 41560, "temperature_c": 50, "utilization_pct": 15, "pstate": "P0", "power_w": 318, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 1}, + {"index": 5, "name": "NVIDIA B30Z", "uuid": "GPU-5", "bdf": "a7:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 28410, "temperature_c": 49, "utilization_pct": 10, "pstate": "P0", "power_w": 307, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 1}, + {"index": 6, "name": "NVIDIA B30Z", "uuid": "GPU-6", "bdf": "ab:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 73320, "temperature_c": 56, "utilization_pct": 29, "pstate": "P0", "power_w": 359, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 1}, + {"index": 7, "name": "NVIDIA B30Z", "uuid": "GPU-7", "bdf": "af:00.0", "driver": "595.71.05", "memory_total_mib": 285184, "memory_used_mib": 22350, "temperature_c": 48, "utilization_pct": 9, "pstate": "P0", "power_w": 301, "power_limit_w": 1100, "power_default_limit_w": 1100, "power_min_limit_w": 200, "power_max_limit_w": 1100, "numa": 1} + ], + "topology": { + "rows": {}, + "nic_legend": { + "NIC1": "mlx5_bond_0", "NIC2": "mlx5_bond_1", "NIC3": "mlx5_bond_2", "NIC4": "mlx5_bond_3", + "NIC5": "mlx5_bond_4", "NIC6": "mlx5_bond_5", "NIC7": "mlx5_bond_6", "NIC8": "mlx5_bond_7" + }, + "gpu_nic": { + "GPU0": [{"slot": "NIC1", "rdma": "mlx5_bond_0", "relation": "PIX"}], + "GPU1": [{"slot": "NIC2", "rdma": "mlx5_bond_1", "relation": "PIX"}], + "GPU2": [{"slot": "NIC3", "rdma": "mlx5_bond_2", "relation": "PIX"}], + "GPU3": [{"slot": "NIC4", "rdma": "mlx5_bond_3", "relation": "PIX"}], + "GPU4": [{"slot": "NIC5", "rdma": "mlx5_bond_4", "relation": "PIX"}], + "GPU5": [{"slot": "NIC6", "rdma": "mlx5_bond_5", "relation": "PIX"}], + "GPU6": [{"slot": "NIC7", "rdma": "mlx5_bond_6", "relation": "PIX"}], + "GPU7": [{"slot": "NIC8", "rdma": "mlx5_bond_7", "relation": "PIX"}] + }, + "gpu_affinity": { + "GPU0": {"cpu_affinity": "0-165", "numa_affinity": "0", "gpu_numa_id": "N/A"}, + "GPU1": {"cpu_affinity": "0-165", "numa_affinity": "0", "gpu_numa_id": "N/A"}, + "GPU2": {"cpu_affinity": "0-165", "numa_affinity": "0", "gpu_numa_id": "N/A"}, + "GPU3": {"cpu_affinity": "0-165", "numa_affinity": "0", "gpu_numa_id": "N/A"}, + "GPU4": {"cpu_affinity": "166-331", "numa_affinity": "1", "gpu_numa_id": "N/A"}, + "GPU5": {"cpu_affinity": "166-331", "numa_affinity": "1", "gpu_numa_id": "N/A"}, + "GPU6": {"cpu_affinity": "166-331", "numa_affinity": "1", "gpu_numa_id": "N/A"}, + "GPU7": {"cpu_affinity": "166-331", "numa_affinity": "1", "gpu_numa_id": "N/A"} + } + }, + "pci_fabric": { + "scope": "guest-visible", + "virtualization": "passthrough-or-virtual-pci", + "bridges": [ + {"bdf": "61:00.0", "parent_bdf": "60:00.0", "numa": 0, "description": "PCI bridge", "children": ["63:00.0", "64:00.0"]}, + {"bdf": "a1:00.0", "parent_bdf": "a0:00.0", "numa": 1, "description": "PCI bridge", "children": ["a3:00.0", "a4:00.0"]} + ], + "endpoints": [ + {"kind": "gpu", "label": "GPU0", "bdf": "63:00.0", "numa": 0, "path": ["60:00.0", "61:00.0", "63:00.0"], "description": "NVIDIA GPU"}, + {"kind": "nic", "label": "eth1", "bdf": "64:00.0", "numa": 0, "path": ["60:00.0", "61:00.0", "64:00.0"], "description": "Ethernet controller"}, + {"kind": "gpu", "label": "GPU4", "bdf": "a3:00.0", "numa": 1, "path": ["a0:00.0", "a1:00.0", "a3:00.0"], "description": "NVIDIA GPU"}, + {"kind": "nic", "label": "eth9", "bdf": "a4:00.0", "numa": 1, "path": ["a0:00.0", "a1:00.0", "a4:00.0"], "description": "Ethernet controller"} + ] + }, + "nvlink": { + "fabric_type": "NVSwitch Fabric", + "per_gpu": { + "GPU0": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125], + "GPU1": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125], + "GPU2": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125], + "GPU3": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125], + "GPU4": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125], + "GPU5": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125], + "GPU6": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125], + "GPU7": [53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125, 53.125] + }, + "active_links": 144, + "links_per_gpu": 18, + "speed_gbs": 53.125 + }, + "nics": [ + { + "name": "eth0", + "driver": "mlx5_core", + "bdf": "02:00.0", + "numa": 0, + "speed_mbps": 200000, + "state": "up", + "carrier": 1, + "mac": "fa:20:20:2e:4a:3f", + "addresses": ["192.0.2.10/24"], + "default_route": true + } + ], + "bonds": [ + {"name": "bond0", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_0", "slaves": [{"name": "eth1", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "64:00.0", "numa": 0}, {"name": "eth2", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "64:00.1", "numa": 0}]}, + {"name": "bond1", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_1", "slaves": [{"name": "eth3", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "68:00.0", "numa": 0}, {"name": "eth4", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "68:00.1", "numa": 0}]}, + {"name": "bond2", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_2", "slaves": [{"name": "eth5", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "6c:00.0", "numa": 0}, {"name": "eth6", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "6c:00.1", "numa": 0}]}, + {"name": "bond3", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_3", "slaves": [{"name": "eth7", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "70:00.0", "numa": 0}, {"name": "eth8", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "70:00.1", "numa": 0}]}, + {"name": "bond4", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_4", "slaves": [{"name": "eth9", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "a4:00.0", "numa": 1}, {"name": "eth10", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "a4:00.1", "numa": 1}]}, + {"name": "bond5", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_5", "slaves": [{"name": "eth11", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "a8:00.0", "numa": 1}, {"name": "eth12", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "a8:00.1", "numa": 1}]}, + {"name": "bond6", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_6", "slaves": [{"name": "eth13", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "ac:00.0", "numa": 1}, {"name": "eth14", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "ac:00.1", "numa": 1}]}, + {"name": "bond7", "mode": "IEEE 802.3ad Dynamic link aggregation", "hash_policy": "layer3+4 (1)", "status": "up", "rdma": "mlx5_bond_7", "slaves": [{"name": "eth15", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "b0:00.0", "numa": 1}, {"name": "eth16", "status": "up", "speed_mbps": 400000, "failures": 0, "bdf": "b0:00.1", "numa": 1}]} + ], + "ib_devices": [ + {"name": "mlx5_17", "ca_type": "MT4129", "firmware": "28.46.5500", "state": "Active", "physical_state": "LinkUp", "rate_gbps": 100, "lid": "1", "link_layer": "InfiniBand", "bdf": "b3:00.0", "numa": 1}, + {"name": "mlx5_18", "ca_type": "MT4129", "firmware": "28.46.5500", "state": "Active", "physical_state": "LinkUp", "rate_gbps": 100, "lid": "4", "link_layer": "InfiniBand", "bdf": "b4:00.0", "numa": 1}, + {"name": "mlx5_19", "ca_type": "MT4129", "firmware": "28.46.5500", "state": "Active", "physical_state": "LinkUp", "rate_gbps": 100, "lid": "5", "link_layer": "InfiniBand", "bdf": "b5:00.0", "numa": 1}, + {"name": "mlx5_20", "ca_type": "MT4129", "firmware": "28.46.5500", "state": "Active", "physical_state": "LinkUp", "rate_gbps": 100, "lid": "6", "link_layer": "InfiniBand", "bdf": "b6:00.0", "numa": 1} + ], + "bmc": { + "detected": false, + "note": "独立 BMC 带外口未向当前 OS/IPMI 通道暴露" + }, + "rdma_map": {}, + "notes": [] +} diff --git a/scripts/gpu_node_profile/tests/test_gpu_node_profile.py b/scripts/gpu_node_profile/tests/test_gpu_node_profile.py new file mode 100644 index 0000000..9cf8100 --- /dev/null +++ b/scripts/gpu_node_profile/tests/test_gpu_node_profile.py @@ -0,0 +1,430 @@ +import io +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import gpu_node_profile as profile + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / "tests" / "fixtures" / "b300_snapshot.json" + + +class ParserTests(unittest.TestCase): + def test_normalize_bdf_uses_endpoint_from_sysfs_path(self): + path = ( + "/sys/devices/pci0000:a0/0000:a0:00.4/0000:b1:00.0/" + "0000:b2:00.0/0000:b3:00.0" + ) + self.assertEqual(profile.normalize_bdf(path), "b3:00.0") + + def test_cuda_build_version_is_concise(self): + self.assertEqual(profile.concise_version("12.8.20250303"), "12.8") + + def test_parse_topology_and_mapping(self): + text = """ + GPU0 GPU1 NIC0 NIC1 CPU Affinity NUMA Affinity +GPU0 X NV18 PIX SYS 0-165 0 +GPU1 NV18 X SYS PIX 166-331 1 +NIC Legend: + NIC0: mlx5_bond_0 + NIC1: mlx5_bond_1 +""" + parsed = profile.parse_topology(text) + self.assertEqual(parsed["nic_legend"]["NIC0"], "mlx5_bond_0") + self.assertEqual(parsed["gpu_nic"]["GPU0"][0]["relation"], "PIX") + self.assertEqual( + parsed["gpu_affinity"]["GPU0"]["cpu_affinity"], "0-165" + ) + self.assertEqual( + parsed["gpu_affinity"]["GPU1"]["numa_affinity"], "1" + ) + self.assertNotIn("CPU", parsed["rows"]["GPU0"]) + + def test_parse_bond(self): + text = """ +Bonding Mode: IEEE 802.3ad Dynamic link aggregation +Transmit Hash Policy: layer3+4 (1) +MII Status: up +Slave Interface: eth1 +MII Status: up +Speed: 400000 Mbps +Link Failure Count: 0 +""" + bond = profile.parse_bond_text("bond0", text) + self.assertEqual(bond["status"], "up") + self.assertEqual(bond["slaves"][0]["speed_mbps"], 400000) + + def test_nvlink_fabric_type_does_not_assume_nvswitch(self): + direct = {"rows": {"GPU0": {"GPU1": "NV4"}, "GPU1": {"GPU0": "NV4"}}} + self.assertEqual( + profile.infer_nvlink_fabric_type(direct, 2), "NVLink Fabric" + ) + switched = { + "rows": { + f"GPU{source}": { + f"GPU{target}": "NV18" + for target in range(4) + if target != source + } + for source in range(4) + } + } + self.assertEqual( + profile.infer_nvlink_fabric_type(switched, 4), "NVSwitch Fabric" + ) + + def test_render_is_offline_native_html(self): + snapshot = json.loads(FIXTURE.read_text(encoding="utf-8")) + profile.attach_gpu_bond_mapping(snapshot) + report = profile.render_html( + snapshot, {"gpus": 8, "bonds": 8, "ib": 4} + ) + self.assertIn("GPU ↔ RDMA 亲和画像", report) + self.assertIn("系统与软件", report) + self.assertIn("CPU 与内存", report) + self.assertIn("带内与带外管理网络", report) + self.assertIn("eth0", report) + self.assertIn("带外 · BMC", report) + self.assertIn("HEALTH 与验证结果", report) + self.assertIn("CPU / NUMA 亲和", report) + self.assertIn("GPU 功耗遥测", report) + self.assertIn("GPU 瞬时功耗", report) + self.assertIn("2.56 kW / 8.80 kW", report) + self.assertIn("瞬时 / 1.10 kW 上限", report) + self.assertIn(">P0<", report) + self.assertIn("GPU0–3", report) + self.assertIn("推荐 CPU 0-165", report) + self.assertIn("不代表当前训练进程已经绑核", report) + self.assertIn('data-numa-filter="all"', report) + self.assertIn('data-numa-filter="0"', report) + self.assertIn('data-numa-filter="1"', report) + self.assertIn("function setNumaView(value)", report) + self.assertIn(".rail[hidden],.numa-ruler [hidden]", report) + self.assertIn(".ib-card .card-head { align-items:flex-start; flex-wrap:wrap;", report) + self.assertIn("eth0 承载默认路由且链路 Up", report) + self.assertIn("mlx5_bond_7", report) + self.assertIn("NVLINK / NVSWITCH FABRIC", report) + self.assertLess(report.index("系统与软件"), report.index("GPU ↔ RDMA 亲和画像")) + self.assertLess(report.index("带内与带外管理网络"), report.index("HEALTH 与验证结果")) + self.assertNotIn("mermaid", report.lower()) + self.assertNotIn("https://", report.lower()) + self.assertNotIn("http://", report.lower()) + + def test_render_3d_topology_is_offline_and_json_driven(self): + snapshot = json.loads(FIXTURE.read_text(encoding="utf-8")) + profile.attach_gpu_bond_mapping(snapshot) + report = profile.render_topology_3d_html(snapshot) + self.assertIn("GPU Machine 3D CT", report) + self.assertIn('id="machine-data"', report) + self.assertIn('id="topology-canvas"', report) + self.assertIn("function buildScene()", report) + self.assertIn("function deepestCommonPath(paths)", report) + self.assertIn('data-camera="inside"', report) + self.assertIn("function setCameraMode(mode,box=null,animate=true)", report) + self.assertIn("双击壳体逐层进入", report) + self.assertIn("state.cameraMode==='inside'", report) + self.assertIn("kind:'bond'", report) + self.assertIn("kind:'roce-port'", report) + self.assertIn("GPU endpoints only · not NIC", report) + self.assertIn("function drawEdge(edge,scale,selected,overlay=false)", report) + self.assertIn("function scopePath(box)", report) + self.assertIn("function nextScopeToward(box)", report) + self.assertIn("function enterScope(box,animate=true)", report) + self.assertIn('id="camera-back"', report) + self.assertIn("parentId:`pair-${gpu.index}`", report) + self.assertIn("PCIe Root Complex", report) + self.assertIn("GDR candidate", report) + self.assertIn('"scope": "guest-visible"', report) + self.assertNotIn("https://", report.lower()) + self.assertNotIn("http://", report.lower()) + + def test_html_templates_are_separate_source_files(self): + operations = profile.load_html_template("gpu_node_profile.html") + topology = profile.load_html_template("gpu_topology_3d.html") + self.assertIn("@@BODY@@", operations) + self.assertIn("@@MACHINE_DATA@@", topology) + self.assertNotIn("https://", operations.lower()) + self.assertNotIn("https://", topology.lower()) + + def test_build_pci_fabric_preserves_bridge_hierarchy(self): + gpus = [ + { + "index": 0, + "bdf": "63:00.0", + "numa": 0, + "pci_path": ["60:00.0", "61:00.0", "63:00.0"], + } + ] + nics = [ + { + "name": "eth1", + "bdf": "64:00.0", + "numa": 0, + "pci_path": ["60:00.0", "61:00.0", "64:00.0"], + } + ] + with patch.object(profile, "pci_numa", return_value=0): + fabric = profile.build_pci_fabric(gpus, nics, [], {}) + self.assertEqual(fabric["scope"], "guest-visible") + bridge = next(item for item in fabric["bridges"] if item["bdf"] == "61:00.0") + self.assertEqual(bridge["children"], ["63:00.0", "64:00.0"]) + + def test_standalone_roce_endpoints_map_uniquely_to_gpus(self): + nics = [ + { + "name": "eth0", + "bdf": "0b:00.0", + "numa": 0, + "pci_path": ["08:00.0", "0b:00.0"], + "speed_mbps": 100000, + "state": "up", + "carrier": 1, + "default_route": True, + "is_bond": False, + }, + { + "name": "eth1", + "bdf": "5a:00.0", + "numa": 0, + "pci_path": ["4c:00.0", "54:00.0", "5a:00.0"], + "speed_mbps": 100000, + "state": "up", + "carrier": 1, + "default_route": False, + "is_bond": False, + }, + { + "name": "eth2", + "bdf": "5a:00.1", + "numa": 0, + "pci_path": ["4c:00.0", "54:00.0", "5a:00.1"], + "speed_mbps": 100000, + "state": "up", + "carrier": 1, + "default_route": False, + "is_bond": False, + }, + ] + rdma_map = {"mlx5_0": "eth0", "mlx5_1": "eth1", "mlx5_2": "eth2"} + endpoints = profile.build_network_endpoints( + nics, + [], + [], + rdma_map, + { + "mlx5_1": {"state": "ACTIVE"}, + "mlx5_2": {"state": "ACTIVE"}, + }, + ) + self.assertEqual([item["name"] for item in endpoints], ["eth1", "eth2"]) + self.assertTrue(all(item["type"] == "standalone" for item in endpoints)) + + snapshot = { + "gpus": [ + {"index": 0, "pci_path": ["4c:00.0", "53:00.0"], "numa": 0}, + {"index": 1, "pci_path": ["4c:00.0", "58:00.0"], "numa": 0}, + ], + "nics": nics, + "bonds": [], + "ib_devices": [], + "rdma_map": rdma_map, + "network_endpoints": endpoints, + "topology": { + "gpu_affinity": {}, + "gpu_nic": { + "GPU0": [ + {"rdma": "mlx5_1", "relation": "PXB"}, + {"rdma": "mlx5_2", "relation": "PXB"}, + ], + "GPU1": [ + {"rdma": "mlx5_1", "relation": "PXB"}, + {"rdma": "mlx5_2", "relation": "PXB"}, + ], + }, + }, + } + profile.attach_gpu_network_mapping(snapshot) + primary_ids = { + gpu.get("primary_network_id") for gpu in snapshot["gpus"] + } + self.assertEqual(primary_ids, {"rdma:mlx5_1", "rdma:mlx5_2"}) + self.assertTrue( + all(gpu.get("network_relation") == "PXB" for gpu in snapshot["gpus"]) + ) + health = profile.evaluate( + snapshot, {"gpus": 2, "bonds": None, "ib": None} + ) + affinity = next( + item for item in health["checks"] if item["title"] == "GPU ↔ RDMA 亲和" + ) + self.assertEqual(affinity["level"], "ok") + self.assertIn("PXB 2", affinity["detail"]) + operations = profile.render_html( + snapshot, {"gpus": 2, "bonds": None, "ib": None} + ) + topology = profile.render_topology_3d_html(snapshot) + self.assertEqual(operations.count("ROCE · DIRECT"), 2) + self.assertNotIn("未发现对应网络路径", operations) + self.assertIn("Mode:'Standalone RDMA'", topology) + self.assertIn("endpoint.topology_visibility", topology) + + def test_unique_mapping_uses_global_minimum_not_greedy_first_match(self): + endpoints = [ + { + "id": "rdma:mlx5_1", + "type": "standalone", + "protocol": "RoCE", + "name": "eth1", + "rdma": "mlx5_1", + "role": "compute", + "members": [{"name": "eth1", "pci_path": []}], + }, + { + "id": "rdma:mlx5_2", + "type": "standalone", + "protocol": "RoCE", + "name": "eth2", + "rdma": "mlx5_2", + "role": "compute", + "members": [{"name": "eth2", "pci_path": []}], + }, + ] + snapshot = { + "gpus": [ + {"index": 0, "pci_path": [], "numa": 0}, + {"index": 1, "pci_path": [], "numa": 0}, + ], + "bonds": [], + "network_endpoints": endpoints, + "topology": { + "gpu_affinity": {}, + "gpu_nic": { + "GPU0": [ + {"rdma": "mlx5_1", "relation": "PIX"}, + {"rdma": "mlx5_2", "relation": "PIX"}, + ], + "GPU1": [ + {"rdma": "mlx5_1", "relation": "PIX"}, + {"rdma": "mlx5_2", "relation": "PHB"}, + ], + }, + }, + } + profile.attach_gpu_network_mapping(snapshot) + self.assertEqual(snapshot["gpus"][0]["network"], "eth2") + self.assertEqual(snapshot["gpus"][1]["network"], "eth1") + self.assertTrue( + all(gpu["network_relation"] == "PIX" for gpu in snapshot["gpus"]) + ) + + def test_non_rdma_bond_is_not_an_accelerator_endpoint(self): + management_bond = { + "name": "bond-mgmt", + "rdma": "", + "status": "up", + "slaves": [{"name": "eno1", "status": "up"}], + } + endpoints = profile.build_network_endpoints( + [], + [management_bond], + [], + {}, + ) + self.assertEqual(endpoints, []) + health = profile.evaluate( + { + "gpus": [], + "bonds": [management_bond], + "ib_devices": [], + "network_endpoints": [], + "topology": {}, + }, + {"gpus": None, "bonds": 0, "ib": None}, + ) + self.assertFalse( + any(item["title"] == "Bond 数量不符合预期" for item in health["checks"]) + ) + + def test_power_telemetry_health_does_not_flag_p0_as_an_error(self): + snapshot = json.loads(FIXTURE.read_text(encoding="utf-8")) + profile.attach_gpu_bond_mapping(snapshot) + health = profile.evaluate(snapshot, {"gpus": 8, "bonds": 8, "ib": 4}) + power_check = next( + item for item in health["checks"] if item["title"] == "GPU 功耗遥测" + ) + self.assertEqual(power_check["level"], "ok") + self.assertIn("P0 8/8", power_check["detail"]) + + def test_collect_gpus_keeps_power_when_optional_range_is_unsupported(self): + core = ( + "0, NVIDIA Test, GPU-0, 00000000:63:00.0, 595.71.05, " + "275040, 1024, 42, 75" + ) + power = "0, P0, 875.5, 1100" + with ( + patch.object(profile, "run", side_effect=[core, power, ""]), + patch.object(profile, "pci_numa", return_value=0), + ): + gpus = profile.collect_gpus() + self.assertEqual(len(gpus), 1) + self.assertEqual(gpus[0]["pstate"], "P0") + self.assertEqual(gpus[0]["power_w"], 875.5) + self.assertEqual(gpus[0]["power_limit_w"], 1100) + self.assertNotIn("power_max_limit_w", gpus[0]) + + def test_incomplete_power_telemetry_is_a_warning(self): + snapshot = json.loads(FIXTURE.read_text(encoding="utf-8")) + snapshot["gpus"][0].pop("power_limit_w") + profile.attach_gpu_bond_mapping(snapshot) + health = profile.evaluate(snapshot, {"gpus": 8, "bonds": 8, "ib": 4}) + power_check = next( + item for item in health["checks"] if item["title"] == "GPU功耗信息不完整" + ) + self.assertEqual(power_check["level"], "warning") + report = profile.render_html(snapshot, {"gpus": 8, "bonds": 8, "ib": 4}) + self.assertIn("遥测不完整", report) + + def test_cli_renders_snapshot(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "profile.html" + topology_output = Path(directory) / "topology-3d.html" + result = profile.main( + [ + "--snapshot-in", + str(FIXTURE), + "--output", + str(output), + "--topology-output", + str(topology_output), + "--expected-gpus", + "8", + "--expected-bonds", + "8", + "--expected-ib", + "4", + ] + ) + self.assertEqual(result, 0) + self.assertTrue(output.exists()) + self.assertGreater(output.stat().st_size, 30_000) + self.assertTrue(topology_output.exists()) + self.assertIn( + 'id="topology-canvas"', + topology_output.read_text(encoding="utf-8"), + ) + + def test_cli_can_emit_structured_snapshot_only(self): + buffer = io.StringIO() + with patch.object(profile.sys, "stdout", buffer): + result = profile.main(["--snapshot-in", str(FIXTURE), "--snapshot-stdout"]) + self.assertEqual(result, 0) + snapshot = json.loads(buffer.getvalue()) + self.assertEqual(snapshot["schema_version"], 1) + self.assertEqual(snapshot["pci_fabric"]["scope"], "guest-visible") + + +if __name__ == "__main__": + unittest.main()