hongshuai.dong 6ea2acf2d6 feat: 新增 GPU 节点画像与 3D 拓扑工具
- 新增只读 GPU 节点采集器,以统一 JSON 描述系统、CPU/NUMA、GPU、NVLink、PCIe、RDMA、网络与功耗信息
- 统一 Bond、独立 RoCE 和 InfiniBand 端点模型,支持 GPU 到 RDMA 端点的全局最优唯一匹配
- 亲和关系严格记录 nvidia-smi topo -m 证据,未暴露的 IB 设备不推测 GPU 连线
- 新增原生 HTML 运维画像和 Canvas 3D CT,可按 NUMA、PCIe 分组及 Bond 逐层查看物理成员端口
- 展示 CPU 亲和、P-State、当前功耗与 Power Limit,并提供 RDMA、Bond、IB、管理网络及整机健康检查
- 排除 CNI/veth 等虚拟接口对管理网络画像的干扰,普通管理 Bond 不计入 RDMA Bond 验收
- 提供脱敏 B300 真机示例、离线自包含页面、使用文档及 17 项解析和渲染回归测试
- 修正生成物忽略规则,确保两个 HTML 源模板能够随工具提交
2026-07-25 13:47:36 +08:00

2156 lines
79 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""Generate a self-contained, native HTML profile for a GPU node.
The collector only executes read-only commands. The generated report has no
Mermaid, JavaScript framework, external font, or CDN dependency.
"""
from __future__ import annotations
import argparse
import csv
import datetime as dt
import html
import json
import os
import platform
import re
import shutil
import socket
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any, Iterable
VERSION = "2.1.0"
ANSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
PCI_RE = re.compile(r"(?:00000000:|0000:)?([0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7])")
def run(command: list[str], timeout: int = 12) -> str:
"""Run a read-only command and return stdout without raising."""
if not command or shutil.which(command[0]) is None:
return ""
try:
result = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
timeout=timeout,
env={**os.environ, "LC_ALL": "C"},
)
except (OSError, subprocess.SubprocessError):
return ""
return ANSI_RE.sub("", result.stdout).strip()
def read_text(path: str | Path) -> str:
try:
return Path(path).read_text(encoding="utf-8", errors="replace").strip()
except OSError:
return ""
def read_int(path: str | Path) -> int | None:
value = read_text(path)
try:
return int(value)
except (TypeError, ValueError):
return None
def normalize_bdf(value: str) -> str:
matches = PCI_RE.findall(value or "")
return matches[-1].lower() if matches else ""
def pci_numa(bdf: str) -> int | None:
bdf = normalize_bdf(bdf)
return read_int(f"/sys/bus/pci/devices/0000:{bdf}/numa_node") if bdf else None
def pci_path(bdf: str) -> list[str]:
"""Return the guest-visible PCI ancestor path ending at the endpoint."""
bdf = normalize_bdf(bdf)
if not bdf:
return []
try:
resolved = str(Path(f"/sys/bus/pci/devices/0000:{bdf}").resolve())
except OSError:
return [bdf]
path: list[str] = []
for match in PCI_RE.findall(resolved):
normalized = normalize_bdf(match)
if normalized and normalized not in path:
path.append(normalized)
if bdf not in path:
path.append(bdf)
return path
def dmi_value(name: str) -> str:
return read_text(f"/sys/class/dmi/id/{name}")
def parse_csv_rows(text: str) -> list[list[str]]:
if not text:
return []
return [[field.strip() for field in row] for row in csv.reader(text.splitlines())]
def collect_gpus() -> list[dict[str, Any]]:
core_fields = [
"index",
"name",
"uuid",
"pci.bus_id",
"driver_version",
"memory.total",
"memory.used",
"temperature.gpu",
"utilization.gpu",
]
output = run(
[
"nvidia-smi",
f"--query-gpu={','.join(core_fields)}",
"--format=csv,noheader,nounits",
]
)
power_fields = [
"index",
"pstate",
"power.draw",
"power.limit",
]
power_output = run(
[
"nvidia-smi",
f"--query-gpu={','.join(power_fields)}",
"--format=csv,noheader,nounits",
]
)
power_by_index: dict[int, dict[str, Any]] = {}
for row in parse_csv_rows(power_output):
if len(row) < len(power_fields) or not row[0].isdigit():
continue
power_by_index[int(row[0])] = {
"pstate": row[1] if row[1] not in {"", "N/A", "[N/A]"} else "",
"power_w": number(row[2]),
"power_limit_w": number(row[3]),
}
range_fields = [
"index",
"power.default_limit",
"power.min_limit",
"power.max_limit",
]
range_output = run(
[
"nvidia-smi",
f"--query-gpu={','.join(range_fields)}",
"--format=csv,noheader,nounits",
]
)
for row in parse_csv_rows(range_output):
if len(row) < len(range_fields) or not row[0].isdigit():
continue
power_by_index.setdefault(int(row[0]), {}).update(
{
"power_default_limit_w": number(row[1]),
"power_min_limit_w": number(row[2]),
"power_max_limit_w": number(row[3]),
}
)
gpus: list[dict[str, Any]] = []
for row in parse_csv_rows(output):
if len(row) < len(core_fields):
continue
bdf = normalize_bdf(row[3])
index = int(row[0]) if row[0].isdigit() else len(gpus)
gpus.append(
{
"index": index,
"name": row[1],
"uuid": row[2],
"bdf": bdf,
"driver": row[4],
"memory_total_mib": number(row[5]),
"memory_used_mib": number(row[6]),
"temperature_c": number(row[7]),
"utilization_pct": number(row[8]),
"numa": pci_numa(bdf),
"pci_path": pci_path(bdf),
**power_by_index.get(index, {}),
}
)
return sorted(gpus, key=lambda item: item["index"])
def number(value: Any) -> float | None:
try:
return float(str(value).strip())
except (TypeError, ValueError):
return None
def parse_topology(text: str) -> dict[str, Any]:
"""Parse nvidia-smi topo -m into GPU relationships and NIC legend."""
text = ANSI_RE.sub("", text or "")
lines = [line.rstrip() for line in text.splitlines()]
header: list[str] = []
rows: dict[str, dict[str, str]] = {}
nic_legend: dict[str, str] = {}
gpu_affinity: dict[str, dict[str, str]] = {}
for line in lines:
stripped = line.strip()
if not stripped:
continue
legend_match = re.match(r"(NIC\d+):\s+(\S+)", stripped)
if legend_match:
nic_legend[legend_match.group(1)] = legend_match.group(2)
continue
fields = re.split(r"\s+", stripped)
if not header and fields and fields[0] == "GPU0":
# Only retain the square GPU/NIC matrix. Labels such as
# "CPU Affinity" contain spaces while row values do not, so keeping
# them in the same whitespace-split header would shift columns.
matrix_end = fields.index("CPU") if "CPU" in fields else len(fields)
header = fields[:matrix_end]
continue
if header and fields and re.fullmatch(r"GPU\d+", fields[0]):
matrix_end = 1 + len(header)
values = fields[1:matrix_end]
gpu_name = fields[0]
rows[gpu_name] = dict(zip(header, values))
suffix = fields[matrix_end:]
gpu_affinity[gpu_name] = {
"cpu_affinity": suffix[0] if len(suffix) > 0 else "",
"numa_affinity": suffix[1] if len(suffix) > 1 else "",
"gpu_numa_id": suffix[2] if len(suffix) > 2 else "",
}
gpu_nic: dict[str, list[dict[str, str]]] = {}
for gpu_name, relations in rows.items():
links: list[dict[str, str]] = []
for column, relation in relations.items():
if column.startswith("NIC"):
links.append(
{
"slot": column,
"rdma": nic_legend.get(column, column),
"relation": relation,
}
)
gpu_nic[gpu_name] = links
return {
"rows": rows,
"nic_legend": nic_legend,
"gpu_nic": gpu_nic,
"gpu_affinity": gpu_affinity,
}
def parse_nvlink_status(text: str) -> dict[str, Any]:
current = ""
links: dict[str, list[float]] = {}
for raw_line in (text or "").splitlines():
line = raw_line.strip()
gpu_match = re.match(r"GPU\s+(\d+):", line)
if gpu_match:
current = f"GPU{gpu_match.group(1)}"
links[current] = []
continue
link_match = re.match(r"Link\s+\d+:\s+([\d.]+)\s+GB/s", line)
if current and link_match:
links[current].append(float(link_match.group(1)))
counts = [len(values) for values in links.values()]
speeds = [speed for values in links.values() for speed in values]
return {
"per_gpu": links,
"active_links": sum(counts),
"links_per_gpu": min(counts) if counts else 0,
"speed_gbs": min(speeds) if speeds else None,
}
def infer_nvlink_fabric_type(topology: dict[str, Any], gpu_count: int) -> str:
"""Avoid calling every NVLink configuration an NVSwitch fabric."""
if gpu_count < 2:
return ""
nv_pairs = 0
for row_name, relations in topology.get("rows", {}).items():
if not str(row_name).startswith("GPU"):
continue
nv_pairs += sum(
str(peer).startswith("GPU") and str(relation).startswith("NV")
for peer, relation in relations.items()
)
if gpu_count > 2 and nv_pairs >= gpu_count * (gpu_count - 1):
return "NVSwitch Fabric"
return "NVLink Fabric"
def parse_bond_text(name: str, text: str) -> dict[str, Any]:
bond: dict[str, Any] = {
"name": name,
"mode": "",
"hash_policy": "",
"status": "",
"slaves": [],
}
current: dict[str, Any] | None = None
for raw_line in (text or "").splitlines():
line = raw_line.strip()
if line.startswith("Bonding Mode:"):
bond["mode"] = line.split(":", 1)[1].strip()
elif line.startswith("Transmit Hash Policy:"):
bond["hash_policy"] = line.split(":", 1)[1].strip()
elif line.startswith("MII Status:") and current is None:
bond["status"] = line.split(":", 1)[1].strip().lower()
elif line.startswith("Slave Interface:"):
current = {"name": line.split(":", 1)[1].strip()}
bond["slaves"].append(current)
elif current is not None and ":" in line:
key, value = [part.strip() for part in line.split(":", 1)]
if key == "MII Status":
current["status"] = value.lower()
elif key == "Speed":
current["speed_mbps"] = int(value.split()[0]) if value.split()[0].isdigit() else None
elif key == "Duplex":
current["duplex"] = value
elif key == "Link Failure Count":
current["failures"] = int(value) if value.isdigit() else None
elif key == "Permanent HW addr":
current["mac"] = value
return bond
def parse_ibdev2netdev(text: str) -> tuple[dict[str, str], dict[str, str]]:
rdma_to_net: dict[str, str] = {}
net_to_rdma: dict[str, str] = {}
pattern = re.compile(r"(\S+)\s+port\s+\d+\s+==>\s+(\S+)\s+\(([^)]+)\)")
for line in (text or "").splitlines():
match = pattern.search(line)
if match:
rdma, netdev = match.group(1), match.group(2)
rdma_to_net[rdma] = netdev
net_to_rdma[netdev] = rdma
return rdma_to_net, net_to_rdma
def parse_rdma_links(text: str) -> dict[str, dict[str, Any]]:
links: dict[str, dict[str, Any]] = {}
for line in (text or "").splitlines():
match = re.match(r"link\s+([^/]+)/(\d+)\s+(.*)", line.strip())
if not match:
continue
name, port, tail = match.groups()
links[name] = {
"port": int(port),
"state": capture_token(tail, "state"),
"physical_state": capture_token(tail, "physical_state"),
"netdev": capture_token(tail, "netdev"),
}
return links
def capture_token(text: str, key: str) -> str:
match = re.search(rf"(?:^|\s){re.escape(key)}\s+(\S+)", text)
return match.group(1) if match else ""
def parse_ibstat(text: str) -> list[dict[str, Any]]:
devices: list[dict[str, Any]] = []
current: dict[str, Any] | None = None
in_port = False
for raw_line in (text or "").splitlines():
line = raw_line.strip()
ca_match = re.match(r"CA '([^']+)'", line)
if ca_match:
current = {"name": ca_match.group(1)}
devices.append(current)
in_port = False
continue
if current is None:
continue
if line.startswith("CA type:"):
current["ca_type"] = line.split(":", 1)[1].strip()
elif line.startswith("Firmware version:"):
current["firmware"] = line.split(":", 1)[1].strip()
elif line.startswith("Port 1:"):
in_port = True
elif in_port and line.startswith("State:"):
current["state"] = line.split(":", 1)[1].strip()
elif in_port and line.startswith("Physical state:"):
current["physical_state"] = line.split(":", 1)[1].strip()
elif in_port and line.startswith("Rate:"):
current["rate_gbps"] = number(line.split(":", 1)[1].strip())
elif in_port and line.startswith("Base lid:"):
current["lid"] = line.split(":", 1)[1].strip()
elif in_port and line.startswith("Link layer:"):
current["link_layer"] = line.split(":", 1)[1].strip()
return devices
def collect_nics() -> list[dict[str, Any]]:
base = Path("/sys/class/net")
if not base.exists():
return []
nics: list[dict[str, Any]] = []
for entry in sorted(base.iterdir(), key=lambda item: natural_key(item.name)):
name = entry.name
if name == "lo":
continue
is_bond = (entry / "bonding").exists()
driver_info = run(["ethtool", "-i", name])
driver = field_from_lines(driver_info, "driver") or ("bonding" if is_bond else "")
bus = normalize_bdf(field_from_lines(driver_info, "bus-info"))
if not driver and not bus:
continue
speed_text = field_from_lines(run(["ethtool", name]), "Speed")
speed_match = re.search(r"(\d+)", speed_text)
nics.append(
{
"name": name,
"driver": driver,
"bdf": bus,
"numa": pci_numa(bus),
"pci_path": pci_path(bus),
"speed_mbps": int(speed_match.group(1)) if speed_match else None,
"state": read_text(entry / "operstate").lower(),
"carrier": read_int(entry / "carrier"),
"mac": read_text(entry / "address"),
"is_bond": is_bond,
}
)
attach_network_context(nics)
return nics
def attach_network_context(nics: list[dict[str, Any]]) -> None:
"""Attach addresses and default-route roles to OS-visible interfaces."""
addresses: dict[str, list[str]] = {}
defaults: set[str] = set()
try:
address_rows = json.loads(run(["ip", "-j", "address", "show"]) or "[]")
for row in address_rows:
values = []
for item in row.get("addr_info", []):
local = item.get("local")
prefix = item.get("prefixlen")
if local and item.get("scope") != "link":
values.append(f"{local}/{prefix}" if prefix is not None else str(local))
addresses[str(row.get("ifname", ""))] = values
route_rows = json.loads(run(["ip", "-j", "route", "show", "default"]) or "[]")
defaults = {str(row.get("dev")) for row in route_rows if row.get("dev")}
except (TypeError, ValueError, json.JSONDecodeError):
pass
for nic in nics:
name = nic["name"]
nic["addresses"] = addresses.get(name, [])
nic["default_route"] = name in defaults
def collect_bmc() -> dict[str, Any]:
"""Read common BMC LAN channels when IPMI is locally available."""
if shutil.which("ipmitool") is None:
return {
"detected": False,
"note": "独立 BMC 带外口未向当前 OS/IPMI 通道暴露",
}
last_values: dict[str, Any] = {}
for channel in (1, 8):
output = run(["ipmitool", "lan", "print", str(channel)], timeout=8)
values: dict[str, Any] = {"channel": channel}
wanted = {
"IP Address Source": "source",
"IP Address": "ip",
"Subnet Mask": "subnet",
"MAC Address": "mac",
"Default Gateway IP": "gateway",
"802.1q VLAN ID": "vlan",
}
for line in output.splitlines():
if ":" not in line:
continue
key, value = [part.strip() for part in line.split(":", 1)]
if key in wanted:
values[wanted[key]] = value
last_values = values
if values.get("ip") and values["ip"] not in {"0.0.0.0", "0.0.0.0/0"}:
return {"detected": True, **values, "note": ""}
return {
"detected": False,
**last_values,
"note": "IPMI 可访问,但常用 LAN 通道未读取到有效地址",
}
def field_from_lines(text: str, field: str) -> str:
prefix = f"{field}:"
for line in (text or "").splitlines():
if line.strip().startswith(prefix):
return line.split(":", 1)[1].strip()
return ""
def natural_key(value: str) -> list[Any]:
return [int(part) if part.isdigit() else part for part in re.split(r"(\d+)", value)]
def collect_bonds(net_to_rdma: dict[str, str], nics: list[dict[str, Any]]) -> list[dict[str, Any]]:
nic_map = {nic["name"]: nic for nic in nics}
bond_dir = Path("/proc/net/bonding")
if not bond_dir.exists():
return []
bonds: list[dict[str, Any]] = []
for path in sorted(bond_dir.iterdir(), key=lambda item: natural_key(item.name)):
bond = parse_bond_text(path.name, read_text(path))
bond["rdma"] = net_to_rdma.get(path.name, "")
for slave in bond["slaves"]:
slave.update(
{
key: value
for key, value in nic_map.get(slave["name"], {}).items()
if key in {"bdf", "numa", "driver", "carrier", "pci_path"}
}
)
bonds.append(bond)
return bonds
def build_network_endpoints(
nics: list[dict[str, Any]],
bonds: list[dict[str, Any]],
ib_devices: list[dict[str, Any]],
rdma_to_net: dict[str, str],
rdma_links: dict[str, dict[str, Any]] | None = None,
) -> list[dict[str, Any]]:
"""Normalize bonded RoCE, standalone RoCE and native IB as RDMA endpoints."""
rdma_links = rdma_links or {}
nic_by_name = {str(nic.get("name")): nic for nic in nics}
consumed_rdma: set[str] = set()
consumed_netdevs: set[str] = set()
endpoints: list[dict[str, Any]] = []
for bond in bonds:
rdma = str(bond.get("rdma") or "")
# A host may also use ordinary management/storage bonds. Only a bond
# exposed by the RDMA stack belongs in the accelerator topology.
if not rdma:
continue
members = [dict(member) for member in bond.get("slaves", [])]
consumed_rdma.add(rdma)
consumed_netdevs.update(str(member.get("name")) for member in members)
endpoint = {
"id": f"bond:{bond.get('name')}",
"type": "bond",
"protocol": "RoCE",
"name": bond.get("name"),
"rdma": rdma,
"role": "compute",
"state": str(bond.get("status") or "").upper(),
"numa": next(
(
member.get("numa")
for member in members
if member.get("numa") is not None
),
None,
),
"capacity_mbps": sum(
number(member.get("speed_mbps")) or 0 for member in members
),
"members": members,
"gpu_links": [],
}
endpoints.append(endpoint)
for device in ib_devices:
rdma = str(device.get("name") or "")
consumed_rdma.add(rdma)
endpoints.append(
{
"id": f"ib:{rdma}",
"type": "standalone",
"protocol": "InfiniBand",
"name": rdma,
"rdma": rdma,
"role": "storage",
"state": str(device.get("state") or "").upper(),
"physical_state": str(device.get("physical_state") or ""),
"numa": device.get("numa"),
"bdf": device.get("bdf") or "",
"pci_path": device.get("pci_path") or [],
"capacity_mbps": (
(number(device.get("rate_gbps")) or 0) * 1000
),
"members": [
{
"name": rdma,
"rdma": rdma,
"bdf": device.get("bdf") or "",
"numa": device.get("numa"),
"pci_path": device.get("pci_path") or [],
"speed_mbps": (
(number(device.get("rate_gbps")) or 0) * 1000
),
"status": str(device.get("state") or "").lower(),
}
],
"gpu_links": [],
}
)
for rdma, netdev in sorted(
rdma_to_net.items(), key=lambda item: natural_key(item[0])
):
if rdma in consumed_rdma or netdev in consumed_netdevs:
continue
nic = nic_by_name.get(netdev)
if (
not nic
or not nic.get("bdf")
or nic.get("default_route")
or nic.get("is_bond")
):
continue
rdma_state = rdma_links.get(rdma, {})
state = str(rdma_state.get("state") or nic.get("state") or "").upper()
member = {
**nic,
"rdma": rdma,
"status": str(nic.get("state") or "").lower(),
}
endpoints.append(
{
"id": f"rdma:{rdma}",
"type": "standalone",
"protocol": "RoCE",
"name": netdev,
"rdma": rdma,
"role": "compute",
"state": state,
"physical_state": str(
rdma_state.get("physical_state") or ""
),
"numa": nic.get("numa"),
"bdf": nic.get("bdf") or "",
"pci_path": nic.get("pci_path") or [],
"capacity_mbps": number(nic.get("speed_mbps")) or 0,
"members": [member],
"gpu_links": [],
}
)
return sorted(
endpoints,
key=lambda endpoint: (
0 if endpoint.get("role") == "compute" else 1,
str(endpoint.get("numa")),
natural_key(str(endpoint.get("name") or "")),
),
)
def parse_lspci_devices(text: str) -> dict[str, dict[str, str]]:
devices: dict[str, dict[str, str]] = {}
pattern = re.compile(r"^[0-9a-fA-F]{4}:([0-9a-fA-F:.]+)\s+(.+)$")
for line in (text or "").splitlines():
match = pattern.match(line.strip())
if not match:
continue
bdf, description = match.groups()
devices[normalize_bdf(bdf)] = {
"bdf": normalize_bdf(bdf),
"description": description,
}
return devices
def parse_lspci_network(text: str) -> dict[str, dict[str, str]]:
return {
bdf: device
for bdf, device in parse_lspci_devices(text).items()
if re.search(
r"Ethernet|Infiniband|Network controller",
device.get("description", ""),
re.I,
)
}
def build_pci_fabric(
gpus: list[dict[str, Any]],
nics: list[dict[str, Any]],
ib_devices: list[dict[str, Any]],
pci_devices: dict[str, dict[str, str]],
) -> dict[str, Any]:
"""Build a device-neutral PCI graph for both 2D and 3D renderers."""
endpoints: list[dict[str, Any]] = []
seen: set[tuple[str, str]] = set()
def add_endpoint(
kind: str,
label: str,
bdf: str,
numa: Any,
path: list[str] | None,
) -> None:
bdf = normalize_bdf(bdf)
key = (kind, bdf)
if not bdf or key in seen:
return
seen.add(key)
endpoints.append(
{
"kind": kind,
"label": label,
"bdf": bdf,
"numa": numa,
"path": path or pci_path(bdf),
"description": pci_devices.get(bdf, {}).get("description", ""),
}
)
for gpu in gpus:
add_endpoint(
"gpu",
f"GPU{gpu.get('index')}",
str(gpu.get("bdf") or ""),
gpu.get("numa"),
gpu.get("pci_path"),
)
for nic in nics:
add_endpoint(
"nic",
str(nic.get("name") or "NIC"),
str(nic.get("bdf") or ""),
nic.get("numa"),
nic.get("pci_path"),
)
for device in ib_devices:
add_endpoint(
"infiniband",
str(device.get("name") or "IB"),
str(device.get("bdf") or ""),
device.get("numa"),
device.get("pci_path"),
)
bridges: dict[str, dict[str, Any]] = {}
for endpoint in endpoints:
path = endpoint.get("path") or []
for index, bdf in enumerate(path[:-1]):
bridge = bridges.setdefault(
bdf,
{
"bdf": bdf,
"parent_bdf": path[index - 1] if index else "",
"numa": pci_numa(bdf),
"description": pci_devices.get(bdf, {}).get(
"description", "PCI bridge"
),
"children": [],
},
)
child = path[index + 1]
if child not in bridge["children"]:
bridge["children"].append(child)
return {
"scope": "guest-visible",
"virtualization": "passthrough-or-virtual-pci",
"bridges": sorted(bridges.values(), key=lambda item: natural_key(item["bdf"])),
"endpoints": sorted(
endpoints,
key=lambda item: (str(item.get("numa")), item["kind"], natural_key(item["label"])),
),
}
def collect_cuda_toolkit() -> str:
for executable in ("nvcc", "/usr/local/cuda/bin/nvcc"):
output = run([executable, "--version"])
match = re.search(r"release\s+([\d.]+)", output)
if match:
return concise_version(match.group(1))
for path in ("/usr/local/cuda/version.json", "/usr/local/cuda/version.txt"):
text = read_text(path)
match = re.search(r"(?:CUDA Version|\"version\"\s*:)\s*\"?([\d.]+)", text)
if match:
return concise_version(match.group(1))
return ""
def concise_version(value: str) -> str:
match = re.match(r"(\d+\.\d+)", value or "")
return match.group(1) if match else value
def collect_snapshot() -> dict[str, Any]:
topo_text = run(["nvidia-smi", "topo", "-m"])
topology = parse_topology(topo_text)
rdma_to_net, net_to_rdma = parse_ibdev2netdev(run(["ibdev2netdev"]))
rdma_links = parse_rdma_links(run(["rdma", "link", "show"]))
nics = collect_nics()
bonds = collect_bonds(net_to_rdma, nics)
lspci_text = run(["lspci", "-Dnn"])
pci_devices = parse_lspci_devices(lspci_text)
pci_network = parse_lspci_network(lspci_text)
ib_devices = []
bonded_rdma = {bond.get("rdma") for bond in bonds}
for device in parse_ibstat(run(["ibstat"])):
if (
device["name"] in bonded_rdma
or str(device.get("link_layer", "")).lower() != "infiniband"
):
continue
bdf = ""
device_path = Path(f"/sys/class/infiniband/{device['name']}/device")
try:
bdf = normalize_bdf(str(device_path.resolve()))
except OSError:
pass
device.update(
{
"bdf": bdf,
"numa": pci_numa(bdf),
"pci_path": pci_path(bdf),
"description": pci_network.get(bdf, {}).get("description", ""),
"rdma_state": rdma_links.get(device["name"], {}),
}
)
ib_devices.append(device)
network_endpoints = build_network_endpoints(
nics,
bonds,
ib_devices,
rdma_to_net,
rdma_links,
)
system = {
"hostname": socket.gethostname(),
"os": os_release(),
"kernel": platform.release(),
"architecture": platform.machine(),
"manufacturer": dmi_value("sys_vendor"),
"product": dmi_value("product_name"),
"product_version": dmi_value("product_version"),
"virtualization": run(["systemd-detect-virt"]),
"cpu_model": cpu_model(),
"numa_nodes": numa_count(),
**collect_cpu_topology(),
**collect_memory(),
}
gpus = collect_gpus()
pci_fabric = build_pci_fabric(gpus, nics, ib_devices, pci_devices)
driver = next((gpu.get("driver") for gpu in gpus if gpu.get("driver")), "")
nvlink = parse_nvlink_status(run(["nvidia-smi", "nvlink", "--status"]))
nvlink["fabric_type"] = infer_nvlink_fabric_type(topology, len(gpus))
snapshot = {
"schema_version": 1,
"generated_at": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
"collector_version": VERSION,
"system": system,
"software": {
"nvidia_driver": driver,
"cuda_toolkit": collect_cuda_toolkit(),
"cuda_note": "Toolkit selected by nvcc; application/container runtime may differ.",
},
"gpus": gpus,
"topology": topology,
"pci_fabric": pci_fabric,
"nvlink": nvlink,
"nics": nics,
"bonds": bonds,
"ib_devices": ib_devices,
"rdma_links": rdma_links,
"network_endpoints": network_endpoints,
"bmc": collect_bmc(),
"rdma_map": rdma_to_net,
"notes": [],
}
attach_gpu_network_mapping(snapshot)
return snapshot
def os_release() -> str:
for line in read_text("/etc/os-release").splitlines():
if line.startswith("PRETTY_NAME="):
return line.split("=", 1)[1].strip().strip('"')
return platform.platform()
def cpu_model() -> str:
for line in read_text("/proc/cpuinfo").splitlines():
if line.lower().startswith("model name"):
return line.split(":", 1)[1].strip()
return ""
def numa_count() -> int:
nodes = list(Path("/sys/devices/system/node").glob("node[0-9]*"))
return len(nodes)
def collect_cpu_topology() -> dict[str, Any]:
values: dict[str, str] = {}
output = run(["lscpu", "-J"])
try:
for item in json.loads(output or "{}").get("lscpu", []):
key = str(item.get("field", "")).rstrip(":")
values[key] = str(item.get("data", "")).strip()
except (TypeError, ValueError, json.JSONDecodeError):
pass
def integer(key: str) -> int | None:
match = re.search(r"\d+", values.get(key, ""))
return int(match.group(0)) if match else None
sockets = integer("Socket(s)")
cores_per_socket = integer("Core(s) per socket")
return {
"cpu_logical": integer("CPU(s)") or os.cpu_count(),
"cpu_sockets": sockets,
"cpu_cores_per_socket": cores_per_socket,
"cpu_physical_cores": (
sockets * cores_per_socket
if sockets is not None and cores_per_socket is not None
else None
),
"cpu_threads_per_core": integer("Thread(s) per core"),
}
def collect_memory() -> dict[str, Any]:
values: dict[str, int] = {}
for line in read_text("/proc/meminfo").splitlines():
if ":" not in line:
continue
key, tail = line.split(":", 1)
match = re.search(r"\d+", tail)
if match:
values[key] = int(match.group(0))
total = values.get("MemTotal")
available = values.get("MemAvailable")
return {
"memory_total_kib": total,
"memory_available_kib": available,
"memory_used_kib": (
total - available
if total is not None and available is not None
else None
),
}
def common_path_depth(left: list[str], right: list[str]) -> int:
depth = 0
for left_item, right_item in zip(left or [], right or []):
if left_item != right_item:
break
depth += 1
return depth
def hungarian_assignment(costs: list[list[int]]) -> list[tuple[int, int]]:
"""Return minimum-cost unique row/column pairs for a rectangular matrix."""
if not costs:
return []
row_count = len(costs)
column_count = len(costs[0])
if row_count > column_count:
raise ValueError("Hungarian assignment requires rows <= columns")
potentials_rows = [0] * (row_count + 1)
potentials_columns = [0] * (column_count + 1)
column_match = [0] * (column_count + 1)
path = [0] * (column_count + 1)
for row in range(1, row_count + 1):
column_match[0] = row
minimums = [10**18] * (column_count + 1)
used = [False] * (column_count + 1)
column = 0
while True:
used[column] = True
current_row = column_match[column]
delta = 10**18
next_column = 0
for candidate in range(1, column_count + 1):
if used[candidate]:
continue
reduced = (
costs[current_row - 1][candidate - 1]
- potentials_rows[current_row]
- potentials_columns[candidate]
)
if reduced < minimums[candidate]:
minimums[candidate] = reduced
path[candidate] = column
if minimums[candidate] < delta:
delta = minimums[candidate]
next_column = candidate
for candidate in range(column_count + 1):
if used[candidate]:
potentials_rows[column_match[candidate]] += delta
potentials_columns[candidate] -= delta
else:
minimums[candidate] -= delta
column = next_column
if column_match[column] == 0:
break
while True:
previous = path[column]
column_match[column] = column_match[previous]
column = previous
if column == 0:
break
return sorted(
(row - 1, column - 1)
for column, row in enumerate(column_match[1:], start=1)
if row
)
def attach_gpu_network_mapping(snapshot: dict[str, Any]) -> None:
"""Map GPUs to bonded or standalone RDMA endpoints using topology distance."""
endpoints = snapshot.get("network_endpoints")
if not isinstance(endpoints, list) or (
not endpoints
and (
snapshot.get("bonds")
or snapshot.get("ib_devices")
or snapshot.get("rdma_map")
)
):
endpoints = build_network_endpoints(
snapshot.get("nics", []),
snapshot.get("bonds", []),
snapshot.get("ib_devices", []),
snapshot.get("rdma_map", {}),
snapshot.get("rdma_links", {}),
)
snapshot["network_endpoints"] = endpoints
snapshot["normalized_by_version"] = VERSION
endpoint_by_id = {
str(endpoint.get("id")): endpoint
for endpoint in endpoints
if endpoint.get("id")
}
bond_by_name = {
str(bond.get("name")): bond for bond in snapshot.get("bonds", [])
}
for endpoint in endpoints:
endpoint["gpu_links"] = []
endpoint["topology_visibility"] = "not-exposed"
for bond in snapshot.get("bonds", []):
bond.pop("gpu_index", None)
bond.pop("relation", None)
topology = snapshot.get("topology", {}).get("gpu_nic", {})
affinity = snapshot.get("topology", {}).get("gpu_affinity", {})
gpus = snapshot.get("gpus", [])
relations: dict[tuple[int, str], str] = {}
for gpu in gpus:
for key in (
"network_relation",
"rdma",
"bond",
"network",
"primary_network_id",
):
gpu.pop(key, None)
gpu["network_links"] = []
gpu_key = f"GPU{gpu['index']}"
gpu.update(
{
key: value
for key, value in affinity.get(gpu_key, {}).items()
if value
}
)
by_rdma = {
str(item.get("rdma")): str(item.get("relation") or "")
for item in topology.get(gpu_key, [])
}
for endpoint in endpoints:
relation = by_rdma.get(str(endpoint.get("rdma") or ""), "")
if relation:
relations[(int(gpu["index"]), str(endpoint["id"]))] = relation
endpoint["topology_visibility"] = "nvidia-smi-topo"
primary_pool = [
endpoint for endpoint in endpoints if endpoint.get("role") == "compute"
]
if not primary_pool:
primary_pool = list(endpoints)
sorted_gpus = sorted(gpus, key=lambda item: int(item["index"]))
sorted_endpoints = sorted(
primary_pool, key=lambda item: natural_key(str(item.get("name") or ""))
)
def pair_cost(gpu: dict[str, Any], endpoint: dict[str, Any]) -> int:
gpu_index = int(gpu["index"])
endpoint_id = str(endpoint["id"])
rank = relation_rank(relations.get((gpu_index, endpoint_id), ""))
if rank >= 9:
return 10**9
member_paths = [
member.get("pci_path") or []
for member in endpoint.get("members", [])
] or [endpoint.get("pci_path") or []]
proximity = max(
(
common_path_depth(gpu.get("pci_path") or [], path)
for path in member_paths
),
default=0,
)
# Topology relation is authoritative. PCI path only breaks equal-distance
# ties, followed by stable natural ordering of GPUs and endpoints.
return rank * 1_000_000 + (100 - min(100, proximity)) * 1_000
primary_assignments: dict[int, tuple[dict[str, Any], str]] = {}
if sorted_gpus and sorted_endpoints:
if len(sorted_gpus) <= len(sorted_endpoints):
costs = [
[
pair_cost(gpu, endpoint) + endpoint_index
for endpoint_index, endpoint in enumerate(sorted_endpoints)
]
for gpu in sorted_gpus
]
unique_pairs = [
(sorted_gpus[gpu_index], sorted_endpoints[endpoint_index])
for gpu_index, endpoint_index in hungarian_assignment(costs)
if costs[gpu_index][endpoint_index] < 10**9
]
else:
costs = [
[
pair_cost(gpu, endpoint) + gpu_index
for gpu_index, gpu in enumerate(sorted_gpus)
]
for endpoint in sorted_endpoints
]
unique_pairs = [
(sorted_gpus[gpu_index], sorted_endpoints[endpoint_index])
for endpoint_index, gpu_index in hungarian_assignment(costs)
if costs[endpoint_index][gpu_index] < 10**9
]
for gpu, endpoint in unique_pairs:
gpu_index = int(gpu["index"])
endpoint_id = str(endpoint["id"])
primary_assignments[gpu_index] = (
endpoint,
relations[(gpu_index, endpoint_id)],
)
for gpu in gpus:
gpu_index = int(gpu["index"])
if gpu_index in primary_assignments:
continue
candidates = [
(
relation_rank(relations.get((gpu_index, str(endpoint["id"])), "")),
str(endpoint.get("name") or ""),
endpoint,
)
for endpoint in primary_pool
if relation_rank(
relations.get((gpu_index, str(endpoint["id"])), "")
)
< 9
]
if candidates:
_, _, endpoint = min(candidates)
primary_assignments[gpu_index] = (
endpoint,
relations[(gpu_index, str(endpoint["id"]))],
)
def attach_link(
gpu: dict[str, Any],
endpoint: dict[str, Any],
relation: str,
role: str,
) -> None:
link = {
"endpoint_id": endpoint["id"],
"name": endpoint.get("name"),
"rdma": endpoint.get("rdma"),
"protocol": endpoint.get("protocol"),
"type": endpoint.get("type"),
"relation": relation,
"role": role,
"evidence": "nvidia-smi topo -m",
}
gpu["network_links"].append(link)
endpoint["gpu_links"].append(
{
"gpu_index": gpu["index"],
"relation": relation,
"role": role,
"evidence": "nvidia-smi topo -m",
}
)
for gpu in gpus:
gpu_index = int(gpu["index"])
primary = primary_assignments.get(gpu_index)
if primary:
endpoint, relation = primary
gpu["primary_network_id"] = endpoint["id"]
gpu["network_relation"] = relation
gpu["rdma"] = endpoint.get("rdma")
gpu["network"] = endpoint.get("name")
if endpoint.get("type") == "bond":
gpu["bond"] = endpoint.get("name")
bond = bond_by_name.get(str(endpoint.get("name")))
if bond is not None:
bond["gpu_index"] = gpu["index"]
bond["relation"] = relation
attach_link(gpu, endpoint, relation, "primary")
ib_candidates = [
(
relation_rank(relations.get((gpu_index, str(endpoint["id"])), "")),
str(endpoint.get("name") or ""),
endpoint,
)
for endpoint in endpoints
if endpoint.get("protocol") == "InfiniBand"
and relation_rank(
relations.get((gpu_index, str(endpoint["id"])), "")
)
< 9
and (not primary or endpoint["id"] != primary[0]["id"])
]
if ib_candidates:
_, _, endpoint = min(ib_candidates)
attach_link(
gpu,
endpoint,
relations[(gpu_index, str(endpoint["id"]))],
"nearest",
)
def attach_gpu_bond_mapping(snapshot: dict[str, Any]) -> None:
"""Backward-compatible alias for callers using the former function name."""
attach_gpu_network_mapping(snapshot)
def relation_rank(relation: str) -> int:
return {"PIX": 0, "PXB": 1, "PHB": 2, "NODE": 3, "SYS": 4}.get(relation, 9)
def evaluate(snapshot: dict[str, Any], expected: dict[str, int | None]) -> dict[str, Any]:
checks: list[dict[str, str]] = []
def add(level: str, title: str, detail: str) -> None:
checks.append({"level": level, "title": title, "detail": detail})
gpus = snapshot.get("gpus", [])
bonds = snapshot.get("bonds", [])
rdma_bonds = [bond for bond in bonds if bond.get("rdma")]
ib_devices = snapshot.get("ib_devices", [])
network_endpoints = snapshot.get("network_endpoints", [])
if not gpus:
add("critical", "未发现 NVIDIA GPU", "nvidia-smi 无可解析输出")
elif expected.get("gpus") is not None and len(gpus) != expected["gpus"]:
add("warning", "GPU 数量不符合预期", f"发现 {len(gpus)},预期 {expected['gpus']}")
else:
add("ok", "GPU 数量", f"发现 {len(gpus)} 张 GPU")
unhealthy_bonds = [
bond
for bond in rdma_bonds
if bond.get("status") != "up"
or any(slave.get("status") != "up" for slave in bond["slaves"])
]
if unhealthy_bonds:
add("critical", "Bond 链路异常", "、".join(bond["name"] for bond in unhealthy_bonds))
elif rdma_bonds:
add(
"ok",
"Bond 链路",
f"{len(rdma_bonds)} 个 RDMA Bond 及全部成员均为 Up",
)
if (
expected.get("bonds") is not None
and len(rdma_bonds) != expected["bonds"]
):
add(
"warning",
"Bond 数量不符合预期",
f"发现 {len(rdma_bonds)} 个 RDMA Bond,预期 {expected['bonds']}",
)
failures = sum(
int(slave.get("failures") or 0)
for bond in rdma_bonds
for slave in bond.get("slaves", [])
)
if failures:
add("warning", "存在历史链路失败", f"Bond 成员累计 Link Failure Count = {failures}")
elif rdma_bonds:
add("ok", "链路失败计数", "全部 Bond 成员为 0")
def endpoint_active(endpoint: dict[str, Any]) -> bool:
state_ok = str(endpoint.get("state") or "").upper() in {"UP", "ACTIVE"}
physical = (
str(endpoint.get("physical_state") or "")
.upper()
.replace("_", "")
)
physical_ok = not physical or physical in {"LINKUP", "ACTIVE", "UP"}
members_ok = all(
not member.get("status")
or str(member.get("status")).lower() in {"up", "active"}
for member in endpoint.get("members", [])
)
return state_ok and physical_ok and members_ok
inactive_endpoints = [
endpoint for endpoint in network_endpoints if not endpoint_active(endpoint)
]
if inactive_endpoints:
add(
"warning",
"RDMA 端点未全部激活",
"、".join(str(endpoint.get("name")) for endpoint in inactive_endpoints),
)
elif network_endpoints:
roce_count = sum(
endpoint.get("protocol") == "RoCE"
for endpoint in network_endpoints
)
ib_endpoint_count = sum(
endpoint.get("protocol") == "InfiniBand"
for endpoint in network_endpoints
)
add(
"ok",
"RDMA 网络端点",
f"RoCE {roce_count} · InfiniBand {ib_endpoint_count},均为 Active/Up",
)
elif snapshot.get("topology", {}).get("nic_legend"):
add(
"warning",
"RDMA 端点建模不完整",
"拓扑矩阵包含 NIC,但未构建 Bond、独立 RoCE 或 IB 端点",
)
mapped_gpus = [
gpu for gpu in gpus if gpu.get("primary_network_id")
]
relation_counts: dict[str, int] = {}
for gpu in mapped_gpus:
relation = str(gpu.get("network_relation") or "N/A")
relation_counts[relation] = relation_counts.get(relation, 0) + 1
relation_detail = " / ".join(
f"{name} {relation_counts[name]}"
for name in ("PIX", "PXB", "PHB", "NODE", "SYS")
if relation_counts.get(name)
)
if gpus and len(mapped_gpus) != len(gpus):
add(
"warning",
"GPU ↔ RDMA 映射不完整",
f"仅 {len(mapped_gpus)}/{len(gpus)} 张 GPU 有最近网络端点",
)
elif mapped_gpus and relation_counts.get("SYS"):
add(
"warning",
"GPU ↔ RDMA 存在跨 NUMA 路径",
relation_detail,
)
elif mapped_gpus:
add(
"ok",
"GPU ↔ RDMA 亲和",
f"{len(mapped_gpus)}/{len(gpus)} 已映射 · {relation_detail}",
)
affinity_gpus = [
gpu
for gpu in gpus
if gpu.get("cpu_affinity") and gpu.get("cpu_affinity") != "N/A"
]
affinity_mismatches = []
for gpu in affinity_gpus:
gpu_numa = gpu.get("numa")
affinity_numa = number(gpu.get("numa_affinity"))
if (
isinstance(gpu_numa, int)
and gpu_numa >= 0
and affinity_numa is not None
and gpu_numa != int(affinity_numa)
):
affinity_mismatches.append(f"GPU{gpu.get('index')}")
if affinity_mismatches:
add(
"warning",
"CPU / NUMA 亲和冲突",
"、".join(affinity_mismatches) + " 的PCI NUMA与内存亲和不一致",
)
elif gpus and len(affinity_gpus) == len(gpus):
add("ok", "CPU / NUMA 亲和", f"{len(gpus)}/{len(gpus)} 张GPU均有推荐CPU范围")
elif gpus:
add(
"warning",
"CPU亲和信息不完整",
f"仅 {len(affinity_gpus)}/{len(gpus)} 张GPU可解析",
)
power_gpus = [
gpu
for gpu in gpus
if number(gpu.get("power_w")) is not None
and number(gpu.get("power_limit_w")) is not None
and bool(gpu.get("pstate"))
]
if gpus and len(power_gpus) == len(gpus):
total_power = sum(number(gpu.get("power_w")) or 0 for gpu in power_gpus)
total_limit = sum(number(gpu.get("power_limit_w")) or 0 for gpu in power_gpus)
p0_count = sum(str(gpu.get("pstate", "")).upper() == "P0" for gpu in gpus)
add(
"ok",
"GPU 功耗遥测",
f"瞬时 {fmt_power(total_power)} / 上限 {fmt_power(total_limit)} · P0 {p0_count}/{len(gpus)}",
)
elif gpus:
add(
"warning",
"GPU功耗信息不完整",
f"仅 {len(power_gpus)}/{len(gpus)} 张GPU可读取当前功耗、上限和P-State",
)
active_ib = sum(str(device.get("state", "")).lower() == "active" for device in ib_devices)
if ib_devices and active_ib != len(ib_devices):
add("warning", "IB 端口未全部激活", f"{active_ib}/{len(ib_devices)} Active")
elif ib_devices:
add("ok", "IB 端口", f"{active_ib}/{len(ib_devices)} Active")
if expected.get("ib") is not None and len(ib_devices) != expected["ib"]:
add("warning", "IB 数量不符合预期", f"发现 {len(ib_devices)},预期 {expected['ib']}")
default_route_nics = [
nic for nic in snapshot.get("nics", []) if nic.get("default_route")
]
unhealthy_management = [
nic
for nic in default_route_nics
if str(nic.get("state", "")).lower() != "up" or nic.get("carrier") == 0
]
if unhealthy_management:
add(
"warning",
"带内管理网异常",
"、".join(str(nic.get("name")) for nic in unhealthy_management),
)
elif default_route_nics:
add(
"ok",
"带内管理网",
"、".join(str(nic.get("name")) for nic in default_route_nics) + " 承载默认路由且链路 Up",
)
nvlink = snapshot.get("nvlink", {})
if len(gpus) > 1 and not nvlink.get("active_links"):
add("warning", "未解析到 NVLink", "无法确认机内 NVLink 状态")
elif nvlink.get("active_links"):
add(
"ok",
"NVLink",
f"共 {nvlink['active_links']} 条 Active,单 GPU 至少 {nvlink.get('links_per_gpu', 0)} 条",
)
severity = {"ok": 0, "warning": 1, "critical": 2}
worst = max((severity[item["level"]] for item in checks), default=1)
return {
"status": ("healthy", "attention", "critical")[worst],
"checks": checks,
}
def esc(value: Any) -> str:
return html.escape("" if value is None else str(value), quote=True)
def fmt_rate(mbps: Any) -> str:
value = number(mbps)
if value is None:
return "未知"
if value >= 1000:
gbps = value / 1000
return f"{gbps:g}G"
return f"{value:g}M"
def fmt_mib(mib: Any) -> str:
value = number(mib)
if value is None:
return "未知"
return f"{value / 1024:.0f} GiB"
def fmt_kib(kib: Any) -> str:
value = number(kib)
if value is None:
return "未知"
return f"{value / 1024 / 1024:.1f} GiB"
def fmt_power(watts: Any) -> str:
value = number(watts)
if value is None:
return "未知"
if value >= 1000:
return f"{value / 1000:.2f} kW"
return f"{value:.0f} W"
def fmt_number(value: Any, fallback: str = "—") -> str:
parsed = number(value)
return f"{parsed:g}" if parsed is not None else fallback
def compact_gpu_indices(indices: list[int]) -> str:
if not indices:
return "GPU?"
ordered = sorted(indices)
if ordered == list(range(ordered[0], ordered[-1] + 1)):
return f"GPU{ordered[0]}" if len(ordered) == 1 else f"GPU{ordered[0]}–{ordered[-1]}"
return ",".join(f"GPU{index}" for index in ordered)
def status_label(value: str) -> str:
return {"healthy": "健康", "attention": "需关注", "critical": "异常"}.get(value, "未知")
def badge(text: str, tone: str = "neutral") -> str:
return f'<span class="badge {esc(tone)}">{esc(text)}</span>'
def render_gpu_lane(
gpu: dict[str, Any], endpoint: dict[str, Any] | None
) -> str:
utilization = gpu.get("utilization_pct")
memory_total = gpu.get("memory_total_mib")
memory_used = gpu.get("memory_used_mib")
memory_pct = (
100 * memory_used / memory_total
if number(memory_used) is not None and number(memory_total)
else 0
)
relation = gpu.get("network_relation", "未映射")
relation_tone = "ok" if relation == "PIX" else "warn"
memory_numa = gpu.get("numa_affinity")
if memory_numa in {None, "", "N/A"}:
memory_numa = gpu.get("numa")
power_w = number(gpu.get("power_w"))
power_limit_w = number(gpu.get("power_limit_w"))
power_pct = (
100 * power_w / power_limit_w
if power_w is not None and power_limit_w
else 0
)
pstate = str(gpu.get("pstate") or "—").upper()
if endpoint and endpoint.get("type") == "bond":
members = endpoint.get("members", [])
slaves = "".join(
f"""
<div class="port {'up' if slave.get('status') == 'up' else 'down'}">
<span class="port-dot"></span>
<span><b>{esc(slave.get('name'))}</b><small>{esc(fmt_rate(slave.get('speed_mbps')))} · {esc(slave.get('bdf'))}</small></span>
</div>
"""
for slave in members
)
aggregate = number(endpoint.get("capacity_mbps")) or 0
endpoint_up = str(endpoint.get("state") or "").upper() in {
"UP",
"ACTIVE",
}
network_html = f"""
<div class="relation">{badge(relation, relation_tone)}<span>PCIe 亲和</span></div>
<article class="nic-card">
<div class="card-head">
<span class="eyebrow">ROCE LAG</span>
{badge("UP" if endpoint_up else "DOWN", "ok" if endpoint_up else "bad")}
</div>
<h4>{esc(endpoint.get("name"))}</h4>
<p>{esc(endpoint.get("rdma") or "无 RDMA 映射")}</p>
<div class="bond-capacity"><b>{esc(fmt_rate(aggregate))}</b><span>聚合容量</span></div>
<div class="ports">{slaves}</div>
</article>
"""
elif endpoint:
member = (endpoint.get("members") or [{}])[0]
endpoint_up = str(endpoint.get("state") or "").upper() in {
"UP",
"ACTIVE",
}
protocol = str(endpoint.get("protocol") or "RDMA")
network_html = f"""
<div class="relation">{badge(relation, relation_tone)}<span>PCIe 亲和</span></div>
<article class="nic-card">
<div class="card-head">
<span class="eyebrow">{esc(protocol.upper())} · DIRECT</span>
{badge("ACTIVE" if endpoint_up else (endpoint.get("state") or "UNKNOWN"), "ok" if endpoint_up else "bad")}
</div>
<h4>{esc(endpoint.get("name"))}</h4>
<p>{esc(endpoint.get("rdma") or "无 RDMA 映射")}</p>
<div class="bond-capacity"><b>{esc(fmt_rate(endpoint.get("capacity_mbps")))}</b><span>物理端口</span></div>
<div class="ports">
<div class="port {'up' if endpoint_up else 'down'}">
<span class="port-dot"></span>
<span><b>{esc(member.get('name') or endpoint.get('name'))}</b><small>{esc(member.get('bdf') or endpoint.get('bdf') or '—')} · NUMA {esc(member.get('numa') if member.get('numa') is not None else '—')}</small></span>
</div>
</div>
</article>
"""
else:
network_html = """
<div class="relation">未发现对应网络路径</div>
<article class="nic-card empty"><p>没有可映射的 Bond、独立 RoCE 或 IB 端点</p></article>
"""
return f"""
<div class="rail" data-numa="{esc(gpu.get('numa'))}">
<article class="gpu-card">
<div class="card-head">
<span class="eyebrow">GPU {esc(gpu.get('index'))}</span>
{badge(f"NUMA {gpu.get('numa')}" if gpu.get('numa') is not None else "NUMA ?", "neutral")}
</div>
<h3>{esc(gpu.get('name') or 'Unknown GPU')}</h3>
<p class="mono">{esc(gpu.get('bdf'))}</p>
<div class="gpu-affinity" title="拓扑推荐亲和,不代表当前进程已绑核">
<span><small>CPU</small><b>{esc(gpu.get('cpu_affinity') or '—')}</b></span>
<span><small>MEM</small><b>N{esc(memory_numa if memory_numa is not None else '—')}</b></span>
</div>
<div class="gpu-stats">
<span><b>{esc(fmt_mib(memory_total))}</b><small>显存</small></span>
<span><b>{esc(fmt_number(gpu.get('temperature_c')))}°</b><small>温度</small></span>
<span><b>{esc(fmt_number(utilization))}%</b><small>利用率</small></span>
</div>
<div class="meter memory-usage-meter" title="显存使用率"><i style="width:{max(0, min(100, memory_pct)):.1f}%"></i></div>
<div class="gpu-power">
<div>
<span><b>{esc(fmt_power(power_w))}</b><small>瞬时 / {esc(fmt_power(power_limit_w))} 上限</small></span>
<em>{esc(pstate)}</em>
</div>
<div class="power-meter" title="瞬时功耗占当前 Power Limit 的比例"><i style="width:{max(0, min(100, power_pct)):.1f}%"></i></div>
</div>
</article>
{network_html}
</div>
"""
def render_ib_card(device: dict[str, Any]) -> str:
active = str(device.get("state", "")).lower() == "active"
return f"""
<article class="ib-card">
<div class="card-head">
<span class="eyebrow">INFINIBAND</span>
{badge("ACTIVE" if active else (device.get("state") or "UNKNOWN"), "ok" if active else "warn")}
</div>
<h4>{esc(device.get('name'))}</h4>
<div class="ib-rate">{esc(fmt_number(device.get('rate_gbps')))}<small>Gb/s</small></div>
<dl>
<div><dt>PCI</dt><dd>{esc(device.get('bdf') or '—')}</dd></div>
<div><dt>NUMA</dt><dd>{esc(device.get('numa') if device.get('numa') is not None else '—')}</dd></div>
<div><dt>CA</dt><dd>{esc(device.get('ca_type') or '—')}</dd></div>
</dl>
</article>
"""
def render_management_card(nic: dict[str, Any]) -> str:
state = str(nic.get("state", "")).lower()
up = state == "up" and nic.get("carrier") != 0
addresses = nic.get("addresses") or []
if nic.get("is_bond"):
role = "带内 · 管理 Bond"
elif nic.get("default_route"):
role = "带内 · 默认路由"
elif addresses:
role = "带内 · OS 网络"
else:
role = "其他 OS 网卡"
address_html = "<br>".join(esc(value) for value in addresses) or "未配置三层地址"
return f"""
<article class="management-card">
<div class="card-head">
<span class="eyebrow">{esc(role)}</span>
{badge("UP" if up else (state.upper() or "UNKNOWN"), "ok" if up else "warn")}
</div>
<div class="management-title"><h4>{esc(nic.get('name'))}</h4><b>{esc(fmt_rate(nic.get('speed_mbps')))}</b></div>
<p class="address">{address_html}</p>
<dl>
<div><dt>Driver</dt><dd>{esc(nic.get('driver') or '—')}</dd></div>
<div><dt>PCI</dt><dd>{esc(nic.get('bdf') or '—')}</dd></div>
<div><dt>NUMA</dt><dd>{esc(nic.get('numa') if nic.get('numa') is not None else '—')}</dd></div>
<div><dt>MAC</dt><dd>{esc(nic.get('mac') or '—')}</dd></div>
</dl>
</article>
"""
def render_bmc_card(bmc: dict[str, Any]) -> str:
if bmc.get("detected"):
return f"""
<article class="management-card bmc-card">
<div class="card-head">
<span class="eyebrow">带外 · BMC</span>{badge("DETECTED", "ok")}
</div>
<div class="management-title"><h4>独立管理口</h4><b>{esc(bmc.get('source') or 'IPMI')}</b></div>
<p class="address">{esc(bmc.get('ip') or '—')}</p>
<dl>
<div><dt>Subnet</dt><dd>{esc(bmc.get('subnet') or '—')}</dd></div>
<div><dt>Gateway</dt><dd>{esc(bmc.get('gateway') or '—')}</dd></div>
<div><dt>Channel / VLAN</dt><dd>{esc(bmc.get('channel') or '—')} / {esc(bmc.get('vlan') or '—')}</dd></div>
<div><dt>MAC</dt><dd>{esc(bmc.get('mac') or '—')}</dd></div>
</dl>
</article>
"""
return f"""
<article class="management-card unavailable">
<div class="card-head">
<span class="eyebrow">带外 · BMC</span>{badge("OS 不可见", "neutral")}
</div>
<div class="management-title"><h4>独立管理口</h4><b>—</b></div>
<p class="address">{esc(bmc.get('note') or '未发现可读取的 BMC LAN 信息')}</p>
<p class="card-note">独立带外口通常不进入宿主 OS 网络命名空间;需要在 BMC/IPMI 或云厂商控制台侧验证。</p>
</article>
"""
def render_check(item: dict[str, str]) -> str:
icon = {"ok": "✓", "warning": "!", "critical": "×"}.get(item["level"], "·")
return f"""
<li class="check {esc(item['level'])}">
<span class="check-icon">{icon}</span>
<span><b>{esc(item['title'])}</b><small>{esc(item['detail'])}</small></span>
</li>
"""
def render_html(snapshot: dict[str, Any], expected: dict[str, int | None]) -> str:
health = evaluate(snapshot, expected)
system = snapshot.get("system", {})
gpus = snapshot.get("gpus", [])
bonds = snapshot.get("bonds", [])
ib_devices = snapshot.get("ib_devices", [])
network_endpoints = snapshot.get("network_endpoints", [])
endpoint_by_id = {
endpoint.get("id"): endpoint
for endpoint in network_endpoints
if endpoint.get("id")
}
lanes = "".join(
render_gpu_lane(
gpu, endpoint_by_id.get(gpu.get("primary_network_id"))
)
for gpu in gpus
)
numa_groups: list[tuple[Any, int, str]] = []
for gpu in gpus:
numa = gpu.get("numa")
cpu_range = str(gpu.get("cpu_affinity") or "—")
if (
not numa_groups
or numa_groups[-1][0] != numa
or numa_groups[-1][2] != cpu_range
):
numa_groups.append((numa, 1, cpu_range))
else:
previous_numa, previous_count, previous_cpu_range = numa_groups[-1]
numa_groups[-1] = (
previous_numa,
previous_count + 1,
previous_cpu_range,
)
numa_columns = " ".join(f"{count}fr" for _, count, _ in numa_groups) or "1fr"
numa_ruler = "".join(
f'<span data-numa="{esc(numa if numa is not None else "?")}" data-count="{count}">'
f"{esc(f'NUMA {numa}' if numa is not None else 'NUMA ?')} "
f"<small>推荐 CPU {esc(cpu_range)}</small></span>"
for numa, count, cpu_range in numa_groups
) or "<span>NUMA 未知</span>"
numa_values = list(
dict.fromkeys(
str(gpu.get("numa"))
for gpu in gpus
if gpu.get("numa") is not None
)
)
numa_switches = "".join(
f'<button type="button" class="numa-switch" data-numa-filter="{esc(numa)}" '
f'aria-pressed="false">NUMA {esc(numa)}</button>'
for numa in numa_values
)
rail_count = max(1, len(gpus))
rail_min_width = max(0, rail_count * 132 + max(0, rail_count - 1) * 10)
affinity_groups: dict[tuple[Any, str], list[int]] = {}
for gpu in gpus:
affinity_numa = gpu.get("numa_affinity")
key = (
affinity_numa if affinity_numa not in (None, "") else gpu.get("numa"),
str(gpu.get("cpu_affinity") or "—"),
)
affinity_groups.setdefault(key, []).append(int(gpu.get("index", 0)))
affinity_summary = "".join(
f"<span><b>{esc(compact_gpu_indices(indices))}</b><small>CPU {esc(cpu_range)} · MEM NUMA {esc(numa)}</small></span>"
for (numa, cpu_range), indices in affinity_groups.items()
)
ib_cards = "".join(render_ib_card(device) for device in ib_devices)
checks = "".join(render_check(item) for item in health["checks"])
accelerator_nic_names = {
member.get("name")
for endpoint in network_endpoints
for member in endpoint.get("members", [])
if member.get("name")
}
accelerator_nic_names.update(
endpoint.get("name")
for endpoint in network_endpoints
if endpoint.get("type") == "bond"
)
management_nics = [
nic
for nic in snapshot.get("nics", [])
if nic.get("name") not in accelerator_nic_names
and (
nic.get("bdf")
or nic.get("default_route")
or nic.get("is_bond")
)
]
management_cards = "".join(render_management_card(nic) for nic in management_nics)
bmc = snapshot.get("bmc") or {
"detected": False,
"note": "快照中没有 BMC/IPMI 信息",
}
management_cards += render_bmc_card(bmc)
total_memory = sum(number(gpu.get("memory_total_mib")) or 0 for gpu in gpus)
roce_mbps = sum(
number(endpoint.get("capacity_mbps")) or 0
for endpoint in network_endpoints
if endpoint.get("protocol") == "RoCE"
and str(endpoint.get("state") or "").upper() in {"UP", "ACTIVE"}
)
ib_gbps = sum(
number(device.get("rate_gbps")) or 0
for device in ib_devices
if str(device.get("state", "")).lower() == "active"
)
power_data_complete = bool(gpus) and all(
number(gpu.get("power_w")) is not None
and number(gpu.get("power_limit_w")) is not None
and bool(gpu.get("pstate"))
for gpu in gpus
)
total_power_w = (
sum(number(gpu.get("power_w")) or 0 for gpu in gpus)
if power_data_complete
else 0
)
total_power_limit_w = (
sum(number(gpu.get("power_limit_w")) or 0 for gpu in gpus)
if power_data_complete
else 0
)
power_pct = (
100 * total_power_w / total_power_limit_w
if total_power_limit_w
else 0
)
p0_count = sum(str(gpu.get("pstate", "")).upper() == "P0" for gpu in gpus)
nvlink = snapshot.get("nvlink", {})
software = snapshot.get("software", {})
generated = snapshot.get("generated_at", "")
memory_total_kib = number(system.get("memory_total_kib"))
memory_used_kib = number(system.get("memory_used_kib"))
memory_pct = (
100 * memory_used_kib / memory_total_kib
if memory_used_kib is not None and memory_total_kib
else 0
)
cpu_count = system.get("cpu_physical_cores") or system.get("cpu_logical") or "—"
cpu_count_label = "物理核心" if system.get("cpu_physical_cores") else "逻辑 CPU"
payload = json.dumps(snapshot, ensure_ascii=False).replace("</", "<\\/")
body = f"""
<header class="hero">
<div>
<span class="kicker">GPU NODE · HARDWARE PORTRAIT</span>
<h1>{esc(system.get('hostname') or 'GPU Node')}</h1>
<p>{esc(system.get('manufacturer'))} {esc(system.get('product'))} · {esc(system.get('os'))}</p>
</div>
<div class="hero-actions">
<span class="overall {esc(health['status'])}"><i></i>{esc(status_label(health['status']))}</span>
<button type="button" onclick="window.print()">打印 / PDF</button>
</div>
</header>
<main>
<section class="overview-grid" aria-label="节点概览">
<article class="overview-card system-overview">
<div class="overview-head">
<div><span class="kicker">SYSTEM / SOFTWARE</span><h2>系统与软件</h2></div>
<button class="ghost compact" type="button" data-open-raw>原始快照</button>
</div>
<p class="overview-lead">{esc(system.get('manufacturer'))} {esc(system.get('product'))}</p>
<dl class="overview-facts">
<div><dt>操作系统</dt><dd>{esc(system.get('os') or '—')}</dd></div>
<div><dt>内核</dt><dd>{esc(system.get('kernel') or '—')}</dd></div>
<div><dt>Driver</dt><dd>{esc(software.get('nvidia_driver') or '—')}</dd></div>
<div><dt>CUDA Toolkit</dt><dd>{esc(software.get('cuda_toolkit') or '—')} <small>nvcc</small></dd></div>
<div><dt>虚拟化</dt><dd>{esc(system.get('virtualization') or 'Bare metal / 未识别')}</dd></div>
<div><dt>生成时间</dt><dd>{esc(generated)}</dd></div>
</dl>
</article>
<article class="overview-card gpu-overview">
<div class="overview-head"><div><span class="kicker">ACCELERATOR</span><h2>GPU</h2></div>{badge("NVLINK ACTIVE" if nvlink.get('active_links') else "NVLINK 未知", "ok" if nvlink.get('active_links') else "warn")}</div>
<p class="overview-lead">{esc(gpus[0].get('name') if gpus else '未发现 NVIDIA GPU')}</p>
<div class="overview-primary"><b>{len(gpus)}</b><span>GPU</span></div>
<div class="gpu-power-summary">
<div><span>GPU 瞬时功耗</span><b>{esc(fmt_power(total_power_w) if power_data_complete else '遥测不完整')} / {esc(fmt_power(total_power_limit_w) if power_data_complete else '—')}</b></div>
<div class="power-meter"><i style="width:{max(0, min(100, power_pct)):.1f}%"></i></div>
<small>Power Limit 上限 · {p0_count}/{len(gpus)} 张处于 P0 · 单次瞬时采样</small>
</div>
<div class="overview-metrics">
<span><b>{esc(fmt_mib(total_memory))}</b><small>GPU 显存</small></span>
<span><b>{esc(fmt_rate(roce_mbps))}</b><small>RoCE 物理容量</small></span>
<span><b>{esc(nvlink.get('links_per_gpu') or '—')}</b><small>NVLink / GPU</small></span>
<span><b>{ib_gbps:g}G</b><small>InfiniBand</small></span>
</div>
</article>
<article class="overview-card cpu-overview">
<div class="overview-head"><div><span class="kicker">HOST COMPUTE</span><h2>CPU 与内存</h2></div>{badge(f"{system.get('numa_nodes') or '—'} NUMA", "neutral")}</div>
<p class="overview-lead">{esc(system.get('cpu_model') or '未知 CPU')}</p>
<div class="overview-primary"><b>{esc(cpu_count)}</b><span>{esc(cpu_count_label)}</span></div>
<div class="memory-row"><span>系统内存</span><b>{esc(fmt_kib(memory_used_kib))} / {esc(fmt_kib(memory_total_kib))}</b></div>
<div class="memory-meter"><i style="width:{max(0, min(100, memory_pct)):.1f}%"></i></div>
<div class="overview-metrics cpu-metrics">
<span><b>{esc(system.get('cpu_sockets') or '—')}</b><small>CPU Sockets</small></span>
<span><b>{esc(system.get('cpu_logical') or '—')}</b><small>逻辑 CPU</small></span>
<span><b>{memory_pct:.1f}%</b><small>内存使用率</small></span>
</div>
<div class="cpu-affinity-summary">
{affinity_summary or '<span><b>未解析</b><small>CPU / NUMA 推荐亲和</small></span>'}
</div>
</article>
</section>
<section class="panel topology-panel">
<div class="section-head">
<div><span class="kicker">TOPOLOGY</span><h2>GPU ↔ RDMA 亲和画像</h2></div>
<div class="legend">
{badge("PIX 最佳", "ok")}{badge("PXB / PHB", "warn")}{badge("SYS 跨 NUMA", "bad")}
</div>
</div>
<div class="topology-viewbar">
<div>
<b>NUMA 视图</b>
<small>总览显示完整拓扑,分览聚焦单个 CPU / 内存域</small>
</div>
<div class="numa-switcher" role="group" aria-label="NUMA 拓扑视图">
<button type="button" class="numa-switch active" data-numa-filter="all" aria-pressed="true">总览</button>
{numa_switches}
</div>
</div>
<div class="numa-ruler" style="grid-template-columns:{esc(numa_columns)}">
{numa_ruler}
</div>
<p class="affinity-note">CPU范围来自 NVIDIA 拓扑亲和,表示更近的CPU/内存路径;不代表当前训练进程已经绑核。</p>
<div class="nvlink-fabric">
<span>NVLINK / NVSWITCH FABRIC</span>
<b>{esc(nvlink.get('active_links') or '—')} Active Links</b>
<small>{esc(nvlink.get('speed_gbs') or '—')} GB/s / Link</small>
</div>
<div class="rails" style="grid-template-columns:repeat({rail_count},minmax(132px,1fr));min-width:{rail_min_width}px">{lanes or '<div class="empty-state">未采集到 GPU 数据</div>'}</div>
<p class="capacity-note">有 Bond 时展示聚合容器与物理成员;无 Bond 时展示独立 RoCE/IB 端点。LACP 容量不代表单连接带宽。</p>
</section>
<div class="network-grid">
<section class="panel">
<div class="section-head">
<div><span class="kicker">STORAGE / FABRIC</span><h2>InfiniBand</h2></div>
</div>
<div class="ib-grid">{ib_cards or '<div class="empty-state">未发现独立 InfiniBand 设备</div>'}</div>
</section>
<section class="panel">
<div class="section-head">
<div><span class="kicker">MANAGEMENT NETWORK</span><h2>带内与带外管理网络</h2></div>
</div>
<div class="management-grid">{management_cards}</div>
</section>
</div>
<section class="panel health-panel">
<div class="section-head">
<div><span class="kicker">HEALTH / VALIDATION</span><h2>HEALTH 与验证结果</h2></div>
<span class="overall {esc(health['status'])}"><i></i>{esc(status_label(health['status']))}</span>
</div>
<ul class="checks">{checks}</ul>
</section>
</main>
<dialog id="raw-dialog">
<div class="dialog-head"><div><span class="kicker">RAW DATA</span><h2>JSON Snapshot</h2></div><button type="button" data-close-raw>关闭</button></div>
<pre>{esc(json.dumps(snapshot, ensure_ascii=False, indent=2))}</pre>
</dialog>
<footer>
<span>GPU Node Profile v{esc(VERSION)}</span>
<span>只读采集 · 原生 HTML · 无外部依赖</span>
</footer>
<script type="application/json" id="snapshot">{payload}</script>
"""
title = f"{system.get('hostname') or 'GPU Node'} · GPU Hardware Portrait"
template = load_html_template("gpu_node_profile.html")
return clean_generated_text(
template.replace("@@TITLE@@", esc(title)).replace("@@BODY@@", body)
)
def clean_generated_text(content: str) -> str:
"""Remove template indentation residue without changing line structure."""
cleaned = "\n".join(line.rstrip() for line in content.splitlines())
return cleaned + ("\n" if content.endswith("\n") else "")
def load_html_template(filename: str) -> str:
"""Load a renderer template beside the collector script."""
template_path = Path(__file__).resolve().parent / "templates" / filename
try:
return template_path.read_text(encoding="utf-8")
except OSError as exc:
raise OSError(
f"HTML template not found: {template_path}. "
"Keep gpu_node_profile.py and templates/ together."
) from exc
def render_topology_3d_html(snapshot: dict[str, Any]) -> str:
payload = json.dumps(snapshot, ensure_ascii=False).replace("</", "<\\/")
system = snapshot.get("system", {})
title = f"{system.get('hostname') or 'GPU Node'} · 3D Topology CT"
template = load_html_template("gpu_topology_3d.html")
return clean_generated_text(
template.replace("@@TITLE@@", esc(title))
.replace("@@MACHINE_DATA@@", payload)
.replace("@@VERSION@@", esc(VERSION))
)
def atomic_write(path: Path, content: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
try:
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
handle.write(content)
os.replace(temp_name, path)
except Exception:
try:
os.unlink(temp_name)
except OSError:
pass
raise
def load_snapshot(path: Path) -> dict[str, Any]:
data = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(data, dict) or data.get("schema_version") != 1:
raise ValueError("snapshot schema_version must be 1")
nvlink = data.setdefault("nvlink", {})
nvlink.setdefault(
"fabric_type",
infer_nvlink_fabric_type(data.get("topology", {}), len(data.get("gpus", []))),
)
attach_gpu_bond_mapping(data)
return data
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Generate a native, self-contained HTML portrait for a GPU node."
)
parser.add_argument("-o", "--output", type=Path, default=Path("gpu_node_profile.html"))
parser.add_argument(
"--topology-output",
type=Path,
help="Generate the standalone 3D topology CT HTML from the same snapshot",
)
parser.add_argument("--snapshot-in", type=Path, help="Render an existing JSON snapshot")
parser.add_argument("--snapshot-out", type=Path, help="Save collected JSON snapshot")
parser.add_argument("--expected-gpus", type=int)
parser.add_argument("--expected-bonds", type=int)
parser.add_argument("--expected-ib", type=int)
output_group = parser.add_mutually_exclusive_group()
output_group.add_argument("--stdout", action="store_true", help="Write operations HTML to stdout")
output_group.add_argument(
"--snapshot-stdout",
action="store_true",
help="Write collected structured JSON to stdout and skip HTML rendering",
)
parser.add_argument("--version", action="version", version=f"%(prog)s {VERSION}")
return parser
def main(argv: Iterable[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
snapshot = load_snapshot(args.snapshot_in) if args.snapshot_in else collect_snapshot()
if args.snapshot_stdout:
sys.stdout.write(json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n")
return 0
if args.snapshot_out:
atomic_write(
args.snapshot_out.expanduser().resolve(),
json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n",
)
expected = {
"gpus": args.expected_gpus,
"bonds": args.expected_bonds,
"ib": args.expected_ib,
}
report = render_html(snapshot, expected)
if args.stdout:
sys.stdout.write(report)
else:
output = args.output.expanduser().resolve()
atomic_write(output, report)
print(f"GPU node profile generated: {output}")
if args.topology_output:
topology_output = args.topology_output.expanduser().resolve()
atomic_write(topology_output, render_topology_3d_html(snapshot))
if not args.stdout:
print(f"GPU 3D topology generated: {topology_output}")
except (OSError, ValueError, json.JSONDecodeError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
return 0
if __name__ == "__main__":
raise SystemExit(main())