- 新增只读 GPU 节点采集器,以统一 JSON 描述系统、CPU/NUMA、GPU、NVLink、PCIe、RDMA、网络与功耗信息 - 统一 Bond、独立 RoCE 和 InfiniBand 端点模型,支持 GPU 到 RDMA 端点的全局最优唯一匹配 - 亲和关系严格记录 nvidia-smi topo -m 证据,未暴露的 IB 设备不推测 GPU 连线 - 新增原生 HTML 运维画像和 Canvas 3D CT,可按 NUMA、PCIe 分组及 Bond 逐层查看物理成员端口 - 展示 CPU 亲和、P-State、当前功耗与 Power Limit,并提供 RDMA、Bond、IB、管理网络及整机健康检查 - 排除 CNI/veth 等虚拟接口对管理网络画像的干扰,普通管理 Bond 不计入 RDMA Bond 验收 - 提供脱敏 B300 真机示例、离线自包含页面、使用文档及 17 项解析和渲染回归测试 - 修正生成物忽略规则,确保两个 HTML 源模板能够随工具提交
2156 lines
79 KiB
Python
Executable File
2156 lines
79 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Generate a self-contained, native HTML profile for a GPU node.
|
||
|
||
The collector only executes read-only commands. The generated report has no
|
||
Mermaid, JavaScript framework, external font, or CDN dependency.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import datetime as dt
|
||
import html
|
||
import json
|
||
import os
|
||
import platform
|
||
import re
|
||
import shutil
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
from pathlib import Path
|
||
from typing import Any, Iterable
|
||
|
||
|
||
VERSION = "2.1.0"
|
||
ANSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
|
||
PCI_RE = re.compile(r"(?:00000000:|0000:)?([0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7])")
|
||
|
||
|
||
def run(command: list[str], timeout: int = 12) -> str:
|
||
"""Run a read-only command and return stdout without raising."""
|
||
if not command or shutil.which(command[0]) is None:
|
||
return ""
|
||
try:
|
||
result = subprocess.run(
|
||
command,
|
||
check=False,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=timeout,
|
||
env={**os.environ, "LC_ALL": "C"},
|
||
)
|
||
except (OSError, subprocess.SubprocessError):
|
||
return ""
|
||
return ANSI_RE.sub("", result.stdout).strip()
|
||
|
||
|
||
def read_text(path: str | Path) -> str:
|
||
try:
|
||
return Path(path).read_text(encoding="utf-8", errors="replace").strip()
|
||
except OSError:
|
||
return ""
|
||
|
||
|
||
def read_int(path: str | Path) -> int | None:
|
||
value = read_text(path)
|
||
try:
|
||
return int(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def normalize_bdf(value: str) -> str:
|
||
matches = PCI_RE.findall(value or "")
|
||
return matches[-1].lower() if matches else ""
|
||
|
||
|
||
def pci_numa(bdf: str) -> int | None:
|
||
bdf = normalize_bdf(bdf)
|
||
return read_int(f"/sys/bus/pci/devices/0000:{bdf}/numa_node") if bdf else None
|
||
|
||
|
||
def pci_path(bdf: str) -> list[str]:
|
||
"""Return the guest-visible PCI ancestor path ending at the endpoint."""
|
||
bdf = normalize_bdf(bdf)
|
||
if not bdf:
|
||
return []
|
||
try:
|
||
resolved = str(Path(f"/sys/bus/pci/devices/0000:{bdf}").resolve())
|
||
except OSError:
|
||
return [bdf]
|
||
path: list[str] = []
|
||
for match in PCI_RE.findall(resolved):
|
||
normalized = normalize_bdf(match)
|
||
if normalized and normalized not in path:
|
||
path.append(normalized)
|
||
if bdf not in path:
|
||
path.append(bdf)
|
||
return path
|
||
|
||
|
||
def dmi_value(name: str) -> str:
|
||
return read_text(f"/sys/class/dmi/id/{name}")
|
||
|
||
|
||
def parse_csv_rows(text: str) -> list[list[str]]:
|
||
if not text:
|
||
return []
|
||
return [[field.strip() for field in row] for row in csv.reader(text.splitlines())]
|
||
|
||
|
||
def collect_gpus() -> list[dict[str, Any]]:
|
||
core_fields = [
|
||
"index",
|
||
"name",
|
||
"uuid",
|
||
"pci.bus_id",
|
||
"driver_version",
|
||
"memory.total",
|
||
"memory.used",
|
||
"temperature.gpu",
|
||
"utilization.gpu",
|
||
]
|
||
output = run(
|
||
[
|
||
"nvidia-smi",
|
||
f"--query-gpu={','.join(core_fields)}",
|
||
"--format=csv,noheader,nounits",
|
||
]
|
||
)
|
||
power_fields = [
|
||
"index",
|
||
"pstate",
|
||
"power.draw",
|
||
"power.limit",
|
||
]
|
||
power_output = run(
|
||
[
|
||
"nvidia-smi",
|
||
f"--query-gpu={','.join(power_fields)}",
|
||
"--format=csv,noheader,nounits",
|
||
]
|
||
)
|
||
power_by_index: dict[int, dict[str, Any]] = {}
|
||
for row in parse_csv_rows(power_output):
|
||
if len(row) < len(power_fields) or not row[0].isdigit():
|
||
continue
|
||
power_by_index[int(row[0])] = {
|
||
"pstate": row[1] if row[1] not in {"", "N/A", "[N/A]"} else "",
|
||
"power_w": number(row[2]),
|
||
"power_limit_w": number(row[3]),
|
||
}
|
||
range_fields = [
|
||
"index",
|
||
"power.default_limit",
|
||
"power.min_limit",
|
||
"power.max_limit",
|
||
]
|
||
range_output = run(
|
||
[
|
||
"nvidia-smi",
|
||
f"--query-gpu={','.join(range_fields)}",
|
||
"--format=csv,noheader,nounits",
|
||
]
|
||
)
|
||
for row in parse_csv_rows(range_output):
|
||
if len(row) < len(range_fields) or not row[0].isdigit():
|
||
continue
|
||
power_by_index.setdefault(int(row[0]), {}).update(
|
||
{
|
||
"power_default_limit_w": number(row[1]),
|
||
"power_min_limit_w": number(row[2]),
|
||
"power_max_limit_w": number(row[3]),
|
||
}
|
||
)
|
||
gpus: list[dict[str, Any]] = []
|
||
for row in parse_csv_rows(output):
|
||
if len(row) < len(core_fields):
|
||
continue
|
||
bdf = normalize_bdf(row[3])
|
||
index = int(row[0]) if row[0].isdigit() else len(gpus)
|
||
gpus.append(
|
||
{
|
||
"index": index,
|
||
"name": row[1],
|
||
"uuid": row[2],
|
||
"bdf": bdf,
|
||
"driver": row[4],
|
||
"memory_total_mib": number(row[5]),
|
||
"memory_used_mib": number(row[6]),
|
||
"temperature_c": number(row[7]),
|
||
"utilization_pct": number(row[8]),
|
||
"numa": pci_numa(bdf),
|
||
"pci_path": pci_path(bdf),
|
||
**power_by_index.get(index, {}),
|
||
}
|
||
)
|
||
return sorted(gpus, key=lambda item: item["index"])
|
||
|
||
|
||
def number(value: Any) -> float | None:
|
||
try:
|
||
return float(str(value).strip())
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def parse_topology(text: str) -> dict[str, Any]:
|
||
"""Parse nvidia-smi topo -m into GPU relationships and NIC legend."""
|
||
text = ANSI_RE.sub("", text or "")
|
||
lines = [line.rstrip() for line in text.splitlines()]
|
||
header: list[str] = []
|
||
rows: dict[str, dict[str, str]] = {}
|
||
nic_legend: dict[str, str] = {}
|
||
gpu_affinity: dict[str, dict[str, str]] = {}
|
||
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
if not stripped:
|
||
continue
|
||
legend_match = re.match(r"(NIC\d+):\s+(\S+)", stripped)
|
||
if legend_match:
|
||
nic_legend[legend_match.group(1)] = legend_match.group(2)
|
||
continue
|
||
fields = re.split(r"\s+", stripped)
|
||
if not header and fields and fields[0] == "GPU0":
|
||
# Only retain the square GPU/NIC matrix. Labels such as
|
||
# "CPU Affinity" contain spaces while row values do not, so keeping
|
||
# them in the same whitespace-split header would shift columns.
|
||
matrix_end = fields.index("CPU") if "CPU" in fields else len(fields)
|
||
header = fields[:matrix_end]
|
||
continue
|
||
if header and fields and re.fullmatch(r"GPU\d+", fields[0]):
|
||
matrix_end = 1 + len(header)
|
||
values = fields[1:matrix_end]
|
||
gpu_name = fields[0]
|
||
rows[gpu_name] = dict(zip(header, values))
|
||
suffix = fields[matrix_end:]
|
||
gpu_affinity[gpu_name] = {
|
||
"cpu_affinity": suffix[0] if len(suffix) > 0 else "",
|
||
"numa_affinity": suffix[1] if len(suffix) > 1 else "",
|
||
"gpu_numa_id": suffix[2] if len(suffix) > 2 else "",
|
||
}
|
||
|
||
gpu_nic: dict[str, list[dict[str, str]]] = {}
|
||
for gpu_name, relations in rows.items():
|
||
links: list[dict[str, str]] = []
|
||
for column, relation in relations.items():
|
||
if column.startswith("NIC"):
|
||
links.append(
|
||
{
|
||
"slot": column,
|
||
"rdma": nic_legend.get(column, column),
|
||
"relation": relation,
|
||
}
|
||
)
|
||
gpu_nic[gpu_name] = links
|
||
return {
|
||
"rows": rows,
|
||
"nic_legend": nic_legend,
|
||
"gpu_nic": gpu_nic,
|
||
"gpu_affinity": gpu_affinity,
|
||
}
|
||
|
||
|
||
def parse_nvlink_status(text: str) -> dict[str, Any]:
|
||
current = ""
|
||
links: dict[str, list[float]] = {}
|
||
for raw_line in (text or "").splitlines():
|
||
line = raw_line.strip()
|
||
gpu_match = re.match(r"GPU\s+(\d+):", line)
|
||
if gpu_match:
|
||
current = f"GPU{gpu_match.group(1)}"
|
||
links[current] = []
|
||
continue
|
||
link_match = re.match(r"Link\s+\d+:\s+([\d.]+)\s+GB/s", line)
|
||
if current and link_match:
|
||
links[current].append(float(link_match.group(1)))
|
||
counts = [len(values) for values in links.values()]
|
||
speeds = [speed for values in links.values() for speed in values]
|
||
return {
|
||
"per_gpu": links,
|
||
"active_links": sum(counts),
|
||
"links_per_gpu": min(counts) if counts else 0,
|
||
"speed_gbs": min(speeds) if speeds else None,
|
||
}
|
||
|
||
|
||
def infer_nvlink_fabric_type(topology: dict[str, Any], gpu_count: int) -> str:
|
||
"""Avoid calling every NVLink configuration an NVSwitch fabric."""
|
||
if gpu_count < 2:
|
||
return ""
|
||
nv_pairs = 0
|
||
for row_name, relations in topology.get("rows", {}).items():
|
||
if not str(row_name).startswith("GPU"):
|
||
continue
|
||
nv_pairs += sum(
|
||
str(peer).startswith("GPU") and str(relation).startswith("NV")
|
||
for peer, relation in relations.items()
|
||
)
|
||
if gpu_count > 2 and nv_pairs >= gpu_count * (gpu_count - 1):
|
||
return "NVSwitch Fabric"
|
||
return "NVLink Fabric"
|
||
|
||
|
||
def parse_bond_text(name: str, text: str) -> dict[str, Any]:
|
||
bond: dict[str, Any] = {
|
||
"name": name,
|
||
"mode": "",
|
||
"hash_policy": "",
|
||
"status": "",
|
||
"slaves": [],
|
||
}
|
||
current: dict[str, Any] | None = None
|
||
for raw_line in (text or "").splitlines():
|
||
line = raw_line.strip()
|
||
if line.startswith("Bonding Mode:"):
|
||
bond["mode"] = line.split(":", 1)[1].strip()
|
||
elif line.startswith("Transmit Hash Policy:"):
|
||
bond["hash_policy"] = line.split(":", 1)[1].strip()
|
||
elif line.startswith("MII Status:") and current is None:
|
||
bond["status"] = line.split(":", 1)[1].strip().lower()
|
||
elif line.startswith("Slave Interface:"):
|
||
current = {"name": line.split(":", 1)[1].strip()}
|
||
bond["slaves"].append(current)
|
||
elif current is not None and ":" in line:
|
||
key, value = [part.strip() for part in line.split(":", 1)]
|
||
if key == "MII Status":
|
||
current["status"] = value.lower()
|
||
elif key == "Speed":
|
||
current["speed_mbps"] = int(value.split()[0]) if value.split()[0].isdigit() else None
|
||
elif key == "Duplex":
|
||
current["duplex"] = value
|
||
elif key == "Link Failure Count":
|
||
current["failures"] = int(value) if value.isdigit() else None
|
||
elif key == "Permanent HW addr":
|
||
current["mac"] = value
|
||
return bond
|
||
|
||
|
||
def parse_ibdev2netdev(text: str) -> tuple[dict[str, str], dict[str, str]]:
|
||
rdma_to_net: dict[str, str] = {}
|
||
net_to_rdma: dict[str, str] = {}
|
||
pattern = re.compile(r"(\S+)\s+port\s+\d+\s+==>\s+(\S+)\s+\(([^)]+)\)")
|
||
for line in (text or "").splitlines():
|
||
match = pattern.search(line)
|
||
if match:
|
||
rdma, netdev = match.group(1), match.group(2)
|
||
rdma_to_net[rdma] = netdev
|
||
net_to_rdma[netdev] = rdma
|
||
return rdma_to_net, net_to_rdma
|
||
|
||
|
||
def parse_rdma_links(text: str) -> dict[str, dict[str, Any]]:
|
||
links: dict[str, dict[str, Any]] = {}
|
||
for line in (text or "").splitlines():
|
||
match = re.match(r"link\s+([^/]+)/(\d+)\s+(.*)", line.strip())
|
||
if not match:
|
||
continue
|
||
name, port, tail = match.groups()
|
||
links[name] = {
|
||
"port": int(port),
|
||
"state": capture_token(tail, "state"),
|
||
"physical_state": capture_token(tail, "physical_state"),
|
||
"netdev": capture_token(tail, "netdev"),
|
||
}
|
||
return links
|
||
|
||
|
||
def capture_token(text: str, key: str) -> str:
|
||
match = re.search(rf"(?:^|\s){re.escape(key)}\s+(\S+)", text)
|
||
return match.group(1) if match else ""
|
||
|
||
|
||
def parse_ibstat(text: str) -> list[dict[str, Any]]:
|
||
devices: list[dict[str, Any]] = []
|
||
current: dict[str, Any] | None = None
|
||
in_port = False
|
||
for raw_line in (text or "").splitlines():
|
||
line = raw_line.strip()
|
||
ca_match = re.match(r"CA '([^']+)'", line)
|
||
if ca_match:
|
||
current = {"name": ca_match.group(1)}
|
||
devices.append(current)
|
||
in_port = False
|
||
continue
|
||
if current is None:
|
||
continue
|
||
if line.startswith("CA type:"):
|
||
current["ca_type"] = line.split(":", 1)[1].strip()
|
||
elif line.startswith("Firmware version:"):
|
||
current["firmware"] = line.split(":", 1)[1].strip()
|
||
elif line.startswith("Port 1:"):
|
||
in_port = True
|
||
elif in_port and line.startswith("State:"):
|
||
current["state"] = line.split(":", 1)[1].strip()
|
||
elif in_port and line.startswith("Physical state:"):
|
||
current["physical_state"] = line.split(":", 1)[1].strip()
|
||
elif in_port and line.startswith("Rate:"):
|
||
current["rate_gbps"] = number(line.split(":", 1)[1].strip())
|
||
elif in_port and line.startswith("Base lid:"):
|
||
current["lid"] = line.split(":", 1)[1].strip()
|
||
elif in_port and line.startswith("Link layer:"):
|
||
current["link_layer"] = line.split(":", 1)[1].strip()
|
||
return devices
|
||
|
||
|
||
def collect_nics() -> list[dict[str, Any]]:
|
||
base = Path("/sys/class/net")
|
||
if not base.exists():
|
||
return []
|
||
nics: list[dict[str, Any]] = []
|
||
for entry in sorted(base.iterdir(), key=lambda item: natural_key(item.name)):
|
||
name = entry.name
|
||
if name == "lo":
|
||
continue
|
||
is_bond = (entry / "bonding").exists()
|
||
driver_info = run(["ethtool", "-i", name])
|
||
driver = field_from_lines(driver_info, "driver") or ("bonding" if is_bond else "")
|
||
bus = normalize_bdf(field_from_lines(driver_info, "bus-info"))
|
||
if not driver and not bus:
|
||
continue
|
||
speed_text = field_from_lines(run(["ethtool", name]), "Speed")
|
||
speed_match = re.search(r"(\d+)", speed_text)
|
||
nics.append(
|
||
{
|
||
"name": name,
|
||
"driver": driver,
|
||
"bdf": bus,
|
||
"numa": pci_numa(bus),
|
||
"pci_path": pci_path(bus),
|
||
"speed_mbps": int(speed_match.group(1)) if speed_match else None,
|
||
"state": read_text(entry / "operstate").lower(),
|
||
"carrier": read_int(entry / "carrier"),
|
||
"mac": read_text(entry / "address"),
|
||
"is_bond": is_bond,
|
||
}
|
||
)
|
||
attach_network_context(nics)
|
||
return nics
|
||
|
||
|
||
def attach_network_context(nics: list[dict[str, Any]]) -> None:
|
||
"""Attach addresses and default-route roles to OS-visible interfaces."""
|
||
addresses: dict[str, list[str]] = {}
|
||
defaults: set[str] = set()
|
||
try:
|
||
address_rows = json.loads(run(["ip", "-j", "address", "show"]) or "[]")
|
||
for row in address_rows:
|
||
values = []
|
||
for item in row.get("addr_info", []):
|
||
local = item.get("local")
|
||
prefix = item.get("prefixlen")
|
||
if local and item.get("scope") != "link":
|
||
values.append(f"{local}/{prefix}" if prefix is not None else str(local))
|
||
addresses[str(row.get("ifname", ""))] = values
|
||
route_rows = json.loads(run(["ip", "-j", "route", "show", "default"]) or "[]")
|
||
defaults = {str(row.get("dev")) for row in route_rows if row.get("dev")}
|
||
except (TypeError, ValueError, json.JSONDecodeError):
|
||
pass
|
||
for nic in nics:
|
||
name = nic["name"]
|
||
nic["addresses"] = addresses.get(name, [])
|
||
nic["default_route"] = name in defaults
|
||
|
||
|
||
def collect_bmc() -> dict[str, Any]:
|
||
"""Read common BMC LAN channels when IPMI is locally available."""
|
||
if shutil.which("ipmitool") is None:
|
||
return {
|
||
"detected": False,
|
||
"note": "独立 BMC 带外口未向当前 OS/IPMI 通道暴露",
|
||
}
|
||
last_values: dict[str, Any] = {}
|
||
for channel in (1, 8):
|
||
output = run(["ipmitool", "lan", "print", str(channel)], timeout=8)
|
||
values: dict[str, Any] = {"channel": channel}
|
||
wanted = {
|
||
"IP Address Source": "source",
|
||
"IP Address": "ip",
|
||
"Subnet Mask": "subnet",
|
||
"MAC Address": "mac",
|
||
"Default Gateway IP": "gateway",
|
||
"802.1q VLAN ID": "vlan",
|
||
}
|
||
for line in output.splitlines():
|
||
if ":" not in line:
|
||
continue
|
||
key, value = [part.strip() for part in line.split(":", 1)]
|
||
if key in wanted:
|
||
values[wanted[key]] = value
|
||
last_values = values
|
||
if values.get("ip") and values["ip"] not in {"0.0.0.0", "0.0.0.0/0"}:
|
||
return {"detected": True, **values, "note": ""}
|
||
return {
|
||
"detected": False,
|
||
**last_values,
|
||
"note": "IPMI 可访问,但常用 LAN 通道未读取到有效地址",
|
||
}
|
||
|
||
|
||
def field_from_lines(text: str, field: str) -> str:
|
||
prefix = f"{field}:"
|
||
for line in (text or "").splitlines():
|
||
if line.strip().startswith(prefix):
|
||
return line.split(":", 1)[1].strip()
|
||
return ""
|
||
|
||
|
||
def natural_key(value: str) -> list[Any]:
|
||
return [int(part) if part.isdigit() else part for part in re.split(r"(\d+)", value)]
|
||
|
||
|
||
def collect_bonds(net_to_rdma: dict[str, str], nics: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
nic_map = {nic["name"]: nic for nic in nics}
|
||
bond_dir = Path("/proc/net/bonding")
|
||
if not bond_dir.exists():
|
||
return []
|
||
bonds: list[dict[str, Any]] = []
|
||
for path in sorted(bond_dir.iterdir(), key=lambda item: natural_key(item.name)):
|
||
bond = parse_bond_text(path.name, read_text(path))
|
||
bond["rdma"] = net_to_rdma.get(path.name, "")
|
||
for slave in bond["slaves"]:
|
||
slave.update(
|
||
{
|
||
key: value
|
||
for key, value in nic_map.get(slave["name"], {}).items()
|
||
if key in {"bdf", "numa", "driver", "carrier", "pci_path"}
|
||
}
|
||
)
|
||
bonds.append(bond)
|
||
return bonds
|
||
|
||
|
||
def build_network_endpoints(
|
||
nics: list[dict[str, Any]],
|
||
bonds: list[dict[str, Any]],
|
||
ib_devices: list[dict[str, Any]],
|
||
rdma_to_net: dict[str, str],
|
||
rdma_links: dict[str, dict[str, Any]] | None = None,
|
||
) -> list[dict[str, Any]]:
|
||
"""Normalize bonded RoCE, standalone RoCE and native IB as RDMA endpoints."""
|
||
rdma_links = rdma_links or {}
|
||
nic_by_name = {str(nic.get("name")): nic for nic in nics}
|
||
consumed_rdma: set[str] = set()
|
||
consumed_netdevs: set[str] = set()
|
||
endpoints: list[dict[str, Any]] = []
|
||
|
||
for bond in bonds:
|
||
rdma = str(bond.get("rdma") or "")
|
||
# A host may also use ordinary management/storage bonds. Only a bond
|
||
# exposed by the RDMA stack belongs in the accelerator topology.
|
||
if not rdma:
|
||
continue
|
||
members = [dict(member) for member in bond.get("slaves", [])]
|
||
consumed_rdma.add(rdma)
|
||
consumed_netdevs.update(str(member.get("name")) for member in members)
|
||
endpoint = {
|
||
"id": f"bond:{bond.get('name')}",
|
||
"type": "bond",
|
||
"protocol": "RoCE",
|
||
"name": bond.get("name"),
|
||
"rdma": rdma,
|
||
"role": "compute",
|
||
"state": str(bond.get("status") or "").upper(),
|
||
"numa": next(
|
||
(
|
||
member.get("numa")
|
||
for member in members
|
||
if member.get("numa") is not None
|
||
),
|
||
None,
|
||
),
|
||
"capacity_mbps": sum(
|
||
number(member.get("speed_mbps")) or 0 for member in members
|
||
),
|
||
"members": members,
|
||
"gpu_links": [],
|
||
}
|
||
endpoints.append(endpoint)
|
||
|
||
for device in ib_devices:
|
||
rdma = str(device.get("name") or "")
|
||
consumed_rdma.add(rdma)
|
||
endpoints.append(
|
||
{
|
||
"id": f"ib:{rdma}",
|
||
"type": "standalone",
|
||
"protocol": "InfiniBand",
|
||
"name": rdma,
|
||
"rdma": rdma,
|
||
"role": "storage",
|
||
"state": str(device.get("state") or "").upper(),
|
||
"physical_state": str(device.get("physical_state") or ""),
|
||
"numa": device.get("numa"),
|
||
"bdf": device.get("bdf") or "",
|
||
"pci_path": device.get("pci_path") or [],
|
||
"capacity_mbps": (
|
||
(number(device.get("rate_gbps")) or 0) * 1000
|
||
),
|
||
"members": [
|
||
{
|
||
"name": rdma,
|
||
"rdma": rdma,
|
||
"bdf": device.get("bdf") or "",
|
||
"numa": device.get("numa"),
|
||
"pci_path": device.get("pci_path") or [],
|
||
"speed_mbps": (
|
||
(number(device.get("rate_gbps")) or 0) * 1000
|
||
),
|
||
"status": str(device.get("state") or "").lower(),
|
||
}
|
||
],
|
||
"gpu_links": [],
|
||
}
|
||
)
|
||
|
||
for rdma, netdev in sorted(
|
||
rdma_to_net.items(), key=lambda item: natural_key(item[0])
|
||
):
|
||
if rdma in consumed_rdma or netdev in consumed_netdevs:
|
||
continue
|
||
nic = nic_by_name.get(netdev)
|
||
if (
|
||
not nic
|
||
or not nic.get("bdf")
|
||
or nic.get("default_route")
|
||
or nic.get("is_bond")
|
||
):
|
||
continue
|
||
rdma_state = rdma_links.get(rdma, {})
|
||
state = str(rdma_state.get("state") or nic.get("state") or "").upper()
|
||
member = {
|
||
**nic,
|
||
"rdma": rdma,
|
||
"status": str(nic.get("state") or "").lower(),
|
||
}
|
||
endpoints.append(
|
||
{
|
||
"id": f"rdma:{rdma}",
|
||
"type": "standalone",
|
||
"protocol": "RoCE",
|
||
"name": netdev,
|
||
"rdma": rdma,
|
||
"role": "compute",
|
||
"state": state,
|
||
"physical_state": str(
|
||
rdma_state.get("physical_state") or ""
|
||
),
|
||
"numa": nic.get("numa"),
|
||
"bdf": nic.get("bdf") or "",
|
||
"pci_path": nic.get("pci_path") or [],
|
||
"capacity_mbps": number(nic.get("speed_mbps")) or 0,
|
||
"members": [member],
|
||
"gpu_links": [],
|
||
}
|
||
)
|
||
|
||
return sorted(
|
||
endpoints,
|
||
key=lambda endpoint: (
|
||
0 if endpoint.get("role") == "compute" else 1,
|
||
str(endpoint.get("numa")),
|
||
natural_key(str(endpoint.get("name") or "")),
|
||
),
|
||
)
|
||
|
||
|
||
def parse_lspci_devices(text: str) -> dict[str, dict[str, str]]:
|
||
devices: dict[str, dict[str, str]] = {}
|
||
pattern = re.compile(r"^[0-9a-fA-F]{4}:([0-9a-fA-F:.]+)\s+(.+)$")
|
||
for line in (text or "").splitlines():
|
||
match = pattern.match(line.strip())
|
||
if not match:
|
||
continue
|
||
bdf, description = match.groups()
|
||
devices[normalize_bdf(bdf)] = {
|
||
"bdf": normalize_bdf(bdf),
|
||
"description": description,
|
||
}
|
||
return devices
|
||
|
||
|
||
def parse_lspci_network(text: str) -> dict[str, dict[str, str]]:
|
||
return {
|
||
bdf: device
|
||
for bdf, device in parse_lspci_devices(text).items()
|
||
if re.search(
|
||
r"Ethernet|Infiniband|Network controller",
|
||
device.get("description", ""),
|
||
re.I,
|
||
)
|
||
}
|
||
|
||
|
||
def build_pci_fabric(
|
||
gpus: list[dict[str, Any]],
|
||
nics: list[dict[str, Any]],
|
||
ib_devices: list[dict[str, Any]],
|
||
pci_devices: dict[str, dict[str, str]],
|
||
) -> dict[str, Any]:
|
||
"""Build a device-neutral PCI graph for both 2D and 3D renderers."""
|
||
endpoints: list[dict[str, Any]] = []
|
||
seen: set[tuple[str, str]] = set()
|
||
|
||
def add_endpoint(
|
||
kind: str,
|
||
label: str,
|
||
bdf: str,
|
||
numa: Any,
|
||
path: list[str] | None,
|
||
) -> None:
|
||
bdf = normalize_bdf(bdf)
|
||
key = (kind, bdf)
|
||
if not bdf or key in seen:
|
||
return
|
||
seen.add(key)
|
||
endpoints.append(
|
||
{
|
||
"kind": kind,
|
||
"label": label,
|
||
"bdf": bdf,
|
||
"numa": numa,
|
||
"path": path or pci_path(bdf),
|
||
"description": pci_devices.get(bdf, {}).get("description", ""),
|
||
}
|
||
)
|
||
|
||
for gpu in gpus:
|
||
add_endpoint(
|
||
"gpu",
|
||
f"GPU{gpu.get('index')}",
|
||
str(gpu.get("bdf") or ""),
|
||
gpu.get("numa"),
|
||
gpu.get("pci_path"),
|
||
)
|
||
for nic in nics:
|
||
add_endpoint(
|
||
"nic",
|
||
str(nic.get("name") or "NIC"),
|
||
str(nic.get("bdf") or ""),
|
||
nic.get("numa"),
|
||
nic.get("pci_path"),
|
||
)
|
||
for device in ib_devices:
|
||
add_endpoint(
|
||
"infiniband",
|
||
str(device.get("name") or "IB"),
|
||
str(device.get("bdf") or ""),
|
||
device.get("numa"),
|
||
device.get("pci_path"),
|
||
)
|
||
|
||
bridges: dict[str, dict[str, Any]] = {}
|
||
for endpoint in endpoints:
|
||
path = endpoint.get("path") or []
|
||
for index, bdf in enumerate(path[:-1]):
|
||
bridge = bridges.setdefault(
|
||
bdf,
|
||
{
|
||
"bdf": bdf,
|
||
"parent_bdf": path[index - 1] if index else "",
|
||
"numa": pci_numa(bdf),
|
||
"description": pci_devices.get(bdf, {}).get(
|
||
"description", "PCI bridge"
|
||
),
|
||
"children": [],
|
||
},
|
||
)
|
||
child = path[index + 1]
|
||
if child not in bridge["children"]:
|
||
bridge["children"].append(child)
|
||
|
||
return {
|
||
"scope": "guest-visible",
|
||
"virtualization": "passthrough-or-virtual-pci",
|
||
"bridges": sorted(bridges.values(), key=lambda item: natural_key(item["bdf"])),
|
||
"endpoints": sorted(
|
||
endpoints,
|
||
key=lambda item: (str(item.get("numa")), item["kind"], natural_key(item["label"])),
|
||
),
|
||
}
|
||
|
||
|
||
def collect_cuda_toolkit() -> str:
|
||
for executable in ("nvcc", "/usr/local/cuda/bin/nvcc"):
|
||
output = run([executable, "--version"])
|
||
match = re.search(r"release\s+([\d.]+)", output)
|
||
if match:
|
||
return concise_version(match.group(1))
|
||
for path in ("/usr/local/cuda/version.json", "/usr/local/cuda/version.txt"):
|
||
text = read_text(path)
|
||
match = re.search(r"(?:CUDA Version|\"version\"\s*:)\s*\"?([\d.]+)", text)
|
||
if match:
|
||
return concise_version(match.group(1))
|
||
return ""
|
||
|
||
|
||
def concise_version(value: str) -> str:
|
||
match = re.match(r"(\d+\.\d+)", value or "")
|
||
return match.group(1) if match else value
|
||
|
||
|
||
def collect_snapshot() -> dict[str, Any]:
|
||
topo_text = run(["nvidia-smi", "topo", "-m"])
|
||
topology = parse_topology(topo_text)
|
||
rdma_to_net, net_to_rdma = parse_ibdev2netdev(run(["ibdev2netdev"]))
|
||
rdma_links = parse_rdma_links(run(["rdma", "link", "show"]))
|
||
nics = collect_nics()
|
||
bonds = collect_bonds(net_to_rdma, nics)
|
||
lspci_text = run(["lspci", "-Dnn"])
|
||
pci_devices = parse_lspci_devices(lspci_text)
|
||
pci_network = parse_lspci_network(lspci_text)
|
||
|
||
ib_devices = []
|
||
bonded_rdma = {bond.get("rdma") for bond in bonds}
|
||
for device in parse_ibstat(run(["ibstat"])):
|
||
if (
|
||
device["name"] in bonded_rdma
|
||
or str(device.get("link_layer", "")).lower() != "infiniband"
|
||
):
|
||
continue
|
||
bdf = ""
|
||
device_path = Path(f"/sys/class/infiniband/{device['name']}/device")
|
||
try:
|
||
bdf = normalize_bdf(str(device_path.resolve()))
|
||
except OSError:
|
||
pass
|
||
device.update(
|
||
{
|
||
"bdf": bdf,
|
||
"numa": pci_numa(bdf),
|
||
"pci_path": pci_path(bdf),
|
||
"description": pci_network.get(bdf, {}).get("description", ""),
|
||
"rdma_state": rdma_links.get(device["name"], {}),
|
||
}
|
||
)
|
||
ib_devices.append(device)
|
||
network_endpoints = build_network_endpoints(
|
||
nics,
|
||
bonds,
|
||
ib_devices,
|
||
rdma_to_net,
|
||
rdma_links,
|
||
)
|
||
|
||
system = {
|
||
"hostname": socket.gethostname(),
|
||
"os": os_release(),
|
||
"kernel": platform.release(),
|
||
"architecture": platform.machine(),
|
||
"manufacturer": dmi_value("sys_vendor"),
|
||
"product": dmi_value("product_name"),
|
||
"product_version": dmi_value("product_version"),
|
||
"virtualization": run(["systemd-detect-virt"]),
|
||
"cpu_model": cpu_model(),
|
||
"numa_nodes": numa_count(),
|
||
**collect_cpu_topology(),
|
||
**collect_memory(),
|
||
}
|
||
gpus = collect_gpus()
|
||
pci_fabric = build_pci_fabric(gpus, nics, ib_devices, pci_devices)
|
||
driver = next((gpu.get("driver") for gpu in gpus if gpu.get("driver")), "")
|
||
nvlink = parse_nvlink_status(run(["nvidia-smi", "nvlink", "--status"]))
|
||
nvlink["fabric_type"] = infer_nvlink_fabric_type(topology, len(gpus))
|
||
snapshot = {
|
||
"schema_version": 1,
|
||
"generated_at": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
|
||
"collector_version": VERSION,
|
||
"system": system,
|
||
"software": {
|
||
"nvidia_driver": driver,
|
||
"cuda_toolkit": collect_cuda_toolkit(),
|
||
"cuda_note": "Toolkit selected by nvcc; application/container runtime may differ.",
|
||
},
|
||
"gpus": gpus,
|
||
"topology": topology,
|
||
"pci_fabric": pci_fabric,
|
||
"nvlink": nvlink,
|
||
"nics": nics,
|
||
"bonds": bonds,
|
||
"ib_devices": ib_devices,
|
||
"rdma_links": rdma_links,
|
||
"network_endpoints": network_endpoints,
|
||
"bmc": collect_bmc(),
|
||
"rdma_map": rdma_to_net,
|
||
"notes": [],
|
||
}
|
||
attach_gpu_network_mapping(snapshot)
|
||
return snapshot
|
||
|
||
|
||
def os_release() -> str:
|
||
for line in read_text("/etc/os-release").splitlines():
|
||
if line.startswith("PRETTY_NAME="):
|
||
return line.split("=", 1)[1].strip().strip('"')
|
||
return platform.platform()
|
||
|
||
|
||
def cpu_model() -> str:
|
||
for line in read_text("/proc/cpuinfo").splitlines():
|
||
if line.lower().startswith("model name"):
|
||
return line.split(":", 1)[1].strip()
|
||
return ""
|
||
|
||
|
||
def numa_count() -> int:
|
||
nodes = list(Path("/sys/devices/system/node").glob("node[0-9]*"))
|
||
return len(nodes)
|
||
|
||
|
||
def collect_cpu_topology() -> dict[str, Any]:
|
||
values: dict[str, str] = {}
|
||
output = run(["lscpu", "-J"])
|
||
try:
|
||
for item in json.loads(output or "{}").get("lscpu", []):
|
||
key = str(item.get("field", "")).rstrip(":")
|
||
values[key] = str(item.get("data", "")).strip()
|
||
except (TypeError, ValueError, json.JSONDecodeError):
|
||
pass
|
||
|
||
def integer(key: str) -> int | None:
|
||
match = re.search(r"\d+", values.get(key, ""))
|
||
return int(match.group(0)) if match else None
|
||
|
||
sockets = integer("Socket(s)")
|
||
cores_per_socket = integer("Core(s) per socket")
|
||
return {
|
||
"cpu_logical": integer("CPU(s)") or os.cpu_count(),
|
||
"cpu_sockets": sockets,
|
||
"cpu_cores_per_socket": cores_per_socket,
|
||
"cpu_physical_cores": (
|
||
sockets * cores_per_socket
|
||
if sockets is not None and cores_per_socket is not None
|
||
else None
|
||
),
|
||
"cpu_threads_per_core": integer("Thread(s) per core"),
|
||
}
|
||
|
||
|
||
def collect_memory() -> dict[str, Any]:
|
||
values: dict[str, int] = {}
|
||
for line in read_text("/proc/meminfo").splitlines():
|
||
if ":" not in line:
|
||
continue
|
||
key, tail = line.split(":", 1)
|
||
match = re.search(r"\d+", tail)
|
||
if match:
|
||
values[key] = int(match.group(0))
|
||
total = values.get("MemTotal")
|
||
available = values.get("MemAvailable")
|
||
return {
|
||
"memory_total_kib": total,
|
||
"memory_available_kib": available,
|
||
"memory_used_kib": (
|
||
total - available
|
||
if total is not None and available is not None
|
||
else None
|
||
),
|
||
}
|
||
|
||
|
||
def common_path_depth(left: list[str], right: list[str]) -> int:
|
||
depth = 0
|
||
for left_item, right_item in zip(left or [], right or []):
|
||
if left_item != right_item:
|
||
break
|
||
depth += 1
|
||
return depth
|
||
|
||
|
||
def hungarian_assignment(costs: list[list[int]]) -> list[tuple[int, int]]:
|
||
"""Return minimum-cost unique row/column pairs for a rectangular matrix."""
|
||
if not costs:
|
||
return []
|
||
row_count = len(costs)
|
||
column_count = len(costs[0])
|
||
if row_count > column_count:
|
||
raise ValueError("Hungarian assignment requires rows <= columns")
|
||
potentials_rows = [0] * (row_count + 1)
|
||
potentials_columns = [0] * (column_count + 1)
|
||
column_match = [0] * (column_count + 1)
|
||
path = [0] * (column_count + 1)
|
||
for row in range(1, row_count + 1):
|
||
column_match[0] = row
|
||
minimums = [10**18] * (column_count + 1)
|
||
used = [False] * (column_count + 1)
|
||
column = 0
|
||
while True:
|
||
used[column] = True
|
||
current_row = column_match[column]
|
||
delta = 10**18
|
||
next_column = 0
|
||
for candidate in range(1, column_count + 1):
|
||
if used[candidate]:
|
||
continue
|
||
reduced = (
|
||
costs[current_row - 1][candidate - 1]
|
||
- potentials_rows[current_row]
|
||
- potentials_columns[candidate]
|
||
)
|
||
if reduced < minimums[candidate]:
|
||
minimums[candidate] = reduced
|
||
path[candidate] = column
|
||
if minimums[candidate] < delta:
|
||
delta = minimums[candidate]
|
||
next_column = candidate
|
||
for candidate in range(column_count + 1):
|
||
if used[candidate]:
|
||
potentials_rows[column_match[candidate]] += delta
|
||
potentials_columns[candidate] -= delta
|
||
else:
|
||
minimums[candidate] -= delta
|
||
column = next_column
|
||
if column_match[column] == 0:
|
||
break
|
||
while True:
|
||
previous = path[column]
|
||
column_match[column] = column_match[previous]
|
||
column = previous
|
||
if column == 0:
|
||
break
|
||
return sorted(
|
||
(row - 1, column - 1)
|
||
for column, row in enumerate(column_match[1:], start=1)
|
||
if row
|
||
)
|
||
|
||
|
||
def attach_gpu_network_mapping(snapshot: dict[str, Any]) -> None:
|
||
"""Map GPUs to bonded or standalone RDMA endpoints using topology distance."""
|
||
endpoints = snapshot.get("network_endpoints")
|
||
if not isinstance(endpoints, list) or (
|
||
not endpoints
|
||
and (
|
||
snapshot.get("bonds")
|
||
or snapshot.get("ib_devices")
|
||
or snapshot.get("rdma_map")
|
||
)
|
||
):
|
||
endpoints = build_network_endpoints(
|
||
snapshot.get("nics", []),
|
||
snapshot.get("bonds", []),
|
||
snapshot.get("ib_devices", []),
|
||
snapshot.get("rdma_map", {}),
|
||
snapshot.get("rdma_links", {}),
|
||
)
|
||
snapshot["network_endpoints"] = endpoints
|
||
snapshot["normalized_by_version"] = VERSION
|
||
|
||
endpoint_by_id = {
|
||
str(endpoint.get("id")): endpoint
|
||
for endpoint in endpoints
|
||
if endpoint.get("id")
|
||
}
|
||
bond_by_name = {
|
||
str(bond.get("name")): bond for bond in snapshot.get("bonds", [])
|
||
}
|
||
for endpoint in endpoints:
|
||
endpoint["gpu_links"] = []
|
||
endpoint["topology_visibility"] = "not-exposed"
|
||
for bond in snapshot.get("bonds", []):
|
||
bond.pop("gpu_index", None)
|
||
bond.pop("relation", None)
|
||
|
||
topology = snapshot.get("topology", {}).get("gpu_nic", {})
|
||
affinity = snapshot.get("topology", {}).get("gpu_affinity", {})
|
||
gpus = snapshot.get("gpus", [])
|
||
relations: dict[tuple[int, str], str] = {}
|
||
|
||
for gpu in gpus:
|
||
for key in (
|
||
"network_relation",
|
||
"rdma",
|
||
"bond",
|
||
"network",
|
||
"primary_network_id",
|
||
):
|
||
gpu.pop(key, None)
|
||
gpu["network_links"] = []
|
||
gpu_key = f"GPU{gpu['index']}"
|
||
gpu.update(
|
||
{
|
||
key: value
|
||
for key, value in affinity.get(gpu_key, {}).items()
|
||
if value
|
||
}
|
||
)
|
||
by_rdma = {
|
||
str(item.get("rdma")): str(item.get("relation") or "")
|
||
for item in topology.get(gpu_key, [])
|
||
}
|
||
for endpoint in endpoints:
|
||
relation = by_rdma.get(str(endpoint.get("rdma") or ""), "")
|
||
if relation:
|
||
relations[(int(gpu["index"]), str(endpoint["id"]))] = relation
|
||
endpoint["topology_visibility"] = "nvidia-smi-topo"
|
||
|
||
primary_pool = [
|
||
endpoint for endpoint in endpoints if endpoint.get("role") == "compute"
|
||
]
|
||
if not primary_pool:
|
||
primary_pool = list(endpoints)
|
||
|
||
sorted_gpus = sorted(gpus, key=lambda item: int(item["index"]))
|
||
sorted_endpoints = sorted(
|
||
primary_pool, key=lambda item: natural_key(str(item.get("name") or ""))
|
||
)
|
||
|
||
def pair_cost(gpu: dict[str, Any], endpoint: dict[str, Any]) -> int:
|
||
gpu_index = int(gpu["index"])
|
||
endpoint_id = str(endpoint["id"])
|
||
rank = relation_rank(relations.get((gpu_index, endpoint_id), ""))
|
||
if rank >= 9:
|
||
return 10**9
|
||
member_paths = [
|
||
member.get("pci_path") or []
|
||
for member in endpoint.get("members", [])
|
||
] or [endpoint.get("pci_path") or []]
|
||
proximity = max(
|
||
(
|
||
common_path_depth(gpu.get("pci_path") or [], path)
|
||
for path in member_paths
|
||
),
|
||
default=0,
|
||
)
|
||
# Topology relation is authoritative. PCI path only breaks equal-distance
|
||
# ties, followed by stable natural ordering of GPUs and endpoints.
|
||
return rank * 1_000_000 + (100 - min(100, proximity)) * 1_000
|
||
|
||
primary_assignments: dict[int, tuple[dict[str, Any], str]] = {}
|
||
if sorted_gpus and sorted_endpoints:
|
||
if len(sorted_gpus) <= len(sorted_endpoints):
|
||
costs = [
|
||
[
|
||
pair_cost(gpu, endpoint) + endpoint_index
|
||
for endpoint_index, endpoint in enumerate(sorted_endpoints)
|
||
]
|
||
for gpu in sorted_gpus
|
||
]
|
||
unique_pairs = [
|
||
(sorted_gpus[gpu_index], sorted_endpoints[endpoint_index])
|
||
for gpu_index, endpoint_index in hungarian_assignment(costs)
|
||
if costs[gpu_index][endpoint_index] < 10**9
|
||
]
|
||
else:
|
||
costs = [
|
||
[
|
||
pair_cost(gpu, endpoint) + gpu_index
|
||
for gpu_index, gpu in enumerate(sorted_gpus)
|
||
]
|
||
for endpoint in sorted_endpoints
|
||
]
|
||
unique_pairs = [
|
||
(sorted_gpus[gpu_index], sorted_endpoints[endpoint_index])
|
||
for endpoint_index, gpu_index in hungarian_assignment(costs)
|
||
if costs[endpoint_index][gpu_index] < 10**9
|
||
]
|
||
for gpu, endpoint in unique_pairs:
|
||
gpu_index = int(gpu["index"])
|
||
endpoint_id = str(endpoint["id"])
|
||
primary_assignments[gpu_index] = (
|
||
endpoint,
|
||
relations[(gpu_index, endpoint_id)],
|
||
)
|
||
|
||
for gpu in gpus:
|
||
gpu_index = int(gpu["index"])
|
||
if gpu_index in primary_assignments:
|
||
continue
|
||
candidates = [
|
||
(
|
||
relation_rank(relations.get((gpu_index, str(endpoint["id"])), "")),
|
||
str(endpoint.get("name") or ""),
|
||
endpoint,
|
||
)
|
||
for endpoint in primary_pool
|
||
if relation_rank(
|
||
relations.get((gpu_index, str(endpoint["id"])), "")
|
||
)
|
||
< 9
|
||
]
|
||
if candidates:
|
||
_, _, endpoint = min(candidates)
|
||
primary_assignments[gpu_index] = (
|
||
endpoint,
|
||
relations[(gpu_index, str(endpoint["id"]))],
|
||
)
|
||
|
||
def attach_link(
|
||
gpu: dict[str, Any],
|
||
endpoint: dict[str, Any],
|
||
relation: str,
|
||
role: str,
|
||
) -> None:
|
||
link = {
|
||
"endpoint_id": endpoint["id"],
|
||
"name": endpoint.get("name"),
|
||
"rdma": endpoint.get("rdma"),
|
||
"protocol": endpoint.get("protocol"),
|
||
"type": endpoint.get("type"),
|
||
"relation": relation,
|
||
"role": role,
|
||
"evidence": "nvidia-smi topo -m",
|
||
}
|
||
gpu["network_links"].append(link)
|
||
endpoint["gpu_links"].append(
|
||
{
|
||
"gpu_index": gpu["index"],
|
||
"relation": relation,
|
||
"role": role,
|
||
"evidence": "nvidia-smi topo -m",
|
||
}
|
||
)
|
||
|
||
for gpu in gpus:
|
||
gpu_index = int(gpu["index"])
|
||
primary = primary_assignments.get(gpu_index)
|
||
if primary:
|
||
endpoint, relation = primary
|
||
gpu["primary_network_id"] = endpoint["id"]
|
||
gpu["network_relation"] = relation
|
||
gpu["rdma"] = endpoint.get("rdma")
|
||
gpu["network"] = endpoint.get("name")
|
||
if endpoint.get("type") == "bond":
|
||
gpu["bond"] = endpoint.get("name")
|
||
bond = bond_by_name.get(str(endpoint.get("name")))
|
||
if bond is not None:
|
||
bond["gpu_index"] = gpu["index"]
|
||
bond["relation"] = relation
|
||
attach_link(gpu, endpoint, relation, "primary")
|
||
|
||
ib_candidates = [
|
||
(
|
||
relation_rank(relations.get((gpu_index, str(endpoint["id"])), "")),
|
||
str(endpoint.get("name") or ""),
|
||
endpoint,
|
||
)
|
||
for endpoint in endpoints
|
||
if endpoint.get("protocol") == "InfiniBand"
|
||
and relation_rank(
|
||
relations.get((gpu_index, str(endpoint["id"])), "")
|
||
)
|
||
< 9
|
||
and (not primary or endpoint["id"] != primary[0]["id"])
|
||
]
|
||
if ib_candidates:
|
||
_, _, endpoint = min(ib_candidates)
|
||
attach_link(
|
||
gpu,
|
||
endpoint,
|
||
relations[(gpu_index, str(endpoint["id"]))],
|
||
"nearest",
|
||
)
|
||
|
||
|
||
def attach_gpu_bond_mapping(snapshot: dict[str, Any]) -> None:
|
||
"""Backward-compatible alias for callers using the former function name."""
|
||
attach_gpu_network_mapping(snapshot)
|
||
|
||
|
||
def relation_rank(relation: str) -> int:
|
||
return {"PIX": 0, "PXB": 1, "PHB": 2, "NODE": 3, "SYS": 4}.get(relation, 9)
|
||
|
||
|
||
def evaluate(snapshot: dict[str, Any], expected: dict[str, int | None]) -> dict[str, Any]:
|
||
checks: list[dict[str, str]] = []
|
||
|
||
def add(level: str, title: str, detail: str) -> None:
|
||
checks.append({"level": level, "title": title, "detail": detail})
|
||
|
||
gpus = snapshot.get("gpus", [])
|
||
bonds = snapshot.get("bonds", [])
|
||
rdma_bonds = [bond for bond in bonds if bond.get("rdma")]
|
||
ib_devices = snapshot.get("ib_devices", [])
|
||
network_endpoints = snapshot.get("network_endpoints", [])
|
||
if not gpus:
|
||
add("critical", "未发现 NVIDIA GPU", "nvidia-smi 无可解析输出")
|
||
elif expected.get("gpus") is not None and len(gpus) != expected["gpus"]:
|
||
add("warning", "GPU 数量不符合预期", f"发现 {len(gpus)},预期 {expected['gpus']}")
|
||
else:
|
||
add("ok", "GPU 数量", f"发现 {len(gpus)} 张 GPU")
|
||
|
||
unhealthy_bonds = [
|
||
bond
|
||
for bond in rdma_bonds
|
||
if bond.get("status") != "up"
|
||
or any(slave.get("status") != "up" for slave in bond["slaves"])
|
||
]
|
||
if unhealthy_bonds:
|
||
add("critical", "Bond 链路异常", "、".join(bond["name"] for bond in unhealthy_bonds))
|
||
elif rdma_bonds:
|
||
add(
|
||
"ok",
|
||
"Bond 链路",
|
||
f"{len(rdma_bonds)} 个 RDMA Bond 及全部成员均为 Up",
|
||
)
|
||
if (
|
||
expected.get("bonds") is not None
|
||
and len(rdma_bonds) != expected["bonds"]
|
||
):
|
||
add(
|
||
"warning",
|
||
"Bond 数量不符合预期",
|
||
f"发现 {len(rdma_bonds)} 个 RDMA Bond,预期 {expected['bonds']}",
|
||
)
|
||
|
||
failures = sum(
|
||
int(slave.get("failures") or 0)
|
||
for bond in rdma_bonds
|
||
for slave in bond.get("slaves", [])
|
||
)
|
||
if failures:
|
||
add("warning", "存在历史链路失败", f"Bond 成员累计 Link Failure Count = {failures}")
|
||
elif rdma_bonds:
|
||
add("ok", "链路失败计数", "全部 Bond 成员为 0")
|
||
|
||
def endpoint_active(endpoint: dict[str, Any]) -> bool:
|
||
state_ok = str(endpoint.get("state") or "").upper() in {"UP", "ACTIVE"}
|
||
physical = (
|
||
str(endpoint.get("physical_state") or "")
|
||
.upper()
|
||
.replace("_", "")
|
||
)
|
||
physical_ok = not physical or physical in {"LINKUP", "ACTIVE", "UP"}
|
||
members_ok = all(
|
||
not member.get("status")
|
||
or str(member.get("status")).lower() in {"up", "active"}
|
||
for member in endpoint.get("members", [])
|
||
)
|
||
return state_ok and physical_ok and members_ok
|
||
|
||
inactive_endpoints = [
|
||
endpoint for endpoint in network_endpoints if not endpoint_active(endpoint)
|
||
]
|
||
if inactive_endpoints:
|
||
add(
|
||
"warning",
|
||
"RDMA 端点未全部激活",
|
||
"、".join(str(endpoint.get("name")) for endpoint in inactive_endpoints),
|
||
)
|
||
elif network_endpoints:
|
||
roce_count = sum(
|
||
endpoint.get("protocol") == "RoCE"
|
||
for endpoint in network_endpoints
|
||
)
|
||
ib_endpoint_count = sum(
|
||
endpoint.get("protocol") == "InfiniBand"
|
||
for endpoint in network_endpoints
|
||
)
|
||
add(
|
||
"ok",
|
||
"RDMA 网络端点",
|
||
f"RoCE {roce_count} · InfiniBand {ib_endpoint_count},均为 Active/Up",
|
||
)
|
||
elif snapshot.get("topology", {}).get("nic_legend"):
|
||
add(
|
||
"warning",
|
||
"RDMA 端点建模不完整",
|
||
"拓扑矩阵包含 NIC,但未构建 Bond、独立 RoCE 或 IB 端点",
|
||
)
|
||
|
||
mapped_gpus = [
|
||
gpu for gpu in gpus if gpu.get("primary_network_id")
|
||
]
|
||
relation_counts: dict[str, int] = {}
|
||
for gpu in mapped_gpus:
|
||
relation = str(gpu.get("network_relation") or "N/A")
|
||
relation_counts[relation] = relation_counts.get(relation, 0) + 1
|
||
relation_detail = " / ".join(
|
||
f"{name} {relation_counts[name]}"
|
||
for name in ("PIX", "PXB", "PHB", "NODE", "SYS")
|
||
if relation_counts.get(name)
|
||
)
|
||
if gpus and len(mapped_gpus) != len(gpus):
|
||
add(
|
||
"warning",
|
||
"GPU ↔ RDMA 映射不完整",
|
||
f"仅 {len(mapped_gpus)}/{len(gpus)} 张 GPU 有最近网络端点",
|
||
)
|
||
elif mapped_gpus and relation_counts.get("SYS"):
|
||
add(
|
||
"warning",
|
||
"GPU ↔ RDMA 存在跨 NUMA 路径",
|
||
relation_detail,
|
||
)
|
||
elif mapped_gpus:
|
||
add(
|
||
"ok",
|
||
"GPU ↔ RDMA 亲和",
|
||
f"{len(mapped_gpus)}/{len(gpus)} 已映射 · {relation_detail}",
|
||
)
|
||
|
||
affinity_gpus = [
|
||
gpu
|
||
for gpu in gpus
|
||
if gpu.get("cpu_affinity") and gpu.get("cpu_affinity") != "N/A"
|
||
]
|
||
affinity_mismatches = []
|
||
for gpu in affinity_gpus:
|
||
gpu_numa = gpu.get("numa")
|
||
affinity_numa = number(gpu.get("numa_affinity"))
|
||
if (
|
||
isinstance(gpu_numa, int)
|
||
and gpu_numa >= 0
|
||
and affinity_numa is not None
|
||
and gpu_numa != int(affinity_numa)
|
||
):
|
||
affinity_mismatches.append(f"GPU{gpu.get('index')}")
|
||
if affinity_mismatches:
|
||
add(
|
||
"warning",
|
||
"CPU / NUMA 亲和冲突",
|
||
"、".join(affinity_mismatches) + " 的PCI NUMA与内存亲和不一致",
|
||
)
|
||
elif gpus and len(affinity_gpus) == len(gpus):
|
||
add("ok", "CPU / NUMA 亲和", f"{len(gpus)}/{len(gpus)} 张GPU均有推荐CPU范围")
|
||
elif gpus:
|
||
add(
|
||
"warning",
|
||
"CPU亲和信息不完整",
|
||
f"仅 {len(affinity_gpus)}/{len(gpus)} 张GPU可解析",
|
||
)
|
||
|
||
power_gpus = [
|
||
gpu
|
||
for gpu in gpus
|
||
if number(gpu.get("power_w")) is not None
|
||
and number(gpu.get("power_limit_w")) is not None
|
||
and bool(gpu.get("pstate"))
|
||
]
|
||
if gpus and len(power_gpus) == len(gpus):
|
||
total_power = sum(number(gpu.get("power_w")) or 0 for gpu in power_gpus)
|
||
total_limit = sum(number(gpu.get("power_limit_w")) or 0 for gpu in power_gpus)
|
||
p0_count = sum(str(gpu.get("pstate", "")).upper() == "P0" for gpu in gpus)
|
||
add(
|
||
"ok",
|
||
"GPU 功耗遥测",
|
||
f"瞬时 {fmt_power(total_power)} / 上限 {fmt_power(total_limit)} · P0 {p0_count}/{len(gpus)}",
|
||
)
|
||
elif gpus:
|
||
add(
|
||
"warning",
|
||
"GPU功耗信息不完整",
|
||
f"仅 {len(power_gpus)}/{len(gpus)} 张GPU可读取当前功耗、上限和P-State",
|
||
)
|
||
|
||
active_ib = sum(str(device.get("state", "")).lower() == "active" for device in ib_devices)
|
||
if ib_devices and active_ib != len(ib_devices):
|
||
add("warning", "IB 端口未全部激活", f"{active_ib}/{len(ib_devices)} Active")
|
||
elif ib_devices:
|
||
add("ok", "IB 端口", f"{active_ib}/{len(ib_devices)} Active")
|
||
if expected.get("ib") is not None and len(ib_devices) != expected["ib"]:
|
||
add("warning", "IB 数量不符合预期", f"发现 {len(ib_devices)},预期 {expected['ib']}")
|
||
|
||
default_route_nics = [
|
||
nic for nic in snapshot.get("nics", []) if nic.get("default_route")
|
||
]
|
||
unhealthy_management = [
|
||
nic
|
||
for nic in default_route_nics
|
||
if str(nic.get("state", "")).lower() != "up" or nic.get("carrier") == 0
|
||
]
|
||
if unhealthy_management:
|
||
add(
|
||
"warning",
|
||
"带内管理网异常",
|
||
"、".join(str(nic.get("name")) for nic in unhealthy_management),
|
||
)
|
||
elif default_route_nics:
|
||
add(
|
||
"ok",
|
||
"带内管理网",
|
||
"、".join(str(nic.get("name")) for nic in default_route_nics) + " 承载默认路由且链路 Up",
|
||
)
|
||
|
||
nvlink = snapshot.get("nvlink", {})
|
||
if len(gpus) > 1 and not nvlink.get("active_links"):
|
||
add("warning", "未解析到 NVLink", "无法确认机内 NVLink 状态")
|
||
elif nvlink.get("active_links"):
|
||
add(
|
||
"ok",
|
||
"NVLink",
|
||
f"共 {nvlink['active_links']} 条 Active,单 GPU 至少 {nvlink.get('links_per_gpu', 0)} 条",
|
||
)
|
||
|
||
severity = {"ok": 0, "warning": 1, "critical": 2}
|
||
worst = max((severity[item["level"]] for item in checks), default=1)
|
||
return {
|
||
"status": ("healthy", "attention", "critical")[worst],
|
||
"checks": checks,
|
||
}
|
||
|
||
|
||
def esc(value: Any) -> str:
|
||
return html.escape("" if value is None else str(value), quote=True)
|
||
|
||
|
||
def fmt_rate(mbps: Any) -> str:
|
||
value = number(mbps)
|
||
if value is None:
|
||
return "未知"
|
||
if value >= 1000:
|
||
gbps = value / 1000
|
||
return f"{gbps:g}G"
|
||
return f"{value:g}M"
|
||
|
||
|
||
def fmt_mib(mib: Any) -> str:
|
||
value = number(mib)
|
||
if value is None:
|
||
return "未知"
|
||
return f"{value / 1024:.0f} GiB"
|
||
|
||
|
||
def fmt_kib(kib: Any) -> str:
|
||
value = number(kib)
|
||
if value is None:
|
||
return "未知"
|
||
return f"{value / 1024 / 1024:.1f} GiB"
|
||
|
||
|
||
def fmt_power(watts: Any) -> str:
|
||
value = number(watts)
|
||
if value is None:
|
||
return "未知"
|
||
if value >= 1000:
|
||
return f"{value / 1000:.2f} kW"
|
||
return f"{value:.0f} W"
|
||
|
||
|
||
def fmt_number(value: Any, fallback: str = "—") -> str:
|
||
parsed = number(value)
|
||
return f"{parsed:g}" if parsed is not None else fallback
|
||
|
||
|
||
def compact_gpu_indices(indices: list[int]) -> str:
|
||
if not indices:
|
||
return "GPU?"
|
||
ordered = sorted(indices)
|
||
if ordered == list(range(ordered[0], ordered[-1] + 1)):
|
||
return f"GPU{ordered[0]}" if len(ordered) == 1 else f"GPU{ordered[0]}–{ordered[-1]}"
|
||
return ",".join(f"GPU{index}" for index in ordered)
|
||
|
||
|
||
def status_label(value: str) -> str:
|
||
return {"healthy": "健康", "attention": "需关注", "critical": "异常"}.get(value, "未知")
|
||
|
||
|
||
def badge(text: str, tone: str = "neutral") -> str:
|
||
return f'<span class="badge {esc(tone)}">{esc(text)}</span>'
|
||
|
||
|
||
def render_gpu_lane(
|
||
gpu: dict[str, Any], endpoint: dict[str, Any] | None
|
||
) -> str:
|
||
utilization = gpu.get("utilization_pct")
|
||
memory_total = gpu.get("memory_total_mib")
|
||
memory_used = gpu.get("memory_used_mib")
|
||
memory_pct = (
|
||
100 * memory_used / memory_total
|
||
if number(memory_used) is not None and number(memory_total)
|
||
else 0
|
||
)
|
||
relation = gpu.get("network_relation", "未映射")
|
||
relation_tone = "ok" if relation == "PIX" else "warn"
|
||
memory_numa = gpu.get("numa_affinity")
|
||
if memory_numa in {None, "", "N/A"}:
|
||
memory_numa = gpu.get("numa")
|
||
power_w = number(gpu.get("power_w"))
|
||
power_limit_w = number(gpu.get("power_limit_w"))
|
||
power_pct = (
|
||
100 * power_w / power_limit_w
|
||
if power_w is not None and power_limit_w
|
||
else 0
|
||
)
|
||
pstate = str(gpu.get("pstate") or "—").upper()
|
||
if endpoint and endpoint.get("type") == "bond":
|
||
members = endpoint.get("members", [])
|
||
slaves = "".join(
|
||
f"""
|
||
<div class="port {'up' if slave.get('status') == 'up' else 'down'}">
|
||
<span class="port-dot"></span>
|
||
<span><b>{esc(slave.get('name'))}</b><small>{esc(fmt_rate(slave.get('speed_mbps')))} · {esc(slave.get('bdf'))}</small></span>
|
||
</div>
|
||
"""
|
||
for slave in members
|
||
)
|
||
aggregate = number(endpoint.get("capacity_mbps")) or 0
|
||
endpoint_up = str(endpoint.get("state") or "").upper() in {
|
||
"UP",
|
||
"ACTIVE",
|
||
}
|
||
network_html = f"""
|
||
<div class="relation">{badge(relation, relation_tone)}<span>PCIe 亲和</span></div>
|
||
<article class="nic-card">
|
||
<div class="card-head">
|
||
<span class="eyebrow">ROCE LAG</span>
|
||
{badge("UP" if endpoint_up else "DOWN", "ok" if endpoint_up else "bad")}
|
||
</div>
|
||
<h4>{esc(endpoint.get("name"))}</h4>
|
||
<p>{esc(endpoint.get("rdma") or "无 RDMA 映射")}</p>
|
||
<div class="bond-capacity"><b>{esc(fmt_rate(aggregate))}</b><span>聚合容量</span></div>
|
||
<div class="ports">{slaves}</div>
|
||
</article>
|
||
"""
|
||
elif endpoint:
|
||
member = (endpoint.get("members") or [{}])[0]
|
||
endpoint_up = str(endpoint.get("state") or "").upper() in {
|
||
"UP",
|
||
"ACTIVE",
|
||
}
|
||
protocol = str(endpoint.get("protocol") or "RDMA")
|
||
network_html = f"""
|
||
<div class="relation">{badge(relation, relation_tone)}<span>PCIe 亲和</span></div>
|
||
<article class="nic-card">
|
||
<div class="card-head">
|
||
<span class="eyebrow">{esc(protocol.upper())} · DIRECT</span>
|
||
{badge("ACTIVE" if endpoint_up else (endpoint.get("state") or "UNKNOWN"), "ok" if endpoint_up else "bad")}
|
||
</div>
|
||
<h4>{esc(endpoint.get("name"))}</h4>
|
||
<p>{esc(endpoint.get("rdma") or "无 RDMA 映射")}</p>
|
||
<div class="bond-capacity"><b>{esc(fmt_rate(endpoint.get("capacity_mbps")))}</b><span>物理端口</span></div>
|
||
<div class="ports">
|
||
<div class="port {'up' if endpoint_up else 'down'}">
|
||
<span class="port-dot"></span>
|
||
<span><b>{esc(member.get('name') or endpoint.get('name'))}</b><small>{esc(member.get('bdf') or endpoint.get('bdf') or '—')} · NUMA {esc(member.get('numa') if member.get('numa') is not None else '—')}</small></span>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
"""
|
||
else:
|
||
network_html = """
|
||
<div class="relation">未发现对应网络路径</div>
|
||
<article class="nic-card empty"><p>没有可映射的 Bond、独立 RoCE 或 IB 端点</p></article>
|
||
"""
|
||
return f"""
|
||
<div class="rail" data-numa="{esc(gpu.get('numa'))}">
|
||
<article class="gpu-card">
|
||
<div class="card-head">
|
||
<span class="eyebrow">GPU {esc(gpu.get('index'))}</span>
|
||
{badge(f"NUMA {gpu.get('numa')}" if gpu.get('numa') is not None else "NUMA ?", "neutral")}
|
||
</div>
|
||
<h3>{esc(gpu.get('name') or 'Unknown GPU')}</h3>
|
||
<p class="mono">{esc(gpu.get('bdf'))}</p>
|
||
<div class="gpu-affinity" title="拓扑推荐亲和,不代表当前进程已绑核">
|
||
<span><small>CPU</small><b>{esc(gpu.get('cpu_affinity') or '—')}</b></span>
|
||
<span><small>MEM</small><b>N{esc(memory_numa if memory_numa is not None else '—')}</b></span>
|
||
</div>
|
||
<div class="gpu-stats">
|
||
<span><b>{esc(fmt_mib(memory_total))}</b><small>显存</small></span>
|
||
<span><b>{esc(fmt_number(gpu.get('temperature_c')))}°</b><small>温度</small></span>
|
||
<span><b>{esc(fmt_number(utilization))}%</b><small>利用率</small></span>
|
||
</div>
|
||
<div class="meter memory-usage-meter" title="显存使用率"><i style="width:{max(0, min(100, memory_pct)):.1f}%"></i></div>
|
||
<div class="gpu-power">
|
||
<div>
|
||
<span><b>{esc(fmt_power(power_w))}</b><small>瞬时 / {esc(fmt_power(power_limit_w))} 上限</small></span>
|
||
<em>{esc(pstate)}</em>
|
||
</div>
|
||
<div class="power-meter" title="瞬时功耗占当前 Power Limit 的比例"><i style="width:{max(0, min(100, power_pct)):.1f}%"></i></div>
|
||
</div>
|
||
</article>
|
||
{network_html}
|
||
</div>
|
||
"""
|
||
|
||
|
||
def render_ib_card(device: dict[str, Any]) -> str:
|
||
active = str(device.get("state", "")).lower() == "active"
|
||
return f"""
|
||
<article class="ib-card">
|
||
<div class="card-head">
|
||
<span class="eyebrow">INFINIBAND</span>
|
||
{badge("ACTIVE" if active else (device.get("state") or "UNKNOWN"), "ok" if active else "warn")}
|
||
</div>
|
||
<h4>{esc(device.get('name'))}</h4>
|
||
<div class="ib-rate">{esc(fmt_number(device.get('rate_gbps')))}<small>Gb/s</small></div>
|
||
<dl>
|
||
<div><dt>PCI</dt><dd>{esc(device.get('bdf') or '—')}</dd></div>
|
||
<div><dt>NUMA</dt><dd>{esc(device.get('numa') if device.get('numa') is not None else '—')}</dd></div>
|
||
<div><dt>CA</dt><dd>{esc(device.get('ca_type') or '—')}</dd></div>
|
||
</dl>
|
||
</article>
|
||
"""
|
||
|
||
|
||
def render_management_card(nic: dict[str, Any]) -> str:
|
||
state = str(nic.get("state", "")).lower()
|
||
up = state == "up" and nic.get("carrier") != 0
|
||
addresses = nic.get("addresses") or []
|
||
if nic.get("is_bond"):
|
||
role = "带内 · 管理 Bond"
|
||
elif nic.get("default_route"):
|
||
role = "带内 · 默认路由"
|
||
elif addresses:
|
||
role = "带内 · OS 网络"
|
||
else:
|
||
role = "其他 OS 网卡"
|
||
address_html = "<br>".join(esc(value) for value in addresses) or "未配置三层地址"
|
||
return f"""
|
||
<article class="management-card">
|
||
<div class="card-head">
|
||
<span class="eyebrow">{esc(role)}</span>
|
||
{badge("UP" if up else (state.upper() or "UNKNOWN"), "ok" if up else "warn")}
|
||
</div>
|
||
<div class="management-title"><h4>{esc(nic.get('name'))}</h4><b>{esc(fmt_rate(nic.get('speed_mbps')))}</b></div>
|
||
<p class="address">{address_html}</p>
|
||
<dl>
|
||
<div><dt>Driver</dt><dd>{esc(nic.get('driver') or '—')}</dd></div>
|
||
<div><dt>PCI</dt><dd>{esc(nic.get('bdf') or '—')}</dd></div>
|
||
<div><dt>NUMA</dt><dd>{esc(nic.get('numa') if nic.get('numa') is not None else '—')}</dd></div>
|
||
<div><dt>MAC</dt><dd>{esc(nic.get('mac') or '—')}</dd></div>
|
||
</dl>
|
||
</article>
|
||
"""
|
||
|
||
|
||
def render_bmc_card(bmc: dict[str, Any]) -> str:
|
||
if bmc.get("detected"):
|
||
return f"""
|
||
<article class="management-card bmc-card">
|
||
<div class="card-head">
|
||
<span class="eyebrow">带外 · BMC</span>{badge("DETECTED", "ok")}
|
||
</div>
|
||
<div class="management-title"><h4>独立管理口</h4><b>{esc(bmc.get('source') or 'IPMI')}</b></div>
|
||
<p class="address">{esc(bmc.get('ip') or '—')}</p>
|
||
<dl>
|
||
<div><dt>Subnet</dt><dd>{esc(bmc.get('subnet') or '—')}</dd></div>
|
||
<div><dt>Gateway</dt><dd>{esc(bmc.get('gateway') or '—')}</dd></div>
|
||
<div><dt>Channel / VLAN</dt><dd>{esc(bmc.get('channel') or '—')} / {esc(bmc.get('vlan') or '—')}</dd></div>
|
||
<div><dt>MAC</dt><dd>{esc(bmc.get('mac') or '—')}</dd></div>
|
||
</dl>
|
||
</article>
|
||
"""
|
||
return f"""
|
||
<article class="management-card unavailable">
|
||
<div class="card-head">
|
||
<span class="eyebrow">带外 · BMC</span>{badge("OS 不可见", "neutral")}
|
||
</div>
|
||
<div class="management-title"><h4>独立管理口</h4><b>—</b></div>
|
||
<p class="address">{esc(bmc.get('note') or '未发现可读取的 BMC LAN 信息')}</p>
|
||
<p class="card-note">独立带外口通常不进入宿主 OS 网络命名空间;需要在 BMC/IPMI 或云厂商控制台侧验证。</p>
|
||
</article>
|
||
"""
|
||
|
||
|
||
def render_check(item: dict[str, str]) -> str:
|
||
icon = {"ok": "✓", "warning": "!", "critical": "×"}.get(item["level"], "·")
|
||
return f"""
|
||
<li class="check {esc(item['level'])}">
|
||
<span class="check-icon">{icon}</span>
|
||
<span><b>{esc(item['title'])}</b><small>{esc(item['detail'])}</small></span>
|
||
</li>
|
||
"""
|
||
|
||
|
||
def render_html(snapshot: dict[str, Any], expected: dict[str, int | None]) -> str:
|
||
health = evaluate(snapshot, expected)
|
||
system = snapshot.get("system", {})
|
||
gpus = snapshot.get("gpus", [])
|
||
bonds = snapshot.get("bonds", [])
|
||
ib_devices = snapshot.get("ib_devices", [])
|
||
network_endpoints = snapshot.get("network_endpoints", [])
|
||
endpoint_by_id = {
|
||
endpoint.get("id"): endpoint
|
||
for endpoint in network_endpoints
|
||
if endpoint.get("id")
|
||
}
|
||
lanes = "".join(
|
||
render_gpu_lane(
|
||
gpu, endpoint_by_id.get(gpu.get("primary_network_id"))
|
||
)
|
||
for gpu in gpus
|
||
)
|
||
numa_groups: list[tuple[Any, int, str]] = []
|
||
for gpu in gpus:
|
||
numa = gpu.get("numa")
|
||
cpu_range = str(gpu.get("cpu_affinity") or "—")
|
||
if (
|
||
not numa_groups
|
||
or numa_groups[-1][0] != numa
|
||
or numa_groups[-1][2] != cpu_range
|
||
):
|
||
numa_groups.append((numa, 1, cpu_range))
|
||
else:
|
||
previous_numa, previous_count, previous_cpu_range = numa_groups[-1]
|
||
numa_groups[-1] = (
|
||
previous_numa,
|
||
previous_count + 1,
|
||
previous_cpu_range,
|
||
)
|
||
numa_columns = " ".join(f"{count}fr" for _, count, _ in numa_groups) or "1fr"
|
||
numa_ruler = "".join(
|
||
f'<span data-numa="{esc(numa if numa is not None else "?")}" data-count="{count}">'
|
||
f"{esc(f'NUMA {numa}' if numa is not None else 'NUMA ?')} "
|
||
f"<small>推荐 CPU {esc(cpu_range)}</small></span>"
|
||
for numa, count, cpu_range in numa_groups
|
||
) or "<span>NUMA 未知</span>"
|
||
numa_values = list(
|
||
dict.fromkeys(
|
||
str(gpu.get("numa"))
|
||
for gpu in gpus
|
||
if gpu.get("numa") is not None
|
||
)
|
||
)
|
||
numa_switches = "".join(
|
||
f'<button type="button" class="numa-switch" data-numa-filter="{esc(numa)}" '
|
||
f'aria-pressed="false">NUMA {esc(numa)}</button>'
|
||
for numa in numa_values
|
||
)
|
||
rail_count = max(1, len(gpus))
|
||
rail_min_width = max(0, rail_count * 132 + max(0, rail_count - 1) * 10)
|
||
affinity_groups: dict[tuple[Any, str], list[int]] = {}
|
||
for gpu in gpus:
|
||
affinity_numa = gpu.get("numa_affinity")
|
||
key = (
|
||
affinity_numa if affinity_numa not in (None, "") else gpu.get("numa"),
|
||
str(gpu.get("cpu_affinity") or "—"),
|
||
)
|
||
affinity_groups.setdefault(key, []).append(int(gpu.get("index", 0)))
|
||
affinity_summary = "".join(
|
||
f"<span><b>{esc(compact_gpu_indices(indices))}</b><small>CPU {esc(cpu_range)} · MEM NUMA {esc(numa)}</small></span>"
|
||
for (numa, cpu_range), indices in affinity_groups.items()
|
||
)
|
||
ib_cards = "".join(render_ib_card(device) for device in ib_devices)
|
||
checks = "".join(render_check(item) for item in health["checks"])
|
||
accelerator_nic_names = {
|
||
member.get("name")
|
||
for endpoint in network_endpoints
|
||
for member in endpoint.get("members", [])
|
||
if member.get("name")
|
||
}
|
||
accelerator_nic_names.update(
|
||
endpoint.get("name")
|
||
for endpoint in network_endpoints
|
||
if endpoint.get("type") == "bond"
|
||
)
|
||
management_nics = [
|
||
nic
|
||
for nic in snapshot.get("nics", [])
|
||
if nic.get("name") not in accelerator_nic_names
|
||
and (
|
||
nic.get("bdf")
|
||
or nic.get("default_route")
|
||
or nic.get("is_bond")
|
||
)
|
||
]
|
||
management_cards = "".join(render_management_card(nic) for nic in management_nics)
|
||
bmc = snapshot.get("bmc") or {
|
||
"detected": False,
|
||
"note": "快照中没有 BMC/IPMI 信息",
|
||
}
|
||
management_cards += render_bmc_card(bmc)
|
||
|
||
total_memory = sum(number(gpu.get("memory_total_mib")) or 0 for gpu in gpus)
|
||
roce_mbps = sum(
|
||
number(endpoint.get("capacity_mbps")) or 0
|
||
for endpoint in network_endpoints
|
||
if endpoint.get("protocol") == "RoCE"
|
||
and str(endpoint.get("state") or "").upper() in {"UP", "ACTIVE"}
|
||
)
|
||
ib_gbps = sum(
|
||
number(device.get("rate_gbps")) or 0
|
||
for device in ib_devices
|
||
if str(device.get("state", "")).lower() == "active"
|
||
)
|
||
power_data_complete = bool(gpus) and all(
|
||
number(gpu.get("power_w")) is not None
|
||
and number(gpu.get("power_limit_w")) is not None
|
||
and bool(gpu.get("pstate"))
|
||
for gpu in gpus
|
||
)
|
||
total_power_w = (
|
||
sum(number(gpu.get("power_w")) or 0 for gpu in gpus)
|
||
if power_data_complete
|
||
else 0
|
||
)
|
||
total_power_limit_w = (
|
||
sum(number(gpu.get("power_limit_w")) or 0 for gpu in gpus)
|
||
if power_data_complete
|
||
else 0
|
||
)
|
||
power_pct = (
|
||
100 * total_power_w / total_power_limit_w
|
||
if total_power_limit_w
|
||
else 0
|
||
)
|
||
p0_count = sum(str(gpu.get("pstate", "")).upper() == "P0" for gpu in gpus)
|
||
nvlink = snapshot.get("nvlink", {})
|
||
software = snapshot.get("software", {})
|
||
generated = snapshot.get("generated_at", "")
|
||
memory_total_kib = number(system.get("memory_total_kib"))
|
||
memory_used_kib = number(system.get("memory_used_kib"))
|
||
memory_pct = (
|
||
100 * memory_used_kib / memory_total_kib
|
||
if memory_used_kib is not None and memory_total_kib
|
||
else 0
|
||
)
|
||
cpu_count = system.get("cpu_physical_cores") or system.get("cpu_logical") or "—"
|
||
cpu_count_label = "物理核心" if system.get("cpu_physical_cores") else "逻辑 CPU"
|
||
|
||
payload = json.dumps(snapshot, ensure_ascii=False).replace("</", "<\\/")
|
||
body = f"""
|
||
<header class="hero">
|
||
<div>
|
||
<span class="kicker">GPU NODE · HARDWARE PORTRAIT</span>
|
||
<h1>{esc(system.get('hostname') or 'GPU Node')}</h1>
|
||
<p>{esc(system.get('manufacturer'))} {esc(system.get('product'))} · {esc(system.get('os'))}</p>
|
||
</div>
|
||
<div class="hero-actions">
|
||
<span class="overall {esc(health['status'])}"><i></i>{esc(status_label(health['status']))}</span>
|
||
<button type="button" onclick="window.print()">打印 / PDF</button>
|
||
</div>
|
||
</header>
|
||
|
||
<main>
|
||
<section class="overview-grid" aria-label="节点概览">
|
||
<article class="overview-card system-overview">
|
||
<div class="overview-head">
|
||
<div><span class="kicker">SYSTEM / SOFTWARE</span><h2>系统与软件</h2></div>
|
||
<button class="ghost compact" type="button" data-open-raw>原始快照</button>
|
||
</div>
|
||
<p class="overview-lead">{esc(system.get('manufacturer'))} {esc(system.get('product'))}</p>
|
||
<dl class="overview-facts">
|
||
<div><dt>操作系统</dt><dd>{esc(system.get('os') or '—')}</dd></div>
|
||
<div><dt>内核</dt><dd>{esc(system.get('kernel') or '—')}</dd></div>
|
||
<div><dt>Driver</dt><dd>{esc(software.get('nvidia_driver') or '—')}</dd></div>
|
||
<div><dt>CUDA Toolkit</dt><dd>{esc(software.get('cuda_toolkit') or '—')} <small>nvcc</small></dd></div>
|
||
<div><dt>虚拟化</dt><dd>{esc(system.get('virtualization') or 'Bare metal / 未识别')}</dd></div>
|
||
<div><dt>生成时间</dt><dd>{esc(generated)}</dd></div>
|
||
</dl>
|
||
</article>
|
||
|
||
<article class="overview-card gpu-overview">
|
||
<div class="overview-head"><div><span class="kicker">ACCELERATOR</span><h2>GPU</h2></div>{badge("NVLINK ACTIVE" if nvlink.get('active_links') else "NVLINK 未知", "ok" if nvlink.get('active_links') else "warn")}</div>
|
||
<p class="overview-lead">{esc(gpus[0].get('name') if gpus else '未发现 NVIDIA GPU')}</p>
|
||
<div class="overview-primary"><b>{len(gpus)}</b><span>GPU</span></div>
|
||
<div class="gpu-power-summary">
|
||
<div><span>GPU 瞬时功耗</span><b>{esc(fmt_power(total_power_w) if power_data_complete else '遥测不完整')} / {esc(fmt_power(total_power_limit_w) if power_data_complete else '—')}</b></div>
|
||
<div class="power-meter"><i style="width:{max(0, min(100, power_pct)):.1f}%"></i></div>
|
||
<small>Power Limit 上限 · {p0_count}/{len(gpus)} 张处于 P0 · 单次瞬时采样</small>
|
||
</div>
|
||
<div class="overview-metrics">
|
||
<span><b>{esc(fmt_mib(total_memory))}</b><small>GPU 显存</small></span>
|
||
<span><b>{esc(fmt_rate(roce_mbps))}</b><small>RoCE 物理容量</small></span>
|
||
<span><b>{esc(nvlink.get('links_per_gpu') or '—')}</b><small>NVLink / GPU</small></span>
|
||
<span><b>{ib_gbps:g}G</b><small>InfiniBand</small></span>
|
||
</div>
|
||
</article>
|
||
|
||
<article class="overview-card cpu-overview">
|
||
<div class="overview-head"><div><span class="kicker">HOST COMPUTE</span><h2>CPU 与内存</h2></div>{badge(f"{system.get('numa_nodes') or '—'} NUMA", "neutral")}</div>
|
||
<p class="overview-lead">{esc(system.get('cpu_model') or '未知 CPU')}</p>
|
||
<div class="overview-primary"><b>{esc(cpu_count)}</b><span>{esc(cpu_count_label)}</span></div>
|
||
<div class="memory-row"><span>系统内存</span><b>{esc(fmt_kib(memory_used_kib))} / {esc(fmt_kib(memory_total_kib))}</b></div>
|
||
<div class="memory-meter"><i style="width:{max(0, min(100, memory_pct)):.1f}%"></i></div>
|
||
<div class="overview-metrics cpu-metrics">
|
||
<span><b>{esc(system.get('cpu_sockets') or '—')}</b><small>CPU Sockets</small></span>
|
||
<span><b>{esc(system.get('cpu_logical') or '—')}</b><small>逻辑 CPU</small></span>
|
||
<span><b>{memory_pct:.1f}%</b><small>内存使用率</small></span>
|
||
</div>
|
||
<div class="cpu-affinity-summary">
|
||
{affinity_summary or '<span><b>未解析</b><small>CPU / NUMA 推荐亲和</small></span>'}
|
||
</div>
|
||
</article>
|
||
</section>
|
||
|
||
<section class="panel topology-panel">
|
||
<div class="section-head">
|
||
<div><span class="kicker">TOPOLOGY</span><h2>GPU ↔ RDMA 亲和画像</h2></div>
|
||
<div class="legend">
|
||
{badge("PIX 最佳", "ok")}{badge("PXB / PHB", "warn")}{badge("SYS 跨 NUMA", "bad")}
|
||
</div>
|
||
</div>
|
||
<div class="topology-viewbar">
|
||
<div>
|
||
<b>NUMA 视图</b>
|
||
<small>总览显示完整拓扑,分览聚焦单个 CPU / 内存域</small>
|
||
</div>
|
||
<div class="numa-switcher" role="group" aria-label="NUMA 拓扑视图">
|
||
<button type="button" class="numa-switch active" data-numa-filter="all" aria-pressed="true">总览</button>
|
||
{numa_switches}
|
||
</div>
|
||
</div>
|
||
<div class="numa-ruler" style="grid-template-columns:{esc(numa_columns)}">
|
||
{numa_ruler}
|
||
</div>
|
||
<p class="affinity-note">CPU范围来自 NVIDIA 拓扑亲和,表示更近的CPU/内存路径;不代表当前训练进程已经绑核。</p>
|
||
<div class="nvlink-fabric">
|
||
<span>NVLINK / NVSWITCH FABRIC</span>
|
||
<b>{esc(nvlink.get('active_links') or '—')} Active Links</b>
|
||
<small>{esc(nvlink.get('speed_gbs') or '—')} GB/s / Link</small>
|
||
</div>
|
||
<div class="rails" style="grid-template-columns:repeat({rail_count},minmax(132px,1fr));min-width:{rail_min_width}px">{lanes or '<div class="empty-state">未采集到 GPU 数据</div>'}</div>
|
||
<p class="capacity-note">有 Bond 时展示聚合容器与物理成员;无 Bond 时展示独立 RoCE/IB 端点。LACP 容量不代表单连接带宽。</p>
|
||
</section>
|
||
|
||
<div class="network-grid">
|
||
<section class="panel">
|
||
<div class="section-head">
|
||
<div><span class="kicker">STORAGE / FABRIC</span><h2>InfiniBand</h2></div>
|
||
</div>
|
||
<div class="ib-grid">{ib_cards or '<div class="empty-state">未发现独立 InfiniBand 设备</div>'}</div>
|
||
</section>
|
||
<section class="panel">
|
||
<div class="section-head">
|
||
<div><span class="kicker">MANAGEMENT NETWORK</span><h2>带内与带外管理网络</h2></div>
|
||
</div>
|
||
<div class="management-grid">{management_cards}</div>
|
||
</section>
|
||
</div>
|
||
|
||
<section class="panel health-panel">
|
||
<div class="section-head">
|
||
<div><span class="kicker">HEALTH / VALIDATION</span><h2>HEALTH 与验证结果</h2></div>
|
||
<span class="overall {esc(health['status'])}"><i></i>{esc(status_label(health['status']))}</span>
|
||
</div>
|
||
<ul class="checks">{checks}</ul>
|
||
</section>
|
||
</main>
|
||
|
||
<dialog id="raw-dialog">
|
||
<div class="dialog-head"><div><span class="kicker">RAW DATA</span><h2>JSON Snapshot</h2></div><button type="button" data-close-raw>关闭</button></div>
|
||
<pre>{esc(json.dumps(snapshot, ensure_ascii=False, indent=2))}</pre>
|
||
</dialog>
|
||
|
||
<footer>
|
||
<span>GPU Node Profile v{esc(VERSION)}</span>
|
||
<span>只读采集 · 原生 HTML · 无外部依赖</span>
|
||
</footer>
|
||
<script type="application/json" id="snapshot">{payload}</script>
|
||
"""
|
||
title = f"{system.get('hostname') or 'GPU Node'} · GPU Hardware Portrait"
|
||
template = load_html_template("gpu_node_profile.html")
|
||
return clean_generated_text(
|
||
template.replace("@@TITLE@@", esc(title)).replace("@@BODY@@", body)
|
||
)
|
||
|
||
|
||
def clean_generated_text(content: str) -> str:
|
||
"""Remove template indentation residue without changing line structure."""
|
||
cleaned = "\n".join(line.rstrip() for line in content.splitlines())
|
||
return cleaned + ("\n" if content.endswith("\n") else "")
|
||
|
||
|
||
def load_html_template(filename: str) -> str:
|
||
"""Load a renderer template beside the collector script."""
|
||
template_path = Path(__file__).resolve().parent / "templates" / filename
|
||
try:
|
||
return template_path.read_text(encoding="utf-8")
|
||
except OSError as exc:
|
||
raise OSError(
|
||
f"HTML template not found: {template_path}. "
|
||
"Keep gpu_node_profile.py and templates/ together."
|
||
) from exc
|
||
|
||
|
||
def render_topology_3d_html(snapshot: dict[str, Any]) -> str:
|
||
payload = json.dumps(snapshot, ensure_ascii=False).replace("</", "<\\/")
|
||
system = snapshot.get("system", {})
|
||
title = f"{system.get('hostname') or 'GPU Node'} · 3D Topology CT"
|
||
template = load_html_template("gpu_topology_3d.html")
|
||
return clean_generated_text(
|
||
template.replace("@@TITLE@@", esc(title))
|
||
.replace("@@MACHINE_DATA@@", payload)
|
||
.replace("@@VERSION@@", esc(VERSION))
|
||
)
|
||
|
||
|
||
|
||
|
||
def atomic_write(path: Path, content: str) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||
try:
|
||
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
|
||
handle.write(content)
|
||
os.replace(temp_name, path)
|
||
except Exception:
|
||
try:
|
||
os.unlink(temp_name)
|
||
except OSError:
|
||
pass
|
||
raise
|
||
|
||
|
||
def load_snapshot(path: Path) -> dict[str, Any]:
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
if not isinstance(data, dict) or data.get("schema_version") != 1:
|
||
raise ValueError("snapshot schema_version must be 1")
|
||
nvlink = data.setdefault("nvlink", {})
|
||
nvlink.setdefault(
|
||
"fabric_type",
|
||
infer_nvlink_fabric_type(data.get("topology", {}), len(data.get("gpus", []))),
|
||
)
|
||
attach_gpu_bond_mapping(data)
|
||
return data
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(
|
||
description="Generate a native, self-contained HTML portrait for a GPU node."
|
||
)
|
||
parser.add_argument("-o", "--output", type=Path, default=Path("gpu_node_profile.html"))
|
||
parser.add_argument(
|
||
"--topology-output",
|
||
type=Path,
|
||
help="Generate the standalone 3D topology CT HTML from the same snapshot",
|
||
)
|
||
parser.add_argument("--snapshot-in", type=Path, help="Render an existing JSON snapshot")
|
||
parser.add_argument("--snapshot-out", type=Path, help="Save collected JSON snapshot")
|
||
parser.add_argument("--expected-gpus", type=int)
|
||
parser.add_argument("--expected-bonds", type=int)
|
||
parser.add_argument("--expected-ib", type=int)
|
||
output_group = parser.add_mutually_exclusive_group()
|
||
output_group.add_argument("--stdout", action="store_true", help="Write operations HTML to stdout")
|
||
output_group.add_argument(
|
||
"--snapshot-stdout",
|
||
action="store_true",
|
||
help="Write collected structured JSON to stdout and skip HTML rendering",
|
||
)
|
||
parser.add_argument("--version", action="version", version=f"%(prog)s {VERSION}")
|
||
return parser
|
||
|
||
|
||
def main(argv: Iterable[str] | None = None) -> int:
|
||
args = build_parser().parse_args(argv)
|
||
try:
|
||
snapshot = load_snapshot(args.snapshot_in) if args.snapshot_in else collect_snapshot()
|
||
if args.snapshot_stdout:
|
||
sys.stdout.write(json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n")
|
||
return 0
|
||
if args.snapshot_out:
|
||
atomic_write(
|
||
args.snapshot_out.expanduser().resolve(),
|
||
json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n",
|
||
)
|
||
expected = {
|
||
"gpus": args.expected_gpus,
|
||
"bonds": args.expected_bonds,
|
||
"ib": args.expected_ib,
|
||
}
|
||
report = render_html(snapshot, expected)
|
||
if args.stdout:
|
||
sys.stdout.write(report)
|
||
else:
|
||
output = args.output.expanduser().resolve()
|
||
atomic_write(output, report)
|
||
print(f"GPU node profile generated: {output}")
|
||
if args.topology_output:
|
||
topology_output = args.topology_output.expanduser().resolve()
|
||
atomic_write(topology_output, render_topology_3d_html(snapshot))
|
||
if not args.stdout:
|
||
print(f"GPU 3D topology generated: {topology_output}")
|
||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||
print(f"error: {exc}", file=sys.stderr)
|
||
return 2
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|