{esc(endpoint.get("name"))}
{esc(endpoint.get("rdma") or "无 RDMA 映射")}
#!/usr/bin/env python3 """Generate a self-contained, native HTML profile for a GPU node. The collector only executes read-only commands. The generated report has no Mermaid, JavaScript framework, external font, or CDN dependency. """ from __future__ import annotations import argparse import csv import datetime as dt import html import json import os import platform import re import shutil import socket import subprocess import sys import tempfile from pathlib import Path from typing import Any, Iterable VERSION = "2.1.0" ANSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") PCI_RE = re.compile(r"(?:00000000:|0000:)?([0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7])") def run(command: list[str], timeout: int = 12) -> str: """Run a read-only command and return stdout without raising.""" if not command or shutil.which(command[0]) is None: return "" try: result = subprocess.run( command, check=False, capture_output=True, text=True, timeout=timeout, env={**os.environ, "LC_ALL": "C"}, ) except (OSError, subprocess.SubprocessError): return "" return ANSI_RE.sub("", result.stdout).strip() def read_text(path: str | Path) -> str: try: return Path(path).read_text(encoding="utf-8", errors="replace").strip() except OSError: return "" def read_int(path: str | Path) -> int | None: value = read_text(path) try: return int(value) except (TypeError, ValueError): return None def normalize_bdf(value: str) -> str: matches = PCI_RE.findall(value or "") return matches[-1].lower() if matches else "" def pci_numa(bdf: str) -> int | None: bdf = normalize_bdf(bdf) return read_int(f"/sys/bus/pci/devices/0000:{bdf}/numa_node") if bdf else None def pci_path(bdf: str) -> list[str]: """Return the guest-visible PCI ancestor path ending at the endpoint.""" bdf = normalize_bdf(bdf) if not bdf: return [] try: resolved = str(Path(f"/sys/bus/pci/devices/0000:{bdf}").resolve()) except OSError: return [bdf] path: list[str] = [] for match in PCI_RE.findall(resolved): normalized = normalize_bdf(match) if normalized and normalized not in path: path.append(normalized) if bdf not in path: path.append(bdf) return path def dmi_value(name: str) -> str: return read_text(f"/sys/class/dmi/id/{name}") def parse_csv_rows(text: str) -> list[list[str]]: if not text: return [] return [[field.strip() for field in row] for row in csv.reader(text.splitlines())] def collect_gpus() -> list[dict[str, Any]]: core_fields = [ "index", "name", "uuid", "pci.bus_id", "driver_version", "memory.total", "memory.used", "temperature.gpu", "utilization.gpu", ] output = run( [ "nvidia-smi", f"--query-gpu={','.join(core_fields)}", "--format=csv,noheader,nounits", ] ) power_fields = [ "index", "pstate", "power.draw", "power.limit", ] power_output = run( [ "nvidia-smi", f"--query-gpu={','.join(power_fields)}", "--format=csv,noheader,nounits", ] ) power_by_index: dict[int, dict[str, Any]] = {} for row in parse_csv_rows(power_output): if len(row) < len(power_fields) or not row[0].isdigit(): continue power_by_index[int(row[0])] = { "pstate": row[1] if row[1] not in {"", "N/A", "[N/A]"} else "", "power_w": number(row[2]), "power_limit_w": number(row[3]), } range_fields = [ "index", "power.default_limit", "power.min_limit", "power.max_limit", ] range_output = run( [ "nvidia-smi", f"--query-gpu={','.join(range_fields)}", "--format=csv,noheader,nounits", ] ) for row in parse_csv_rows(range_output): if len(row) < len(range_fields) or not row[0].isdigit(): continue power_by_index.setdefault(int(row[0]), {}).update( { "power_default_limit_w": number(row[1]), "power_min_limit_w": number(row[2]), "power_max_limit_w": number(row[3]), } ) gpus: list[dict[str, Any]] = [] for row in parse_csv_rows(output): if len(row) < len(core_fields): continue bdf = normalize_bdf(row[3]) index = int(row[0]) if row[0].isdigit() else len(gpus) gpus.append( { "index": index, "name": row[1], "uuid": row[2], "bdf": bdf, "driver": row[4], "memory_total_mib": number(row[5]), "memory_used_mib": number(row[6]), "temperature_c": number(row[7]), "utilization_pct": number(row[8]), "numa": pci_numa(bdf), "pci_path": pci_path(bdf), **power_by_index.get(index, {}), } ) return sorted(gpus, key=lambda item: item["index"]) def number(value: Any) -> float | None: try: return float(str(value).strip()) except (TypeError, ValueError): return None def parse_topology(text: str) -> dict[str, Any]: """Parse nvidia-smi topo -m into GPU relationships and NIC legend.""" text = ANSI_RE.sub("", text or "") lines = [line.rstrip() for line in text.splitlines()] header: list[str] = [] rows: dict[str, dict[str, str]] = {} nic_legend: dict[str, str] = {} gpu_affinity: dict[str, dict[str, str]] = {} for line in lines: stripped = line.strip() if not stripped: continue legend_match = re.match(r"(NIC\d+):\s+(\S+)", stripped) if legend_match: nic_legend[legend_match.group(1)] = legend_match.group(2) continue fields = re.split(r"\s+", stripped) if not header and fields and fields[0] == "GPU0": # Only retain the square GPU/NIC matrix. Labels such as # "CPU Affinity" contain spaces while row values do not, so keeping # them in the same whitespace-split header would shift columns. matrix_end = fields.index("CPU") if "CPU" in fields else len(fields) header = fields[:matrix_end] continue if header and fields and re.fullmatch(r"GPU\d+", fields[0]): matrix_end = 1 + len(header) values = fields[1:matrix_end] gpu_name = fields[0] rows[gpu_name] = dict(zip(header, values)) suffix = fields[matrix_end:] gpu_affinity[gpu_name] = { "cpu_affinity": suffix[0] if len(suffix) > 0 else "", "numa_affinity": suffix[1] if len(suffix) > 1 else "", "gpu_numa_id": suffix[2] if len(suffix) > 2 else "", } gpu_nic: dict[str, list[dict[str, str]]] = {} for gpu_name, relations in rows.items(): links: list[dict[str, str]] = [] for column, relation in relations.items(): if column.startswith("NIC"): links.append( { "slot": column, "rdma": nic_legend.get(column, column), "relation": relation, } ) gpu_nic[gpu_name] = links return { "rows": rows, "nic_legend": nic_legend, "gpu_nic": gpu_nic, "gpu_affinity": gpu_affinity, } def parse_nvlink_status(text: str) -> dict[str, Any]: current = "" links: dict[str, list[float]] = {} for raw_line in (text or "").splitlines(): line = raw_line.strip() gpu_match = re.match(r"GPU\s+(\d+):", line) if gpu_match: current = f"GPU{gpu_match.group(1)}" links[current] = [] continue link_match = re.match(r"Link\s+\d+:\s+([\d.]+)\s+GB/s", line) if current and link_match: links[current].append(float(link_match.group(1))) counts = [len(values) for values in links.values()] speeds = [speed for values in links.values() for speed in values] return { "per_gpu": links, "active_links": sum(counts), "links_per_gpu": min(counts) if counts else 0, "speed_gbs": min(speeds) if speeds else None, } def infer_nvlink_fabric_type(topology: dict[str, Any], gpu_count: int) -> str: """Avoid calling every NVLink configuration an NVSwitch fabric.""" if gpu_count < 2: return "" nv_pairs = 0 for row_name, relations in topology.get("rows", {}).items(): if not str(row_name).startswith("GPU"): continue nv_pairs += sum( str(peer).startswith("GPU") and str(relation).startswith("NV") for peer, relation in relations.items() ) if gpu_count > 2 and nv_pairs >= gpu_count * (gpu_count - 1): return "NVSwitch Fabric" return "NVLink Fabric" def parse_bond_text(name: str, text: str) -> dict[str, Any]: bond: dict[str, Any] = { "name": name, "mode": "", "hash_policy": "", "status": "", "slaves": [], } current: dict[str, Any] | None = None for raw_line in (text or "").splitlines(): line = raw_line.strip() if line.startswith("Bonding Mode:"): bond["mode"] = line.split(":", 1)[1].strip() elif line.startswith("Transmit Hash Policy:"): bond["hash_policy"] = line.split(":", 1)[1].strip() elif line.startswith("MII Status:") and current is None: bond["status"] = line.split(":", 1)[1].strip().lower() elif line.startswith("Slave Interface:"): current = {"name": line.split(":", 1)[1].strip()} bond["slaves"].append(current) elif current is not None and ":" in line: key, value = [part.strip() for part in line.split(":", 1)] if key == "MII Status": current["status"] = value.lower() elif key == "Speed": current["speed_mbps"] = int(value.split()[0]) if value.split()[0].isdigit() else None elif key == "Duplex": current["duplex"] = value elif key == "Link Failure Count": current["failures"] = int(value) if value.isdigit() else None elif key == "Permanent HW addr": current["mac"] = value return bond def parse_ibdev2netdev(text: str) -> tuple[dict[str, str], dict[str, str]]: rdma_to_net: dict[str, str] = {} net_to_rdma: dict[str, str] = {} pattern = re.compile(r"(\S+)\s+port\s+\d+\s+==>\s+(\S+)\s+\(([^)]+)\)") for line in (text or "").splitlines(): match = pattern.search(line) if match: rdma, netdev = match.group(1), match.group(2) rdma_to_net[rdma] = netdev net_to_rdma[netdev] = rdma return rdma_to_net, net_to_rdma def parse_rdma_links(text: str) -> dict[str, dict[str, Any]]: links: dict[str, dict[str, Any]] = {} for line in (text or "").splitlines(): match = re.match(r"link\s+([^/]+)/(\d+)\s+(.*)", line.strip()) if not match: continue name, port, tail = match.groups() links[name] = { "port": int(port), "state": capture_token(tail, "state"), "physical_state": capture_token(tail, "physical_state"), "netdev": capture_token(tail, "netdev"), } return links def capture_token(text: str, key: str) -> str: match = re.search(rf"(?:^|\s){re.escape(key)}\s+(\S+)", text) return match.group(1) if match else "" def parse_ibstat(text: str) -> list[dict[str, Any]]: devices: list[dict[str, Any]] = [] current: dict[str, Any] | None = None in_port = False for raw_line in (text or "").splitlines(): line = raw_line.strip() ca_match = re.match(r"CA '([^']+)'", line) if ca_match: current = {"name": ca_match.group(1)} devices.append(current) in_port = False continue if current is None: continue if line.startswith("CA type:"): current["ca_type"] = line.split(":", 1)[1].strip() elif line.startswith("Firmware version:"): current["firmware"] = line.split(":", 1)[1].strip() elif line.startswith("Port 1:"): in_port = True elif in_port and line.startswith("State:"): current["state"] = line.split(":", 1)[1].strip() elif in_port and line.startswith("Physical state:"): current["physical_state"] = line.split(":", 1)[1].strip() elif in_port and line.startswith("Rate:"): current["rate_gbps"] = number(line.split(":", 1)[1].strip()) elif in_port and line.startswith("Base lid:"): current["lid"] = line.split(":", 1)[1].strip() elif in_port and line.startswith("Link layer:"): current["link_layer"] = line.split(":", 1)[1].strip() return devices def collect_nics() -> list[dict[str, Any]]: base = Path("/sys/class/net") if not base.exists(): return [] nics: list[dict[str, Any]] = [] for entry in sorted(base.iterdir(), key=lambda item: natural_key(item.name)): name = entry.name if name == "lo": continue is_bond = (entry / "bonding").exists() driver_info = run(["ethtool", "-i", name]) driver = field_from_lines(driver_info, "driver") or ("bonding" if is_bond else "") bus = normalize_bdf(field_from_lines(driver_info, "bus-info")) if not driver and not bus: continue speed_text = field_from_lines(run(["ethtool", name]), "Speed") speed_match = re.search(r"(\d+)", speed_text) nics.append( { "name": name, "driver": driver, "bdf": bus, "numa": pci_numa(bus), "pci_path": pci_path(bus), "speed_mbps": int(speed_match.group(1)) if speed_match else None, "state": read_text(entry / "operstate").lower(), "carrier": read_int(entry / "carrier"), "mac": read_text(entry / "address"), "is_bond": is_bond, } ) attach_network_context(nics) return nics def attach_network_context(nics: list[dict[str, Any]]) -> None: """Attach addresses and default-route roles to OS-visible interfaces.""" addresses: dict[str, list[str]] = {} defaults: set[str] = set() try: address_rows = json.loads(run(["ip", "-j", "address", "show"]) or "[]") for row in address_rows: values = [] for item in row.get("addr_info", []): local = item.get("local") prefix = item.get("prefixlen") if local and item.get("scope") != "link": values.append(f"{local}/{prefix}" if prefix is not None else str(local)) addresses[str(row.get("ifname", ""))] = values route_rows = json.loads(run(["ip", "-j", "route", "show", "default"]) or "[]") defaults = {str(row.get("dev")) for row in route_rows if row.get("dev")} except (TypeError, ValueError, json.JSONDecodeError): pass for nic in nics: name = nic["name"] nic["addresses"] = addresses.get(name, []) nic["default_route"] = name in defaults def collect_bmc() -> dict[str, Any]: """Read common BMC LAN channels when IPMI is locally available.""" if shutil.which("ipmitool") is None: return { "detected": False, "note": "独立 BMC 带外口未向当前 OS/IPMI 通道暴露", } last_values: dict[str, Any] = {} for channel in (1, 8): output = run(["ipmitool", "lan", "print", str(channel)], timeout=8) values: dict[str, Any] = {"channel": channel} wanted = { "IP Address Source": "source", "IP Address": "ip", "Subnet Mask": "subnet", "MAC Address": "mac", "Default Gateway IP": "gateway", "802.1q VLAN ID": "vlan", } for line in output.splitlines(): if ":" not in line: continue key, value = [part.strip() for part in line.split(":", 1)] if key in wanted: values[wanted[key]] = value last_values = values if values.get("ip") and values["ip"] not in {"0.0.0.0", "0.0.0.0/0"}: return {"detected": True, **values, "note": ""} return { "detected": False, **last_values, "note": "IPMI 可访问,但常用 LAN 通道未读取到有效地址", } def field_from_lines(text: str, field: str) -> str: prefix = f"{field}:" for line in (text or "").splitlines(): if line.strip().startswith(prefix): return line.split(":", 1)[1].strip() return "" def natural_key(value: str) -> list[Any]: return [int(part) if part.isdigit() else part for part in re.split(r"(\d+)", value)] def collect_bonds(net_to_rdma: dict[str, str], nics: list[dict[str, Any]]) -> list[dict[str, Any]]: nic_map = {nic["name"]: nic for nic in nics} bond_dir = Path("/proc/net/bonding") if not bond_dir.exists(): return [] bonds: list[dict[str, Any]] = [] for path in sorted(bond_dir.iterdir(), key=lambda item: natural_key(item.name)): bond = parse_bond_text(path.name, read_text(path)) bond["rdma"] = net_to_rdma.get(path.name, "") for slave in bond["slaves"]: slave.update( { key: value for key, value in nic_map.get(slave["name"], {}).items() if key in {"bdf", "numa", "driver", "carrier", "pci_path"} } ) bonds.append(bond) return bonds def build_network_endpoints( nics: list[dict[str, Any]], bonds: list[dict[str, Any]], ib_devices: list[dict[str, Any]], rdma_to_net: dict[str, str], rdma_links: dict[str, dict[str, Any]] | None = None, ) -> list[dict[str, Any]]: """Normalize bonded RoCE, standalone RoCE and native IB as RDMA endpoints.""" rdma_links = rdma_links or {} nic_by_name = {str(nic.get("name")): nic for nic in nics} consumed_rdma: set[str] = set() consumed_netdevs: set[str] = set() endpoints: list[dict[str, Any]] = [] for bond in bonds: rdma = str(bond.get("rdma") or "") # A host may also use ordinary management/storage bonds. Only a bond # exposed by the RDMA stack belongs in the accelerator topology. if not rdma: continue members = [dict(member) for member in bond.get("slaves", [])] consumed_rdma.add(rdma) consumed_netdevs.update(str(member.get("name")) for member in members) endpoint = { "id": f"bond:{bond.get('name')}", "type": "bond", "protocol": "RoCE", "name": bond.get("name"), "rdma": rdma, "role": "compute", "state": str(bond.get("status") or "").upper(), "numa": next( ( member.get("numa") for member in members if member.get("numa") is not None ), None, ), "capacity_mbps": sum( number(member.get("speed_mbps")) or 0 for member in members ), "members": members, "gpu_links": [], } endpoints.append(endpoint) for device in ib_devices: rdma = str(device.get("name") or "") consumed_rdma.add(rdma) endpoints.append( { "id": f"ib:{rdma}", "type": "standalone", "protocol": "InfiniBand", "name": rdma, "rdma": rdma, "role": "storage", "state": str(device.get("state") or "").upper(), "physical_state": str(device.get("physical_state") or ""), "numa": device.get("numa"), "bdf": device.get("bdf") or "", "pci_path": device.get("pci_path") or [], "capacity_mbps": ( (number(device.get("rate_gbps")) or 0) * 1000 ), "members": [ { "name": rdma, "rdma": rdma, "bdf": device.get("bdf") or "", "numa": device.get("numa"), "pci_path": device.get("pci_path") or [], "speed_mbps": ( (number(device.get("rate_gbps")) or 0) * 1000 ), "status": str(device.get("state") or "").lower(), } ], "gpu_links": [], } ) for rdma, netdev in sorted( rdma_to_net.items(), key=lambda item: natural_key(item[0]) ): if rdma in consumed_rdma or netdev in consumed_netdevs: continue nic = nic_by_name.get(netdev) if ( not nic or not nic.get("bdf") or nic.get("default_route") or nic.get("is_bond") ): continue rdma_state = rdma_links.get(rdma, {}) state = str(rdma_state.get("state") or nic.get("state") or "").upper() member = { **nic, "rdma": rdma, "status": str(nic.get("state") or "").lower(), } endpoints.append( { "id": f"rdma:{rdma}", "type": "standalone", "protocol": "RoCE", "name": netdev, "rdma": rdma, "role": "compute", "state": state, "physical_state": str( rdma_state.get("physical_state") or "" ), "numa": nic.get("numa"), "bdf": nic.get("bdf") or "", "pci_path": nic.get("pci_path") or [], "capacity_mbps": number(nic.get("speed_mbps")) or 0, "members": [member], "gpu_links": [], } ) return sorted( endpoints, key=lambda endpoint: ( 0 if endpoint.get("role") == "compute" else 1, str(endpoint.get("numa")), natural_key(str(endpoint.get("name") or "")), ), ) def parse_lspci_devices(text: str) -> dict[str, dict[str, str]]: devices: dict[str, dict[str, str]] = {} pattern = re.compile(r"^[0-9a-fA-F]{4}:([0-9a-fA-F:.]+)\s+(.+)$") for line in (text or "").splitlines(): match = pattern.match(line.strip()) if not match: continue bdf, description = match.groups() devices[normalize_bdf(bdf)] = { "bdf": normalize_bdf(bdf), "description": description, } return devices def parse_lspci_network(text: str) -> dict[str, dict[str, str]]: return { bdf: device for bdf, device in parse_lspci_devices(text).items() if re.search( r"Ethernet|Infiniband|Network controller", device.get("description", ""), re.I, ) } def build_pci_fabric( gpus: list[dict[str, Any]], nics: list[dict[str, Any]], ib_devices: list[dict[str, Any]], pci_devices: dict[str, dict[str, str]], ) -> dict[str, Any]: """Build a device-neutral PCI graph for both 2D and 3D renderers.""" endpoints: list[dict[str, Any]] = [] seen: set[tuple[str, str]] = set() def add_endpoint( kind: str, label: str, bdf: str, numa: Any, path: list[str] | None, ) -> None: bdf = normalize_bdf(bdf) key = (kind, bdf) if not bdf or key in seen: return seen.add(key) endpoints.append( { "kind": kind, "label": label, "bdf": bdf, "numa": numa, "path": path or pci_path(bdf), "description": pci_devices.get(bdf, {}).get("description", ""), } ) for gpu in gpus: add_endpoint( "gpu", f"GPU{gpu.get('index')}", str(gpu.get("bdf") or ""), gpu.get("numa"), gpu.get("pci_path"), ) for nic in nics: add_endpoint( "nic", str(nic.get("name") or "NIC"), str(nic.get("bdf") or ""), nic.get("numa"), nic.get("pci_path"), ) for device in ib_devices: add_endpoint( "infiniband", str(device.get("name") or "IB"), str(device.get("bdf") or ""), device.get("numa"), device.get("pci_path"), ) bridges: dict[str, dict[str, Any]] = {} for endpoint in endpoints: path = endpoint.get("path") or [] for index, bdf in enumerate(path[:-1]): bridge = bridges.setdefault( bdf, { "bdf": bdf, "parent_bdf": path[index - 1] if index else "", "numa": pci_numa(bdf), "description": pci_devices.get(bdf, {}).get( "description", "PCI bridge" ), "children": [], }, ) child = path[index + 1] if child not in bridge["children"]: bridge["children"].append(child) return { "scope": "guest-visible", "virtualization": "passthrough-or-virtual-pci", "bridges": sorted(bridges.values(), key=lambda item: natural_key(item["bdf"])), "endpoints": sorted( endpoints, key=lambda item: (str(item.get("numa")), item["kind"], natural_key(item["label"])), ), } def collect_cuda_toolkit() -> str: for executable in ("nvcc", "/usr/local/cuda/bin/nvcc"): output = run([executable, "--version"]) match = re.search(r"release\s+([\d.]+)", output) if match: return concise_version(match.group(1)) for path in ("/usr/local/cuda/version.json", "/usr/local/cuda/version.txt"): text = read_text(path) match = re.search(r"(?:CUDA Version|\"version\"\s*:)\s*\"?([\d.]+)", text) if match: return concise_version(match.group(1)) return "" def concise_version(value: str) -> str: match = re.match(r"(\d+\.\d+)", value or "") return match.group(1) if match else value def collect_snapshot() -> dict[str, Any]: topo_text = run(["nvidia-smi", "topo", "-m"]) topology = parse_topology(topo_text) rdma_to_net, net_to_rdma = parse_ibdev2netdev(run(["ibdev2netdev"])) rdma_links = parse_rdma_links(run(["rdma", "link", "show"])) nics = collect_nics() bonds = collect_bonds(net_to_rdma, nics) lspci_text = run(["lspci", "-Dnn"]) pci_devices = parse_lspci_devices(lspci_text) pci_network = parse_lspci_network(lspci_text) ib_devices = [] bonded_rdma = {bond.get("rdma") for bond in bonds} for device in parse_ibstat(run(["ibstat"])): if ( device["name"] in bonded_rdma or str(device.get("link_layer", "")).lower() != "infiniband" ): continue bdf = "" device_path = Path(f"/sys/class/infiniband/{device['name']}/device") try: bdf = normalize_bdf(str(device_path.resolve())) except OSError: pass device.update( { "bdf": bdf, "numa": pci_numa(bdf), "pci_path": pci_path(bdf), "description": pci_network.get(bdf, {}).get("description", ""), "rdma_state": rdma_links.get(device["name"], {}), } ) ib_devices.append(device) network_endpoints = build_network_endpoints( nics, bonds, ib_devices, rdma_to_net, rdma_links, ) system = { "hostname": socket.gethostname(), "os": os_release(), "kernel": platform.release(), "architecture": platform.machine(), "manufacturer": dmi_value("sys_vendor"), "product": dmi_value("product_name"), "product_version": dmi_value("product_version"), "virtualization": run(["systemd-detect-virt"]), "cpu_model": cpu_model(), "numa_nodes": numa_count(), **collect_cpu_topology(), **collect_memory(), } gpus = collect_gpus() pci_fabric = build_pci_fabric(gpus, nics, ib_devices, pci_devices) driver = next((gpu.get("driver") for gpu in gpus if gpu.get("driver")), "") nvlink = parse_nvlink_status(run(["nvidia-smi", "nvlink", "--status"])) nvlink["fabric_type"] = infer_nvlink_fabric_type(topology, len(gpus)) snapshot = { "schema_version": 1, "generated_at": dt.datetime.now().astimezone().isoformat(timespec="seconds"), "collector_version": VERSION, "system": system, "software": { "nvidia_driver": driver, "cuda_toolkit": collect_cuda_toolkit(), "cuda_note": "Toolkit selected by nvcc; application/container runtime may differ.", }, "gpus": gpus, "topology": topology, "pci_fabric": pci_fabric, "nvlink": nvlink, "nics": nics, "bonds": bonds, "ib_devices": ib_devices, "rdma_links": rdma_links, "network_endpoints": network_endpoints, "bmc": collect_bmc(), "rdma_map": rdma_to_net, "notes": [], } attach_gpu_network_mapping(snapshot) return snapshot def os_release() -> str: for line in read_text("/etc/os-release").splitlines(): if line.startswith("PRETTY_NAME="): return line.split("=", 1)[1].strip().strip('"') return platform.platform() def cpu_model() -> str: for line in read_text("/proc/cpuinfo").splitlines(): if line.lower().startswith("model name"): return line.split(":", 1)[1].strip() return "" def numa_count() -> int: nodes = list(Path("/sys/devices/system/node").glob("node[0-9]*")) return len(nodes) def collect_cpu_topology() -> dict[str, Any]: values: dict[str, str] = {} output = run(["lscpu", "-J"]) try: for item in json.loads(output or "{}").get("lscpu", []): key = str(item.get("field", "")).rstrip(":") values[key] = str(item.get("data", "")).strip() except (TypeError, ValueError, json.JSONDecodeError): pass def integer(key: str) -> int | None: match = re.search(r"\d+", values.get(key, "")) return int(match.group(0)) if match else None sockets = integer("Socket(s)") cores_per_socket = integer("Core(s) per socket") return { "cpu_logical": integer("CPU(s)") or os.cpu_count(), "cpu_sockets": sockets, "cpu_cores_per_socket": cores_per_socket, "cpu_physical_cores": ( sockets * cores_per_socket if sockets is not None and cores_per_socket is not None else None ), "cpu_threads_per_core": integer("Thread(s) per core"), } def collect_memory() -> dict[str, Any]: values: dict[str, int] = {} for line in read_text("/proc/meminfo").splitlines(): if ":" not in line: continue key, tail = line.split(":", 1) match = re.search(r"\d+", tail) if match: values[key] = int(match.group(0)) total = values.get("MemTotal") available = values.get("MemAvailable") return { "memory_total_kib": total, "memory_available_kib": available, "memory_used_kib": ( total - available if total is not None and available is not None else None ), } def common_path_depth(left: list[str], right: list[str]) -> int: depth = 0 for left_item, right_item in zip(left or [], right or []): if left_item != right_item: break depth += 1 return depth def hungarian_assignment(costs: list[list[int]]) -> list[tuple[int, int]]: """Return minimum-cost unique row/column pairs for a rectangular matrix.""" if not costs: return [] row_count = len(costs) column_count = len(costs[0]) if row_count > column_count: raise ValueError("Hungarian assignment requires rows <= columns") potentials_rows = [0] * (row_count + 1) potentials_columns = [0] * (column_count + 1) column_match = [0] * (column_count + 1) path = [0] * (column_count + 1) for row in range(1, row_count + 1): column_match[0] = row minimums = [10**18] * (column_count + 1) used = [False] * (column_count + 1) column = 0 while True: used[column] = True current_row = column_match[column] delta = 10**18 next_column = 0 for candidate in range(1, column_count + 1): if used[candidate]: continue reduced = ( costs[current_row - 1][candidate - 1] - potentials_rows[current_row] - potentials_columns[candidate] ) if reduced < minimums[candidate]: minimums[candidate] = reduced path[candidate] = column if minimums[candidate] < delta: delta = minimums[candidate] next_column = candidate for candidate in range(column_count + 1): if used[candidate]: potentials_rows[column_match[candidate]] += delta potentials_columns[candidate] -= delta else: minimums[candidate] -= delta column = next_column if column_match[column] == 0: break while True: previous = path[column] column_match[column] = column_match[previous] column = previous if column == 0: break return sorted( (row - 1, column - 1) for column, row in enumerate(column_match[1:], start=1) if row ) def attach_gpu_network_mapping(snapshot: dict[str, Any]) -> None: """Map GPUs to bonded or standalone RDMA endpoints using topology distance.""" endpoints = snapshot.get("network_endpoints") if not isinstance(endpoints, list) or ( not endpoints and ( snapshot.get("bonds") or snapshot.get("ib_devices") or snapshot.get("rdma_map") ) ): endpoints = build_network_endpoints( snapshot.get("nics", []), snapshot.get("bonds", []), snapshot.get("ib_devices", []), snapshot.get("rdma_map", {}), snapshot.get("rdma_links", {}), ) snapshot["network_endpoints"] = endpoints snapshot["normalized_by_version"] = VERSION endpoint_by_id = { str(endpoint.get("id")): endpoint for endpoint in endpoints if endpoint.get("id") } bond_by_name = { str(bond.get("name")): bond for bond in snapshot.get("bonds", []) } for endpoint in endpoints: endpoint["gpu_links"] = [] endpoint["topology_visibility"] = "not-exposed" for bond in snapshot.get("bonds", []): bond.pop("gpu_index", None) bond.pop("relation", None) topology = snapshot.get("topology", {}).get("gpu_nic", {}) affinity = snapshot.get("topology", {}).get("gpu_affinity", {}) gpus = snapshot.get("gpus", []) relations: dict[tuple[int, str], str] = {} for gpu in gpus: for key in ( "network_relation", "rdma", "bond", "network", "primary_network_id", ): gpu.pop(key, None) gpu["network_links"] = [] gpu_key = f"GPU{gpu['index']}" gpu.update( { key: value for key, value in affinity.get(gpu_key, {}).items() if value } ) by_rdma = { str(item.get("rdma")): str(item.get("relation") or "") for item in topology.get(gpu_key, []) } for endpoint in endpoints: relation = by_rdma.get(str(endpoint.get("rdma") or ""), "") if relation: relations[(int(gpu["index"]), str(endpoint["id"]))] = relation endpoint["topology_visibility"] = "nvidia-smi-topo" primary_pool = [ endpoint for endpoint in endpoints if endpoint.get("role") == "compute" ] if not primary_pool: primary_pool = list(endpoints) sorted_gpus = sorted(gpus, key=lambda item: int(item["index"])) sorted_endpoints = sorted( primary_pool, key=lambda item: natural_key(str(item.get("name") or "")) ) def pair_cost(gpu: dict[str, Any], endpoint: dict[str, Any]) -> int: gpu_index = int(gpu["index"]) endpoint_id = str(endpoint["id"]) rank = relation_rank(relations.get((gpu_index, endpoint_id), "")) if rank >= 9: return 10**9 member_paths = [ member.get("pci_path") or [] for member in endpoint.get("members", []) ] or [endpoint.get("pci_path") or []] proximity = max( ( common_path_depth(gpu.get("pci_path") or [], path) for path in member_paths ), default=0, ) # Topology relation is authoritative. PCI path only breaks equal-distance # ties, followed by stable natural ordering of GPUs and endpoints. return rank * 1_000_000 + (100 - min(100, proximity)) * 1_000 primary_assignments: dict[int, tuple[dict[str, Any], str]] = {} if sorted_gpus and sorted_endpoints: if len(sorted_gpus) <= len(sorted_endpoints): costs = [ [ pair_cost(gpu, endpoint) + endpoint_index for endpoint_index, endpoint in enumerate(sorted_endpoints) ] for gpu in sorted_gpus ] unique_pairs = [ (sorted_gpus[gpu_index], sorted_endpoints[endpoint_index]) for gpu_index, endpoint_index in hungarian_assignment(costs) if costs[gpu_index][endpoint_index] < 10**9 ] else: costs = [ [ pair_cost(gpu, endpoint) + gpu_index for gpu_index, gpu in enumerate(sorted_gpus) ] for endpoint in sorted_endpoints ] unique_pairs = [ (sorted_gpus[gpu_index], sorted_endpoints[endpoint_index]) for endpoint_index, gpu_index in hungarian_assignment(costs) if costs[endpoint_index][gpu_index] < 10**9 ] for gpu, endpoint in unique_pairs: gpu_index = int(gpu["index"]) endpoint_id = str(endpoint["id"]) primary_assignments[gpu_index] = ( endpoint, relations[(gpu_index, endpoint_id)], ) for gpu in gpus: gpu_index = int(gpu["index"]) if gpu_index in primary_assignments: continue candidates = [ ( relation_rank(relations.get((gpu_index, str(endpoint["id"])), "")), str(endpoint.get("name") or ""), endpoint, ) for endpoint in primary_pool if relation_rank( relations.get((gpu_index, str(endpoint["id"])), "") ) < 9 ] if candidates: _, _, endpoint = min(candidates) primary_assignments[gpu_index] = ( endpoint, relations[(gpu_index, str(endpoint["id"]))], ) def attach_link( gpu: dict[str, Any], endpoint: dict[str, Any], relation: str, role: str, ) -> None: link = { "endpoint_id": endpoint["id"], "name": endpoint.get("name"), "rdma": endpoint.get("rdma"), "protocol": endpoint.get("protocol"), "type": endpoint.get("type"), "relation": relation, "role": role, "evidence": "nvidia-smi topo -m", } gpu["network_links"].append(link) endpoint["gpu_links"].append( { "gpu_index": gpu["index"], "relation": relation, "role": role, "evidence": "nvidia-smi topo -m", } ) for gpu in gpus: gpu_index = int(gpu["index"]) primary = primary_assignments.get(gpu_index) if primary: endpoint, relation = primary gpu["primary_network_id"] = endpoint["id"] gpu["network_relation"] = relation gpu["rdma"] = endpoint.get("rdma") gpu["network"] = endpoint.get("name") if endpoint.get("type") == "bond": gpu["bond"] = endpoint.get("name") bond = bond_by_name.get(str(endpoint.get("name"))) if bond is not None: bond["gpu_index"] = gpu["index"] bond["relation"] = relation attach_link(gpu, endpoint, relation, "primary") ib_candidates = [ ( relation_rank(relations.get((gpu_index, str(endpoint["id"])), "")), str(endpoint.get("name") or ""), endpoint, ) for endpoint in endpoints if endpoint.get("protocol") == "InfiniBand" and relation_rank( relations.get((gpu_index, str(endpoint["id"])), "") ) < 9 and (not primary or endpoint["id"] != primary[0]["id"]) ] if ib_candidates: _, _, endpoint = min(ib_candidates) attach_link( gpu, endpoint, relations[(gpu_index, str(endpoint["id"]))], "nearest", ) def attach_gpu_bond_mapping(snapshot: dict[str, Any]) -> None: """Backward-compatible alias for callers using the former function name.""" attach_gpu_network_mapping(snapshot) def relation_rank(relation: str) -> int: return {"PIX": 0, "PXB": 1, "PHB": 2, "NODE": 3, "SYS": 4}.get(relation, 9) def evaluate(snapshot: dict[str, Any], expected: dict[str, int | None]) -> dict[str, Any]: checks: list[dict[str, str]] = [] def add(level: str, title: str, detail: str) -> None: checks.append({"level": level, "title": title, "detail": detail}) gpus = snapshot.get("gpus", []) bonds = snapshot.get("bonds", []) rdma_bonds = [bond for bond in bonds if bond.get("rdma")] ib_devices = snapshot.get("ib_devices", []) network_endpoints = snapshot.get("network_endpoints", []) if not gpus: add("critical", "未发现 NVIDIA GPU", "nvidia-smi 无可解析输出") elif expected.get("gpus") is not None and len(gpus) != expected["gpus"]: add("warning", "GPU 数量不符合预期", f"发现 {len(gpus)},预期 {expected['gpus']}") else: add("ok", "GPU 数量", f"发现 {len(gpus)} 张 GPU") unhealthy_bonds = [ bond for bond in rdma_bonds if bond.get("status") != "up" or any(slave.get("status") != "up" for slave in bond["slaves"]) ] if unhealthy_bonds: add("critical", "Bond 链路异常", "、".join(bond["name"] for bond in unhealthy_bonds)) elif rdma_bonds: add( "ok", "Bond 链路", f"{len(rdma_bonds)} 个 RDMA Bond 及全部成员均为 Up", ) if ( expected.get("bonds") is not None and len(rdma_bonds) != expected["bonds"] ): add( "warning", "Bond 数量不符合预期", f"发现 {len(rdma_bonds)} 个 RDMA Bond,预期 {expected['bonds']}", ) failures = sum( int(slave.get("failures") or 0) for bond in rdma_bonds for slave in bond.get("slaves", []) ) if failures: add("warning", "存在历史链路失败", f"Bond 成员累计 Link Failure Count = {failures}") elif rdma_bonds: add("ok", "链路失败计数", "全部 Bond 成员为 0") def endpoint_active(endpoint: dict[str, Any]) -> bool: state_ok = str(endpoint.get("state") or "").upper() in {"UP", "ACTIVE"} physical = ( str(endpoint.get("physical_state") or "") .upper() .replace("_", "") ) physical_ok = not physical or physical in {"LINKUP", "ACTIVE", "UP"} members_ok = all( not member.get("status") or str(member.get("status")).lower() in {"up", "active"} for member in endpoint.get("members", []) ) return state_ok and physical_ok and members_ok inactive_endpoints = [ endpoint for endpoint in network_endpoints if not endpoint_active(endpoint) ] if inactive_endpoints: add( "warning", "RDMA 端点未全部激活", "、".join(str(endpoint.get("name")) for endpoint in inactive_endpoints), ) elif network_endpoints: roce_count = sum( endpoint.get("protocol") == "RoCE" for endpoint in network_endpoints ) ib_endpoint_count = sum( endpoint.get("protocol") == "InfiniBand" for endpoint in network_endpoints ) add( "ok", "RDMA 网络端点", f"RoCE {roce_count} · InfiniBand {ib_endpoint_count},均为 Active/Up", ) elif snapshot.get("topology", {}).get("nic_legend"): add( "warning", "RDMA 端点建模不完整", "拓扑矩阵包含 NIC,但未构建 Bond、独立 RoCE 或 IB 端点", ) mapped_gpus = [ gpu for gpu in gpus if gpu.get("primary_network_id") ] relation_counts: dict[str, int] = {} for gpu in mapped_gpus: relation = str(gpu.get("network_relation") or "N/A") relation_counts[relation] = relation_counts.get(relation, 0) + 1 relation_detail = " / ".join( f"{name} {relation_counts[name]}" for name in ("PIX", "PXB", "PHB", "NODE", "SYS") if relation_counts.get(name) ) if gpus and len(mapped_gpus) != len(gpus): add( "warning", "GPU ↔ RDMA 映射不完整", f"仅 {len(mapped_gpus)}/{len(gpus)} 张 GPU 有最近网络端点", ) elif mapped_gpus and relation_counts.get("SYS"): add( "warning", "GPU ↔ RDMA 存在跨 NUMA 路径", relation_detail, ) elif mapped_gpus: add( "ok", "GPU ↔ RDMA 亲和", f"{len(mapped_gpus)}/{len(gpus)} 已映射 · {relation_detail}", ) affinity_gpus = [ gpu for gpu in gpus if gpu.get("cpu_affinity") and gpu.get("cpu_affinity") != "N/A" ] affinity_mismatches = [] for gpu in affinity_gpus: gpu_numa = gpu.get("numa") affinity_numa = number(gpu.get("numa_affinity")) if ( isinstance(gpu_numa, int) and gpu_numa >= 0 and affinity_numa is not None and gpu_numa != int(affinity_numa) ): affinity_mismatches.append(f"GPU{gpu.get('index')}") if affinity_mismatches: add( "warning", "CPU / NUMA 亲和冲突", "、".join(affinity_mismatches) + " 的PCI NUMA与内存亲和不一致", ) elif gpus and len(affinity_gpus) == len(gpus): add("ok", "CPU / NUMA 亲和", f"{len(gpus)}/{len(gpus)} 张GPU均有推荐CPU范围") elif gpus: add( "warning", "CPU亲和信息不完整", f"仅 {len(affinity_gpus)}/{len(gpus)} 张GPU可解析", ) power_gpus = [ gpu for gpu in gpus if number(gpu.get("power_w")) is not None and number(gpu.get("power_limit_w")) is not None and bool(gpu.get("pstate")) ] if gpus and len(power_gpus) == len(gpus): total_power = sum(number(gpu.get("power_w")) or 0 for gpu in power_gpus) total_limit = sum(number(gpu.get("power_limit_w")) or 0 for gpu in power_gpus) p0_count = sum(str(gpu.get("pstate", "")).upper() == "P0" for gpu in gpus) add( "ok", "GPU 功耗遥测", f"瞬时 {fmt_power(total_power)} / 上限 {fmt_power(total_limit)} · P0 {p0_count}/{len(gpus)}", ) elif gpus: add( "warning", "GPU功耗信息不完整", f"仅 {len(power_gpus)}/{len(gpus)} 张GPU可读取当前功耗、上限和P-State", ) active_ib = sum(str(device.get("state", "")).lower() == "active" for device in ib_devices) if ib_devices and active_ib != len(ib_devices): add("warning", "IB 端口未全部激活", f"{active_ib}/{len(ib_devices)} Active") elif ib_devices: add("ok", "IB 端口", f"{active_ib}/{len(ib_devices)} Active") if expected.get("ib") is not None and len(ib_devices) != expected["ib"]: add("warning", "IB 数量不符合预期", f"发现 {len(ib_devices)},预期 {expected['ib']}") default_route_nics = [ nic for nic in snapshot.get("nics", []) if nic.get("default_route") ] unhealthy_management = [ nic for nic in default_route_nics if str(nic.get("state", "")).lower() != "up" or nic.get("carrier") == 0 ] if unhealthy_management: add( "warning", "带内管理网异常", "、".join(str(nic.get("name")) for nic in unhealthy_management), ) elif default_route_nics: add( "ok", "带内管理网", "、".join(str(nic.get("name")) for nic in default_route_nics) + " 承载默认路由且链路 Up", ) nvlink = snapshot.get("nvlink", {}) if len(gpus) > 1 and not nvlink.get("active_links"): add("warning", "未解析到 NVLink", "无法确认机内 NVLink 状态") elif nvlink.get("active_links"): add( "ok", "NVLink", f"共 {nvlink['active_links']} 条 Active,单 GPU 至少 {nvlink.get('links_per_gpu', 0)} 条", ) severity = {"ok": 0, "warning": 1, "critical": 2} worst = max((severity[item["level"]] for item in checks), default=1) return { "status": ("healthy", "attention", "critical")[worst], "checks": checks, } def esc(value: Any) -> str: return html.escape("" if value is None else str(value), quote=True) def fmt_rate(mbps: Any) -> str: value = number(mbps) if value is None: return "未知" if value >= 1000: gbps = value / 1000 return f"{gbps:g}G" return f"{value:g}M" def fmt_mib(mib: Any) -> str: value = number(mib) if value is None: return "未知" return f"{value / 1024:.0f} GiB" def fmt_kib(kib: Any) -> str: value = number(kib) if value is None: return "未知" return f"{value / 1024 / 1024:.1f} GiB" def fmt_power(watts: Any) -> str: value = number(watts) if value is None: return "未知" if value >= 1000: return f"{value / 1000:.2f} kW" return f"{value:.0f} W" def fmt_number(value: Any, fallback: str = "—") -> str: parsed = number(value) return f"{parsed:g}" if parsed is not None else fallback def compact_gpu_indices(indices: list[int]) -> str: if not indices: return "GPU?" ordered = sorted(indices) if ordered == list(range(ordered[0], ordered[-1] + 1)): return f"GPU{ordered[0]}" if len(ordered) == 1 else f"GPU{ordered[0]}–{ordered[-1]}" return ",".join(f"GPU{index}" for index in ordered) def status_label(value: str) -> str: return {"healthy": "健康", "attention": "需关注", "critical": "异常"}.get(value, "未知") def badge(text: str, tone: str = "neutral") -> str: return f'{esc(text)}' def render_gpu_lane( gpu: dict[str, Any], endpoint: dict[str, Any] | None ) -> str: utilization = gpu.get("utilization_pct") memory_total = gpu.get("memory_total_mib") memory_used = gpu.get("memory_used_mib") memory_pct = ( 100 * memory_used / memory_total if number(memory_used) is not None and number(memory_total) else 0 ) relation = gpu.get("network_relation", "未映射") relation_tone = "ok" if relation == "PIX" else "warn" memory_numa = gpu.get("numa_affinity") if memory_numa in {None, "", "N/A"}: memory_numa = gpu.get("numa") power_w = number(gpu.get("power_w")) power_limit_w = number(gpu.get("power_limit_w")) power_pct = ( 100 * power_w / power_limit_w if power_w is not None and power_limit_w else 0 ) pstate = str(gpu.get("pstate") or "—").upper() if endpoint and endpoint.get("type") == "bond": members = endpoint.get("members", []) slaves = "".join( f"""
{esc(endpoint.get("rdma") or "无 RDMA 映射")}
{esc(endpoint.get("rdma") or "无 RDMA 映射")}
没有可映射的 Bond、独立 RoCE 或 IB 端点
{esc(gpu.get('bdf'))}
{address_html}
{esc(bmc.get('ip') or '—')}
{esc(bmc.get('note') or '未发现可读取的 BMC LAN 信息')}
独立带外口通常不进入宿主 OS 网络命名空间;需要在 BMC/IPMI 或云厂商控制台侧验证。
{esc(system.get('manufacturer'))} {esc(system.get('product'))} · {esc(system.get('os'))}
{esc(system.get('manufacturer'))} {esc(system.get('product'))}
{esc(gpus[0].get('name') if gpus else '未发现 NVIDIA GPU')}
{esc(system.get('cpu_model') or '未知 CPU')}
CPU范围来自 NVIDIA 拓扑亲和,表示更近的CPU/内存路径;不代表当前训练进程已经绑核。
有 Bond 时展示聚合容器与物理成员;无 Bond 时展示独立 RoCE/IB 端点。LACP 容量不代表单连接带宽。