test_gpu_scripts/modules/hygon_smi.py

191 lines
6.7 KiB
Python

"""Helpers for querying Hygon DCU devices through ``hy-smi``."""
import os
import re
import shutil
import subprocess
from typing import Dict, List, Optional
def find_hy_smi() -> Optional[str]:
"""Return the hy-smi executable path when available."""
path = shutil.which("hy-smi")
if path:
return path
path = "/opt/hyhal/bin/hy-smi"
return path if os.path.isfile(path) and os.access(path, os.X_OK) else None
def run_hy_smi(*args: str, timeout: int = 30) -> Optional[str]:
"""Run hy-smi and return stdout, or ``None`` on failure."""
binary = find_hy_smi()
if not binary:
return None
try:
result = subprocess.run(
[binary, *args], capture_output=True, text=True, timeout=timeout
)
except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
return None
return result.stdout.strip() if result.returncode == 0 else None
def _indexed_values(text: Optional[str], pattern: str) -> Dict[int, str]:
if not text:
return {}
values: Dict[int, str] = {}
for match in re.finditer(pattern, text, re.MULTILINE | re.IGNORECASE):
values[int(match.group(1))] = match.group(2).strip()
return values
def _number(value, default=0.0):
try:
return float(value)
except (TypeError, ValueError):
return default
def _pcie_gen(gt_per_second: float) -> int:
if gt_per_second >= 32:
return 5
if gt_per_second >= 16:
return 4
if gt_per_second >= 8:
return 3
if gt_per_second >= 5:
return 2
return 1 if gt_per_second > 0 else 0
def collect_hygon_info(include_topology: bool = True) -> dict:
"""Collect normalized per-device information for GPUInfo and HealthCheck."""
summary = run_hy_smi()
if not summary:
return {"error": "hy-smi query failed", "gpu_count": 0, "gpus": []}
rows: Dict[int, dict] = {}
row_re = re.compile(
r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W\s+(\S+)\s+"
r"([\d.]+)W\s+(\d+)%\s+([\d.]+)%",
re.MULTILINE,
)
for match in row_re.finditer(summary):
index = int(match.group(1))
rows[index] = {
"index": index,
"temperature": int(round(float(match.group(2)))),
"power_draw": float(match.group(3)),
"performance_level": match.group(4),
"power_limit": float(match.group(5)),
"vram_used_pct": int(match.group(6)),
"utilization_pct": float(match.group(7)),
}
product = run_hy_smi("--showproductname")
driver = run_hy_smi("--showdriverversion")
memory = run_hy_smi("--showmemavailable")
bus = run_hy_smi("--showbus")
unique = run_hy_smi("--showuniqueid")
serial = run_hy_smi("--showserial")
clocks = run_hy_smi("--showclocks")
ecc = run_hy_smi("--showmemeccinfo")
xgmi = run_hy_smi("--showxgmierr")
replay = run_hy_smi("--showreplaycount")
health = run_hy_smi("--healthcheck")
throttle = run_hy_smi("--showfdr")
names = _indexed_values(product, r"HCU\[(\d+)\].*Card Series:\s*([^\s]+)")
memories = _indexed_values(memory, r"HCU\[(\d+)\].*Available memory size \(MiB\):\s*(\d+)")
buses = _indexed_values(bus, r"HCU\[(\d+)\].*PCI Bus:\s*([0-9a-fA-F:.]+)")
unique_ids = _indexed_values(unique, r"HCU\[(\d+)\].*Unique ID:\s*(\S+)")
serials = _indexed_values(serial, r"HCU\[(\d+)\].*Serial Number:\s*(\S+)")
sclk = _indexed_values(clocks, r"HCU\[(\d+)\].*sclk clock level:.*\((\d+)Mhz\)")
mclk = _indexed_values(clocks, r"HCU\[(\d+)\].*mclk clock level:.*\((\d+)Mhz\)")
pcie = {}
if clocks:
for match in re.finditer(
r"HCU\[(\d+)\].*pcie clock level:.*\(([\d.]+)GT/s,\s*x(\d+)", clocks
):
pcie[int(match.group(1))] = (float(match.group(2)), int(match.group(3)))
ecc_enabled = {
idx: value.upper() == "ON"
for idx, value in _indexed_values(
ecc, r"HCU\[(\d+)\].*Vram memory ECC status:\s*(\S+)"
).items()
}
xgmi_errors = {
idx: int(value)
for idx, value in _indexed_values(
xgmi, r"HCU\[(\d+)\].*XGMI Error count\s+(\d+)"
).items()
}
replay_counts = {
idx: int(value)
for idx, value in _indexed_values(
replay, r"HCU\[(\d+)\].*PCIe Replay Count:\s*(\d+)"
).items()
}
healthy = {
idx: value.lower() == "healthy"
for idx, value in _indexed_values(
health, r"HCU\[(\d+)\].*:\s*(Healthy|Unhealthy)\s*$"
).items()
}
throttle_status = _indexed_values(
throttle, r"HCU\[(\d+)\].*Throttler Status:\s*(.+?)\s*$"
)
driver_match = re.search(r"Driver Version:\s*(\S+)", driver or "")
driver_version = driver_match.group(1) if driver_match else "N/A"
gpu_count = max(rows.keys(), default=-1) + 1
gpus: List[dict] = []
for index in range(gpu_count):
row = rows.get(index, {"index": index})
total_mb = int(memories.get(index, 0))
used_mb = int(total_mb * int(row.get("vram_used_pct", 0)) / 100)
gt_s, width = pcie.get(index, (0.0, 0))
throttle_text = throttle_status.get(index, "")
real_throttle = bool(
throttle_text
and any(token not in {"IDLE", "ACTIVE", "NONE"}
for token in re.findall(r"[A-Z_]+", throttle_text.upper()))
)
gpus.append({
**row,
"name": names.get(index, "Hygon DCU"),
"uuid": unique_ids.get(index, "N/A"),
"pci_bus_id": buses.get(index, "N/A"),
"pcie_link_gen": _pcie_gen(gt_s),
"pcie_link_width": width,
"pcie_gt_s": gt_s,
"vram_total_mb": total_mb,
"vram_used_mb": used_mb,
"vram_free_mb": max(total_mb - used_mb, 0),
"clock_sm": int(_number(sclk.get(index))),
"clock_mem": int(_number(mclk.get(index))),
"fan_speed": 0,
"persistence_mode": True,
"compute_mode": row.get("performance_level", "auto"),
"serial_number": serials.get(index, "N/A"),
"ecc_enabled": ecc_enabled.get(index, False),
"ecc_errors_single": 0,
"ecc_errors_double": 0,
"xgmi_errors": xgmi_errors.get(index, 0),
"pcie_replay_count": replay_counts.get(index, 0),
"healthy": healthy.get(index, False),
"throttling_active": real_throttle,
"throttling_text": throttle_text,
})
return {
"driver_version": driver_version,
"runtime_name": "DTK",
"runtime_version": os.environ.get("DTK_VERSION", "N/A"),
"gpu_count": gpu_count,
"gpus": gpus,
"topology": run_hy_smi("--showtopo") if include_topology else "",
"vendor": "hygon",
}