191 lines
6.7 KiB
Python
191 lines
6.7 KiB
Python
"""Helpers for querying Hygon DCU devices through ``hy-smi``."""
|
|
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
from typing import Dict, List, Optional
|
|
|
|
|
|
def find_hy_smi() -> Optional[str]:
|
|
"""Return the hy-smi executable path when available."""
|
|
path = shutil.which("hy-smi")
|
|
if path:
|
|
return path
|
|
path = "/opt/hyhal/bin/hy-smi"
|
|
return path if os.path.isfile(path) and os.access(path, os.X_OK) else None
|
|
|
|
|
|
def run_hy_smi(*args: str, timeout: int = 30) -> Optional[str]:
|
|
"""Run hy-smi and return stdout, or ``None`` on failure."""
|
|
binary = find_hy_smi()
|
|
if not binary:
|
|
return None
|
|
try:
|
|
result = subprocess.run(
|
|
[binary, *args], capture_output=True, text=True, timeout=timeout
|
|
)
|
|
except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
|
|
return None
|
|
return result.stdout.strip() if result.returncode == 0 else None
|
|
|
|
|
|
def _indexed_values(text: Optional[str], pattern: str) -> Dict[int, str]:
|
|
if not text:
|
|
return {}
|
|
values: Dict[int, str] = {}
|
|
for match in re.finditer(pattern, text, re.MULTILINE | re.IGNORECASE):
|
|
values[int(match.group(1))] = match.group(2).strip()
|
|
return values
|
|
|
|
|
|
def _number(value, default=0.0):
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def _pcie_gen(gt_per_second: float) -> int:
|
|
if gt_per_second >= 32:
|
|
return 5
|
|
if gt_per_second >= 16:
|
|
return 4
|
|
if gt_per_second >= 8:
|
|
return 3
|
|
if gt_per_second >= 5:
|
|
return 2
|
|
return 1 if gt_per_second > 0 else 0
|
|
|
|
|
|
def collect_hygon_info(include_topology: bool = True) -> dict:
|
|
"""Collect normalized per-device information for GPUInfo and HealthCheck."""
|
|
summary = run_hy_smi()
|
|
if not summary:
|
|
return {"error": "hy-smi query failed", "gpu_count": 0, "gpus": []}
|
|
|
|
rows: Dict[int, dict] = {}
|
|
row_re = re.compile(
|
|
r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W\s+(\S+)\s+"
|
|
r"([\d.]+)W\s+(\d+)%\s+([\d.]+)%",
|
|
re.MULTILINE,
|
|
)
|
|
for match in row_re.finditer(summary):
|
|
index = int(match.group(1))
|
|
rows[index] = {
|
|
"index": index,
|
|
"temperature": int(round(float(match.group(2)))),
|
|
"power_draw": float(match.group(3)),
|
|
"performance_level": match.group(4),
|
|
"power_limit": float(match.group(5)),
|
|
"vram_used_pct": int(match.group(6)),
|
|
"utilization_pct": float(match.group(7)),
|
|
}
|
|
|
|
product = run_hy_smi("--showproductname")
|
|
driver = run_hy_smi("--showdriverversion")
|
|
memory = run_hy_smi("--showmemavailable")
|
|
bus = run_hy_smi("--showbus")
|
|
unique = run_hy_smi("--showuniqueid")
|
|
serial = run_hy_smi("--showserial")
|
|
clocks = run_hy_smi("--showclocks")
|
|
ecc = run_hy_smi("--showmemeccinfo")
|
|
xgmi = run_hy_smi("--showxgmierr")
|
|
replay = run_hy_smi("--showreplaycount")
|
|
health = run_hy_smi("--healthcheck")
|
|
throttle = run_hy_smi("--showfdr")
|
|
|
|
names = _indexed_values(product, r"HCU\[(\d+)\].*Card Series:\s*([^\s]+)")
|
|
memories = _indexed_values(memory, r"HCU\[(\d+)\].*Available memory size \(MiB\):\s*(\d+)")
|
|
buses = _indexed_values(bus, r"HCU\[(\d+)\].*PCI Bus:\s*([0-9a-fA-F:.]+)")
|
|
unique_ids = _indexed_values(unique, r"HCU\[(\d+)\].*Unique ID:\s*(\S+)")
|
|
serials = _indexed_values(serial, r"HCU\[(\d+)\].*Serial Number:\s*(\S+)")
|
|
sclk = _indexed_values(clocks, r"HCU\[(\d+)\].*sclk clock level:.*\((\d+)Mhz\)")
|
|
mclk = _indexed_values(clocks, r"HCU\[(\d+)\].*mclk clock level:.*\((\d+)Mhz\)")
|
|
pcie = {}
|
|
if clocks:
|
|
for match in re.finditer(
|
|
r"HCU\[(\d+)\].*pcie clock level:.*\(([\d.]+)GT/s,\s*x(\d+)", clocks
|
|
):
|
|
pcie[int(match.group(1))] = (float(match.group(2)), int(match.group(3)))
|
|
ecc_enabled = {
|
|
idx: value.upper() == "ON"
|
|
for idx, value in _indexed_values(
|
|
ecc, r"HCU\[(\d+)\].*Vram memory ECC status:\s*(\S+)"
|
|
).items()
|
|
}
|
|
xgmi_errors = {
|
|
idx: int(value)
|
|
for idx, value in _indexed_values(
|
|
xgmi, r"HCU\[(\d+)\].*XGMI Error count\s+(\d+)"
|
|
).items()
|
|
}
|
|
replay_counts = {
|
|
idx: int(value)
|
|
for idx, value in _indexed_values(
|
|
replay, r"HCU\[(\d+)\].*PCIe Replay Count:\s*(\d+)"
|
|
).items()
|
|
}
|
|
healthy = {
|
|
idx: value.lower() == "healthy"
|
|
for idx, value in _indexed_values(
|
|
health, r"HCU\[(\d+)\].*:\s*(Healthy|Unhealthy)\s*$"
|
|
).items()
|
|
}
|
|
throttle_status = _indexed_values(
|
|
throttle, r"HCU\[(\d+)\].*Throttler Status:\s*(.+?)\s*$"
|
|
)
|
|
|
|
driver_match = re.search(r"Driver Version:\s*(\S+)", driver or "")
|
|
driver_version = driver_match.group(1) if driver_match else "N/A"
|
|
|
|
gpu_count = max(rows.keys(), default=-1) + 1
|
|
gpus: List[dict] = []
|
|
for index in range(gpu_count):
|
|
row = rows.get(index, {"index": index})
|
|
total_mb = int(memories.get(index, 0))
|
|
used_mb = int(total_mb * int(row.get("vram_used_pct", 0)) / 100)
|
|
gt_s, width = pcie.get(index, (0.0, 0))
|
|
throttle_text = throttle_status.get(index, "")
|
|
real_throttle = bool(
|
|
throttle_text
|
|
and any(token not in {"IDLE", "ACTIVE", "NONE"}
|
|
for token in re.findall(r"[A-Z_]+", throttle_text.upper()))
|
|
)
|
|
gpus.append({
|
|
**row,
|
|
"name": names.get(index, "Hygon DCU"),
|
|
"uuid": unique_ids.get(index, "N/A"),
|
|
"pci_bus_id": buses.get(index, "N/A"),
|
|
"pcie_link_gen": _pcie_gen(gt_s),
|
|
"pcie_link_width": width,
|
|
"pcie_gt_s": gt_s,
|
|
"vram_total_mb": total_mb,
|
|
"vram_used_mb": used_mb,
|
|
"vram_free_mb": max(total_mb - used_mb, 0),
|
|
"clock_sm": int(_number(sclk.get(index))),
|
|
"clock_mem": int(_number(mclk.get(index))),
|
|
"fan_speed": 0,
|
|
"persistence_mode": True,
|
|
"compute_mode": row.get("performance_level", "auto"),
|
|
"serial_number": serials.get(index, "N/A"),
|
|
"ecc_enabled": ecc_enabled.get(index, False),
|
|
"ecc_errors_single": 0,
|
|
"ecc_errors_double": 0,
|
|
"xgmi_errors": xgmi_errors.get(index, 0),
|
|
"pcie_replay_count": replay_counts.get(index, 0),
|
|
"healthy": healthy.get(index, False),
|
|
"throttling_active": real_throttle,
|
|
"throttling_text": throttle_text,
|
|
})
|
|
|
|
return {
|
|
"driver_version": driver_version,
|
|
"runtime_name": "DTK",
|
|
"runtime_version": os.environ.get("DTK_VERSION", "N/A"),
|
|
"gpu_count": gpu_count,
|
|
"gpus": gpus,
|
|
"topology": run_hy_smi("--showtopo") if include_topology else "",
|
|
"vendor": "hygon",
|
|
}
|