"""Helpers for querying Hygon DCU devices through ``hy-smi``.""" import os import re import shutil import subprocess from typing import Dict, List, Optional def find_hy_smi() -> Optional[str]: """Return the hy-smi executable path when available.""" path = shutil.which("hy-smi") if path: return path path = "/opt/hyhal/bin/hy-smi" return path if os.path.isfile(path) and os.access(path, os.X_OK) else None def run_hy_smi(*args: str, timeout: int = 30) -> Optional[str]: """Run hy-smi and return stdout, or ``None`` on failure.""" binary = find_hy_smi() if not binary: return None try: result = subprocess.run( [binary, *args], capture_output=True, text=True, timeout=timeout ) except (FileNotFoundError, OSError, subprocess.TimeoutExpired): return None return result.stdout.strip() if result.returncode == 0 else None def _indexed_values(text: Optional[str], pattern: str) -> Dict[int, str]: if not text: return {} values: Dict[int, str] = {} for match in re.finditer(pattern, text, re.MULTILINE | re.IGNORECASE): values[int(match.group(1))] = match.group(2).strip() return values def _number(value, default=0.0): try: return float(value) except (TypeError, ValueError): return default def _pcie_gen(gt_per_second: float) -> int: if gt_per_second >= 32: return 5 if gt_per_second >= 16: return 4 if gt_per_second >= 8: return 3 if gt_per_second >= 5: return 2 return 1 if gt_per_second > 0 else 0 def collect_hygon_info(include_topology: bool = True) -> dict: """Collect normalized per-device information for GPUInfo and HealthCheck.""" summary = run_hy_smi() if not summary: return {"error": "hy-smi query failed", "gpu_count": 0, "gpus": []} rows: Dict[int, dict] = {} row_re = re.compile( r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W\s+(\S+)\s+" r"([\d.]+)W\s+(\d+)%\s+([\d.]+)%", re.MULTILINE, ) for match in row_re.finditer(summary): index = int(match.group(1)) rows[index] = { "index": index, "temperature": int(round(float(match.group(2)))), "power_draw": float(match.group(3)), "performance_level": match.group(4), "power_limit": float(match.group(5)), "vram_used_pct": int(match.group(6)), "utilization_pct": float(match.group(7)), } product = run_hy_smi("--showproductname") driver = run_hy_smi("--showdriverversion") memory = run_hy_smi("--showmemavailable") bus = run_hy_smi("--showbus") unique = run_hy_smi("--showuniqueid") serial = run_hy_smi("--showserial") clocks = run_hy_smi("--showclocks") ecc = run_hy_smi("--showmemeccinfo") xgmi = run_hy_smi("--showxgmierr") replay = run_hy_smi("--showreplaycount") health = run_hy_smi("--healthcheck") throttle = run_hy_smi("--showfdr") names = _indexed_values(product, r"HCU\[(\d+)\].*Card Series:\s*([^\s]+)") memories = _indexed_values(memory, r"HCU\[(\d+)\].*Available memory size \(MiB\):\s*(\d+)") buses = _indexed_values(bus, r"HCU\[(\d+)\].*PCI Bus:\s*([0-9a-fA-F:.]+)") unique_ids = _indexed_values(unique, r"HCU\[(\d+)\].*Unique ID:\s*(\S+)") serials = _indexed_values(serial, r"HCU\[(\d+)\].*Serial Number:\s*(\S+)") sclk = _indexed_values(clocks, r"HCU\[(\d+)\].*sclk clock level:.*\((\d+)Mhz\)") mclk = _indexed_values(clocks, r"HCU\[(\d+)\].*mclk clock level:.*\((\d+)Mhz\)") pcie = {} if clocks: for match in re.finditer( r"HCU\[(\d+)\].*pcie clock level:.*\(([\d.]+)GT/s,\s*x(\d+)", clocks ): pcie[int(match.group(1))] = (float(match.group(2)), int(match.group(3))) ecc_enabled = { idx: value.upper() == "ON" for idx, value in _indexed_values( ecc, r"HCU\[(\d+)\].*Vram memory ECC status:\s*(\S+)" ).items() } xgmi_errors = { idx: int(value) for idx, value in _indexed_values( xgmi, r"HCU\[(\d+)\].*XGMI Error count\s+(\d+)" ).items() } replay_counts = { idx: int(value) for idx, value in _indexed_values( replay, r"HCU\[(\d+)\].*PCIe Replay Count:\s*(\d+)" ).items() } healthy = { idx: value.lower() == "healthy" for idx, value in _indexed_values( health, r"HCU\[(\d+)\].*:\s*(Healthy|Unhealthy)\s*$" ).items() } throttle_status = _indexed_values( throttle, r"HCU\[(\d+)\].*Throttler Status:\s*(.+?)\s*$" ) driver_match = re.search(r"Driver Version:\s*(\S+)", driver or "") driver_version = driver_match.group(1) if driver_match else "N/A" gpu_count = max(rows.keys(), default=-1) + 1 gpus: List[dict] = [] for index in range(gpu_count): row = rows.get(index, {"index": index}) total_mb = int(memories.get(index, 0)) used_mb = int(total_mb * int(row.get("vram_used_pct", 0)) / 100) gt_s, width = pcie.get(index, (0.0, 0)) throttle_text = throttle_status.get(index, "") real_throttle = bool( throttle_text and any(token not in {"IDLE", "ACTIVE", "NONE"} for token in re.findall(r"[A-Z_]+", throttle_text.upper())) ) gpus.append({ **row, "name": names.get(index, "Hygon DCU"), "uuid": unique_ids.get(index, "N/A"), "pci_bus_id": buses.get(index, "N/A"), "pcie_link_gen": _pcie_gen(gt_s), "pcie_link_width": width, "pcie_gt_s": gt_s, "vram_total_mb": total_mb, "vram_used_mb": used_mb, "vram_free_mb": max(total_mb - used_mb, 0), "clock_sm": int(_number(sclk.get(index))), "clock_mem": int(_number(mclk.get(index))), "fan_speed": 0, "persistence_mode": True, "compute_mode": row.get("performance_level", "auto"), "serial_number": serials.get(index, "N/A"), "ecc_enabled": ecc_enabled.get(index, False), "ecc_errors_single": 0, "ecc_errors_double": 0, "xgmi_errors": xgmi_errors.get(index, 0), "pcie_replay_count": replay_counts.get(index, 0), "healthy": healthy.get(index, False), "throttling_active": real_throttle, "throttling_text": throttle_text, }) return { "driver_version": driver_version, "runtime_name": "DTK", "runtime_version": os.environ.get("DTK_VERSION", "N/A"), "gpu_count": gpu_count, "gpus": gpus, "topology": run_hy_smi("--showtopo") if include_topology else "", "vendor": "hygon", }