"""Accelerator information detection for NVIDIA GPUs and Hygon DCUs.""" import subprocess import shutil from datetime import datetime from typing import Optional from rich.console import Console from rich.table import Table from rich.panel import Panel from rich.text import Text from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label from modules.hygon_smi import collect_hygon_info, find_hy_smi class GPUInfo: def __init__(self, config: dict): self.config = config self.console = Console() configured = config.get("gpu_type", "auto") self.gpu_type = configured if configured != "auto" else detect_gpu_type() self.specs = get_gpu_specs(self.gpu_type) self.gpu_label = get_gpu_label(self.gpu_type) def _run_smi(self, query: str, fmt: str = "csv,noheader,nounits") -> Optional[str]: if not shutil.which("nvidia-smi"): return None try: r = subprocess.run( ["nvidia-smi", f"--query-gpu={query}", f"--format={fmt}"], capture_output=True, text=True, timeout=30, ) return r.stdout.strip() if r.returncode == 0 else None except (subprocess.TimeoutExpired, FileNotFoundError): return None def run(self) -> dict: if self.gpu_type == "bw1000" or (not shutil.which("nvidia-smi") and find_hy_smi()): result = collect_hygon_info(include_topology=True) runtime_cfg = self.config.get("runtime", {}) or {} result.update({ "runtime_name": runtime_cfg.get("name", result.get("runtime_name", "DTK")), "runtime_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")), "cuda_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")), "timestamp": datetime.now().isoformat(), "detected_gpu_type": "bw1000", "gpu_label": get_gpu_label("bw1000"), }) return result if not shutil.which("nvidia-smi"): self.console.print("[bold red]nvidia-smi not found![/bold red]") return {"error": "nvidia-smi not found", "gpu_count": 0} gpu_count_str = self._run_smi("count") if not gpu_count_str: return {"error": "nvidia-smi query failed", "gpu_count": 0} gpu_count = int(gpu_count_str.strip().split("\n")[0]) names = self._run_smi("name").split("\n") if self._run_smi("name") else [] uuids = self._run_smi("uuid").split("\n") if self._run_smi("uuid") else [] pcie_bus = self._run_smi("pci.bus_id").split("\n") if self._run_smi("pci.bus_id") else [] pcie_gen = self._run_smi("pcie.link.gen.current").split("\n") if self._run_smi("pcie.link.gen.current") else [] pcie_width = self._run_smi("pcie.link.width.current").split("\n") if self._run_smi("pcie.link.width.current") else [] vram_total = self._run_smi("memory.total").split("\n") if self._run_smi("memory.total") else [] vram_used = self._run_smi("memory.used").split("\n") if self._run_smi("memory.used") else [] vram_free = self._run_smi("memory.free").split("\n") if self._run_smi("memory.free") else [] power_draw = self._run_smi("power.draw").split("\n") if self._run_smi("power.draw") else [] power_limit = self._run_smi("power.limit").split("\n") if self._run_smi("power.limit") else [] clock_sm = self._run_smi("clocks.sm").split("\n") if self._run_smi("clocks.sm") else [] clock_mem = self._run_smi("clocks.mem").split("\n") if self._run_smi("clocks.mem") else [] temperature = self._run_smi("temperature.gpu").split("\n") if self._run_smi("temperature.gpu") else [] fan_speed = self._run_smi("fan.speed").split("\n") if self._run_smi("fan.speed") else [] persistence = self._run_smi("persistence_mode").split("\n") if self._run_smi("persistence_mode") else [] compute_mode = self._run_smi("compute_mode").split("\n") if self._run_smi("compute_mode") else [] serial = self._run_smi("serial").split("\n") if self._run_smi("serial") else [] ecc_single = self._run_smi("ecc.errors.single_bit.total.volatile").split("\n") if self._run_smi("ecc.errors.single_bit.total.volatile") else [] ecc_double = self._run_smi("ecc.errors.double_bit.total.volatile").split("\n") if self._run_smi("ecc.errors.double_bit.total.volatile") else [] driver_info = self._run_smi("driver_version", "csv,noheader") cuda_info = self._get_cuda_version() def safe_get(lst, idx, default="N/A"): try: return lst[idx].strip() if idx < len(lst) else default except (IndexError, ValueError): return default def safe_int(val, default=0): try: return int(val) if val not in ("N/A", "", "[N/A]") else default except (ValueError, TypeError): return default def safe_float(val, default=0.0): try: return float(val) if val not in ("N/A", "", "[N/A]") else default except (ValueError, TypeError): return default gpus = [] for i in range(gpu_count): gpus.append({ "index": i, "name": safe_get(names, i), "uuid": safe_get(uuids, i), "pci_bus_id": safe_get(pcie_bus, i), "pcie_link_gen": safe_int(safe_get(pcie_gen, i)), "pcie_link_width": safe_int(safe_get(pcie_width, i)), "vram_total_mb": safe_int(safe_get(vram_total, i)), "vram_used_mb": safe_int(safe_get(vram_used, i)), "vram_free_mb": safe_int(safe_get(vram_free, i)), "power_draw": safe_float(safe_get(power_draw, i)), "power_limit": safe_float(safe_get(power_limit, i)), "clock_sm": safe_int(safe_get(clock_sm, i)), "clock_mem": safe_int(safe_get(clock_mem, i)), "temperature": safe_int(safe_get(temperature, i)), "fan_speed": safe_int(safe_get(fan_speed, i)), "persistence_mode": safe_get(persistence, i) == "Enabled", "compute_mode": safe_get(compute_mode, i), "serial_number": safe_get(serial, i), "ecc_errors_single": safe_int(safe_get(ecc_single, i)), "ecc_errors_double": safe_int(safe_get(ecc_double, i)), }) topology = self._get_topology() return { "driver_version": safe_get(driver_info.split("\n"), 0) if driver_info else "N/A", "cuda_version": cuda_info or "N/A", "gpu_count": gpu_count, "gpus": gpus, "topology": topology, "timestamp": datetime.now().isoformat(), "detected_gpu_type": self.gpu_type, "gpu_label": self.gpu_label, } def _get_cuda_version(self) -> Optional[str]: """Parse CUDA version from nvidia-smi header output (query-gpu field removed in newer drivers).""" try: r = subprocess.run( ["nvidia-smi"], capture_output=True, text=True, timeout=15, ) if r.returncode == 0: import re m = re.search(r"CUDA Version:\s+([\d.]+)", r.stdout) if m: return m.group(1) except (subprocess.TimeoutExpired, FileNotFoundError): pass return None def _get_topology(self) -> str: try: r = subprocess.run( ["nvidia-smi", "topo", "-m"], capture_output=True, text=True, timeout=15, ) return r.stdout if r.returncode == 0 else "Unavailable" except (subprocess.TimeoutExpired, FileNotFoundError): return "Unavailable" @staticmethod def print_results(results: dict, console: Console = None): c = console or Console() if "error" in results: c.print(f"[bold red]Error: {results['error']}[/bold red]") return c.print(f"\n[bold cyan]GPU Information[/bold cyan]") c.print(f" Driver Version : {results.get('driver_version', 'N/A')}") runtime_name = results.get("runtime_name", "CUDA") runtime_version = results.get("runtime_version", results.get("cuda_version", "N/A")) c.print(f" {runtime_name} Version{' ' * max(0, 4-len(runtime_name))}: {runtime_version}") c.print(f" GPU Count : {results.get('gpu_count', 0)}") c.print(f" Detected GPU : {results.get('gpu_label', 'Unknown')} ({results.get('detected_gpu_type', 'unknown')})") c.print(f" Timestamp : {results.get('timestamp', 'N/A')}") gpus = results.get("gpus", []) if not gpus: return table = Table(title="GPU Details", box=None, padding=(0, 1), show_lines=False) table.add_column("GPU", style="bold cyan", width=5) table.add_column("Model", width=18) table.add_column("VRAM", justify="right", width=14) table.add_column("Temp", justify="right", width=6) table.add_column("Power", justify="right", width=12) table.add_column("SM Clk", justify="right", width=8) table.add_column("Mem Clk", justify="right", width=8) table.add_column("PCIe", width=10) table.add_column("Persist", width=8) for g in gpus: name = g["name"] if any(k in name for k in ("A100", "A800", "H100", "H200", "B200", "B300", "BW1000")): name = f"[bold green]{name}[/bold green]" vram = f"{g['vram_used_mb']}/{g['vram_total_mb']} MB" temp = f"{g['temperature']}°C" temp_color = "red" if g["temperature"] > 85 else ("yellow" if g["temperature"] > 75 else "green") temp = f"[{temp_color}]{temp}[/{temp_color}]" power = f"{g['power_draw']:.0f}/{g['power_limit']:.0f}W" sm_clk = f"{g['clock_sm']} MHz" mem_clk = f"{g['clock_mem']} MHz" pcie = f"Gen{g['pcie_link_gen']} x{g['pcie_link_width']}" persist = "[green]ON[/green]" if g["persistence_mode"] else "[red]OFF[/red]" ecc_warn = "" if g["ecc_errors_double"] > 0: ecc_warn = " [bold red]ECC![/bold red]" table.add_row( str(g["index"]), name, vram, temp, power, sm_clk, mem_clk, pcie, persist + ecc_warn, ) c.print(table) topology = results.get("topology", "") if topology and topology != "Unavailable": title = "HSW/XHCL Topology" if results.get("vendor") == "hygon" else "NVLink/NVSwitch Topology" c.print(Panel(topology, title=f"[bold]{title}[/bold]", border_style="cyan")) warnings = [] for g in gpus: if not g["persistence_mode"]: warnings.append(f"GPU {g['index']}: Persistence mode DISABLED") if g["temperature"] > 85: warnings.append(f"GPU {g['index']}: High temperature {g['temperature']}°C") if g["ecc_errors_double"] > 0: warnings.append(f"GPU {g['index']}: {g['ecc_errors_double']} double-bit ECC errors!") if warnings: c.print("\n[bold yellow]Warnings:[/bold yellow]") for w in warnings: c.print(f" [yellow]⚠ {w}[/yellow]")