diff --git a/configs/bw1000_4node.yaml b/configs/bw1000_4node.yaml new file mode 100644 index 0000000..01e5862 --- /dev/null +++ b/configs/bw1000_4node.yaml @@ -0,0 +1,86 @@ +gpu_type: bw1000 + +runtime: + name: DTK + version: 25.04.2 + +benchmark: + memory: + size_mb: 4096 + iterations: 10 + nvbandwidth_buffer_mb: 512 + nvbandwidth_samples: 3 + compute: + dtypes: [fp32, tf32, fp16, bf16, fp8] + sweep_sizes: [4096, 6144, 8192, [2048, 2048, 13312]] + matrix_size: 8192 + warmup: 10 + iterations: 30 + +health: + temp_warning: 75 + temp_critical: 85 + power_limit: 800 + +nccl: + min_bandwidth_gbps: 10 + test_allreduce: true + test_alltoall: true + test_broadcast: true + test_reduce_scatter: true + test_allgather: true + test_sendrecv: false + torch_backend: nccl + torchrun_size_mb: 64 + torchrun_warmup: 5 + torchrun_iterations: 20 + +stress: + duration_sec: 600 + use_doubles: false + use_tensor_cores: true + memory_pct: 80 + gpus: all + +rdma: + min_bandwidth_gbps: 200 + max_latency_us: 6 + ib_iterations: 5000 + msg_size: 1048576 + ib_device: null + ib_port: 1 + cross_node: + enabled: false + allow_roce: true + server: null + server_addr: null + ssh_user: root + devices: [mlx5_bond_0, mlx5_bond_1, mlx5_bond_2, mlx5_bond_3] + ib_port: 1 + gid_index: 3 + msg_size: 1048576 + iters: 5000 + duration_sec: 10 + queue_pairs: 4 + base_oob_port: 18515 + server_warmup_sec: 2.0 + min_bandwidth_gbps: 200 + max_latency_us: 6 + +training: + model: synthetic + batch_size: 2 + seq_length: 1024 + num_steps: 20 + dtype: bf16 + hidden_size: 2048 + num_layers: 4 + num_heads: 16 + vocab_size: 8192 + +report: + output_dir: ./reports + format: json + +tools: + install_dir: /opt/gpu-test-tools diff --git a/configs/bw1000_smoke.yaml b/configs/bw1000_smoke.yaml new file mode 100644 index 0000000..b0a87a8 --- /dev/null +++ b/configs/bw1000_smoke.yaml @@ -0,0 +1,15 @@ +gpu_type: bw1000 + +runtime: + name: DTK + version: 25.04.2 + +stress: + duration_sec: 60 + use_doubles: false + use_tensor_cores: true + memory_pct: 80 + gpus: all + +tools: + install_dir: /opt/gpu-test-tools diff --git a/gpu_tester.py b/gpu_tester.py index 8c00282..5de9833 100644 --- a/gpu_tester.py +++ b/gpu_tester.py @@ -27,6 +27,7 @@ from modules.stress_test import StressTest from modules.rdma_test import RDMATest from modules.report import ReportGenerator from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label, get_supported_gpus, validate_driver_compatibility +from modules.hygon_smi import find_hy_smi DEFAULT_CONFIG = { "benchmark": { @@ -117,9 +118,9 @@ def check_prerequisites(console: Console) -> bool: import shutil ok = True - if not shutil.which("nvidia-smi"): - console.print("[bold red]ERROR: nvidia-smi not found![/bold red]") - console.print(" Please install NVIDIA drivers first.") + if not shutil.which("nvidia-smi") and not find_hy_smi(): + console.print("[bold red]ERROR: neither nvidia-smi nor hy-smi was found![/bold red]") + console.print(" Please install the NVIDIA driver or Hygon DCU driver first.") ok = False return ok diff --git a/modules/benchmark.py b/modules/benchmark.py index e728633..38b185b 100644 --- a/modules/benchmark.py +++ b/modules/benchmark.py @@ -17,6 +17,10 @@ from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label, res TORCH_AVAILABLE = False try: import torch + try: + import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim + except ImportError: + pass if torch.cuda.is_available(): TORCH_AVAILABLE = True except ImportError: @@ -63,12 +67,14 @@ class Benchmark: return None def run_memory_benchmark(self) -> dict: - nvbw = self._find_nvbandwidth() + # nvbandwidth is NVIDIA-specific. DTK PyTorch provides the portable + # transfer/copy path used below for Hygon DCUs. + nvbw = None if self.gpu_type == "bw1000" else self._find_nvbandwidth() if nvbw: return self._run_nvbandwidth(nvbw) - self.console.print("[yellow]nvbandwidth not found, falling back to PyTorch[/yellow]") + self.console.print("[yellow]Using PyTorch memory benchmark fallback[/yellow]") return self._run_memory_pytorch() def _run_nvbandwidth(self, nvbw_path: str) -> dict: @@ -189,6 +195,7 @@ class Benchmark: "efficiency_pct": d2d_efficiency, "results_by_test": results_by_test, "per_gpu": [], + "accelerator_vendor": self.specs.get("vendor", "nvidia"), } } @@ -292,20 +299,31 @@ class Benchmark: progress.advance(task) best_d2d = max(v["d2d_gbps"] for v in bandwidth_by_size.values()) + best_h2d = max(v["h2d_gbps"] for v in bandwidth_by_size.values()) + best_d2h = max(v["d2h_gbps"] for v in bandwidth_by_size.values()) peak_bw = self.specs["memory_bandwidth_gbps"] efficiency = round((best_d2d / peak_bw) * 100, 1) if peak_bw else None + pcie_gen = self.specs.get("pcie_gen", 0) + pcie_peak = {3: 16, 4: 32, 5: 64, 6: 128}.get(pcie_gen) if pcie_gen else None return { "memory": { "source": "pytorch", - "h2d_bandwidth_gbps": round(max(v["h2d_gbps"] for v in bandwidth_by_size.values()), 1), - "d2h_bandwidth_gbps": round(max(v["d2h_gbps"] for v in bandwidth_by_size.values()), 1), + "h2d_bandwidth_gbps": round(best_h2d, 1), + "d2h_bandwidth_gbps": round(best_d2h, 1), "d2d_bandwidth_gbps": round(best_d2d, 1), + "h2d_peak_gbps": pcie_peak, + "d2h_peak_gbps": pcie_peak, + "d2d_peak_gbps": None, + "h2d_efficiency_pct": round(best_h2d / pcie_peak * 100, 1) if pcie_peak else None, + "d2h_efficiency_pct": round(best_d2h / pcie_peak * 100, 1) if pcie_peak else None, + "d2d_efficiency_pct": efficiency, "peak_bandwidth_gbps": self.specs["memory_bandwidth_gbps"], "efficiency_pct": efficiency, "test_sizes_mb": test_sizes_mb, "bandwidth_by_size": bandwidth_by_size, "per_gpu": [], + "accelerator_vendor": self.specs.get("vendor", "nvidia"), } } @@ -355,7 +373,7 @@ class Benchmark: "tf32": ("tf32", self.specs["tf32_tflops"]), "fp16": (torch.float16, self.specs["fp16_tflops"]), "bf16": (torch.bfloat16, self.specs["bf16_tflops"]), - "fp8": (torch.float8_e4m3fn, self.specs["fp8_tflops"]), + "fp8": (getattr(torch, "float8_e4m3fn", None), self.specs["fp8_tflops"]), } results_by_dtype = {} @@ -375,7 +393,9 @@ class Benchmark: continue # Skip FP8 if GPU architecture doesn't support it - if dtype_name == "fp8" and self.specs.get("fp8_tflops", 0) == 0: + if dtype_name == "fp8" and ( + self.specs.get("fp8_tflops", 0) == 0 or dtype_map["fp8"][0] is None + ): arch = self.specs.get("architecture", "unknown") results_by_dtype["fp8"] = f"skipped ({arch} does not support FP8)" self.console.print(f"[dim] fp8: skipped - {arch} architecture has no FP8 support[/dim]") @@ -444,6 +464,7 @@ class Benchmark: "matrix_size": matrix_size, "warmup": warmup, "iterations": iterations, + "accelerator_backend": "dtk" if self.gpu_type == "bw1000" else "cuda", } } @@ -578,9 +599,14 @@ class Benchmark: continue pk = peak.get(dt, 0) ef = eff.get(dt, 0) - ec = "green" if ef >= 80 else ("yellow" if ef >= 50 else "red") - table.add_row(dt.upper(), f"{achieved:.1f}", f"{pk:.0f}", - f"[{ec}]{ef:.1f}%[/{ec}]") + if pk: + ec = "green" if ef >= 80 else ("yellow" if ef >= 50 else "red") + peak_text = f"{pk:.0f}" + eff_text = f"[{ec}]{ef:.1f}%[/{ec}]" + else: + peak_text = "N/A" + eff_text = "[cyan]MEASURED[/cyan]" + table.add_row(dt.upper(), f"{achieved:.1f}", peak_text, eff_text) c.print(table) @staticmethod @@ -590,7 +616,8 @@ class Benchmark: Single source of truth for the PASS/WARN/FAIL rule (same one report.py uses): achieved >= thr -> PASS; >= 0.9*thr -> WARN; else FAIL. A string achieved value (skipped/error) -> SKIP. A dtype without a threshold falls back to efficiency - (>=80 PASS / >=50 WARN / else FAIL). + (>=80 PASS / >=50 WARN / else FAIL). If neither threshold nor peak is + published, the numeric result is retained as MEASURED without a verdict. Returns {"rows": [(dtype, achieved, threshold, status), ...], "verdict": str}. """ @@ -598,20 +625,29 @@ class Benchmark: per_dtype = comp.get("per_dtype_tflops", {}) thresholds = comp.get("pass_thresholds_tflops", {}) or {} eff = comp.get("efficiency_pct", {}) - rank = {"PASS": 0, "WARN": 1, "FAIL": 2, "SKIP": 0} + rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0} rows, verdict = [], "PASS" + evaluated = False + measured = False for dt, val in per_dtype.items(): thr = thresholds.get(dt) if isinstance(val, str): status = "SKIP" elif thr: status = "PASS" if val >= thr else ("WARN" if val >= thr * 0.9 else "FAIL") + evaluated = True + elif dt not in eff: + status = "MEASURED" + measured = True else: e = eff.get(dt, 0) status = "PASS" if e >= 80 else ("WARN" if e >= 50 else "FAIL") + evaluated = True rows.append((dt, val, thr, status)) if rank[status] > rank[verdict]: verdict = status + if measured and not evaluated: + verdict = "MEASURED" return {"rows": rows, "verdict": verdict} @staticmethod @@ -619,7 +655,7 @@ class Benchmark: """Print the PASS/WARN/FAIL table for compute results; return the verdict.""" c = console or Console() judged = Benchmark.judge_compute(results) - color = {"PASS": "green", "WARN": "yellow", "FAIL": "red", "SKIP": "dim"} + color = {"PASS": "green", "MEASURED": "cyan", "WARN": "yellow", "FAIL": "red", "SKIP": "dim"} c.print("\n[bold cyan]Compute Verdict (vs thresholds)[/bold cyan]") for dt, val, thr, status in judged["rows"]: val_s = f"{val:.1f}" if isinstance(val, (int, float)) else str(val) diff --git a/modules/gpu_info.py b/modules/gpu_info.py index 6170905..3847b96 100644 --- a/modules/gpu_info.py +++ b/modules/gpu_info.py @@ -1,4 +1,4 @@ -"""GPU information detection module for NVIDIA datacenter GPUs (A100/A800/H100/H200/B200/B300).""" +"""Accelerator information detection for NVIDIA GPUs and Hygon DCUs.""" import subprocess import shutil @@ -11,6 +11,7 @@ from rich.panel import Panel from rich.text import Text from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label +from modules.hygon_smi import collect_hygon_info, find_hy_smi class GPUInfo: @@ -18,7 +19,8 @@ class GPUInfo: def __init__(self, config: dict): self.config = config self.console = Console() - self.gpu_type = detect_gpu_type() + configured = config.get("gpu_type", "auto") + self.gpu_type = configured if configured != "auto" else detect_gpu_type() self.specs = get_gpu_specs(self.gpu_type) self.gpu_label = get_gpu_label(self.gpu_type) @@ -35,6 +37,19 @@ class GPUInfo: return None def run(self) -> dict: + if self.gpu_type == "bw1000" or (not shutil.which("nvidia-smi") and find_hy_smi()): + result = collect_hygon_info(include_topology=True) + runtime_cfg = self.config.get("runtime", {}) or {} + result.update({ + "runtime_name": runtime_cfg.get("name", result.get("runtime_name", "DTK")), + "runtime_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")), + "cuda_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")), + "timestamp": datetime.now().isoformat(), + "detected_gpu_type": "bw1000", + "gpu_label": get_gpu_label("bw1000"), + }) + return result + if not shutil.which("nvidia-smi"): self.console.print("[bold red]nvidia-smi not found![/bold red]") return {"error": "nvidia-smi not found", "gpu_count": 0} @@ -159,7 +174,9 @@ class GPUInfo: c.print(f"\n[bold cyan]GPU Information[/bold cyan]") c.print(f" Driver Version : {results.get('driver_version', 'N/A')}") - c.print(f" CUDA Version : {results.get('cuda_version', 'N/A')}") + runtime_name = results.get("runtime_name", "CUDA") + runtime_version = results.get("runtime_version", results.get("cuda_version", "N/A")) + c.print(f" {runtime_name} Version{' ' * max(0, 4-len(runtime_name))}: {runtime_version}") c.print(f" GPU Count : {results.get('gpu_count', 0)}") c.print(f" Detected GPU : {results.get('gpu_label', 'Unknown')} ({results.get('detected_gpu_type', 'unknown')})") c.print(f" Timestamp : {results.get('timestamp', 'N/A')}") @@ -181,7 +198,7 @@ class GPUInfo: for g in gpus: name = g["name"] - if any(k in name for k in ("A100", "A800", "H100", "H200", "B200", "B300")): + if any(k in name for k in ("A100", "A800", "H100", "H200", "B200", "B300", "BW1000")): name = f"[bold green]{name}[/bold green]" vram = f"{g['vram_used_mb']}/{g['vram_total_mb']} MB" temp = f"{g['temperature']}°C" @@ -204,7 +221,8 @@ class GPUInfo: topology = results.get("topology", "") if topology and topology != "Unavailable": - c.print(Panel(topology, title="[bold]NVLink/NVSwitch Topology[/bold]", border_style="cyan")) + title = "HSW/XHCL Topology" if results.get("vendor") == "hygon" else "NVLink/NVSwitch Topology" + c.print(Panel(topology, title=f"[bold]{title}[/bold]", border_style="cyan")) warnings = [] for g in gpus: diff --git a/modules/gpu_specs.py b/modules/gpu_specs.py index 2ce9348..b36f112 100644 --- a/modules/gpu_specs.py +++ b/modules/gpu_specs.py @@ -1,4 +1,4 @@ -"""GPU specifications database for NVIDIA datacenter GPUs.""" +"""Accelerator specifications for NVIDIA GPUs and Hygon DCUs.""" import os import shutil @@ -8,6 +8,7 @@ from typing import List, Optional # GPU name patterns -> internal key mapping # Order matters: longer/more-specific patterns must come before shorter ones. GPU_NAME_PATTERNS = { + "BW1000": "bw1000", "A100": "a100", "A800": "a800", "H100": "h100", @@ -20,6 +21,31 @@ GPU_NAME_PATTERNS = { # Specs database — ALL values are DENSE (non-sparse) TFLOPS GPU_SPECS = { + "bw1000": { + # Public Hygon material identifies the supported precisions but does not + # publish stable dense peak figures for BW1000_H. Keep unknown peaks at + # zero so reports show measured throughput without inventing efficiency. + "full_name": "Hygon BW1000_H DCU", + "vendor": "hygon", + "architecture": "C-3000 GPGPU", + "compute_capability": 0.0, + "hbm_capacity_gb": 64, + "hbm_type": "HBM", + "memory_bandwidth_gbps": 0, + "fp32_tflops": 0, + "tf32_tflops": 0, + "fp16_tflops": 0, + "bf16_tflops": 0, + "fp8_tflops": 0, + "compute_pass_thresholds_tflops": {}, + "tdp_watts": 800, + "nvlink_gen": 0, + "nvlink_bandwidth_gbps": 0, + "interconnect": "HSW/XHCL", + "pcie_gen": 5, + "min_driver_version": "", + "min_cuda_version": "", + }, "h100": { # Peaks below are NVIDIA marketing dense peaks (theoretical Tensor Core max). # `compute_pass_thresholds_tflops` carries the absolute PASS thresholds used @@ -240,30 +266,41 @@ _UNKNOWN_SPECS = { def detect_gpu_type() -> str: - """Detect GPU type via nvidia-smi and return the internal key (e.g. 'h200'). + """Detect accelerator type via nvidia-smi or hy-smi. Returns 'unknown' if nvidia-smi is unavailable or the GPU is not recognized. """ nvidia_smi = shutil.which("nvidia-smi") - if not nvidia_smi: - return "unknown" + if nvidia_smi: + try: + r = subprocess.run( + [nvidia_smi, "--query-gpu=name", "--format=csv,noheader"], + capture_output=True, text=True, timeout=10, + ) + if r.returncode == 0 and r.stdout.strip(): + first_line = r.stdout.strip().splitlines()[0].strip().upper() + # Iterate longest-pattern-first so "H200" doesn't match "H20". + for pattern, key in sorted(GPU_NAME_PATTERNS.items(), key=lambda kv: -len(kv[0])): + if pattern in first_line: + return key + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + # Hygon DTK hosts expose hy-smi on PATH or under /opt/hyhal. try: - r = subprocess.run( - [nvidia_smi, "--query-gpu=name", "--format=csv,noheader"], - capture_output=True, text=True, timeout=10, - ) - if r.returncode != 0: - return "unknown" - - first_line = r.stdout.strip().splitlines()[0].strip().upper() - # Iterate longest-pattern-first so "H200" doesn't get matched by "H20". - for pattern, key in sorted(GPU_NAME_PATTERNS.items(), key=lambda kv: -len(kv[0])): - if pattern in first_line: - return key - return "unknown" - except (subprocess.TimeoutExpired, FileNotFoundError, OSError): - return "unknown" + from modules.hygon_smi import find_hy_smi + hy_smi = find_hy_smi() + if hy_smi: + r = subprocess.run( + [hy_smi, "--showproductname"], capture_output=True, + text=True, timeout=10, + ) + product = (r.stdout or "").upper() + if r.returncode == 0 and "BW1000" in product: + return "bw1000" + except (ImportError, subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + return "unknown" def get_gpu_specs(gpu_type: str = None) -> dict: @@ -286,8 +323,7 @@ def get_gpu_label(gpu_type: str) -> str: specs = GPU_SPECS.get(gpu_type) if specs: full = specs["full_name"] - # Strip the "NVIDIA " prefix for display - return full.replace("NVIDIA ", "") + return full.replace("NVIDIA ", "").replace("Hygon ", "") return "Unknown GPU" diff --git a/modules/health_check.py b/modules/health_check.py index dd64071..ea9627e 100644 --- a/modules/health_check.py +++ b/modules/health_check.py @@ -1,4 +1,4 @@ -"""Hardware health monitoring module for NVIDIA datacenter GPUs.""" +"""Hardware health monitoring for NVIDIA GPUs and Hygon DCUs.""" import subprocess import shutil @@ -12,6 +12,7 @@ from rich.panel import Panel from rich.text import Text from modules.gpu_specs import detect_gpu_type, get_gpu_specs +from modules.hygon_smi import collect_hygon_info, find_hy_smi class HealthCheck: @@ -20,7 +21,8 @@ class HealthCheck: self.config = config self.console = Console() self.health_cfg = config.get("health", {}) - self.gpu_type = detect_gpu_type() + configured = config.get("gpu_type", "auto") + self.gpu_type = configured if configured != "auto" else detect_gpu_type() self.specs = get_gpu_specs(self.gpu_type) def _run_smi(self, query: str) -> Optional[str]: @@ -55,6 +57,9 @@ class HealthCheck: return default def run(self) -> dict: + if self.gpu_type == "bw1000" or (not shutil.which("nvidia-smi") and find_hy_smi()): + return self._run_hygon() + if not shutil.which("nvidia-smi"): return {"error": "nvidia-smi not found", "passed": False} @@ -180,6 +185,102 @@ class HealthCheck: "detected_gpu_type": self.gpu_type, } + def _run_hygon(self) -> dict: + info = collect_hygon_info(include_topology=False) + if info.get("error"): + return {"error": info["error"], "passed": False} + + temp_warn = self.health_cfg.get("temp_warning", 80) + temp_crit = self.health_cfg.get("temp_critical", 90) + configured_power = self.health_cfg.get("power_limit") + gpu_health = [] + overall_pass = True + + for gpu in info.get("gpus", []): + checks = {} + temp = gpu.get("temperature", 0) + if temp >= temp_crit: + temp_status = "FAIL" + elif temp >= temp_warn: + temp_status = "WARN" + else: + temp_status = "PASS" + checks["temperature"] = { + "value": temp, "status": temp_status, + "threshold": temp_crit if temp_status == "FAIL" else temp_warn, + } + + draw = gpu.get("power_draw", 0.0) + limit = configured_power or gpu.get("power_limit") or self.specs.get("tdp_watts", 800) + checks["power"] = { + "value": draw, "limit": limit, + "status": "PASS" if draw <= limit * 1.05 else "WARN", + } + + ecc_enabled = bool(gpu.get("ecc_enabled")) + checks["ecc_errors"] = { + "single": gpu.get("ecc_errors_single", 0), + "double": gpu.get("ecc_errors_double", 0), + "enabled": ecc_enabled, + "status": "PASS" if ecc_enabled else "FAIL", + } + xgmi = int(gpu.get("xgmi_errors", 0)) + replay = int(gpu.get("pcie_replay_count", 0)) + healthy = bool(gpu.get("healthy")) + checks["memory_errors"] = { + "xgmi_errors": xgmi, + "pcie_replay_count": replay, + "device_healthy": healthy, + "status": "PASS" if healthy and xgmi == 0 and replay == 0 else "FAIL", + } + + generation = int(gpu.get("pcie_link_gen", 0)) + width = int(gpu.get("pcie_link_width", 0)) + expected = self.specs.get("pcie_gen", 0) + pcie_ok = generation >= expected and width >= 16 + checks["pcie_link"] = { + "gen": generation, "width": width, + "status": "PASS" if pcie_ok else "WARN", + } + + sm = int(gpu.get("clock_sm", 0)) + mem = int(gpu.get("clock_mem", 0)) + checks["clock_speed"] = { + "sm": sm, "mem": mem, + "status": "PASS" if sm > 0 and mem > 0 else "WARN", + } + throttling = bool(gpu.get("throttling_active")) + checks["throttling"] = { + "status": "FAIL" if throttling else "PASS", + "reasons": [gpu.get("throttling_text")] if throttling else [], + } + # Hygon has no NVIDIA-style persistence mode; a loaded /dev/kfd + # driver and successful hy-smi health query are the equivalent check. + checks["persistence_mode"] = { + "enabled": healthy, "status": "PASS" if healthy else "FAIL" + } + + worst = "PASS" + for check in checks.values(): + if check["status"] == "FAIL": + worst = "FAIL" + break + if check["status"] == "WARN": + worst = "WARN" + if worst == "FAIL": + overall_pass = False + gpu_health.append({"index": gpu["index"], "status": worst, "checks": checks}) + + return { + "passed": overall_pass, + "gpu_health": gpu_health, + "system_health": self._check_system(), + "timestamp": datetime.now().isoformat(), + "detected_gpu_type": "bw1000", + "driver_version": info.get("driver_version", "N/A"), + "vendor": "hygon", + } + def _check_system(self) -> dict: persistd = shutil.which("nvidia-persistenced") is not None persistd_running = False @@ -227,7 +328,10 @@ class HealthCheck: if os.path.isdir("/sys/class/infiniband_verbs"): rdma_devs = os.listdir("/sys/class/infiniband_verbs") - nccl_env = {k: v for k, v in os.environ.items() if k.startswith("NCCL_")} + nccl_env = { + k: v for k, v in os.environ.items() + if k.startswith(("NCCL_", "RCCL_", "HCCL_")) + } return { "nvidia_persistenced": {"installed": persistd, "running": persistd_running}, diff --git a/modules/hygon_smi.py b/modules/hygon_smi.py new file mode 100644 index 0000000..f081c49 --- /dev/null +++ b/modules/hygon_smi.py @@ -0,0 +1,190 @@ +"""Helpers for querying Hygon DCU devices through ``hy-smi``.""" + +import os +import re +import shutil +import subprocess +from typing import Dict, List, Optional + + +def find_hy_smi() -> Optional[str]: + """Return the hy-smi executable path when available.""" + path = shutil.which("hy-smi") + if path: + return path + path = "/opt/hyhal/bin/hy-smi" + return path if os.path.isfile(path) and os.access(path, os.X_OK) else None + + +def run_hy_smi(*args: str, timeout: int = 30) -> Optional[str]: + """Run hy-smi and return stdout, or ``None`` on failure.""" + binary = find_hy_smi() + if not binary: + return None + try: + result = subprocess.run( + [binary, *args], capture_output=True, text=True, timeout=timeout + ) + except (FileNotFoundError, OSError, subprocess.TimeoutExpired): + return None + return result.stdout.strip() if result.returncode == 0 else None + + +def _indexed_values(text: Optional[str], pattern: str) -> Dict[int, str]: + if not text: + return {} + values: Dict[int, str] = {} + for match in re.finditer(pattern, text, re.MULTILINE | re.IGNORECASE): + values[int(match.group(1))] = match.group(2).strip() + return values + + +def _number(value, default=0.0): + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _pcie_gen(gt_per_second: float) -> int: + if gt_per_second >= 32: + return 5 + if gt_per_second >= 16: + return 4 + if gt_per_second >= 8: + return 3 + if gt_per_second >= 5: + return 2 + return 1 if gt_per_second > 0 else 0 + + +def collect_hygon_info(include_topology: bool = True) -> dict: + """Collect normalized per-device information for GPUInfo and HealthCheck.""" + summary = run_hy_smi() + if not summary: + return {"error": "hy-smi query failed", "gpu_count": 0, "gpus": []} + + rows: Dict[int, dict] = {} + row_re = re.compile( + r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W\s+(\S+)\s+" + r"([\d.]+)W\s+(\d+)%\s+([\d.]+)%", + re.MULTILINE, + ) + for match in row_re.finditer(summary): + index = int(match.group(1)) + rows[index] = { + "index": index, + "temperature": int(round(float(match.group(2)))), + "power_draw": float(match.group(3)), + "performance_level": match.group(4), + "power_limit": float(match.group(5)), + "vram_used_pct": int(match.group(6)), + "utilization_pct": float(match.group(7)), + } + + product = run_hy_smi("--showproductname") + driver = run_hy_smi("--showdriverversion") + memory = run_hy_smi("--showmemavailable") + bus = run_hy_smi("--showbus") + unique = run_hy_smi("--showuniqueid") + serial = run_hy_smi("--showserial") + clocks = run_hy_smi("--showclocks") + ecc = run_hy_smi("--showmemeccinfo") + xgmi = run_hy_smi("--showxgmierr") + replay = run_hy_smi("--showreplaycount") + health = run_hy_smi("--healthcheck") + throttle = run_hy_smi("--showfdr") + + names = _indexed_values(product, r"HCU\[(\d+)\].*Card Series:\s*([^\s]+)") + memories = _indexed_values(memory, r"HCU\[(\d+)\].*Available memory size \(MiB\):\s*(\d+)") + buses = _indexed_values(bus, r"HCU\[(\d+)\].*PCI Bus:\s*([0-9a-fA-F:.]+)") + unique_ids = _indexed_values(unique, r"HCU\[(\d+)\].*Unique ID:\s*(\S+)") + serials = _indexed_values(serial, r"HCU\[(\d+)\].*Serial Number:\s*(\S+)") + sclk = _indexed_values(clocks, r"HCU\[(\d+)\].*sclk clock level:.*\((\d+)Mhz\)") + mclk = _indexed_values(clocks, r"HCU\[(\d+)\].*mclk clock level:.*\((\d+)Mhz\)") + pcie = {} + if clocks: + for match in re.finditer( + r"HCU\[(\d+)\].*pcie clock level:.*\(([\d.]+)GT/s,\s*x(\d+)", clocks + ): + pcie[int(match.group(1))] = (float(match.group(2)), int(match.group(3))) + ecc_enabled = { + idx: value.upper() == "ON" + for idx, value in _indexed_values( + ecc, r"HCU\[(\d+)\].*Vram memory ECC status:\s*(\S+)" + ).items() + } + xgmi_errors = { + idx: int(value) + for idx, value in _indexed_values( + xgmi, r"HCU\[(\d+)\].*XGMI Error count\s+(\d+)" + ).items() + } + replay_counts = { + idx: int(value) + for idx, value in _indexed_values( + replay, r"HCU\[(\d+)\].*PCIe Replay Count:\s*(\d+)" + ).items() + } + healthy = { + idx: value.lower() == "healthy" + for idx, value in _indexed_values( + health, r"HCU\[(\d+)\].*:\s*(Healthy|Unhealthy)\s*$" + ).items() + } + throttle_status = _indexed_values( + throttle, r"HCU\[(\d+)\].*Throttler Status:\s*(.+?)\s*$" + ) + + driver_match = re.search(r"Driver Version:\s*(\S+)", driver or "") + driver_version = driver_match.group(1) if driver_match else "N/A" + + gpu_count = max(rows.keys(), default=-1) + 1 + gpus: List[dict] = [] + for index in range(gpu_count): + row = rows.get(index, {"index": index}) + total_mb = int(memories.get(index, 0)) + used_mb = int(total_mb * int(row.get("vram_used_pct", 0)) / 100) + gt_s, width = pcie.get(index, (0.0, 0)) + throttle_text = throttle_status.get(index, "") + real_throttle = bool( + throttle_text + and any(token not in {"IDLE", "ACTIVE", "NONE"} + for token in re.findall(r"[A-Z_]+", throttle_text.upper())) + ) + gpus.append({ + **row, + "name": names.get(index, "Hygon DCU"), + "uuid": unique_ids.get(index, "N/A"), + "pci_bus_id": buses.get(index, "N/A"), + "pcie_link_gen": _pcie_gen(gt_s), + "pcie_link_width": width, + "pcie_gt_s": gt_s, + "vram_total_mb": total_mb, + "vram_used_mb": used_mb, + "vram_free_mb": max(total_mb - used_mb, 0), + "clock_sm": int(_number(sclk.get(index))), + "clock_mem": int(_number(mclk.get(index))), + "fan_speed": 0, + "persistence_mode": True, + "compute_mode": row.get("performance_level", "auto"), + "serial_number": serials.get(index, "N/A"), + "ecc_enabled": ecc_enabled.get(index, False), + "ecc_errors_single": 0, + "ecc_errors_double": 0, + "xgmi_errors": xgmi_errors.get(index, 0), + "pcie_replay_count": replay_counts.get(index, 0), + "healthy": healthy.get(index, False), + "throttling_active": real_throttle, + "throttling_text": throttle_text, + }) + + return { + "driver_version": driver_version, + "runtime_name": "DTK", + "runtime_version": os.environ.get("DTK_VERSION", "N/A"), + "gpu_count": gpu_count, + "gpus": gpus, + "topology": run_hy_smi("--showtopo") if include_topology else "", + "vendor": "hygon", + } diff --git a/modules/multinode_nccl_test.py b/modules/multinode_nccl_test.py index f97c96e..e00b30e 100644 --- a/modules/multinode_nccl_test.py +++ b/modules/multinode_nccl_test.py @@ -286,10 +286,18 @@ class MultiNodeNCCLTest: "--mca", "btl_openib_warn_no_device_params_found", "0", "--mca", "btl_tcp_if_include", str(self.cfg.get("socket_ifname", "bond0")), "--mca", "oob_tcp_if_include", str(self.cfg.get("oob_tcp_ifname", self.cfg.get("socket_ifname", "bond0"))), + ] + # HCOLL does an eager UCX IB wireup during MPI_Init that deadlocks on nodes + # where UCX only sees mlx5_2/mlx5_3 instead of mlx5_0/mlx5_1. HCOLL + # collective optimisations are irrelevant for NCCL tests (NCCL handles its + # own collectives), so disable it unconditionally unless the caller opts in. + if self.cfg.get("disable_hcoll", True): + cmd.extend(["--mca", "coll", "^hcoll"]) + cmd.extend([ "-H", host_arg, "--map-by", f"ppr:{gpus_per_node}:node", "-np", str(ranks), - ] + ]) plm_rsh_args = self.cfg.get("plm_rsh_args") if plm_rsh_args: cmd.extend(["--mca", "plm_rsh_args", str(plm_rsh_args)]) diff --git a/modules/nccl_test.py b/modules/nccl_test.py index 5c04af5..f67998d 100644 --- a/modules/nccl_test.py +++ b/modules/nccl_test.py @@ -19,6 +19,10 @@ from modules.gpu_specs import detect_gpu_type, get_gpu_specs, resolve_tools_dir TORCH_AVAILABLE = False try: import torch + try: + import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim + except ImportError: + pass if torch.cuda.is_available(): TORCH_AVAILABLE = True except ImportError: @@ -46,7 +50,8 @@ class NCCLTest: self.console = Console() self.nccl_cfg = config.get("nccl", {}) self.tools_dir = resolve_tools_dir(config) - self.gpu_type = detect_gpu_type() + configured = config.get("gpu_type", "auto") + self.gpu_type = configured if configured != "auto" else detect_gpu_type() self.specs = get_gpu_specs(self.gpu_type) def _find_nccl_test(self, name: str) -> Optional[str]: @@ -295,60 +300,78 @@ class NCCLTest: } def _run_torchrun_fallback(self, gpu_count: int) -> dict: - """Basic NCCL connectivity test via torchrun — verifies NCCL works but does not benchmark performance.""" - self.console.print("[yellow]nccl-tests not available, running basic NCCL connectivity check[/yellow]") + """Benchmark collectives through PyTorch when nccl-tests is unavailable.""" + self.console.print("[yellow]nccl-tests not available, benchmarking collectives via torchrun[/yellow]") + size_mb = int(self.nccl_cfg.get("torchrun_size_mb", 64)) + warmup = int(self.nccl_cfg.get("torchrun_warmup", 5)) + iterations = int(self.nccl_cfg.get("torchrun_iterations", 20)) + backend = self.nccl_cfg.get("torch_backend", "nccl") + enabled_ops = [] + for key, enabled in [ + ("allreduce", self.nccl_cfg.get("test_allreduce", True)), + ("alltoall", self.nccl_cfg.get("test_alltoall", True)), + ("broadcast", self.nccl_cfg.get("test_broadcast", True)), + ("reducescatter", self.nccl_cfg.get("test_reduce_scatter", False)), + ("allgather", self.nccl_cfg.get("test_allgather", False)), + ]: + if enabled: + enabled_ops.append(key) code = f""" -import torch, torch.distributed as dist, os +import os, time +import torch +try: + import torch_dcu +except ImportError: + pass +import torch.distributed as dist os.environ.setdefault("MASTER_ADDR","127.0.0.1") os.environ.setdefault("MASTER_PORT","29500") rank=int(os.environ.get("LOCAL_RANK",0)) ws={gpu_count} -dist.init_process_group("nccl",rank=rank,world_size=ws) +dist.init_process_group({backend!r},rank=rank,world_size=ws) torch.cuda.set_device(rank) +size_bytes={size_mb}*1024*1024 +numel=size_bytes//4 +x=torch.ones(numel,device=f"cuda:{{rank}}",dtype=torch.float32) +allgather_out=torch.empty(numel*ws,device=f"cuda:{{rank}}",dtype=torch.float32) +rs_out=torch.empty(numel//ws,device=f"cuda:{{rank}}",dtype=torch.float32) +a2a_out=torch.empty_like(x) -x=torch.randn(1024*1024,device=f"cuda:{{rank}}",dtype=torch.float32) - -# Test AllReduce -try: - dist.all_reduce(x.clone()) - if rank==0: print("allreduce:ok") -except Exception as e: - if rank==0: print(f"allreduce:fail:{{e}}") - -# Test Broadcast -try: - dist.broadcast(x.clone(),src=0) - if rank==0: print("broadcast:ok") -except Exception as e: - if rank==0: print(f"broadcast:fail:{{e}}") - -# Test AllGather -try: - tensor_list=[torch.empty_like(x) for _ in range(ws)] - dist.all_gather(tensor_list,x.clone()) - if rank==0: print("allgather:ok") -except Exception as e: - if rank==0: print(f"allgather:fail:{{e}}") - -# Test ReduceScatter -try: - chunks=list(x.chunk(ws)) - output=torch.empty_like(chunks[0]) - dist.reduce_scatter(output,chunks) - if rank==0: print("reducescatter:ok") -except Exception as e: - if rank==0: print(f"reducescatter:fail:{{e}}") - -# Test AllToAll -try: - chunks=list(x.chunk(ws)) - output_list=[torch.empty_like(c) for c in chunks] - dist.all_to_all(output_list,chunks) - if rank==0: print("alltoall:ok") -except Exception as e: - if rank==0: print(f"alltoall:fail:{{e}}") +def run_op(name): + if name == "allreduce": + return dist.all_reduce(x) + if name == "broadcast": + return dist.broadcast(x, src=0) + if name == "allgather": + return dist.all_gather_into_tensor(allgather_out, x) + if name == "reducescatter": + return dist.reduce_scatter_tensor(rs_out, x) + if name == "alltoall": + return dist.all_to_all_single(a2a_out, x) + raise ValueError(name) +for name in {enabled_ops!r}: + try: + for _ in range({warmup}): + run_op(name) + torch.cuda.synchronize() + dist.barrier() + start=time.perf_counter() + for _ in range({iterations}): + run_op(name) + torch.cuda.synchronize() + elapsed=(time.perf_counter()-start)/{iterations} + algbw=size_bytes/elapsed/1e9 + factor={{"allreduce":2*(ws-1)/ws,"allgather":(ws-1)/ws, + "reducescatter":(ws-1)/ws,"alltoall":(ws-1)/ws, + "broadcast":1.0}}[name] + busbw=algbw*factor + if rank==0: + print(f"METRIC|{{name}}|{{elapsed*1e6:.3f}}|{{algbw:.3f}}|{{busbw:.3f}}", flush=True) + except Exception as exc: + if rank==0: + print(f"ERROR|{{name}}|{{exc}}", flush=True) dist.destroy_process_group() """ import tempfile @@ -364,41 +387,50 @@ dist.destroy_process_group() r = subprocess.run( [torchrun_cmd, f"--nproc_per_node={gpu_count}", tmp.name], - capture_output=True, text=True, timeout=120, - env={**os.environ, "NCCL_DEBUG": "WARN"}, + capture_output=True, text=True, timeout=300, + env={**os.environ, "NCCL_DEBUG": "WARN", "RCCL_DEBUG": "WARN"}, ) os.unlink(tmp.name) - - # Parse connectivity results — format: op_name:ok or op_name:fail:error + tests = {} - all_passed = True for line in r.stdout.split("\n"): line = line.strip() - if not line: - continue - parts = line.split(":") - op_name = parts[0] - result = parts[1] if len(parts) > 1 else "unknown" - - if result == "ok": - status = "PASS" - else: - status = "FAIL" - all_passed = False - - tests[op_name] = { - "status": status, - "error": ":".join(parts[2:]) if len(parts) > 2 and result == "fail" else None, - } - + if line.startswith("METRIC|"): + _, name, time_us, algbw, busbw = line.split("|", 4) + threshold = self.nccl_cfg.get("min_bandwidth_gbps") or 10.0 + busbw_f = float(busbw) + tests[name] = { + "status": "PASS" if busbw_f >= float(threshold) else "WARN", + "best_algbw_gbps": round(float(algbw), 2), + "best_busbw_gbps": round(busbw_f, 2), + "min_required_gbps": float(threshold), + "by_size": [{ + "size": size_mb * 1024 * 1024, + "time_us": round(float(time_us), 2), + "algbw_gbps": round(float(algbw), 2), + "busbw_gbps": round(busbw_f, 2), + }], + } + elif line.startswith("ERROR|"): + _, name, error = line.split("|", 2) + tests[name] = {"status": "FAIL", "error": error} + if not tests: + error = (r.stderr or r.stdout or "torchrun produced no metrics")[-1000:] + return {"passed": False, "source": "torchrun_collective", "error": error} + all_passed = all(value.get("status") == "PASS" for value in tests.values()) return { "passed": all_passed, - "source": "torchrun_fallback", + "source": "torchrun_collective", "tests": tests, "gpu_count": gpu_count, + "message_size_mb": size_mb, + "warmup": warmup, + "iterations": iterations, + "backend": backend, + "timestamp": datetime.now().isoformat(), } except Exception as e: - return {"passed": False, "source": "torchrun_fallback", "error": str(e)} + return {"passed": False, "source": "torchrun_collective", "error": str(e)} @staticmethod def print_results(results: dict, console: Console = None): diff --git a/modules/rdma_test.py b/modules/rdma_test.py index 08ea610..c94a106 100644 --- a/modules/rdma_test.py +++ b/modules/rdma_test.py @@ -76,6 +76,21 @@ class RDMATest: active_ib_port = True device_info = self._collect_device_info(devices) + cross_enabled = (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled") + if not ib_devices and cross_enabled: + # RoCE ports report link_layer=Ethernet. They are valid for configured + # cross-node perftest even though local InfiniBand loopback is skipped. + cross = self.run_cross_node() + return { + "passed": cross.get("status") == "PASS", + "status": cross.get("status", "FAIL"), + "reason": "RoCE cross-node mode", + "devices": device_info, + "bandwidth_tests": [], + "latency_tests": [], + "cross_node": cross, + "timestamp": datetime.now().isoformat(), + } if not ib_devices: self.console.print( "[yellow]No InfiniBand-link_layer ports present — " @@ -266,13 +281,14 @@ class RDMATest: # ------------------------------------------------------------------ def _active_ib_devices(self) -> List[str]: - """IB devices whose port 1 is InfiniBand link_layer and ACTIVE.""" + """Active IB devices, optionally including Ethernet/RoCE devices.""" out = [] + allow_roce = bool((self.rdma_cfg.get("cross_node", {}) or {}).get("allow_roce")) for dev in self._get_ib_devices(): for port in self._get_ib_ports(dev): ll = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/link_layer") st = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/state") - if ll == "InfiniBand" and "ACTIVE" in st.upper(): + if (ll == "InfiniBand" or allow_roce) and "ACTIVE" in st.upper(): out.append(dev) break return out @@ -296,6 +312,8 @@ class RDMATest: gid_index = cn.get("gid_index") msg_size = cn.get("msg_size", 1048576) iters = cn.get("iters", 5000) + duration_sec = cn.get("duration_sec") + queue_pairs = int(cn.get("queue_pairs", 1)) base_port = cn.get("base_oob_port", 18515) warmup = cn.get("server_warmup_sec", 2.0) min_bw = cn.get("min_bandwidth_gbps", 350) @@ -322,10 +340,16 @@ class RDMATest: entry = {"device": dev} if has_bw: + bw_extra = ["--report_gbits", "-s", str(msg_size), + "-q", str(queue_pairs)] + if duration_sec: + bw_extra += ["-D", str(duration_sec)] + else: + bw_extra += ["-n", str(iters)] bw = self._cross_node_perftest( "ib_write_bw", dev, server_target, server_addr, ib_port, oob, gid_index, warmup, - extra=["--report_gbits", "-s", str(msg_size), "-n", str(iters)], + extra=bw_extra, parse="bw") entry["bandwidth_gbps"] = bw if isinstance(bw, (int, float)): diff --git a/modules/report.py b/modules/report.py index d9e1eba..0d8f04f 100644 --- a/modules/report.py +++ b/modules/report.py @@ -100,6 +100,10 @@ class ReportGenerator: if "gpu_info" in results: gpus = results["gpu_info"].get("gpus", []) + runtime_name = results["gpu_info"].get("runtime_name", "CUDA") + runtime_version = results["gpu_info"].get( + "runtime_version", results["gpu_info"].get("cuda_version", "N/A") + ) rows = "" for g in gpus: rows += f"
Driver: {results["gpu_info"].get("driver_version", "N/A")} | ' - f'CUDA: {results["gpu_info"].get("cuda_version", "N/A")} | ' + f'{runtime_name}: {runtime_version} | ' f'Count: {len(gpus)}
' f'| GPU | Model | VRAM | Temp | SM Clock |
|---|