diff --git a/configs/bw1000_4node.yaml b/configs/bw1000_4node.yaml new file mode 100644 index 0000000..01e5862 --- /dev/null +++ b/configs/bw1000_4node.yaml @@ -0,0 +1,86 @@ +gpu_type: bw1000 + +runtime: + name: DTK + version: 25.04.2 + +benchmark: + memory: + size_mb: 4096 + iterations: 10 + nvbandwidth_buffer_mb: 512 + nvbandwidth_samples: 3 + compute: + dtypes: [fp32, tf32, fp16, bf16, fp8] + sweep_sizes: [4096, 6144, 8192, [2048, 2048, 13312]] + matrix_size: 8192 + warmup: 10 + iterations: 30 + +health: + temp_warning: 75 + temp_critical: 85 + power_limit: 800 + +nccl: + min_bandwidth_gbps: 10 + test_allreduce: true + test_alltoall: true + test_broadcast: true + test_reduce_scatter: true + test_allgather: true + test_sendrecv: false + torch_backend: nccl + torchrun_size_mb: 64 + torchrun_warmup: 5 + torchrun_iterations: 20 + +stress: + duration_sec: 600 + use_doubles: false + use_tensor_cores: true + memory_pct: 80 + gpus: all + +rdma: + min_bandwidth_gbps: 200 + max_latency_us: 6 + ib_iterations: 5000 + msg_size: 1048576 + ib_device: null + ib_port: 1 + cross_node: + enabled: false + allow_roce: true + server: null + server_addr: null + ssh_user: root + devices: [mlx5_bond_0, mlx5_bond_1, mlx5_bond_2, mlx5_bond_3] + ib_port: 1 + gid_index: 3 + msg_size: 1048576 + iters: 5000 + duration_sec: 10 + queue_pairs: 4 + base_oob_port: 18515 + server_warmup_sec: 2.0 + min_bandwidth_gbps: 200 + max_latency_us: 6 + +training: + model: synthetic + batch_size: 2 + seq_length: 1024 + num_steps: 20 + dtype: bf16 + hidden_size: 2048 + num_layers: 4 + num_heads: 16 + vocab_size: 8192 + +report: + output_dir: ./reports + format: json + +tools: + install_dir: /opt/gpu-test-tools diff --git a/configs/bw1000_smoke.yaml b/configs/bw1000_smoke.yaml new file mode 100644 index 0000000..b0a87a8 --- /dev/null +++ b/configs/bw1000_smoke.yaml @@ -0,0 +1,15 @@ +gpu_type: bw1000 + +runtime: + name: DTK + version: 25.04.2 + +stress: + duration_sec: 60 + use_doubles: false + use_tensor_cores: true + memory_pct: 80 + gpus: all + +tools: + install_dir: /opt/gpu-test-tools diff --git a/gpu_tester.py b/gpu_tester.py index 8c00282..5de9833 100644 --- a/gpu_tester.py +++ b/gpu_tester.py @@ -27,6 +27,7 @@ from modules.stress_test import StressTest from modules.rdma_test import RDMATest from modules.report import ReportGenerator from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label, get_supported_gpus, validate_driver_compatibility +from modules.hygon_smi import find_hy_smi DEFAULT_CONFIG = { "benchmark": { @@ -117,9 +118,9 @@ def check_prerequisites(console: Console) -> bool: import shutil ok = True - if not shutil.which("nvidia-smi"): - console.print("[bold red]ERROR: nvidia-smi not found![/bold red]") - console.print(" Please install NVIDIA drivers first.") + if not shutil.which("nvidia-smi") and not find_hy_smi(): + console.print("[bold red]ERROR: neither nvidia-smi nor hy-smi was found![/bold red]") + console.print(" Please install the NVIDIA driver or Hygon DCU driver first.") ok = False return ok diff --git a/modules/benchmark.py b/modules/benchmark.py index e728633..38b185b 100644 --- a/modules/benchmark.py +++ b/modules/benchmark.py @@ -17,6 +17,10 @@ from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label, res TORCH_AVAILABLE = False try: import torch + try: + import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim + except ImportError: + pass if torch.cuda.is_available(): TORCH_AVAILABLE = True except ImportError: @@ -63,12 +67,14 @@ class Benchmark: return None def run_memory_benchmark(self) -> dict: - nvbw = self._find_nvbandwidth() + # nvbandwidth is NVIDIA-specific. DTK PyTorch provides the portable + # transfer/copy path used below for Hygon DCUs. + nvbw = None if self.gpu_type == "bw1000" else self._find_nvbandwidth() if nvbw: return self._run_nvbandwidth(nvbw) - self.console.print("[yellow]nvbandwidth not found, falling back to PyTorch[/yellow]") + self.console.print("[yellow]Using PyTorch memory benchmark fallback[/yellow]") return self._run_memory_pytorch() def _run_nvbandwidth(self, nvbw_path: str) -> dict: @@ -189,6 +195,7 @@ class Benchmark: "efficiency_pct": d2d_efficiency, "results_by_test": results_by_test, "per_gpu": [], + "accelerator_vendor": self.specs.get("vendor", "nvidia"), } } @@ -292,20 +299,31 @@ class Benchmark: progress.advance(task) best_d2d = max(v["d2d_gbps"] for v in bandwidth_by_size.values()) + best_h2d = max(v["h2d_gbps"] for v in bandwidth_by_size.values()) + best_d2h = max(v["d2h_gbps"] for v in bandwidth_by_size.values()) peak_bw = self.specs["memory_bandwidth_gbps"] efficiency = round((best_d2d / peak_bw) * 100, 1) if peak_bw else None + pcie_gen = self.specs.get("pcie_gen", 0) + pcie_peak = {3: 16, 4: 32, 5: 64, 6: 128}.get(pcie_gen) if pcie_gen else None return { "memory": { "source": "pytorch", - "h2d_bandwidth_gbps": round(max(v["h2d_gbps"] for v in bandwidth_by_size.values()), 1), - "d2h_bandwidth_gbps": round(max(v["d2h_gbps"] for v in bandwidth_by_size.values()), 1), + "h2d_bandwidth_gbps": round(best_h2d, 1), + "d2h_bandwidth_gbps": round(best_d2h, 1), "d2d_bandwidth_gbps": round(best_d2d, 1), + "h2d_peak_gbps": pcie_peak, + "d2h_peak_gbps": pcie_peak, + "d2d_peak_gbps": None, + "h2d_efficiency_pct": round(best_h2d / pcie_peak * 100, 1) if pcie_peak else None, + "d2h_efficiency_pct": round(best_d2h / pcie_peak * 100, 1) if pcie_peak else None, + "d2d_efficiency_pct": efficiency, "peak_bandwidth_gbps": self.specs["memory_bandwidth_gbps"], "efficiency_pct": efficiency, "test_sizes_mb": test_sizes_mb, "bandwidth_by_size": bandwidth_by_size, "per_gpu": [], + "accelerator_vendor": self.specs.get("vendor", "nvidia"), } } @@ -355,7 +373,7 @@ class Benchmark: "tf32": ("tf32", self.specs["tf32_tflops"]), "fp16": (torch.float16, self.specs["fp16_tflops"]), "bf16": (torch.bfloat16, self.specs["bf16_tflops"]), - "fp8": (torch.float8_e4m3fn, self.specs["fp8_tflops"]), + "fp8": (getattr(torch, "float8_e4m3fn", None), self.specs["fp8_tflops"]), } results_by_dtype = {} @@ -375,7 +393,9 @@ class Benchmark: continue # Skip FP8 if GPU architecture doesn't support it - if dtype_name == "fp8" and self.specs.get("fp8_tflops", 0) == 0: + if dtype_name == "fp8" and ( + self.specs.get("fp8_tflops", 0) == 0 or dtype_map["fp8"][0] is None + ): arch = self.specs.get("architecture", "unknown") results_by_dtype["fp8"] = f"skipped ({arch} does not support FP8)" self.console.print(f"[dim] fp8: skipped - {arch} architecture has no FP8 support[/dim]") @@ -444,6 +464,7 @@ class Benchmark: "matrix_size": matrix_size, "warmup": warmup, "iterations": iterations, + "accelerator_backend": "dtk" if self.gpu_type == "bw1000" else "cuda", } } @@ -578,9 +599,14 @@ class Benchmark: continue pk = peak.get(dt, 0) ef = eff.get(dt, 0) - ec = "green" if ef >= 80 else ("yellow" if ef >= 50 else "red") - table.add_row(dt.upper(), f"{achieved:.1f}", f"{pk:.0f}", - f"[{ec}]{ef:.1f}%[/{ec}]") + if pk: + ec = "green" if ef >= 80 else ("yellow" if ef >= 50 else "red") + peak_text = f"{pk:.0f}" + eff_text = f"[{ec}]{ef:.1f}%[/{ec}]" + else: + peak_text = "N/A" + eff_text = "[cyan]MEASURED[/cyan]" + table.add_row(dt.upper(), f"{achieved:.1f}", peak_text, eff_text) c.print(table) @staticmethod @@ -590,7 +616,8 @@ class Benchmark: Single source of truth for the PASS/WARN/FAIL rule (same one report.py uses): achieved >= thr -> PASS; >= 0.9*thr -> WARN; else FAIL. A string achieved value (skipped/error) -> SKIP. A dtype without a threshold falls back to efficiency - (>=80 PASS / >=50 WARN / else FAIL). + (>=80 PASS / >=50 WARN / else FAIL). If neither threshold nor peak is + published, the numeric result is retained as MEASURED without a verdict. Returns {"rows": [(dtype, achieved, threshold, status), ...], "verdict": str}. """ @@ -598,20 +625,29 @@ class Benchmark: per_dtype = comp.get("per_dtype_tflops", {}) thresholds = comp.get("pass_thresholds_tflops", {}) or {} eff = comp.get("efficiency_pct", {}) - rank = {"PASS": 0, "WARN": 1, "FAIL": 2, "SKIP": 0} + rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0} rows, verdict = [], "PASS" + evaluated = False + measured = False for dt, val in per_dtype.items(): thr = thresholds.get(dt) if isinstance(val, str): status = "SKIP" elif thr: status = "PASS" if val >= thr else ("WARN" if val >= thr * 0.9 else "FAIL") + evaluated = True + elif dt not in eff: + status = "MEASURED" + measured = True else: e = eff.get(dt, 0) status = "PASS" if e >= 80 else ("WARN" if e >= 50 else "FAIL") + evaluated = True rows.append((dt, val, thr, status)) if rank[status] > rank[verdict]: verdict = status + if measured and not evaluated: + verdict = "MEASURED" return {"rows": rows, "verdict": verdict} @staticmethod @@ -619,7 +655,7 @@ class Benchmark: """Print the PASS/WARN/FAIL table for compute results; return the verdict.""" c = console or Console() judged = Benchmark.judge_compute(results) - color = {"PASS": "green", "WARN": "yellow", "FAIL": "red", "SKIP": "dim"} + color = {"PASS": "green", "MEASURED": "cyan", "WARN": "yellow", "FAIL": "red", "SKIP": "dim"} c.print("\n[bold cyan]Compute Verdict (vs thresholds)[/bold cyan]") for dt, val, thr, status in judged["rows"]: val_s = f"{val:.1f}" if isinstance(val, (int, float)) else str(val) diff --git a/modules/gpu_info.py b/modules/gpu_info.py index 6170905..3847b96 100644 --- a/modules/gpu_info.py +++ b/modules/gpu_info.py @@ -1,4 +1,4 @@ -"""GPU information detection module for NVIDIA datacenter GPUs (A100/A800/H100/H200/B200/B300).""" +"""Accelerator information detection for NVIDIA GPUs and Hygon DCUs.""" import subprocess import shutil @@ -11,6 +11,7 @@ from rich.panel import Panel from rich.text import Text from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label +from modules.hygon_smi import collect_hygon_info, find_hy_smi class GPUInfo: @@ -18,7 +19,8 @@ class GPUInfo: def __init__(self, config: dict): self.config = config self.console = Console() - self.gpu_type = detect_gpu_type() + configured = config.get("gpu_type", "auto") + self.gpu_type = configured if configured != "auto" else detect_gpu_type() self.specs = get_gpu_specs(self.gpu_type) self.gpu_label = get_gpu_label(self.gpu_type) @@ -35,6 +37,19 @@ class GPUInfo: return None def run(self) -> dict: + if self.gpu_type == "bw1000" or (not shutil.which("nvidia-smi") and find_hy_smi()): + result = collect_hygon_info(include_topology=True) + runtime_cfg = self.config.get("runtime", {}) or {} + result.update({ + "runtime_name": runtime_cfg.get("name", result.get("runtime_name", "DTK")), + "runtime_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")), + "cuda_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")), + "timestamp": datetime.now().isoformat(), + "detected_gpu_type": "bw1000", + "gpu_label": get_gpu_label("bw1000"), + }) + return result + if not shutil.which("nvidia-smi"): self.console.print("[bold red]nvidia-smi not found![/bold red]") return {"error": "nvidia-smi not found", "gpu_count": 0} @@ -159,7 +174,9 @@ class GPUInfo: c.print(f"\n[bold cyan]GPU Information[/bold cyan]") c.print(f" Driver Version : {results.get('driver_version', 'N/A')}") - c.print(f" CUDA Version : {results.get('cuda_version', 'N/A')}") + runtime_name = results.get("runtime_name", "CUDA") + runtime_version = results.get("runtime_version", results.get("cuda_version", "N/A")) + c.print(f" {runtime_name} Version{' ' * max(0, 4-len(runtime_name))}: {runtime_version}") c.print(f" GPU Count : {results.get('gpu_count', 0)}") c.print(f" Detected GPU : {results.get('gpu_label', 'Unknown')} ({results.get('detected_gpu_type', 'unknown')})") c.print(f" Timestamp : {results.get('timestamp', 'N/A')}") @@ -181,7 +198,7 @@ class GPUInfo: for g in gpus: name = g["name"] - if any(k in name for k in ("A100", "A800", "H100", "H200", "B200", "B300")): + if any(k in name for k in ("A100", "A800", "H100", "H200", "B200", "B300", "BW1000")): name = f"[bold green]{name}[/bold green]" vram = f"{g['vram_used_mb']}/{g['vram_total_mb']} MB" temp = f"{g['temperature']}°C" @@ -204,7 +221,8 @@ class GPUInfo: topology = results.get("topology", "") if topology and topology != "Unavailable": - c.print(Panel(topology, title="[bold]NVLink/NVSwitch Topology[/bold]", border_style="cyan")) + title = "HSW/XHCL Topology" if results.get("vendor") == "hygon" else "NVLink/NVSwitch Topology" + c.print(Panel(topology, title=f"[bold]{title}[/bold]", border_style="cyan")) warnings = [] for g in gpus: diff --git a/modules/gpu_specs.py b/modules/gpu_specs.py index 2ce9348..b36f112 100644 --- a/modules/gpu_specs.py +++ b/modules/gpu_specs.py @@ -1,4 +1,4 @@ -"""GPU specifications database for NVIDIA datacenter GPUs.""" +"""Accelerator specifications for NVIDIA GPUs and Hygon DCUs.""" import os import shutil @@ -8,6 +8,7 @@ from typing import List, Optional # GPU name patterns -> internal key mapping # Order matters: longer/more-specific patterns must come before shorter ones. GPU_NAME_PATTERNS = { + "BW1000": "bw1000", "A100": "a100", "A800": "a800", "H100": "h100", @@ -20,6 +21,31 @@ GPU_NAME_PATTERNS = { # Specs database — ALL values are DENSE (non-sparse) TFLOPS GPU_SPECS = { + "bw1000": { + # Public Hygon material identifies the supported precisions but does not + # publish stable dense peak figures for BW1000_H. Keep unknown peaks at + # zero so reports show measured throughput without inventing efficiency. + "full_name": "Hygon BW1000_H DCU", + "vendor": "hygon", + "architecture": "C-3000 GPGPU", + "compute_capability": 0.0, + "hbm_capacity_gb": 64, + "hbm_type": "HBM", + "memory_bandwidth_gbps": 0, + "fp32_tflops": 0, + "tf32_tflops": 0, + "fp16_tflops": 0, + "bf16_tflops": 0, + "fp8_tflops": 0, + "compute_pass_thresholds_tflops": {}, + "tdp_watts": 800, + "nvlink_gen": 0, + "nvlink_bandwidth_gbps": 0, + "interconnect": "HSW/XHCL", + "pcie_gen": 5, + "min_driver_version": "", + "min_cuda_version": "", + }, "h100": { # Peaks below are NVIDIA marketing dense peaks (theoretical Tensor Core max). # `compute_pass_thresholds_tflops` carries the absolute PASS thresholds used @@ -240,30 +266,41 @@ _UNKNOWN_SPECS = { def detect_gpu_type() -> str: - """Detect GPU type via nvidia-smi and return the internal key (e.g. 'h200'). + """Detect accelerator type via nvidia-smi or hy-smi. Returns 'unknown' if nvidia-smi is unavailable or the GPU is not recognized. """ nvidia_smi = shutil.which("nvidia-smi") - if not nvidia_smi: - return "unknown" + if nvidia_smi: + try: + r = subprocess.run( + [nvidia_smi, "--query-gpu=name", "--format=csv,noheader"], + capture_output=True, text=True, timeout=10, + ) + if r.returncode == 0 and r.stdout.strip(): + first_line = r.stdout.strip().splitlines()[0].strip().upper() + # Iterate longest-pattern-first so "H200" doesn't match "H20". + for pattern, key in sorted(GPU_NAME_PATTERNS.items(), key=lambda kv: -len(kv[0])): + if pattern in first_line: + return key + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + # Hygon DTK hosts expose hy-smi on PATH or under /opt/hyhal. try: - r = subprocess.run( - [nvidia_smi, "--query-gpu=name", "--format=csv,noheader"], - capture_output=True, text=True, timeout=10, - ) - if r.returncode != 0: - return "unknown" - - first_line = r.stdout.strip().splitlines()[0].strip().upper() - # Iterate longest-pattern-first so "H200" doesn't get matched by "H20". - for pattern, key in sorted(GPU_NAME_PATTERNS.items(), key=lambda kv: -len(kv[0])): - if pattern in first_line: - return key - return "unknown" - except (subprocess.TimeoutExpired, FileNotFoundError, OSError): - return "unknown" + from modules.hygon_smi import find_hy_smi + hy_smi = find_hy_smi() + if hy_smi: + r = subprocess.run( + [hy_smi, "--showproductname"], capture_output=True, + text=True, timeout=10, + ) + product = (r.stdout or "").upper() + if r.returncode == 0 and "BW1000" in product: + return "bw1000" + except (ImportError, subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + return "unknown" def get_gpu_specs(gpu_type: str = None) -> dict: @@ -286,8 +323,7 @@ def get_gpu_label(gpu_type: str) -> str: specs = GPU_SPECS.get(gpu_type) if specs: full = specs["full_name"] - # Strip the "NVIDIA " prefix for display - return full.replace("NVIDIA ", "") + return full.replace("NVIDIA ", "").replace("Hygon ", "") return "Unknown GPU" diff --git a/modules/health_check.py b/modules/health_check.py index dd64071..ea9627e 100644 --- a/modules/health_check.py +++ b/modules/health_check.py @@ -1,4 +1,4 @@ -"""Hardware health monitoring module for NVIDIA datacenter GPUs.""" +"""Hardware health monitoring for NVIDIA GPUs and Hygon DCUs.""" import subprocess import shutil @@ -12,6 +12,7 @@ from rich.panel import Panel from rich.text import Text from modules.gpu_specs import detect_gpu_type, get_gpu_specs +from modules.hygon_smi import collect_hygon_info, find_hy_smi class HealthCheck: @@ -20,7 +21,8 @@ class HealthCheck: self.config = config self.console = Console() self.health_cfg = config.get("health", {}) - self.gpu_type = detect_gpu_type() + configured = config.get("gpu_type", "auto") + self.gpu_type = configured if configured != "auto" else detect_gpu_type() self.specs = get_gpu_specs(self.gpu_type) def _run_smi(self, query: str) -> Optional[str]: @@ -55,6 +57,9 @@ class HealthCheck: return default def run(self) -> dict: + if self.gpu_type == "bw1000" or (not shutil.which("nvidia-smi") and find_hy_smi()): + return self._run_hygon() + if not shutil.which("nvidia-smi"): return {"error": "nvidia-smi not found", "passed": False} @@ -180,6 +185,102 @@ class HealthCheck: "detected_gpu_type": self.gpu_type, } + def _run_hygon(self) -> dict: + info = collect_hygon_info(include_topology=False) + if info.get("error"): + return {"error": info["error"], "passed": False} + + temp_warn = self.health_cfg.get("temp_warning", 80) + temp_crit = self.health_cfg.get("temp_critical", 90) + configured_power = self.health_cfg.get("power_limit") + gpu_health = [] + overall_pass = True + + for gpu in info.get("gpus", []): + checks = {} + temp = gpu.get("temperature", 0) + if temp >= temp_crit: + temp_status = "FAIL" + elif temp >= temp_warn: + temp_status = "WARN" + else: + temp_status = "PASS" + checks["temperature"] = { + "value": temp, "status": temp_status, + "threshold": temp_crit if temp_status == "FAIL" else temp_warn, + } + + draw = gpu.get("power_draw", 0.0) + limit = configured_power or gpu.get("power_limit") or self.specs.get("tdp_watts", 800) + checks["power"] = { + "value": draw, "limit": limit, + "status": "PASS" if draw <= limit * 1.05 else "WARN", + } + + ecc_enabled = bool(gpu.get("ecc_enabled")) + checks["ecc_errors"] = { + "single": gpu.get("ecc_errors_single", 0), + "double": gpu.get("ecc_errors_double", 0), + "enabled": ecc_enabled, + "status": "PASS" if ecc_enabled else "FAIL", + } + xgmi = int(gpu.get("xgmi_errors", 0)) + replay = int(gpu.get("pcie_replay_count", 0)) + healthy = bool(gpu.get("healthy")) + checks["memory_errors"] = { + "xgmi_errors": xgmi, + "pcie_replay_count": replay, + "device_healthy": healthy, + "status": "PASS" if healthy and xgmi == 0 and replay == 0 else "FAIL", + } + + generation = int(gpu.get("pcie_link_gen", 0)) + width = int(gpu.get("pcie_link_width", 0)) + expected = self.specs.get("pcie_gen", 0) + pcie_ok = generation >= expected and width >= 16 + checks["pcie_link"] = { + "gen": generation, "width": width, + "status": "PASS" if pcie_ok else "WARN", + } + + sm = int(gpu.get("clock_sm", 0)) + mem = int(gpu.get("clock_mem", 0)) + checks["clock_speed"] = { + "sm": sm, "mem": mem, + "status": "PASS" if sm > 0 and mem > 0 else "WARN", + } + throttling = bool(gpu.get("throttling_active")) + checks["throttling"] = { + "status": "FAIL" if throttling else "PASS", + "reasons": [gpu.get("throttling_text")] if throttling else [], + } + # Hygon has no NVIDIA-style persistence mode; a loaded /dev/kfd + # driver and successful hy-smi health query are the equivalent check. + checks["persistence_mode"] = { + "enabled": healthy, "status": "PASS" if healthy else "FAIL" + } + + worst = "PASS" + for check in checks.values(): + if check["status"] == "FAIL": + worst = "FAIL" + break + if check["status"] == "WARN": + worst = "WARN" + if worst == "FAIL": + overall_pass = False + gpu_health.append({"index": gpu["index"], "status": worst, "checks": checks}) + + return { + "passed": overall_pass, + "gpu_health": gpu_health, + "system_health": self._check_system(), + "timestamp": datetime.now().isoformat(), + "detected_gpu_type": "bw1000", + "driver_version": info.get("driver_version", "N/A"), + "vendor": "hygon", + } + def _check_system(self) -> dict: persistd = shutil.which("nvidia-persistenced") is not None persistd_running = False @@ -227,7 +328,10 @@ class HealthCheck: if os.path.isdir("/sys/class/infiniband_verbs"): rdma_devs = os.listdir("/sys/class/infiniband_verbs") - nccl_env = {k: v for k, v in os.environ.items() if k.startswith("NCCL_")} + nccl_env = { + k: v for k, v in os.environ.items() + if k.startswith(("NCCL_", "RCCL_", "HCCL_")) + } return { "nvidia_persistenced": {"installed": persistd, "running": persistd_running}, diff --git a/modules/hygon_smi.py b/modules/hygon_smi.py new file mode 100644 index 0000000..f081c49 --- /dev/null +++ b/modules/hygon_smi.py @@ -0,0 +1,190 @@ +"""Helpers for querying Hygon DCU devices through ``hy-smi``.""" + +import os +import re +import shutil +import subprocess +from typing import Dict, List, Optional + + +def find_hy_smi() -> Optional[str]: + """Return the hy-smi executable path when available.""" + path = shutil.which("hy-smi") + if path: + return path + path = "/opt/hyhal/bin/hy-smi" + return path if os.path.isfile(path) and os.access(path, os.X_OK) else None + + +def run_hy_smi(*args: str, timeout: int = 30) -> Optional[str]: + """Run hy-smi and return stdout, or ``None`` on failure.""" + binary = find_hy_smi() + if not binary: + return None + try: + result = subprocess.run( + [binary, *args], capture_output=True, text=True, timeout=timeout + ) + except (FileNotFoundError, OSError, subprocess.TimeoutExpired): + return None + return result.stdout.strip() if result.returncode == 0 else None + + +def _indexed_values(text: Optional[str], pattern: str) -> Dict[int, str]: + if not text: + return {} + values: Dict[int, str] = {} + for match in re.finditer(pattern, text, re.MULTILINE | re.IGNORECASE): + values[int(match.group(1))] = match.group(2).strip() + return values + + +def _number(value, default=0.0): + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _pcie_gen(gt_per_second: float) -> int: + if gt_per_second >= 32: + return 5 + if gt_per_second >= 16: + return 4 + if gt_per_second >= 8: + return 3 + if gt_per_second >= 5: + return 2 + return 1 if gt_per_second > 0 else 0 + + +def collect_hygon_info(include_topology: bool = True) -> dict: + """Collect normalized per-device information for GPUInfo and HealthCheck.""" + summary = run_hy_smi() + if not summary: + return {"error": "hy-smi query failed", "gpu_count": 0, "gpus": []} + + rows: Dict[int, dict] = {} + row_re = re.compile( + r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W\s+(\S+)\s+" + r"([\d.]+)W\s+(\d+)%\s+([\d.]+)%", + re.MULTILINE, + ) + for match in row_re.finditer(summary): + index = int(match.group(1)) + rows[index] = { + "index": index, + "temperature": int(round(float(match.group(2)))), + "power_draw": float(match.group(3)), + "performance_level": match.group(4), + "power_limit": float(match.group(5)), + "vram_used_pct": int(match.group(6)), + "utilization_pct": float(match.group(7)), + } + + product = run_hy_smi("--showproductname") + driver = run_hy_smi("--showdriverversion") + memory = run_hy_smi("--showmemavailable") + bus = run_hy_smi("--showbus") + unique = run_hy_smi("--showuniqueid") + serial = run_hy_smi("--showserial") + clocks = run_hy_smi("--showclocks") + ecc = run_hy_smi("--showmemeccinfo") + xgmi = run_hy_smi("--showxgmierr") + replay = run_hy_smi("--showreplaycount") + health = run_hy_smi("--healthcheck") + throttle = run_hy_smi("--showfdr") + + names = _indexed_values(product, r"HCU\[(\d+)\].*Card Series:\s*([^\s]+)") + memories = _indexed_values(memory, r"HCU\[(\d+)\].*Available memory size \(MiB\):\s*(\d+)") + buses = _indexed_values(bus, r"HCU\[(\d+)\].*PCI Bus:\s*([0-9a-fA-F:.]+)") + unique_ids = _indexed_values(unique, r"HCU\[(\d+)\].*Unique ID:\s*(\S+)") + serials = _indexed_values(serial, r"HCU\[(\d+)\].*Serial Number:\s*(\S+)") + sclk = _indexed_values(clocks, r"HCU\[(\d+)\].*sclk clock level:.*\((\d+)Mhz\)") + mclk = _indexed_values(clocks, r"HCU\[(\d+)\].*mclk clock level:.*\((\d+)Mhz\)") + pcie = {} + if clocks: + for match in re.finditer( + r"HCU\[(\d+)\].*pcie clock level:.*\(([\d.]+)GT/s,\s*x(\d+)", clocks + ): + pcie[int(match.group(1))] = (float(match.group(2)), int(match.group(3))) + ecc_enabled = { + idx: value.upper() == "ON" + for idx, value in _indexed_values( + ecc, r"HCU\[(\d+)\].*Vram memory ECC status:\s*(\S+)" + ).items() + } + xgmi_errors = { + idx: int(value) + for idx, value in _indexed_values( + xgmi, r"HCU\[(\d+)\].*XGMI Error count\s+(\d+)" + ).items() + } + replay_counts = { + idx: int(value) + for idx, value in _indexed_values( + replay, r"HCU\[(\d+)\].*PCIe Replay Count:\s*(\d+)" + ).items() + } + healthy = { + idx: value.lower() == "healthy" + for idx, value in _indexed_values( + health, r"HCU\[(\d+)\].*:\s*(Healthy|Unhealthy)\s*$" + ).items() + } + throttle_status = _indexed_values( + throttle, r"HCU\[(\d+)\].*Throttler Status:\s*(.+?)\s*$" + ) + + driver_match = re.search(r"Driver Version:\s*(\S+)", driver or "") + driver_version = driver_match.group(1) if driver_match else "N/A" + + gpu_count = max(rows.keys(), default=-1) + 1 + gpus: List[dict] = [] + for index in range(gpu_count): + row = rows.get(index, {"index": index}) + total_mb = int(memories.get(index, 0)) + used_mb = int(total_mb * int(row.get("vram_used_pct", 0)) / 100) + gt_s, width = pcie.get(index, (0.0, 0)) + throttle_text = throttle_status.get(index, "") + real_throttle = bool( + throttle_text + and any(token not in {"IDLE", "ACTIVE", "NONE"} + for token in re.findall(r"[A-Z_]+", throttle_text.upper())) + ) + gpus.append({ + **row, + "name": names.get(index, "Hygon DCU"), + "uuid": unique_ids.get(index, "N/A"), + "pci_bus_id": buses.get(index, "N/A"), + "pcie_link_gen": _pcie_gen(gt_s), + "pcie_link_width": width, + "pcie_gt_s": gt_s, + "vram_total_mb": total_mb, + "vram_used_mb": used_mb, + "vram_free_mb": max(total_mb - used_mb, 0), + "clock_sm": int(_number(sclk.get(index))), + "clock_mem": int(_number(mclk.get(index))), + "fan_speed": 0, + "persistence_mode": True, + "compute_mode": row.get("performance_level", "auto"), + "serial_number": serials.get(index, "N/A"), + "ecc_enabled": ecc_enabled.get(index, False), + "ecc_errors_single": 0, + "ecc_errors_double": 0, + "xgmi_errors": xgmi_errors.get(index, 0), + "pcie_replay_count": replay_counts.get(index, 0), + "healthy": healthy.get(index, False), + "throttling_active": real_throttle, + "throttling_text": throttle_text, + }) + + return { + "driver_version": driver_version, + "runtime_name": "DTK", + "runtime_version": os.environ.get("DTK_VERSION", "N/A"), + "gpu_count": gpu_count, + "gpus": gpus, + "topology": run_hy_smi("--showtopo") if include_topology else "", + "vendor": "hygon", + } diff --git a/modules/multinode_nccl_test.py b/modules/multinode_nccl_test.py index f97c96e..e00b30e 100644 --- a/modules/multinode_nccl_test.py +++ b/modules/multinode_nccl_test.py @@ -286,10 +286,18 @@ class MultiNodeNCCLTest: "--mca", "btl_openib_warn_no_device_params_found", "0", "--mca", "btl_tcp_if_include", str(self.cfg.get("socket_ifname", "bond0")), "--mca", "oob_tcp_if_include", str(self.cfg.get("oob_tcp_ifname", self.cfg.get("socket_ifname", "bond0"))), + ] + # HCOLL does an eager UCX IB wireup during MPI_Init that deadlocks on nodes + # where UCX only sees mlx5_2/mlx5_3 instead of mlx5_0/mlx5_1. HCOLL + # collective optimisations are irrelevant for NCCL tests (NCCL handles its + # own collectives), so disable it unconditionally unless the caller opts in. + if self.cfg.get("disable_hcoll", True): + cmd.extend(["--mca", "coll", "^hcoll"]) + cmd.extend([ "-H", host_arg, "--map-by", f"ppr:{gpus_per_node}:node", "-np", str(ranks), - ] + ]) plm_rsh_args = self.cfg.get("plm_rsh_args") if plm_rsh_args: cmd.extend(["--mca", "plm_rsh_args", str(plm_rsh_args)]) diff --git a/modules/nccl_test.py b/modules/nccl_test.py index 5c04af5..f67998d 100644 --- a/modules/nccl_test.py +++ b/modules/nccl_test.py @@ -19,6 +19,10 @@ from modules.gpu_specs import detect_gpu_type, get_gpu_specs, resolve_tools_dir TORCH_AVAILABLE = False try: import torch + try: + import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim + except ImportError: + pass if torch.cuda.is_available(): TORCH_AVAILABLE = True except ImportError: @@ -46,7 +50,8 @@ class NCCLTest: self.console = Console() self.nccl_cfg = config.get("nccl", {}) self.tools_dir = resolve_tools_dir(config) - self.gpu_type = detect_gpu_type() + configured = config.get("gpu_type", "auto") + self.gpu_type = configured if configured != "auto" else detect_gpu_type() self.specs = get_gpu_specs(self.gpu_type) def _find_nccl_test(self, name: str) -> Optional[str]: @@ -295,60 +300,78 @@ class NCCLTest: } def _run_torchrun_fallback(self, gpu_count: int) -> dict: - """Basic NCCL connectivity test via torchrun — verifies NCCL works but does not benchmark performance.""" - self.console.print("[yellow]nccl-tests not available, running basic NCCL connectivity check[/yellow]") + """Benchmark collectives through PyTorch when nccl-tests is unavailable.""" + self.console.print("[yellow]nccl-tests not available, benchmarking collectives via torchrun[/yellow]") + size_mb = int(self.nccl_cfg.get("torchrun_size_mb", 64)) + warmup = int(self.nccl_cfg.get("torchrun_warmup", 5)) + iterations = int(self.nccl_cfg.get("torchrun_iterations", 20)) + backend = self.nccl_cfg.get("torch_backend", "nccl") + enabled_ops = [] + for key, enabled in [ + ("allreduce", self.nccl_cfg.get("test_allreduce", True)), + ("alltoall", self.nccl_cfg.get("test_alltoall", True)), + ("broadcast", self.nccl_cfg.get("test_broadcast", True)), + ("reducescatter", self.nccl_cfg.get("test_reduce_scatter", False)), + ("allgather", self.nccl_cfg.get("test_allgather", False)), + ]: + if enabled: + enabled_ops.append(key) code = f""" -import torch, torch.distributed as dist, os +import os, time +import torch +try: + import torch_dcu +except ImportError: + pass +import torch.distributed as dist os.environ.setdefault("MASTER_ADDR","127.0.0.1") os.environ.setdefault("MASTER_PORT","29500") rank=int(os.environ.get("LOCAL_RANK",0)) ws={gpu_count} -dist.init_process_group("nccl",rank=rank,world_size=ws) +dist.init_process_group({backend!r},rank=rank,world_size=ws) torch.cuda.set_device(rank) +size_bytes={size_mb}*1024*1024 +numel=size_bytes//4 +x=torch.ones(numel,device=f"cuda:{{rank}}",dtype=torch.float32) +allgather_out=torch.empty(numel*ws,device=f"cuda:{{rank}}",dtype=torch.float32) +rs_out=torch.empty(numel//ws,device=f"cuda:{{rank}}",dtype=torch.float32) +a2a_out=torch.empty_like(x) -x=torch.randn(1024*1024,device=f"cuda:{{rank}}",dtype=torch.float32) - -# Test AllReduce -try: - dist.all_reduce(x.clone()) - if rank==0: print("allreduce:ok") -except Exception as e: - if rank==0: print(f"allreduce:fail:{{e}}") - -# Test Broadcast -try: - dist.broadcast(x.clone(),src=0) - if rank==0: print("broadcast:ok") -except Exception as e: - if rank==0: print(f"broadcast:fail:{{e}}") - -# Test AllGather -try: - tensor_list=[torch.empty_like(x) for _ in range(ws)] - dist.all_gather(tensor_list,x.clone()) - if rank==0: print("allgather:ok") -except Exception as e: - if rank==0: print(f"allgather:fail:{{e}}") - -# Test ReduceScatter -try: - chunks=list(x.chunk(ws)) - output=torch.empty_like(chunks[0]) - dist.reduce_scatter(output,chunks) - if rank==0: print("reducescatter:ok") -except Exception as e: - if rank==0: print(f"reducescatter:fail:{{e}}") - -# Test AllToAll -try: - chunks=list(x.chunk(ws)) - output_list=[torch.empty_like(c) for c in chunks] - dist.all_to_all(output_list,chunks) - if rank==0: print("alltoall:ok") -except Exception as e: - if rank==0: print(f"alltoall:fail:{{e}}") +def run_op(name): + if name == "allreduce": + return dist.all_reduce(x) + if name == "broadcast": + return dist.broadcast(x, src=0) + if name == "allgather": + return dist.all_gather_into_tensor(allgather_out, x) + if name == "reducescatter": + return dist.reduce_scatter_tensor(rs_out, x) + if name == "alltoall": + return dist.all_to_all_single(a2a_out, x) + raise ValueError(name) +for name in {enabled_ops!r}: + try: + for _ in range({warmup}): + run_op(name) + torch.cuda.synchronize() + dist.barrier() + start=time.perf_counter() + for _ in range({iterations}): + run_op(name) + torch.cuda.synchronize() + elapsed=(time.perf_counter()-start)/{iterations} + algbw=size_bytes/elapsed/1e9 + factor={{"allreduce":2*(ws-1)/ws,"allgather":(ws-1)/ws, + "reducescatter":(ws-1)/ws,"alltoall":(ws-1)/ws, + "broadcast":1.0}}[name] + busbw=algbw*factor + if rank==0: + print(f"METRIC|{{name}}|{{elapsed*1e6:.3f}}|{{algbw:.3f}}|{{busbw:.3f}}", flush=True) + except Exception as exc: + if rank==0: + print(f"ERROR|{{name}}|{{exc}}", flush=True) dist.destroy_process_group() """ import tempfile @@ -364,41 +387,50 @@ dist.destroy_process_group() r = subprocess.run( [torchrun_cmd, f"--nproc_per_node={gpu_count}", tmp.name], - capture_output=True, text=True, timeout=120, - env={**os.environ, "NCCL_DEBUG": "WARN"}, + capture_output=True, text=True, timeout=300, + env={**os.environ, "NCCL_DEBUG": "WARN", "RCCL_DEBUG": "WARN"}, ) os.unlink(tmp.name) - - # Parse connectivity results — format: op_name:ok or op_name:fail:error + tests = {} - all_passed = True for line in r.stdout.split("\n"): line = line.strip() - if not line: - continue - parts = line.split(":") - op_name = parts[0] - result = parts[1] if len(parts) > 1 else "unknown" - - if result == "ok": - status = "PASS" - else: - status = "FAIL" - all_passed = False - - tests[op_name] = { - "status": status, - "error": ":".join(parts[2:]) if len(parts) > 2 and result == "fail" else None, - } - + if line.startswith("METRIC|"): + _, name, time_us, algbw, busbw = line.split("|", 4) + threshold = self.nccl_cfg.get("min_bandwidth_gbps") or 10.0 + busbw_f = float(busbw) + tests[name] = { + "status": "PASS" if busbw_f >= float(threshold) else "WARN", + "best_algbw_gbps": round(float(algbw), 2), + "best_busbw_gbps": round(busbw_f, 2), + "min_required_gbps": float(threshold), + "by_size": [{ + "size": size_mb * 1024 * 1024, + "time_us": round(float(time_us), 2), + "algbw_gbps": round(float(algbw), 2), + "busbw_gbps": round(busbw_f, 2), + }], + } + elif line.startswith("ERROR|"): + _, name, error = line.split("|", 2) + tests[name] = {"status": "FAIL", "error": error} + if not tests: + error = (r.stderr or r.stdout or "torchrun produced no metrics")[-1000:] + return {"passed": False, "source": "torchrun_collective", "error": error} + all_passed = all(value.get("status") == "PASS" for value in tests.values()) return { "passed": all_passed, - "source": "torchrun_fallback", + "source": "torchrun_collective", "tests": tests, "gpu_count": gpu_count, + "message_size_mb": size_mb, + "warmup": warmup, + "iterations": iterations, + "backend": backend, + "timestamp": datetime.now().isoformat(), } except Exception as e: - return {"passed": False, "source": "torchrun_fallback", "error": str(e)} + return {"passed": False, "source": "torchrun_collective", "error": str(e)} @staticmethod def print_results(results: dict, console: Console = None): diff --git a/modules/rdma_test.py b/modules/rdma_test.py index 08ea610..c94a106 100644 --- a/modules/rdma_test.py +++ b/modules/rdma_test.py @@ -76,6 +76,21 @@ class RDMATest: active_ib_port = True device_info = self._collect_device_info(devices) + cross_enabled = (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled") + if not ib_devices and cross_enabled: + # RoCE ports report link_layer=Ethernet. They are valid for configured + # cross-node perftest even though local InfiniBand loopback is skipped. + cross = self.run_cross_node() + return { + "passed": cross.get("status") == "PASS", + "status": cross.get("status", "FAIL"), + "reason": "RoCE cross-node mode", + "devices": device_info, + "bandwidth_tests": [], + "latency_tests": [], + "cross_node": cross, + "timestamp": datetime.now().isoformat(), + } if not ib_devices: self.console.print( "[yellow]No InfiniBand-link_layer ports present — " @@ -266,13 +281,14 @@ class RDMATest: # ------------------------------------------------------------------ def _active_ib_devices(self) -> List[str]: - """IB devices whose port 1 is InfiniBand link_layer and ACTIVE.""" + """Active IB devices, optionally including Ethernet/RoCE devices.""" out = [] + allow_roce = bool((self.rdma_cfg.get("cross_node", {}) or {}).get("allow_roce")) for dev in self._get_ib_devices(): for port in self._get_ib_ports(dev): ll = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/link_layer") st = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/state") - if ll == "InfiniBand" and "ACTIVE" in st.upper(): + if (ll == "InfiniBand" or allow_roce) and "ACTIVE" in st.upper(): out.append(dev) break return out @@ -296,6 +312,8 @@ class RDMATest: gid_index = cn.get("gid_index") msg_size = cn.get("msg_size", 1048576) iters = cn.get("iters", 5000) + duration_sec = cn.get("duration_sec") + queue_pairs = int(cn.get("queue_pairs", 1)) base_port = cn.get("base_oob_port", 18515) warmup = cn.get("server_warmup_sec", 2.0) min_bw = cn.get("min_bandwidth_gbps", 350) @@ -322,10 +340,16 @@ class RDMATest: entry = {"device": dev} if has_bw: + bw_extra = ["--report_gbits", "-s", str(msg_size), + "-q", str(queue_pairs)] + if duration_sec: + bw_extra += ["-D", str(duration_sec)] + else: + bw_extra += ["-n", str(iters)] bw = self._cross_node_perftest( "ib_write_bw", dev, server_target, server_addr, ib_port, oob, gid_index, warmup, - extra=["--report_gbits", "-s", str(msg_size), "-n", str(iters)], + extra=bw_extra, parse="bw") entry["bandwidth_gbps"] = bw if isinstance(bw, (int, float)): diff --git a/modules/report.py b/modules/report.py index d9e1eba..0d8f04f 100644 --- a/modules/report.py +++ b/modules/report.py @@ -100,6 +100,10 @@ class ReportGenerator: if "gpu_info" in results: gpus = results["gpu_info"].get("gpus", []) + runtime_name = results["gpu_info"].get("runtime_name", "CUDA") + runtime_version = results["gpu_info"].get( + "runtime_version", results["gpu_info"].get("cuda_version", "N/A") + ) rows = "" for g in gpus: rows += f"GPU {g['index']}{g['name']}" @@ -109,7 +113,7 @@ class ReportGenerator: sections.append( f'

GPU Information

' f'

Driver: {results["gpu_info"].get("driver_version", "N/A")} | ' - f'CUDA: {results["gpu_info"].get("cuda_version", "N/A")} | ' + f'{runtime_name}: {runtime_version} | ' f'Count: {len(gpus)}

' f'' f'{rows}
GPUModelVRAMTempSM Clock
' @@ -194,8 +198,10 @@ class ReportGenerator: gpus = gpu_info.get("gpus", []) gpu_name = gpus[0]["name"] if gpus else "Unknown" lines.append(f"- **GPU:** {gpu_name} x{gpu_info.get('gpu_count', len(gpus))}") + runtime_name = gpu_info.get("runtime_name", "CUDA") + runtime_version = gpu_info.get("runtime_version", gpu_info.get("cuda_version", "N/A")) lines.append(f"- **Driver:** {gpu_info.get('driver_version', 'N/A')} | " - f"**CUDA:** {gpu_info.get('cuda_version', 'N/A')}") + f"**{runtime_name}:** {runtime_version}") lines.append("") # --- Summary table --- @@ -270,9 +276,16 @@ class ReportGenerator: if not d2d_peak: d2d_peak = mem_data.get("peak_bandwidth_gbps") or 0 d2d_eff = mem_data.get("efficiency_pct") or 0 - lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {h2d_peak:.0f} GB/s | {h2d_eff:.1f}% |") - lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {d2h_peak:.0f} GB/s | {d2h_eff:.1f}% |") - lines.append(f"| D2D (NVLink) | {d2d:.1f} GB/s | {d2d_peak:.0f} GB/s | {d2d_eff:.1f}% |") + def peak_text(value): + return f"{value:.0f} GB/s" if value else "N/A" + + def efficiency_text(value): + return f"{value:.1f}%" if value else "N/A" + + lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {peak_text(h2d_peak)} | {efficiency_text(h2d_eff)} |") + lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {peak_text(d2h_peak)} | {efficiency_text(d2h_eff)} |") + d2d_label = "D2D (HBM copy)" if mem_data.get("accelerator_vendor") == "hygon" else "D2D (NVLink)" + lines.append(f"| {d2d_label} | {d2d:.1f} GB/s | {peak_text(d2d_peak)} | {efficiency_text(d2d_eff)} |") lines.append("") # PyTorch fallback can't accurately measure HBM peak (intra-GPU copy_() # only reaches ~20% of HBM bandwidth). When fallback is used, report @@ -298,6 +311,10 @@ class ReportGenerator: # When present, override the legacy 80%-of-peak rule on a per-dtype basis. pass_thresholds = comp_data.get("pass_thresholds_tflops", {}) or {} use_abs = bool(pass_thresholds) + has_peak_reference = any( + isinstance(value, (int, float)) and value > 0 + for value in peak_tflops.values() + ) if use_abs: lines.append("| DType | Achieved (TFLOPS) | Peak | Threshold | Status |") else: @@ -305,7 +322,7 @@ class ReportGenerator: lines.append("|-------|-------------------|------|------------|--------|") worst_eff = 100.0 overall_status = "PASS" - rank = {"PASS": 0, "WARN": 1, "FAIL": 2, "SKIP": 0} + rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0} for dt, val in per_dtype.items(): if isinstance(val, str): # skipped or error @@ -324,17 +341,22 @@ class ReportGenerator: else: status = "FAIL" lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | >= {thr} | {status} |") - else: + elif has_peak_reference: status = "PASS" if ef >= 80 else ("WARN" if ef >= 50 else "FAIL") lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | {ef:.1f}% | {status} |") + else: + status = "MEASURED" + lines.append(f"| {dt.upper()} | {val:.1f} | N/A | N/A | {status} |") if rank.get(status, 0) > rank.get(overall_status, 0): overall_status = status lines.append("") if use_abs: lines.append(f"**Verdict: {overall_status}** (absolute TFLOPS thresholds; worst efficiency {worst_eff:.1f}%)\n") - else: + elif has_peak_reference: overall_status = "PASS" if worst_eff >= 80 else ("WARN" if worst_eff >= 50 else "FAIL") lines.append(f"**Verdict: {overall_status}** (worst efficiency {worst_eff:.1f}%)\n") + else: + lines.append("**Verdict: MEASURED** (no published peak/acceptance threshold configured)\n") # --- NCCL --- nccl = results.get("nccl") @@ -369,6 +391,17 @@ class ReportGenerator: lines.append(f"- **Source:** {source}") lines.append(f"- **Duration:** {elapsed:.0f}s (requested {duration}s)") lines.append(f"- **Result: {'PASS' if passed else 'FAIL'}**") + max_temp = stress.get("max_temperature_c", {}) or {} + max_power = stress.get("max_power_w", {}) or {} + if max_temp or max_power: + lines.append("") + lines.append("| GPU | Peak Temp | Peak Power |") + lines.append("|-----|-----------|------------|") + for gpu in sorted(set(max_temp) | set(max_power), key=lambda value: int(value)): + lines.append( + f"| {gpu} | {max_temp.get(gpu, 'N/A')} C | " + f"{max_power.get(gpu, 'N/A')} W |" + ) lines.append("") # --- RDMA --- @@ -519,7 +552,8 @@ class ReportGenerator: verdict = "PASS" if worst >= 80 else ("WARN" if worst >= 50 else "FAIL") items.append(("Compute Throughput", f"{verdict} (worst {worst:.1f}%)")) else: - items.append(("Compute Throughput", "N/A")) + measured = [v for v in per_dtype.values() if isinstance(v, (int, float))] + items.append(("Compute Throughput", "MEASURED (no published threshold)" if measured else "N/A")) # NCCL if "nccl" in results: diff --git a/modules/stress_test.py b/modules/stress_test.py index 8b69d1c..583314b 100644 --- a/modules/stress_test.py +++ b/modules/stress_test.py @@ -2,6 +2,7 @@ import glob import os +import re import shutil import subprocess import time @@ -13,6 +14,7 @@ from rich.live import Live from rich.text import Text from modules.gpu_specs import resolve_tools_dir +from modules.hygon_smi import run_hy_smi class StressTest: @@ -56,12 +58,12 @@ class StressTest: if not result.get("passed") and result.get("elapsed_sec", 0) < duration_sec * 0.5: self.console.print("\n[yellow]gpu-burn exited early (possible OOM), switching to PyTorch stress test[/yellow]") self.console.print("[dim]PyTorch mode dynamically adapts to available memory[/dim]\n") - return self._run_pytorch_stress(duration_sec, memory_pct) + return self._run_pytorch_stress(duration_sec, memory_pct, use_tensor_cores) return result self.console.print("[yellow]gpu_burn not found, using PyTorch stress test[/yellow]") - return self._run_pytorch_stress(duration_sec, memory_pct) + return self._run_pytorch_stress(duration_sec, memory_pct, use_tensor_cores) def _run_gpu_burn(self, gpu_burn: str, duration: int, doubles: bool, tensor_cores: bool, target_gpus: str) -> dict: @@ -116,9 +118,26 @@ class StressTest: "timestamp": datetime.now().isoformat(), } - def _run_pytorch_stress(self, duration: int, memory_pct: int = 90) -> dict: + @staticmethod + def _hygon_snapshot() -> dict: + output = run_hy_smi() + temps, powers = {}, {} + if output: + pattern = re.compile(r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W", re.MULTILINE) + for match in pattern.finditer(output): + idx = int(match.group(1)) + temps[idx] = float(match.group(2)) + powers[idx] = float(match.group(3)) + return {"temperatures_c": temps, "power_w": powers} + + def _run_pytorch_stress(self, duration: int, memory_pct: int = 90, + tensor_cores: bool = True) -> dict: try: import torch + try: + import torch_dcu # noqa: F401 + except ImportError: + pass if not torch.cuda.is_available(): return {"error": "pytorch_not_available"} except ImportError: @@ -129,9 +148,10 @@ class StressTest: gpu_status = {} t0 = time.time() + snapshots = [self._hygon_snapshot()] try: - tensors = {} + work_a, work_b, outputs, ballast = {}, {}, {}, {} for i in range(gpu_count): with torch.cuda.device(i): # Get actual free memory (accounting for other processes) @@ -143,16 +163,23 @@ class StressTest: # Cap at actual free memory with 5% safety margin alloc_bytes = min(target_mem, int(free_mem * 0.95)) - # matmul(A, A.T) needs 2x input memory (input + output) - mem_side = int((alloc_bytes / 4 / 2) ** 0.5) - # Cap compute matrix so a single matmul completes in ~2s on H100/H200 - # (FP32 ≈ 67 TFLOPS → 2*4096³/67e12 ≈ 2s). Without this cap, a 141GB - # HBM yields side ≈ 131K → single matmul ~68s × 8 GPUs serial → loop - # overshoots a 60s duration request by 10×+. - MAX_COMPUTE_SIDE = 4096 - side = min(mem_side, MAX_COMPUTE_SIDE) + dtype = torch.bfloat16 if tensor_cores else torch.float32 + elem_size = torch.tensor([], dtype=dtype).element_size() + mem_side = int((alloc_bytes / elem_size / 3) ** 0.5) + max_compute_side = 8192 if tensor_cores else 4096 + side = min(mem_side, max_compute_side) + compute_bytes = side * side * elem_size * 3 + ballast_target = max(0, alloc_bytes - compute_bytes - 1024**3) + ballast[i] = [] + chunk_size = 1024**3 + while ballast_target > 0: + this_chunk = min(chunk_size, ballast_target) + tensor = torch.empty(this_chunk, device=f"cuda:{i}", dtype=torch.uint8) + tensor.fill_(0x5A) + ballast[i].append(tensor) + ballast_target -= this_chunk - actual_mem_mb = side * side * 4 / 1024 / 1024 + actual_mem_mb = (alloc_bytes - max(0, ballast_target)) / 1024 / 1024 total_mem_mb = total_mem / 1024 / 1024 free_mem_mb = free_mem / 1024 / 1024 @@ -161,7 +188,9 @@ class StressTest: f"alloc {actual_mem_mb:.0f}MB ({actual_mem_mb/total_mem_mb*100:.0f}%) - " f"matrix {side}x{side}[/dim]" ) - tensors[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=torch.float32) + work_a[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=dtype) + work_b[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=dtype) + outputs[i] = torch.empty(side, side, device=f"cuda:{i}", dtype=dtype) self.console.print(f"\n[cyan]Starting stress test for {duration} seconds...[/cyan]") @@ -171,7 +200,7 @@ class StressTest: # GPUs, otherwise the 8 GPUs run serially and overshoot the duration. for i in range(gpu_count): with torch.cuda.device(i): - tensors[i] = torch.matmul(tensors[i], tensors[i].T) + outputs[i] = torch.matmul(work_a[i], work_b[i]) # Single sync per pass — waits for all 8 streams concurrently for i in range(gpu_count): with torch.cuda.device(i): @@ -181,6 +210,7 @@ class StressTest: current_elapsed = time.time() - t0 if int(current_elapsed) != int(elapsed_check) and int(current_elapsed) % 10 == 0: self.console.print(f" [dim]Running {int(current_elapsed)}s / {duration}s[/dim]") + snapshots.append(self._hygon_snapshot()) elapsed_check = current_elapsed for i in range(gpu_count): @@ -200,16 +230,28 @@ class StressTest: "gpu_status": gpu_status, } finally: - tensors.clear() + for mapping in (work_a, work_b, outputs, ballast): + mapping.clear() torch.cuda.empty_cache() elapsed = round(time.time() - t0, 1) + snapshots.append(self._hygon_snapshot()) + max_temp = {} + max_power = {} + for snapshot in snapshots: + for idx, value in snapshot.get("temperatures_c", {}).items(): + max_temp[idx] = max(max_temp.get(idx, value), value) + for idx, value in snapshot.get("power_w", {}).items(): + max_power[idx] = max(max_power.get(idx, value), value) return { "source": "pytorch", "passed": True, "duration_sec": duration, "elapsed_sec": elapsed, "gpu_status": gpu_status, + "memory_pct": memory_pct, + "max_temperature_c": max_temp, + "max_power_w": max_power, "timestamp": datetime.now().isoformat(), } @@ -245,5 +287,15 @@ class StressTest: color = "green" if status == "PASS" else "red" c.print(f" GPU {gid}: [{color}]{status}[/{color}]") + max_temp = results.get("max_temperature_c", {}) + max_power = results.get("max_power_w", {}) + if max_temp or max_power: + c.print("\n Peak telemetry:") + for gid in sorted(set(max_temp) | set(max_power)): + c.print( + f" GPU {gid}: {max_temp.get(gid, 'N/A')} C, " + f"{max_power.get(gid, 'N/A')} W" + ) + if results.get("error"): c.print(f" [red]Error: {results['error']}[/red]") diff --git a/modules/training_sim.py b/modules/training_sim.py index dc7f5a3..bd36fab 100644 --- a/modules/training_sim.py +++ b/modules/training_sim.py @@ -13,6 +13,10 @@ from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeEl TORCH_AVAILABLE = False try: import torch + try: + import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim + except ImportError: + pass if torch.cuda.is_available(): TORCH_AVAILABLE = True except ImportError: @@ -49,6 +53,9 @@ class TrainingSim: self.console.print(f" Model: {model_name} | Batch: {batch_size} | Seq: {seq_length} | " f"DType: {dtype_str} | Steps: {num_steps} | GPUs: {gpu_count}") + if model_name == "synthetic": + return self._run_synthetic(gpu_count, batch_size, seq_length, num_steps, dtype) + try: from transformers import AutoModelForCausalLM, AutoTokenizer except ImportError: @@ -145,10 +152,10 @@ class TrainingSim: def _run_synthetic(self, gpu_count, batch_size, seq_length, num_steps, dtype) -> dict: self.console.print(" Running synthetic training benchmark...") - hidden_size = 4096 - num_layers = 6 - num_heads = 32 - vocab_size = 32000 + hidden_size = int(self.train_cfg.get("hidden_size", 4096)) + num_layers = int(self.train_cfg.get("num_layers", 6)) + num_heads = int(self.train_cfg.get("num_heads", 32)) + vocab_size = int(self.train_cfg.get("vocab_size", 32000)) class SyntheticTransformer(torch.nn.Module): def __init__(self): diff --git a/scripts/bw1000_smoke.py b/scripts/bw1000_smoke.py new file mode 100644 index 0000000..fd1cde7 --- /dev/null +++ b/scripts/bw1000_smoke.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Minimal DTK/PyTorch smoke test used before the full BW1000 suite.""" + +import json + +import torch + +try: + import torch_dcu # noqa: F401 +except ImportError: + pass + + +def main() -> None: + count = torch.cuda.device_count() + if not torch.cuda.is_available() or count < 1: + raise SystemExit("DTK PyTorch reports no available accelerator") + + devices = [] + for index in range(count): + props = torch.cuda.get_device_properties(index) + devices.append({ + "index": index, + "name": torch.cuda.get_device_name(index), + "memory_gib": round(props.total_memory / 1024**3, 2), + }) + + a = torch.randn(2048, 2048, device="cuda:0", dtype=torch.bfloat16) + b = torch.randn(2048, 2048, device="cuda:0", dtype=torch.bfloat16) + c = a @ b + torch.cuda.synchronize() + if not torch.isfinite(c).all().item(): + raise SystemExit("BF16 matmul produced non-finite output") + + print(json.dumps({ + "torch_version": torch.__version__, + "cuda_api_available": torch.cuda.is_available(), + "device_count": count, + "devices": devices, + "bf16_matmul": "PASS", + "checksum": round(float(c.float().mean().item()), 6), + }, indent=2)) + + +if __name__ == "__main__": + main()