feat: add Hygon BW1000 compatibility

This commit is contained in:
zulifeng 2026-07-10 17:47:14 +08:00
parent cb5786c319
commit 219065eea1
15 changed files with 836 additions and 147 deletions

86
configs/bw1000_4node.yaml Normal file
View File

@ -0,0 +1,86 @@
gpu_type: bw1000
runtime:
name: DTK
version: 25.04.2
benchmark:
memory:
size_mb: 4096
iterations: 10
nvbandwidth_buffer_mb: 512
nvbandwidth_samples: 3
compute:
dtypes: [fp32, tf32, fp16, bf16, fp8]
sweep_sizes: [4096, 6144, 8192, [2048, 2048, 13312]]
matrix_size: 8192
warmup: 10
iterations: 30
health:
temp_warning: 75
temp_critical: 85
power_limit: 800
nccl:
min_bandwidth_gbps: 10
test_allreduce: true
test_alltoall: true
test_broadcast: true
test_reduce_scatter: true
test_allgather: true
test_sendrecv: false
torch_backend: nccl
torchrun_size_mb: 64
torchrun_warmup: 5
torchrun_iterations: 20
stress:
duration_sec: 600
use_doubles: false
use_tensor_cores: true
memory_pct: 80
gpus: all
rdma:
min_bandwidth_gbps: 200
max_latency_us: 6
ib_iterations: 5000
msg_size: 1048576
ib_device: null
ib_port: 1
cross_node:
enabled: false
allow_roce: true
server: null
server_addr: null
ssh_user: root
devices: [mlx5_bond_0, mlx5_bond_1, mlx5_bond_2, mlx5_bond_3]
ib_port: 1
gid_index: 3
msg_size: 1048576
iters: 5000
duration_sec: 10
queue_pairs: 4
base_oob_port: 18515
server_warmup_sec: 2.0
min_bandwidth_gbps: 200
max_latency_us: 6
training:
model: synthetic
batch_size: 2
seq_length: 1024
num_steps: 20
dtype: bf16
hidden_size: 2048
num_layers: 4
num_heads: 16
vocab_size: 8192
report:
output_dir: ./reports
format: json
tools:
install_dir: /opt/gpu-test-tools

15
configs/bw1000_smoke.yaml Normal file
View File

@ -0,0 +1,15 @@
gpu_type: bw1000
runtime:
name: DTK
version: 25.04.2
stress:
duration_sec: 60
use_doubles: false
use_tensor_cores: true
memory_pct: 80
gpus: all
tools:
install_dir: /opt/gpu-test-tools

View File

@ -27,6 +27,7 @@ from modules.stress_test import StressTest
from modules.rdma_test import RDMATest
from modules.report import ReportGenerator
from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label, get_supported_gpus, validate_driver_compatibility
from modules.hygon_smi import find_hy_smi
DEFAULT_CONFIG = {
"benchmark": {
@ -117,9 +118,9 @@ def check_prerequisites(console: Console) -> bool:
import shutil
ok = True
if not shutil.which("nvidia-smi"):
console.print("[bold red]ERROR: nvidia-smi not found![/bold red]")
console.print(" Please install NVIDIA drivers first.")
if not shutil.which("nvidia-smi") and not find_hy_smi():
console.print("[bold red]ERROR: neither nvidia-smi nor hy-smi was found![/bold red]")
console.print(" Please install the NVIDIA driver or Hygon DCU driver first.")
ok = False
return ok

View File

@ -17,6 +17,10 @@ from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label, res
TORCH_AVAILABLE = False
try:
import torch
try:
import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim
except ImportError:
pass
if torch.cuda.is_available():
TORCH_AVAILABLE = True
except ImportError:
@ -63,12 +67,14 @@ class Benchmark:
return None
def run_memory_benchmark(self) -> dict:
nvbw = self._find_nvbandwidth()
# nvbandwidth is NVIDIA-specific. DTK PyTorch provides the portable
# transfer/copy path used below for Hygon DCUs.
nvbw = None if self.gpu_type == "bw1000" else self._find_nvbandwidth()
if nvbw:
return self._run_nvbandwidth(nvbw)
self.console.print("[yellow]nvbandwidth not found, falling back to PyTorch[/yellow]")
self.console.print("[yellow]Using PyTorch memory benchmark fallback[/yellow]")
return self._run_memory_pytorch()
def _run_nvbandwidth(self, nvbw_path: str) -> dict:
@ -189,6 +195,7 @@ class Benchmark:
"efficiency_pct": d2d_efficiency,
"results_by_test": results_by_test,
"per_gpu": [],
"accelerator_vendor": self.specs.get("vendor", "nvidia"),
}
}
@ -292,20 +299,31 @@ class Benchmark:
progress.advance(task)
best_d2d = max(v["d2d_gbps"] for v in bandwidth_by_size.values())
best_h2d = max(v["h2d_gbps"] for v in bandwidth_by_size.values())
best_d2h = max(v["d2h_gbps"] for v in bandwidth_by_size.values())
peak_bw = self.specs["memory_bandwidth_gbps"]
efficiency = round((best_d2d / peak_bw) * 100, 1) if peak_bw else None
pcie_gen = self.specs.get("pcie_gen", 0)
pcie_peak = {3: 16, 4: 32, 5: 64, 6: 128}.get(pcie_gen) if pcie_gen else None
return {
"memory": {
"source": "pytorch",
"h2d_bandwidth_gbps": round(max(v["h2d_gbps"] for v in bandwidth_by_size.values()), 1),
"d2h_bandwidth_gbps": round(max(v["d2h_gbps"] for v in bandwidth_by_size.values()), 1),
"h2d_bandwidth_gbps": round(best_h2d, 1),
"d2h_bandwidth_gbps": round(best_d2h, 1),
"d2d_bandwidth_gbps": round(best_d2d, 1),
"h2d_peak_gbps": pcie_peak,
"d2h_peak_gbps": pcie_peak,
"d2d_peak_gbps": None,
"h2d_efficiency_pct": round(best_h2d / pcie_peak * 100, 1) if pcie_peak else None,
"d2h_efficiency_pct": round(best_d2h / pcie_peak * 100, 1) if pcie_peak else None,
"d2d_efficiency_pct": efficiency,
"peak_bandwidth_gbps": self.specs["memory_bandwidth_gbps"],
"efficiency_pct": efficiency,
"test_sizes_mb": test_sizes_mb,
"bandwidth_by_size": bandwidth_by_size,
"per_gpu": [],
"accelerator_vendor": self.specs.get("vendor", "nvidia"),
}
}
@ -355,7 +373,7 @@ class Benchmark:
"tf32": ("tf32", self.specs["tf32_tflops"]),
"fp16": (torch.float16, self.specs["fp16_tflops"]),
"bf16": (torch.bfloat16, self.specs["bf16_tflops"]),
"fp8": (torch.float8_e4m3fn, self.specs["fp8_tflops"]),
"fp8": (getattr(torch, "float8_e4m3fn", None), self.specs["fp8_tflops"]),
}
results_by_dtype = {}
@ -375,7 +393,9 @@ class Benchmark:
continue
# Skip FP8 if GPU architecture doesn't support it
if dtype_name == "fp8" and self.specs.get("fp8_tflops", 0) == 0:
if dtype_name == "fp8" and (
self.specs.get("fp8_tflops", 0) == 0 or dtype_map["fp8"][0] is None
):
arch = self.specs.get("architecture", "unknown")
results_by_dtype["fp8"] = f"skipped ({arch} does not support FP8)"
self.console.print(f"[dim] fp8: skipped - {arch} architecture has no FP8 support[/dim]")
@ -444,6 +464,7 @@ class Benchmark:
"matrix_size": matrix_size,
"warmup": warmup,
"iterations": iterations,
"accelerator_backend": "dtk" if self.gpu_type == "bw1000" else "cuda",
}
}
@ -578,9 +599,14 @@ class Benchmark:
continue
pk = peak.get(dt, 0)
ef = eff.get(dt, 0)
ec = "green" if ef >= 80 else ("yellow" if ef >= 50 else "red")
table.add_row(dt.upper(), f"{achieved:.1f}", f"{pk:.0f}",
f"[{ec}]{ef:.1f}%[/{ec}]")
if pk:
ec = "green" if ef >= 80 else ("yellow" if ef >= 50 else "red")
peak_text = f"{pk:.0f}"
eff_text = f"[{ec}]{ef:.1f}%[/{ec}]"
else:
peak_text = "N/A"
eff_text = "[cyan]MEASURED[/cyan]"
table.add_row(dt.upper(), f"{achieved:.1f}", peak_text, eff_text)
c.print(table)
@staticmethod
@ -590,7 +616,8 @@ class Benchmark:
Single source of truth for the PASS/WARN/FAIL rule (same one report.py uses):
achieved >= thr -> PASS; >= 0.9*thr -> WARN; else FAIL. A string achieved value
(skipped/error) -> SKIP. A dtype without a threshold falls back to efficiency
(>=80 PASS / >=50 WARN / else FAIL).
(>=80 PASS / >=50 WARN / else FAIL). If neither threshold nor peak is
published, the numeric result is retained as MEASURED without a verdict.
Returns {"rows": [(dtype, achieved, threshold, status), ...], "verdict": str}.
"""
@ -598,20 +625,29 @@ class Benchmark:
per_dtype = comp.get("per_dtype_tflops", {})
thresholds = comp.get("pass_thresholds_tflops", {}) or {}
eff = comp.get("efficiency_pct", {})
rank = {"PASS": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
rows, verdict = [], "PASS"
evaluated = False
measured = False
for dt, val in per_dtype.items():
thr = thresholds.get(dt)
if isinstance(val, str):
status = "SKIP"
elif thr:
status = "PASS" if val >= thr else ("WARN" if val >= thr * 0.9 else "FAIL")
evaluated = True
elif dt not in eff:
status = "MEASURED"
measured = True
else:
e = eff.get(dt, 0)
status = "PASS" if e >= 80 else ("WARN" if e >= 50 else "FAIL")
evaluated = True
rows.append((dt, val, thr, status))
if rank[status] > rank[verdict]:
verdict = status
if measured and not evaluated:
verdict = "MEASURED"
return {"rows": rows, "verdict": verdict}
@staticmethod
@ -619,7 +655,7 @@ class Benchmark:
"""Print the PASS/WARN/FAIL table for compute results; return the verdict."""
c = console or Console()
judged = Benchmark.judge_compute(results)
color = {"PASS": "green", "WARN": "yellow", "FAIL": "red", "SKIP": "dim"}
color = {"PASS": "green", "MEASURED": "cyan", "WARN": "yellow", "FAIL": "red", "SKIP": "dim"}
c.print("\n[bold cyan]Compute Verdict (vs thresholds)[/bold cyan]")
for dt, val, thr, status in judged["rows"]:
val_s = f"{val:.1f}" if isinstance(val, (int, float)) else str(val)

View File

@ -1,4 +1,4 @@
"""GPU information detection module for NVIDIA datacenter GPUs (A100/A800/H100/H200/B200/B300)."""
"""Accelerator information detection for NVIDIA GPUs and Hygon DCUs."""
import subprocess
import shutil
@ -11,6 +11,7 @@ from rich.panel import Panel
from rich.text import Text
from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label
from modules.hygon_smi import collect_hygon_info, find_hy_smi
class GPUInfo:
@ -18,7 +19,8 @@ class GPUInfo:
def __init__(self, config: dict):
self.config = config
self.console = Console()
self.gpu_type = detect_gpu_type()
configured = config.get("gpu_type", "auto")
self.gpu_type = configured if configured != "auto" else detect_gpu_type()
self.specs = get_gpu_specs(self.gpu_type)
self.gpu_label = get_gpu_label(self.gpu_type)
@ -35,6 +37,19 @@ class GPUInfo:
return None
def run(self) -> dict:
if self.gpu_type == "bw1000" or (not shutil.which("nvidia-smi") and find_hy_smi()):
result = collect_hygon_info(include_topology=True)
runtime_cfg = self.config.get("runtime", {}) or {}
result.update({
"runtime_name": runtime_cfg.get("name", result.get("runtime_name", "DTK")),
"runtime_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")),
"cuda_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")),
"timestamp": datetime.now().isoformat(),
"detected_gpu_type": "bw1000",
"gpu_label": get_gpu_label("bw1000"),
})
return result
if not shutil.which("nvidia-smi"):
self.console.print("[bold red]nvidia-smi not found![/bold red]")
return {"error": "nvidia-smi not found", "gpu_count": 0}
@ -159,7 +174,9 @@ class GPUInfo:
c.print(f"\n[bold cyan]GPU Information[/bold cyan]")
c.print(f" Driver Version : {results.get('driver_version', 'N/A')}")
c.print(f" CUDA Version : {results.get('cuda_version', 'N/A')}")
runtime_name = results.get("runtime_name", "CUDA")
runtime_version = results.get("runtime_version", results.get("cuda_version", "N/A"))
c.print(f" {runtime_name} Version{' ' * max(0, 4-len(runtime_name))}: {runtime_version}")
c.print(f" GPU Count : {results.get('gpu_count', 0)}")
c.print(f" Detected GPU : {results.get('gpu_label', 'Unknown')} ({results.get('detected_gpu_type', 'unknown')})")
c.print(f" Timestamp : {results.get('timestamp', 'N/A')}")
@ -181,7 +198,7 @@ class GPUInfo:
for g in gpus:
name = g["name"]
if any(k in name for k in ("A100", "A800", "H100", "H200", "B200", "B300")):
if any(k in name for k in ("A100", "A800", "H100", "H200", "B200", "B300", "BW1000")):
name = f"[bold green]{name}[/bold green]"
vram = f"{g['vram_used_mb']}/{g['vram_total_mb']} MB"
temp = f"{g['temperature']}°C"
@ -204,7 +221,8 @@ class GPUInfo:
topology = results.get("topology", "")
if topology and topology != "Unavailable":
c.print(Panel(topology, title="[bold]NVLink/NVSwitch Topology[/bold]", border_style="cyan"))
title = "HSW/XHCL Topology" if results.get("vendor") == "hygon" else "NVLink/NVSwitch Topology"
c.print(Panel(topology, title=f"[bold]{title}[/bold]", border_style="cyan"))
warnings = []
for g in gpus:

View File

@ -1,4 +1,4 @@
"""GPU specifications database for NVIDIA datacenter GPUs."""
"""Accelerator specifications for NVIDIA GPUs and Hygon DCUs."""
import os
import shutil
@ -8,6 +8,7 @@ from typing import List, Optional
# GPU name patterns -> internal key mapping
# Order matters: longer/more-specific patterns must come before shorter ones.
GPU_NAME_PATTERNS = {
"BW1000": "bw1000",
"A100": "a100",
"A800": "a800",
"H100": "h100",
@ -20,6 +21,31 @@ GPU_NAME_PATTERNS = {
# Specs database — ALL values are DENSE (non-sparse) TFLOPS
GPU_SPECS = {
"bw1000": {
# Public Hygon material identifies the supported precisions but does not
# publish stable dense peak figures for BW1000_H. Keep unknown peaks at
# zero so reports show measured throughput without inventing efficiency.
"full_name": "Hygon BW1000_H DCU",
"vendor": "hygon",
"architecture": "C-3000 GPGPU",
"compute_capability": 0.0,
"hbm_capacity_gb": 64,
"hbm_type": "HBM",
"memory_bandwidth_gbps": 0,
"fp32_tflops": 0,
"tf32_tflops": 0,
"fp16_tflops": 0,
"bf16_tflops": 0,
"fp8_tflops": 0,
"compute_pass_thresholds_tflops": {},
"tdp_watts": 800,
"nvlink_gen": 0,
"nvlink_bandwidth_gbps": 0,
"interconnect": "HSW/XHCL",
"pcie_gen": 5,
"min_driver_version": "",
"min_cuda_version": "",
},
"h100": {
# Peaks below are NVIDIA marketing dense peaks (theoretical Tensor Core max).
# `compute_pass_thresholds_tflops` carries the absolute PASS thresholds used
@ -240,30 +266,41 @@ _UNKNOWN_SPECS = {
def detect_gpu_type() -> str:
"""Detect GPU type via nvidia-smi and return the internal key (e.g. 'h200').
"""Detect accelerator type via nvidia-smi or hy-smi.
Returns 'unknown' if nvidia-smi is unavailable or the GPU is not recognized.
"""
nvidia_smi = shutil.which("nvidia-smi")
if not nvidia_smi:
return "unknown"
if nvidia_smi:
try:
r = subprocess.run(
[nvidia_smi, "--query-gpu=name", "--format=csv,noheader"],
capture_output=True, text=True, timeout=10,
)
if r.returncode == 0 and r.stdout.strip():
first_line = r.stdout.strip().splitlines()[0].strip().upper()
# Iterate longest-pattern-first so "H200" doesn't match "H20".
for pattern, key in sorted(GPU_NAME_PATTERNS.items(), key=lambda kv: -len(kv[0])):
if pattern in first_line:
return key
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
pass
# Hygon DTK hosts expose hy-smi on PATH or under /opt/hyhal.
try:
r = subprocess.run(
[nvidia_smi, "--query-gpu=name", "--format=csv,noheader"],
capture_output=True, text=True, timeout=10,
)
if r.returncode != 0:
return "unknown"
first_line = r.stdout.strip().splitlines()[0].strip().upper()
# Iterate longest-pattern-first so "H200" doesn't get matched by "H20".
for pattern, key in sorted(GPU_NAME_PATTERNS.items(), key=lambda kv: -len(kv[0])):
if pattern in first_line:
return key
return "unknown"
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
return "unknown"
from modules.hygon_smi import find_hy_smi
hy_smi = find_hy_smi()
if hy_smi:
r = subprocess.run(
[hy_smi, "--showproductname"], capture_output=True,
text=True, timeout=10,
)
product = (r.stdout or "").upper()
if r.returncode == 0 and "BW1000" in product:
return "bw1000"
except (ImportError, subprocess.TimeoutExpired, FileNotFoundError, OSError):
pass
return "unknown"
def get_gpu_specs(gpu_type: str = None) -> dict:
@ -286,8 +323,7 @@ def get_gpu_label(gpu_type: str) -> str:
specs = GPU_SPECS.get(gpu_type)
if specs:
full = specs["full_name"]
# Strip the "NVIDIA " prefix for display
return full.replace("NVIDIA ", "")
return full.replace("NVIDIA ", "").replace("Hygon ", "")
return "Unknown GPU"

View File

@ -1,4 +1,4 @@
"""Hardware health monitoring module for NVIDIA datacenter GPUs."""
"""Hardware health monitoring for NVIDIA GPUs and Hygon DCUs."""
import subprocess
import shutil
@ -12,6 +12,7 @@ from rich.panel import Panel
from rich.text import Text
from modules.gpu_specs import detect_gpu_type, get_gpu_specs
from modules.hygon_smi import collect_hygon_info, find_hy_smi
class HealthCheck:
@ -20,7 +21,8 @@ class HealthCheck:
self.config = config
self.console = Console()
self.health_cfg = config.get("health", {})
self.gpu_type = detect_gpu_type()
configured = config.get("gpu_type", "auto")
self.gpu_type = configured if configured != "auto" else detect_gpu_type()
self.specs = get_gpu_specs(self.gpu_type)
def _run_smi(self, query: str) -> Optional[str]:
@ -55,6 +57,9 @@ class HealthCheck:
return default
def run(self) -> dict:
if self.gpu_type == "bw1000" or (not shutil.which("nvidia-smi") and find_hy_smi()):
return self._run_hygon()
if not shutil.which("nvidia-smi"):
return {"error": "nvidia-smi not found", "passed": False}
@ -180,6 +185,102 @@ class HealthCheck:
"detected_gpu_type": self.gpu_type,
}
def _run_hygon(self) -> dict:
info = collect_hygon_info(include_topology=False)
if info.get("error"):
return {"error": info["error"], "passed": False}
temp_warn = self.health_cfg.get("temp_warning", 80)
temp_crit = self.health_cfg.get("temp_critical", 90)
configured_power = self.health_cfg.get("power_limit")
gpu_health = []
overall_pass = True
for gpu in info.get("gpus", []):
checks = {}
temp = gpu.get("temperature", 0)
if temp >= temp_crit:
temp_status = "FAIL"
elif temp >= temp_warn:
temp_status = "WARN"
else:
temp_status = "PASS"
checks["temperature"] = {
"value": temp, "status": temp_status,
"threshold": temp_crit if temp_status == "FAIL" else temp_warn,
}
draw = gpu.get("power_draw", 0.0)
limit = configured_power or gpu.get("power_limit") or self.specs.get("tdp_watts", 800)
checks["power"] = {
"value": draw, "limit": limit,
"status": "PASS" if draw <= limit * 1.05 else "WARN",
}
ecc_enabled = bool(gpu.get("ecc_enabled"))
checks["ecc_errors"] = {
"single": gpu.get("ecc_errors_single", 0),
"double": gpu.get("ecc_errors_double", 0),
"enabled": ecc_enabled,
"status": "PASS" if ecc_enabled else "FAIL",
}
xgmi = int(gpu.get("xgmi_errors", 0))
replay = int(gpu.get("pcie_replay_count", 0))
healthy = bool(gpu.get("healthy"))
checks["memory_errors"] = {
"xgmi_errors": xgmi,
"pcie_replay_count": replay,
"device_healthy": healthy,
"status": "PASS" if healthy and xgmi == 0 and replay == 0 else "FAIL",
}
generation = int(gpu.get("pcie_link_gen", 0))
width = int(gpu.get("pcie_link_width", 0))
expected = self.specs.get("pcie_gen", 0)
pcie_ok = generation >= expected and width >= 16
checks["pcie_link"] = {
"gen": generation, "width": width,
"status": "PASS" if pcie_ok else "WARN",
}
sm = int(gpu.get("clock_sm", 0))
mem = int(gpu.get("clock_mem", 0))
checks["clock_speed"] = {
"sm": sm, "mem": mem,
"status": "PASS" if sm > 0 and mem > 0 else "WARN",
}
throttling = bool(gpu.get("throttling_active"))
checks["throttling"] = {
"status": "FAIL" if throttling else "PASS",
"reasons": [gpu.get("throttling_text")] if throttling else [],
}
# Hygon has no NVIDIA-style persistence mode; a loaded /dev/kfd
# driver and successful hy-smi health query are the equivalent check.
checks["persistence_mode"] = {
"enabled": healthy, "status": "PASS" if healthy else "FAIL"
}
worst = "PASS"
for check in checks.values():
if check["status"] == "FAIL":
worst = "FAIL"
break
if check["status"] == "WARN":
worst = "WARN"
if worst == "FAIL":
overall_pass = False
gpu_health.append({"index": gpu["index"], "status": worst, "checks": checks})
return {
"passed": overall_pass,
"gpu_health": gpu_health,
"system_health": self._check_system(),
"timestamp": datetime.now().isoformat(),
"detected_gpu_type": "bw1000",
"driver_version": info.get("driver_version", "N/A"),
"vendor": "hygon",
}
def _check_system(self) -> dict:
persistd = shutil.which("nvidia-persistenced") is not None
persistd_running = False
@ -227,7 +328,10 @@ class HealthCheck:
if os.path.isdir("/sys/class/infiniband_verbs"):
rdma_devs = os.listdir("/sys/class/infiniband_verbs")
nccl_env = {k: v for k, v in os.environ.items() if k.startswith("NCCL_")}
nccl_env = {
k: v for k, v in os.environ.items()
if k.startswith(("NCCL_", "RCCL_", "HCCL_"))
}
return {
"nvidia_persistenced": {"installed": persistd, "running": persistd_running},

190
modules/hygon_smi.py Normal file
View File

@ -0,0 +1,190 @@
"""Helpers for querying Hygon DCU devices through ``hy-smi``."""
import os
import re
import shutil
import subprocess
from typing import Dict, List, Optional
def find_hy_smi() -> Optional[str]:
"""Return the hy-smi executable path when available."""
path = shutil.which("hy-smi")
if path:
return path
path = "/opt/hyhal/bin/hy-smi"
return path if os.path.isfile(path) and os.access(path, os.X_OK) else None
def run_hy_smi(*args: str, timeout: int = 30) -> Optional[str]:
"""Run hy-smi and return stdout, or ``None`` on failure."""
binary = find_hy_smi()
if not binary:
return None
try:
result = subprocess.run(
[binary, *args], capture_output=True, text=True, timeout=timeout
)
except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
return None
return result.stdout.strip() if result.returncode == 0 else None
def _indexed_values(text: Optional[str], pattern: str) -> Dict[int, str]:
if not text:
return {}
values: Dict[int, str] = {}
for match in re.finditer(pattern, text, re.MULTILINE | re.IGNORECASE):
values[int(match.group(1))] = match.group(2).strip()
return values
def _number(value, default=0.0):
try:
return float(value)
except (TypeError, ValueError):
return default
def _pcie_gen(gt_per_second: float) -> int:
if gt_per_second >= 32:
return 5
if gt_per_second >= 16:
return 4
if gt_per_second >= 8:
return 3
if gt_per_second >= 5:
return 2
return 1 if gt_per_second > 0 else 0
def collect_hygon_info(include_topology: bool = True) -> dict:
"""Collect normalized per-device information for GPUInfo and HealthCheck."""
summary = run_hy_smi()
if not summary:
return {"error": "hy-smi query failed", "gpu_count": 0, "gpus": []}
rows: Dict[int, dict] = {}
row_re = re.compile(
r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W\s+(\S+)\s+"
r"([\d.]+)W\s+(\d+)%\s+([\d.]+)%",
re.MULTILINE,
)
for match in row_re.finditer(summary):
index = int(match.group(1))
rows[index] = {
"index": index,
"temperature": int(round(float(match.group(2)))),
"power_draw": float(match.group(3)),
"performance_level": match.group(4),
"power_limit": float(match.group(5)),
"vram_used_pct": int(match.group(6)),
"utilization_pct": float(match.group(7)),
}
product = run_hy_smi("--showproductname")
driver = run_hy_smi("--showdriverversion")
memory = run_hy_smi("--showmemavailable")
bus = run_hy_smi("--showbus")
unique = run_hy_smi("--showuniqueid")
serial = run_hy_smi("--showserial")
clocks = run_hy_smi("--showclocks")
ecc = run_hy_smi("--showmemeccinfo")
xgmi = run_hy_smi("--showxgmierr")
replay = run_hy_smi("--showreplaycount")
health = run_hy_smi("--healthcheck")
throttle = run_hy_smi("--showfdr")
names = _indexed_values(product, r"HCU\[(\d+)\].*Card Series:\s*([^\s]+)")
memories = _indexed_values(memory, r"HCU\[(\d+)\].*Available memory size \(MiB\):\s*(\d+)")
buses = _indexed_values(bus, r"HCU\[(\d+)\].*PCI Bus:\s*([0-9a-fA-F:.]+)")
unique_ids = _indexed_values(unique, r"HCU\[(\d+)\].*Unique ID:\s*(\S+)")
serials = _indexed_values(serial, r"HCU\[(\d+)\].*Serial Number:\s*(\S+)")
sclk = _indexed_values(clocks, r"HCU\[(\d+)\].*sclk clock level:.*\((\d+)Mhz\)")
mclk = _indexed_values(clocks, r"HCU\[(\d+)\].*mclk clock level:.*\((\d+)Mhz\)")
pcie = {}
if clocks:
for match in re.finditer(
r"HCU\[(\d+)\].*pcie clock level:.*\(([\d.]+)GT/s,\s*x(\d+)", clocks
):
pcie[int(match.group(1))] = (float(match.group(2)), int(match.group(3)))
ecc_enabled = {
idx: value.upper() == "ON"
for idx, value in _indexed_values(
ecc, r"HCU\[(\d+)\].*Vram memory ECC status:\s*(\S+)"
).items()
}
xgmi_errors = {
idx: int(value)
for idx, value in _indexed_values(
xgmi, r"HCU\[(\d+)\].*XGMI Error count\s+(\d+)"
).items()
}
replay_counts = {
idx: int(value)
for idx, value in _indexed_values(
replay, r"HCU\[(\d+)\].*PCIe Replay Count:\s*(\d+)"
).items()
}
healthy = {
idx: value.lower() == "healthy"
for idx, value in _indexed_values(
health, r"HCU\[(\d+)\].*:\s*(Healthy|Unhealthy)\s*$"
).items()
}
throttle_status = _indexed_values(
throttle, r"HCU\[(\d+)\].*Throttler Status:\s*(.+?)\s*$"
)
driver_match = re.search(r"Driver Version:\s*(\S+)", driver or "")
driver_version = driver_match.group(1) if driver_match else "N/A"
gpu_count = max(rows.keys(), default=-1) + 1
gpus: List[dict] = []
for index in range(gpu_count):
row = rows.get(index, {"index": index})
total_mb = int(memories.get(index, 0))
used_mb = int(total_mb * int(row.get("vram_used_pct", 0)) / 100)
gt_s, width = pcie.get(index, (0.0, 0))
throttle_text = throttle_status.get(index, "")
real_throttle = bool(
throttle_text
and any(token not in {"IDLE", "ACTIVE", "NONE"}
for token in re.findall(r"[A-Z_]+", throttle_text.upper()))
)
gpus.append({
**row,
"name": names.get(index, "Hygon DCU"),
"uuid": unique_ids.get(index, "N/A"),
"pci_bus_id": buses.get(index, "N/A"),
"pcie_link_gen": _pcie_gen(gt_s),
"pcie_link_width": width,
"pcie_gt_s": gt_s,
"vram_total_mb": total_mb,
"vram_used_mb": used_mb,
"vram_free_mb": max(total_mb - used_mb, 0),
"clock_sm": int(_number(sclk.get(index))),
"clock_mem": int(_number(mclk.get(index))),
"fan_speed": 0,
"persistence_mode": True,
"compute_mode": row.get("performance_level", "auto"),
"serial_number": serials.get(index, "N/A"),
"ecc_enabled": ecc_enabled.get(index, False),
"ecc_errors_single": 0,
"ecc_errors_double": 0,
"xgmi_errors": xgmi_errors.get(index, 0),
"pcie_replay_count": replay_counts.get(index, 0),
"healthy": healthy.get(index, False),
"throttling_active": real_throttle,
"throttling_text": throttle_text,
})
return {
"driver_version": driver_version,
"runtime_name": "DTK",
"runtime_version": os.environ.get("DTK_VERSION", "N/A"),
"gpu_count": gpu_count,
"gpus": gpus,
"topology": run_hy_smi("--showtopo") if include_topology else "",
"vendor": "hygon",
}

View File

@ -286,10 +286,18 @@ class MultiNodeNCCLTest:
"--mca", "btl_openib_warn_no_device_params_found", "0",
"--mca", "btl_tcp_if_include", str(self.cfg.get("socket_ifname", "bond0")),
"--mca", "oob_tcp_if_include", str(self.cfg.get("oob_tcp_ifname", self.cfg.get("socket_ifname", "bond0"))),
]
# HCOLL does an eager UCX IB wireup during MPI_Init that deadlocks on nodes
# where UCX only sees mlx5_2/mlx5_3 instead of mlx5_0/mlx5_1. HCOLL
# collective optimisations are irrelevant for NCCL tests (NCCL handles its
# own collectives), so disable it unconditionally unless the caller opts in.
if self.cfg.get("disable_hcoll", True):
cmd.extend(["--mca", "coll", "^hcoll"])
cmd.extend([
"-H", host_arg,
"--map-by", f"ppr:{gpus_per_node}:node",
"-np", str(ranks),
]
])
plm_rsh_args = self.cfg.get("plm_rsh_args")
if plm_rsh_args:
cmd.extend(["--mca", "plm_rsh_args", str(plm_rsh_args)])

View File

@ -19,6 +19,10 @@ from modules.gpu_specs import detect_gpu_type, get_gpu_specs, resolve_tools_dir
TORCH_AVAILABLE = False
try:
import torch
try:
import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim
except ImportError:
pass
if torch.cuda.is_available():
TORCH_AVAILABLE = True
except ImportError:
@ -46,7 +50,8 @@ class NCCLTest:
self.console = Console()
self.nccl_cfg = config.get("nccl", {})
self.tools_dir = resolve_tools_dir(config)
self.gpu_type = detect_gpu_type()
configured = config.get("gpu_type", "auto")
self.gpu_type = configured if configured != "auto" else detect_gpu_type()
self.specs = get_gpu_specs(self.gpu_type)
def _find_nccl_test(self, name: str) -> Optional[str]:
@ -295,60 +300,78 @@ class NCCLTest:
}
def _run_torchrun_fallback(self, gpu_count: int) -> dict:
"""Basic NCCL connectivity test via torchrun — verifies NCCL works but does not benchmark performance."""
self.console.print("[yellow]nccl-tests not available, running basic NCCL connectivity check[/yellow]")
"""Benchmark collectives through PyTorch when nccl-tests is unavailable."""
self.console.print("[yellow]nccl-tests not available, benchmarking collectives via torchrun[/yellow]")
size_mb = int(self.nccl_cfg.get("torchrun_size_mb", 64))
warmup = int(self.nccl_cfg.get("torchrun_warmup", 5))
iterations = int(self.nccl_cfg.get("torchrun_iterations", 20))
backend = self.nccl_cfg.get("torch_backend", "nccl")
enabled_ops = []
for key, enabled in [
("allreduce", self.nccl_cfg.get("test_allreduce", True)),
("alltoall", self.nccl_cfg.get("test_alltoall", True)),
("broadcast", self.nccl_cfg.get("test_broadcast", True)),
("reducescatter", self.nccl_cfg.get("test_reduce_scatter", False)),
("allgather", self.nccl_cfg.get("test_allgather", False)),
]:
if enabled:
enabled_ops.append(key)
code = f"""
import torch, torch.distributed as dist, os
import os, time
import torch
try:
import torch_dcu
except ImportError:
pass
import torch.distributed as dist
os.environ.setdefault("MASTER_ADDR","127.0.0.1")
os.environ.setdefault("MASTER_PORT","29500")
rank=int(os.environ.get("LOCAL_RANK",0))
ws={gpu_count}
dist.init_process_group("nccl",rank=rank,world_size=ws)
dist.init_process_group({backend!r},rank=rank,world_size=ws)
torch.cuda.set_device(rank)
size_bytes={size_mb}*1024*1024
numel=size_bytes//4
x=torch.ones(numel,device=f"cuda:{{rank}}",dtype=torch.float32)
allgather_out=torch.empty(numel*ws,device=f"cuda:{{rank}}",dtype=torch.float32)
rs_out=torch.empty(numel//ws,device=f"cuda:{{rank}}",dtype=torch.float32)
a2a_out=torch.empty_like(x)
x=torch.randn(1024*1024,device=f"cuda:{{rank}}",dtype=torch.float32)
# Test AllReduce
try:
dist.all_reduce(x.clone())
if rank==0: print("allreduce:ok")
except Exception as e:
if rank==0: print(f"allreduce:fail:{{e}}")
# Test Broadcast
try:
dist.broadcast(x.clone(),src=0)
if rank==0: print("broadcast:ok")
except Exception as e:
if rank==0: print(f"broadcast:fail:{{e}}")
# Test AllGather
try:
tensor_list=[torch.empty_like(x) for _ in range(ws)]
dist.all_gather(tensor_list,x.clone())
if rank==0: print("allgather:ok")
except Exception as e:
if rank==0: print(f"allgather:fail:{{e}}")
# Test ReduceScatter
try:
chunks=list(x.chunk(ws))
output=torch.empty_like(chunks[0])
dist.reduce_scatter(output,chunks)
if rank==0: print("reducescatter:ok")
except Exception as e:
if rank==0: print(f"reducescatter:fail:{{e}}")
# Test AllToAll
try:
chunks=list(x.chunk(ws))
output_list=[torch.empty_like(c) for c in chunks]
dist.all_to_all(output_list,chunks)
if rank==0: print("alltoall:ok")
except Exception as e:
if rank==0: print(f"alltoall:fail:{{e}}")
def run_op(name):
if name == "allreduce":
return dist.all_reduce(x)
if name == "broadcast":
return dist.broadcast(x, src=0)
if name == "allgather":
return dist.all_gather_into_tensor(allgather_out, x)
if name == "reducescatter":
return dist.reduce_scatter_tensor(rs_out, x)
if name == "alltoall":
return dist.all_to_all_single(a2a_out, x)
raise ValueError(name)
for name in {enabled_ops!r}:
try:
for _ in range({warmup}):
run_op(name)
torch.cuda.synchronize()
dist.barrier()
start=time.perf_counter()
for _ in range({iterations}):
run_op(name)
torch.cuda.synchronize()
elapsed=(time.perf_counter()-start)/{iterations}
algbw=size_bytes/elapsed/1e9
factor={{"allreduce":2*(ws-1)/ws,"allgather":(ws-1)/ws,
"reducescatter":(ws-1)/ws,"alltoall":(ws-1)/ws,
"broadcast":1.0}}[name]
busbw=algbw*factor
if rank==0:
print(f"METRIC|{{name}}|{{elapsed*1e6:.3f}}|{{algbw:.3f}}|{{busbw:.3f}}", flush=True)
except Exception as exc:
if rank==0:
print(f"ERROR|{{name}}|{{exc}}", flush=True)
dist.destroy_process_group()
"""
import tempfile
@ -364,41 +387,50 @@ dist.destroy_process_group()
r = subprocess.run(
[torchrun_cmd, f"--nproc_per_node={gpu_count}", tmp.name],
capture_output=True, text=True, timeout=120,
env={**os.environ, "NCCL_DEBUG": "WARN"},
capture_output=True, text=True, timeout=300,
env={**os.environ, "NCCL_DEBUG": "WARN", "RCCL_DEBUG": "WARN"},
)
os.unlink(tmp.name)
# Parse connectivity results — format: op_name:ok or op_name:fail:error
tests = {}
all_passed = True
for line in r.stdout.split("\n"):
line = line.strip()
if not line:
continue
parts = line.split(":")
op_name = parts[0]
result = parts[1] if len(parts) > 1 else "unknown"
if result == "ok":
status = "PASS"
else:
status = "FAIL"
all_passed = False
tests[op_name] = {
"status": status,
"error": ":".join(parts[2:]) if len(parts) > 2 and result == "fail" else None,
}
if line.startswith("METRIC|"):
_, name, time_us, algbw, busbw = line.split("|", 4)
threshold = self.nccl_cfg.get("min_bandwidth_gbps") or 10.0
busbw_f = float(busbw)
tests[name] = {
"status": "PASS" if busbw_f >= float(threshold) else "WARN",
"best_algbw_gbps": round(float(algbw), 2),
"best_busbw_gbps": round(busbw_f, 2),
"min_required_gbps": float(threshold),
"by_size": [{
"size": size_mb * 1024 * 1024,
"time_us": round(float(time_us), 2),
"algbw_gbps": round(float(algbw), 2),
"busbw_gbps": round(busbw_f, 2),
}],
}
elif line.startswith("ERROR|"):
_, name, error = line.split("|", 2)
tests[name] = {"status": "FAIL", "error": error}
if not tests:
error = (r.stderr or r.stdout or "torchrun produced no metrics")[-1000:]
return {"passed": False, "source": "torchrun_collective", "error": error}
all_passed = all(value.get("status") == "PASS" for value in tests.values())
return {
"passed": all_passed,
"source": "torchrun_fallback",
"source": "torchrun_collective",
"tests": tests,
"gpu_count": gpu_count,
"message_size_mb": size_mb,
"warmup": warmup,
"iterations": iterations,
"backend": backend,
"timestamp": datetime.now().isoformat(),
}
except Exception as e:
return {"passed": False, "source": "torchrun_fallback", "error": str(e)}
return {"passed": False, "source": "torchrun_collective", "error": str(e)}
@staticmethod
def print_results(results: dict, console: Console = None):

View File

@ -76,6 +76,21 @@ class RDMATest:
active_ib_port = True
device_info = self._collect_device_info(devices)
cross_enabled = (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled")
if not ib_devices and cross_enabled:
# RoCE ports report link_layer=Ethernet. They are valid for configured
# cross-node perftest even though local InfiniBand loopback is skipped.
cross = self.run_cross_node()
return {
"passed": cross.get("status") == "PASS",
"status": cross.get("status", "FAIL"),
"reason": "RoCE cross-node mode",
"devices": device_info,
"bandwidth_tests": [],
"latency_tests": [],
"cross_node": cross,
"timestamp": datetime.now().isoformat(),
}
if not ib_devices:
self.console.print(
"[yellow]No InfiniBand-link_layer ports present — "
@ -266,13 +281,14 @@ class RDMATest:
# ------------------------------------------------------------------
def _active_ib_devices(self) -> List[str]:
"""IB devices whose port 1 is InfiniBand link_layer and ACTIVE."""
"""Active IB devices, optionally including Ethernet/RoCE devices."""
out = []
allow_roce = bool((self.rdma_cfg.get("cross_node", {}) or {}).get("allow_roce"))
for dev in self._get_ib_devices():
for port in self._get_ib_ports(dev):
ll = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/link_layer")
st = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/state")
if ll == "InfiniBand" and "ACTIVE" in st.upper():
if (ll == "InfiniBand" or allow_roce) and "ACTIVE" in st.upper():
out.append(dev)
break
return out
@ -296,6 +312,8 @@ class RDMATest:
gid_index = cn.get("gid_index")
msg_size = cn.get("msg_size", 1048576)
iters = cn.get("iters", 5000)
duration_sec = cn.get("duration_sec")
queue_pairs = int(cn.get("queue_pairs", 1))
base_port = cn.get("base_oob_port", 18515)
warmup = cn.get("server_warmup_sec", 2.0)
min_bw = cn.get("min_bandwidth_gbps", 350)
@ -322,10 +340,16 @@ class RDMATest:
entry = {"device": dev}
if has_bw:
bw_extra = ["--report_gbits", "-s", str(msg_size),
"-q", str(queue_pairs)]
if duration_sec:
bw_extra += ["-D", str(duration_sec)]
else:
bw_extra += ["-n", str(iters)]
bw = self._cross_node_perftest(
"ib_write_bw", dev, server_target, server_addr, ib_port,
oob, gid_index, warmup,
extra=["--report_gbits", "-s", str(msg_size), "-n", str(iters)],
extra=bw_extra,
parse="bw")
entry["bandwidth_gbps"] = bw
if isinstance(bw, (int, float)):

View File

@ -100,6 +100,10 @@ class ReportGenerator:
if "gpu_info" in results:
gpus = results["gpu_info"].get("gpus", [])
runtime_name = results["gpu_info"].get("runtime_name", "CUDA")
runtime_version = results["gpu_info"].get(
"runtime_version", results["gpu_info"].get("cuda_version", "N/A")
)
rows = ""
for g in gpus:
rows += f"<tr><td>GPU {g['index']}</td><td>{g['name']}</td>"
@ -109,7 +113,7 @@ class ReportGenerator:
sections.append(
f'<div class="section"><h2>GPU Information</h2>'
f'<p>Driver: {results["gpu_info"].get("driver_version", "N/A")} | '
f'CUDA: {results["gpu_info"].get("cuda_version", "N/A")} | '
f'{runtime_name}: {runtime_version} | '
f'Count: {len(gpus)}</p>'
f'<table><tr><th>GPU</th><th>Model</th><th>VRAM</th><th>Temp</th><th>SM Clock</th></tr>'
f'{rows}</table></div>'
@ -194,8 +198,10 @@ class ReportGenerator:
gpus = gpu_info.get("gpus", [])
gpu_name = gpus[0]["name"] if gpus else "Unknown"
lines.append(f"- **GPU:** {gpu_name} x{gpu_info.get('gpu_count', len(gpus))}")
runtime_name = gpu_info.get("runtime_name", "CUDA")
runtime_version = gpu_info.get("runtime_version", gpu_info.get("cuda_version", "N/A"))
lines.append(f"- **Driver:** {gpu_info.get('driver_version', 'N/A')} | "
f"**CUDA:** {gpu_info.get('cuda_version', 'N/A')}")
f"**{runtime_name}:** {runtime_version}")
lines.append("")
# --- Summary table ---
@ -270,9 +276,16 @@ class ReportGenerator:
if not d2d_peak:
d2d_peak = mem_data.get("peak_bandwidth_gbps") or 0
d2d_eff = mem_data.get("efficiency_pct") or 0
lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {h2d_peak:.0f} GB/s | {h2d_eff:.1f}% |")
lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {d2h_peak:.0f} GB/s | {d2h_eff:.1f}% |")
lines.append(f"| D2D (NVLink) | {d2d:.1f} GB/s | {d2d_peak:.0f} GB/s | {d2d_eff:.1f}% |")
def peak_text(value):
return f"{value:.0f} GB/s" if value else "N/A"
def efficiency_text(value):
return f"{value:.1f}%" if value else "N/A"
lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {peak_text(h2d_peak)} | {efficiency_text(h2d_eff)} |")
lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {peak_text(d2h_peak)} | {efficiency_text(d2h_eff)} |")
d2d_label = "D2D (HBM copy)" if mem_data.get("accelerator_vendor") == "hygon" else "D2D (NVLink)"
lines.append(f"| {d2d_label} | {d2d:.1f} GB/s | {peak_text(d2d_peak)} | {efficiency_text(d2d_eff)} |")
lines.append("")
# PyTorch fallback can't accurately measure HBM peak (intra-GPU copy_()
# only reaches ~20% of HBM bandwidth). When fallback is used, report
@ -298,6 +311,10 @@ class ReportGenerator:
# When present, override the legacy 80%-of-peak rule on a per-dtype basis.
pass_thresholds = comp_data.get("pass_thresholds_tflops", {}) or {}
use_abs = bool(pass_thresholds)
has_peak_reference = any(
isinstance(value, (int, float)) and value > 0
for value in peak_tflops.values()
)
if use_abs:
lines.append("| DType | Achieved (TFLOPS) | Peak | Threshold | Status |")
else:
@ -305,7 +322,7 @@ class ReportGenerator:
lines.append("|-------|-------------------|------|------------|--------|")
worst_eff = 100.0
overall_status = "PASS"
rank = {"PASS": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
for dt, val in per_dtype.items():
if isinstance(val, str):
# skipped or error
@ -324,17 +341,22 @@ class ReportGenerator:
else:
status = "FAIL"
lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | >= {thr} | {status} |")
else:
elif has_peak_reference:
status = "PASS" if ef >= 80 else ("WARN" if ef >= 50 else "FAIL")
lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | {ef:.1f}% | {status} |")
else:
status = "MEASURED"
lines.append(f"| {dt.upper()} | {val:.1f} | N/A | N/A | {status} |")
if rank.get(status, 0) > rank.get(overall_status, 0):
overall_status = status
lines.append("")
if use_abs:
lines.append(f"**Verdict: {overall_status}** (absolute TFLOPS thresholds; worst efficiency {worst_eff:.1f}%)\n")
else:
elif has_peak_reference:
overall_status = "PASS" if worst_eff >= 80 else ("WARN" if worst_eff >= 50 else "FAIL")
lines.append(f"**Verdict: {overall_status}** (worst efficiency {worst_eff:.1f}%)\n")
else:
lines.append("**Verdict: MEASURED** (no published peak/acceptance threshold configured)\n")
# --- NCCL ---
nccl = results.get("nccl")
@ -369,6 +391,17 @@ class ReportGenerator:
lines.append(f"- **Source:** {source}")
lines.append(f"- **Duration:** {elapsed:.0f}s (requested {duration}s)")
lines.append(f"- **Result: {'PASS' if passed else 'FAIL'}**")
max_temp = stress.get("max_temperature_c", {}) or {}
max_power = stress.get("max_power_w", {}) or {}
if max_temp or max_power:
lines.append("")
lines.append("| GPU | Peak Temp | Peak Power |")
lines.append("|-----|-----------|------------|")
for gpu in sorted(set(max_temp) | set(max_power), key=lambda value: int(value)):
lines.append(
f"| {gpu} | {max_temp.get(gpu, 'N/A')} C | "
f"{max_power.get(gpu, 'N/A')} W |"
)
lines.append("")
# --- RDMA ---
@ -519,7 +552,8 @@ class ReportGenerator:
verdict = "PASS" if worst >= 80 else ("WARN" if worst >= 50 else "FAIL")
items.append(("Compute Throughput", f"{verdict} (worst {worst:.1f}%)"))
else:
items.append(("Compute Throughput", "N/A"))
measured = [v for v in per_dtype.values() if isinstance(v, (int, float))]
items.append(("Compute Throughput", "MEASURED (no published threshold)" if measured else "N/A"))
# NCCL
if "nccl" in results:

View File

@ -2,6 +2,7 @@
import glob
import os
import re
import shutil
import subprocess
import time
@ -13,6 +14,7 @@ from rich.live import Live
from rich.text import Text
from modules.gpu_specs import resolve_tools_dir
from modules.hygon_smi import run_hy_smi
class StressTest:
@ -56,12 +58,12 @@ class StressTest:
if not result.get("passed") and result.get("elapsed_sec", 0) < duration_sec * 0.5:
self.console.print("\n[yellow]gpu-burn exited early (possible OOM), switching to PyTorch stress test[/yellow]")
self.console.print("[dim]PyTorch mode dynamically adapts to available memory[/dim]\n")
return self._run_pytorch_stress(duration_sec, memory_pct)
return self._run_pytorch_stress(duration_sec, memory_pct, use_tensor_cores)
return result
self.console.print("[yellow]gpu_burn not found, using PyTorch stress test[/yellow]")
return self._run_pytorch_stress(duration_sec, memory_pct)
return self._run_pytorch_stress(duration_sec, memory_pct, use_tensor_cores)
def _run_gpu_burn(self, gpu_burn: str, duration: int,
doubles: bool, tensor_cores: bool, target_gpus: str) -> dict:
@ -116,9 +118,26 @@ class StressTest:
"timestamp": datetime.now().isoformat(),
}
def _run_pytorch_stress(self, duration: int, memory_pct: int = 90) -> dict:
@staticmethod
def _hygon_snapshot() -> dict:
output = run_hy_smi()
temps, powers = {}, {}
if output:
pattern = re.compile(r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W", re.MULTILINE)
for match in pattern.finditer(output):
idx = int(match.group(1))
temps[idx] = float(match.group(2))
powers[idx] = float(match.group(3))
return {"temperatures_c": temps, "power_w": powers}
def _run_pytorch_stress(self, duration: int, memory_pct: int = 90,
tensor_cores: bool = True) -> dict:
try:
import torch
try:
import torch_dcu # noqa: F401
except ImportError:
pass
if not torch.cuda.is_available():
return {"error": "pytorch_not_available"}
except ImportError:
@ -129,9 +148,10 @@ class StressTest:
gpu_status = {}
t0 = time.time()
snapshots = [self._hygon_snapshot()]
try:
tensors = {}
work_a, work_b, outputs, ballast = {}, {}, {}, {}
for i in range(gpu_count):
with torch.cuda.device(i):
# Get actual free memory (accounting for other processes)
@ -143,16 +163,23 @@ class StressTest:
# Cap at actual free memory with 5% safety margin
alloc_bytes = min(target_mem, int(free_mem * 0.95))
# matmul(A, A.T) needs 2x input memory (input + output)
mem_side = int((alloc_bytes / 4 / 2) ** 0.5)
# Cap compute matrix so a single matmul completes in ~2s on H100/H200
# (FP32 ≈ 67 TFLOPS → 2*4096³/67e12 ≈ 2s). Without this cap, a 141GB
# HBM yields side ≈ 131K → single matmul ~68s × 8 GPUs serial → loop
# overshoots a 60s duration request by 10×+.
MAX_COMPUTE_SIDE = 4096
side = min(mem_side, MAX_COMPUTE_SIDE)
dtype = torch.bfloat16 if tensor_cores else torch.float32
elem_size = torch.tensor([], dtype=dtype).element_size()
mem_side = int((alloc_bytes / elem_size / 3) ** 0.5)
max_compute_side = 8192 if tensor_cores else 4096
side = min(mem_side, max_compute_side)
compute_bytes = side * side * elem_size * 3
ballast_target = max(0, alloc_bytes - compute_bytes - 1024**3)
ballast[i] = []
chunk_size = 1024**3
while ballast_target > 0:
this_chunk = min(chunk_size, ballast_target)
tensor = torch.empty(this_chunk, device=f"cuda:{i}", dtype=torch.uint8)
tensor.fill_(0x5A)
ballast[i].append(tensor)
ballast_target -= this_chunk
actual_mem_mb = side * side * 4 / 1024 / 1024
actual_mem_mb = (alloc_bytes - max(0, ballast_target)) / 1024 / 1024
total_mem_mb = total_mem / 1024 / 1024
free_mem_mb = free_mem / 1024 / 1024
@ -161,7 +188,9 @@ class StressTest:
f"alloc {actual_mem_mb:.0f}MB ({actual_mem_mb/total_mem_mb*100:.0f}%) - "
f"matrix {side}x{side}[/dim]"
)
tensors[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=torch.float32)
work_a[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=dtype)
work_b[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=dtype)
outputs[i] = torch.empty(side, side, device=f"cuda:{i}", dtype=dtype)
self.console.print(f"\n[cyan]Starting stress test for {duration} seconds...[/cyan]")
@ -171,7 +200,7 @@ class StressTest:
# GPUs, otherwise the 8 GPUs run serially and overshoot the duration.
for i in range(gpu_count):
with torch.cuda.device(i):
tensors[i] = torch.matmul(tensors[i], tensors[i].T)
outputs[i] = torch.matmul(work_a[i], work_b[i])
# Single sync per pass — waits for all 8 streams concurrently
for i in range(gpu_count):
with torch.cuda.device(i):
@ -181,6 +210,7 @@ class StressTest:
current_elapsed = time.time() - t0
if int(current_elapsed) != int(elapsed_check) and int(current_elapsed) % 10 == 0:
self.console.print(f" [dim]Running {int(current_elapsed)}s / {duration}s[/dim]")
snapshots.append(self._hygon_snapshot())
elapsed_check = current_elapsed
for i in range(gpu_count):
@ -200,16 +230,28 @@ class StressTest:
"gpu_status": gpu_status,
}
finally:
tensors.clear()
for mapping in (work_a, work_b, outputs, ballast):
mapping.clear()
torch.cuda.empty_cache()
elapsed = round(time.time() - t0, 1)
snapshots.append(self._hygon_snapshot())
max_temp = {}
max_power = {}
for snapshot in snapshots:
for idx, value in snapshot.get("temperatures_c", {}).items():
max_temp[idx] = max(max_temp.get(idx, value), value)
for idx, value in snapshot.get("power_w", {}).items():
max_power[idx] = max(max_power.get(idx, value), value)
return {
"source": "pytorch",
"passed": True,
"duration_sec": duration,
"elapsed_sec": elapsed,
"gpu_status": gpu_status,
"memory_pct": memory_pct,
"max_temperature_c": max_temp,
"max_power_w": max_power,
"timestamp": datetime.now().isoformat(),
}
@ -245,5 +287,15 @@ class StressTest:
color = "green" if status == "PASS" else "red"
c.print(f" GPU {gid}: [{color}]{status}[/{color}]")
max_temp = results.get("max_temperature_c", {})
max_power = results.get("max_power_w", {})
if max_temp or max_power:
c.print("\n Peak telemetry:")
for gid in sorted(set(max_temp) | set(max_power)):
c.print(
f" GPU {gid}: {max_temp.get(gid, 'N/A')} C, "
f"{max_power.get(gid, 'N/A')} W"
)
if results.get("error"):
c.print(f" [red]Error: {results['error']}[/red]")

View File

@ -13,6 +13,10 @@ from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeEl
TORCH_AVAILABLE = False
try:
import torch
try:
import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim
except ImportError:
pass
if torch.cuda.is_available():
TORCH_AVAILABLE = True
except ImportError:
@ -49,6 +53,9 @@ class TrainingSim:
self.console.print(f" Model: {model_name} | Batch: {batch_size} | Seq: {seq_length} | "
f"DType: {dtype_str} | Steps: {num_steps} | GPUs: {gpu_count}")
if model_name == "synthetic":
return self._run_synthetic(gpu_count, batch_size, seq_length, num_steps, dtype)
try:
from transformers import AutoModelForCausalLM, AutoTokenizer
except ImportError:
@ -145,10 +152,10 @@ class TrainingSim:
def _run_synthetic(self, gpu_count, batch_size, seq_length, num_steps, dtype) -> dict:
self.console.print(" Running synthetic training benchmark...")
hidden_size = 4096
num_layers = 6
num_heads = 32
vocab_size = 32000
hidden_size = int(self.train_cfg.get("hidden_size", 4096))
num_layers = int(self.train_cfg.get("num_layers", 6))
num_heads = int(self.train_cfg.get("num_heads", 32))
vocab_size = int(self.train_cfg.get("vocab_size", 32000))
class SyntheticTransformer(torch.nn.Module):
def __init__(self):

46
scripts/bw1000_smoke.py Normal file
View File

@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Minimal DTK/PyTorch smoke test used before the full BW1000 suite."""
import json
import torch
try:
import torch_dcu # noqa: F401
except ImportError:
pass
def main() -> None:
count = torch.cuda.device_count()
if not torch.cuda.is_available() or count < 1:
raise SystemExit("DTK PyTorch reports no available accelerator")
devices = []
for index in range(count):
props = torch.cuda.get_device_properties(index)
devices.append({
"index": index,
"name": torch.cuda.get_device_name(index),
"memory_gib": round(props.total_memory / 1024**3, 2),
})
a = torch.randn(2048, 2048, device="cuda:0", dtype=torch.bfloat16)
b = torch.randn(2048, 2048, device="cuda:0", dtype=torch.bfloat16)
c = a @ b
torch.cuda.synchronize()
if not torch.isfinite(c).all().item():
raise SystemExit("BF16 matmul produced non-finite output")
print(json.dumps({
"torch_version": torch.__version__,
"cuda_api_available": torch.cuda.is_available(),
"device_count": count,
"devices": devices,
"bf16_matmul": "PASS",
"checksum": round(float(c.float().mean().item()), 6),
}, indent=2))
if __name__ == "__main__":
main()