feat: add Hygon BW1000 compatibility
This commit is contained in:
parent
cb5786c319
commit
219065eea1
86
configs/bw1000_4node.yaml
Normal file
86
configs/bw1000_4node.yaml
Normal file
@ -0,0 +1,86 @@
|
||||
gpu_type: bw1000
|
||||
|
||||
runtime:
|
||||
name: DTK
|
||||
version: 25.04.2
|
||||
|
||||
benchmark:
|
||||
memory:
|
||||
size_mb: 4096
|
||||
iterations: 10
|
||||
nvbandwidth_buffer_mb: 512
|
||||
nvbandwidth_samples: 3
|
||||
compute:
|
||||
dtypes: [fp32, tf32, fp16, bf16, fp8]
|
||||
sweep_sizes: [4096, 6144, 8192, [2048, 2048, 13312]]
|
||||
matrix_size: 8192
|
||||
warmup: 10
|
||||
iterations: 30
|
||||
|
||||
health:
|
||||
temp_warning: 75
|
||||
temp_critical: 85
|
||||
power_limit: 800
|
||||
|
||||
nccl:
|
||||
min_bandwidth_gbps: 10
|
||||
test_allreduce: true
|
||||
test_alltoall: true
|
||||
test_broadcast: true
|
||||
test_reduce_scatter: true
|
||||
test_allgather: true
|
||||
test_sendrecv: false
|
||||
torch_backend: nccl
|
||||
torchrun_size_mb: 64
|
||||
torchrun_warmup: 5
|
||||
torchrun_iterations: 20
|
||||
|
||||
stress:
|
||||
duration_sec: 600
|
||||
use_doubles: false
|
||||
use_tensor_cores: true
|
||||
memory_pct: 80
|
||||
gpus: all
|
||||
|
||||
rdma:
|
||||
min_bandwidth_gbps: 200
|
||||
max_latency_us: 6
|
||||
ib_iterations: 5000
|
||||
msg_size: 1048576
|
||||
ib_device: null
|
||||
ib_port: 1
|
||||
cross_node:
|
||||
enabled: false
|
||||
allow_roce: true
|
||||
server: null
|
||||
server_addr: null
|
||||
ssh_user: root
|
||||
devices: [mlx5_bond_0, mlx5_bond_1, mlx5_bond_2, mlx5_bond_3]
|
||||
ib_port: 1
|
||||
gid_index: 3
|
||||
msg_size: 1048576
|
||||
iters: 5000
|
||||
duration_sec: 10
|
||||
queue_pairs: 4
|
||||
base_oob_port: 18515
|
||||
server_warmup_sec: 2.0
|
||||
min_bandwidth_gbps: 200
|
||||
max_latency_us: 6
|
||||
|
||||
training:
|
||||
model: synthetic
|
||||
batch_size: 2
|
||||
seq_length: 1024
|
||||
num_steps: 20
|
||||
dtype: bf16
|
||||
hidden_size: 2048
|
||||
num_layers: 4
|
||||
num_heads: 16
|
||||
vocab_size: 8192
|
||||
|
||||
report:
|
||||
output_dir: ./reports
|
||||
format: json
|
||||
|
||||
tools:
|
||||
install_dir: /opt/gpu-test-tools
|
||||
15
configs/bw1000_smoke.yaml
Normal file
15
configs/bw1000_smoke.yaml
Normal file
@ -0,0 +1,15 @@
|
||||
gpu_type: bw1000
|
||||
|
||||
runtime:
|
||||
name: DTK
|
||||
version: 25.04.2
|
||||
|
||||
stress:
|
||||
duration_sec: 60
|
||||
use_doubles: false
|
||||
use_tensor_cores: true
|
||||
memory_pct: 80
|
||||
gpus: all
|
||||
|
||||
tools:
|
||||
install_dir: /opt/gpu-test-tools
|
||||
@ -27,6 +27,7 @@ from modules.stress_test import StressTest
|
||||
from modules.rdma_test import RDMATest
|
||||
from modules.report import ReportGenerator
|
||||
from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label, get_supported_gpus, validate_driver_compatibility
|
||||
from modules.hygon_smi import find_hy_smi
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
"benchmark": {
|
||||
@ -117,9 +118,9 @@ def check_prerequisites(console: Console) -> bool:
|
||||
import shutil
|
||||
|
||||
ok = True
|
||||
if not shutil.which("nvidia-smi"):
|
||||
console.print("[bold red]ERROR: nvidia-smi not found![/bold red]")
|
||||
console.print(" Please install NVIDIA drivers first.")
|
||||
if not shutil.which("nvidia-smi") and not find_hy_smi():
|
||||
console.print("[bold red]ERROR: neither nvidia-smi nor hy-smi was found![/bold red]")
|
||||
console.print(" Please install the NVIDIA driver or Hygon DCU driver first.")
|
||||
ok = False
|
||||
return ok
|
||||
|
||||
|
||||
@ -17,6 +17,10 @@ from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label, res
|
||||
TORCH_AVAILABLE = False
|
||||
try:
|
||||
import torch
|
||||
try:
|
||||
import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim
|
||||
except ImportError:
|
||||
pass
|
||||
if torch.cuda.is_available():
|
||||
TORCH_AVAILABLE = True
|
||||
except ImportError:
|
||||
@ -63,12 +67,14 @@ class Benchmark:
|
||||
return None
|
||||
|
||||
def run_memory_benchmark(self) -> dict:
|
||||
nvbw = self._find_nvbandwidth()
|
||||
# nvbandwidth is NVIDIA-specific. DTK PyTorch provides the portable
|
||||
# transfer/copy path used below for Hygon DCUs.
|
||||
nvbw = None if self.gpu_type == "bw1000" else self._find_nvbandwidth()
|
||||
|
||||
if nvbw:
|
||||
return self._run_nvbandwidth(nvbw)
|
||||
|
||||
self.console.print("[yellow]nvbandwidth not found, falling back to PyTorch[/yellow]")
|
||||
self.console.print("[yellow]Using PyTorch memory benchmark fallback[/yellow]")
|
||||
return self._run_memory_pytorch()
|
||||
|
||||
def _run_nvbandwidth(self, nvbw_path: str) -> dict:
|
||||
@ -189,6 +195,7 @@ class Benchmark:
|
||||
"efficiency_pct": d2d_efficiency,
|
||||
"results_by_test": results_by_test,
|
||||
"per_gpu": [],
|
||||
"accelerator_vendor": self.specs.get("vendor", "nvidia"),
|
||||
}
|
||||
}
|
||||
|
||||
@ -292,20 +299,31 @@ class Benchmark:
|
||||
progress.advance(task)
|
||||
|
||||
best_d2d = max(v["d2d_gbps"] for v in bandwidth_by_size.values())
|
||||
best_h2d = max(v["h2d_gbps"] for v in bandwidth_by_size.values())
|
||||
best_d2h = max(v["d2h_gbps"] for v in bandwidth_by_size.values())
|
||||
peak_bw = self.specs["memory_bandwidth_gbps"]
|
||||
efficiency = round((best_d2d / peak_bw) * 100, 1) if peak_bw else None
|
||||
pcie_gen = self.specs.get("pcie_gen", 0)
|
||||
pcie_peak = {3: 16, 4: 32, 5: 64, 6: 128}.get(pcie_gen) if pcie_gen else None
|
||||
|
||||
return {
|
||||
"memory": {
|
||||
"source": "pytorch",
|
||||
"h2d_bandwidth_gbps": round(max(v["h2d_gbps"] for v in bandwidth_by_size.values()), 1),
|
||||
"d2h_bandwidth_gbps": round(max(v["d2h_gbps"] for v in bandwidth_by_size.values()), 1),
|
||||
"h2d_bandwidth_gbps": round(best_h2d, 1),
|
||||
"d2h_bandwidth_gbps": round(best_d2h, 1),
|
||||
"d2d_bandwidth_gbps": round(best_d2d, 1),
|
||||
"h2d_peak_gbps": pcie_peak,
|
||||
"d2h_peak_gbps": pcie_peak,
|
||||
"d2d_peak_gbps": None,
|
||||
"h2d_efficiency_pct": round(best_h2d / pcie_peak * 100, 1) if pcie_peak else None,
|
||||
"d2h_efficiency_pct": round(best_d2h / pcie_peak * 100, 1) if pcie_peak else None,
|
||||
"d2d_efficiency_pct": efficiency,
|
||||
"peak_bandwidth_gbps": self.specs["memory_bandwidth_gbps"],
|
||||
"efficiency_pct": efficiency,
|
||||
"test_sizes_mb": test_sizes_mb,
|
||||
"bandwidth_by_size": bandwidth_by_size,
|
||||
"per_gpu": [],
|
||||
"accelerator_vendor": self.specs.get("vendor", "nvidia"),
|
||||
}
|
||||
}
|
||||
|
||||
@ -355,7 +373,7 @@ class Benchmark:
|
||||
"tf32": ("tf32", self.specs["tf32_tflops"]),
|
||||
"fp16": (torch.float16, self.specs["fp16_tflops"]),
|
||||
"bf16": (torch.bfloat16, self.specs["bf16_tflops"]),
|
||||
"fp8": (torch.float8_e4m3fn, self.specs["fp8_tflops"]),
|
||||
"fp8": (getattr(torch, "float8_e4m3fn", None), self.specs["fp8_tflops"]),
|
||||
}
|
||||
|
||||
results_by_dtype = {}
|
||||
@ -375,7 +393,9 @@ class Benchmark:
|
||||
continue
|
||||
|
||||
# Skip FP8 if GPU architecture doesn't support it
|
||||
if dtype_name == "fp8" and self.specs.get("fp8_tflops", 0) == 0:
|
||||
if dtype_name == "fp8" and (
|
||||
self.specs.get("fp8_tflops", 0) == 0 or dtype_map["fp8"][0] is None
|
||||
):
|
||||
arch = self.specs.get("architecture", "unknown")
|
||||
results_by_dtype["fp8"] = f"skipped ({arch} does not support FP8)"
|
||||
self.console.print(f"[dim] fp8: skipped - {arch} architecture has no FP8 support[/dim]")
|
||||
@ -444,6 +464,7 @@ class Benchmark:
|
||||
"matrix_size": matrix_size,
|
||||
"warmup": warmup,
|
||||
"iterations": iterations,
|
||||
"accelerator_backend": "dtk" if self.gpu_type == "bw1000" else "cuda",
|
||||
}
|
||||
}
|
||||
|
||||
@ -578,9 +599,14 @@ class Benchmark:
|
||||
continue
|
||||
pk = peak.get(dt, 0)
|
||||
ef = eff.get(dt, 0)
|
||||
ec = "green" if ef >= 80 else ("yellow" if ef >= 50 else "red")
|
||||
table.add_row(dt.upper(), f"{achieved:.1f}", f"{pk:.0f}",
|
||||
f"[{ec}]{ef:.1f}%[/{ec}]")
|
||||
if pk:
|
||||
ec = "green" if ef >= 80 else ("yellow" if ef >= 50 else "red")
|
||||
peak_text = f"{pk:.0f}"
|
||||
eff_text = f"[{ec}]{ef:.1f}%[/{ec}]"
|
||||
else:
|
||||
peak_text = "N/A"
|
||||
eff_text = "[cyan]MEASURED[/cyan]"
|
||||
table.add_row(dt.upper(), f"{achieved:.1f}", peak_text, eff_text)
|
||||
c.print(table)
|
||||
|
||||
@staticmethod
|
||||
@ -590,7 +616,8 @@ class Benchmark:
|
||||
Single source of truth for the PASS/WARN/FAIL rule (same one report.py uses):
|
||||
achieved >= thr -> PASS; >= 0.9*thr -> WARN; else FAIL. A string achieved value
|
||||
(skipped/error) -> SKIP. A dtype without a threshold falls back to efficiency
|
||||
(>=80 PASS / >=50 WARN / else FAIL).
|
||||
(>=80 PASS / >=50 WARN / else FAIL). If neither threshold nor peak is
|
||||
published, the numeric result is retained as MEASURED without a verdict.
|
||||
|
||||
Returns {"rows": [(dtype, achieved, threshold, status), ...], "verdict": str}.
|
||||
"""
|
||||
@ -598,20 +625,29 @@ class Benchmark:
|
||||
per_dtype = comp.get("per_dtype_tflops", {})
|
||||
thresholds = comp.get("pass_thresholds_tflops", {}) or {}
|
||||
eff = comp.get("efficiency_pct", {})
|
||||
rank = {"PASS": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
|
||||
rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
|
||||
rows, verdict = [], "PASS"
|
||||
evaluated = False
|
||||
measured = False
|
||||
for dt, val in per_dtype.items():
|
||||
thr = thresholds.get(dt)
|
||||
if isinstance(val, str):
|
||||
status = "SKIP"
|
||||
elif thr:
|
||||
status = "PASS" if val >= thr else ("WARN" if val >= thr * 0.9 else "FAIL")
|
||||
evaluated = True
|
||||
elif dt not in eff:
|
||||
status = "MEASURED"
|
||||
measured = True
|
||||
else:
|
||||
e = eff.get(dt, 0)
|
||||
status = "PASS" if e >= 80 else ("WARN" if e >= 50 else "FAIL")
|
||||
evaluated = True
|
||||
rows.append((dt, val, thr, status))
|
||||
if rank[status] > rank[verdict]:
|
||||
verdict = status
|
||||
if measured and not evaluated:
|
||||
verdict = "MEASURED"
|
||||
return {"rows": rows, "verdict": verdict}
|
||||
|
||||
@staticmethod
|
||||
@ -619,7 +655,7 @@ class Benchmark:
|
||||
"""Print the PASS/WARN/FAIL table for compute results; return the verdict."""
|
||||
c = console or Console()
|
||||
judged = Benchmark.judge_compute(results)
|
||||
color = {"PASS": "green", "WARN": "yellow", "FAIL": "red", "SKIP": "dim"}
|
||||
color = {"PASS": "green", "MEASURED": "cyan", "WARN": "yellow", "FAIL": "red", "SKIP": "dim"}
|
||||
c.print("\n[bold cyan]Compute Verdict (vs thresholds)[/bold cyan]")
|
||||
for dt, val, thr, status in judged["rows"]:
|
||||
val_s = f"{val:.1f}" if isinstance(val, (int, float)) else str(val)
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
"""GPU information detection module for NVIDIA datacenter GPUs (A100/A800/H100/H200/B200/B300)."""
|
||||
"""Accelerator information detection for NVIDIA GPUs and Hygon DCUs."""
|
||||
|
||||
import subprocess
|
||||
import shutil
|
||||
@ -11,6 +11,7 @@ from rich.panel import Panel
|
||||
from rich.text import Text
|
||||
|
||||
from modules.gpu_specs import detect_gpu_type, get_gpu_specs, get_gpu_label
|
||||
from modules.hygon_smi import collect_hygon_info, find_hy_smi
|
||||
|
||||
|
||||
class GPUInfo:
|
||||
@ -18,7 +19,8 @@ class GPUInfo:
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
self.console = Console()
|
||||
self.gpu_type = detect_gpu_type()
|
||||
configured = config.get("gpu_type", "auto")
|
||||
self.gpu_type = configured if configured != "auto" else detect_gpu_type()
|
||||
self.specs = get_gpu_specs(self.gpu_type)
|
||||
self.gpu_label = get_gpu_label(self.gpu_type)
|
||||
|
||||
@ -35,6 +37,19 @@ class GPUInfo:
|
||||
return None
|
||||
|
||||
def run(self) -> dict:
|
||||
if self.gpu_type == "bw1000" or (not shutil.which("nvidia-smi") and find_hy_smi()):
|
||||
result = collect_hygon_info(include_topology=True)
|
||||
runtime_cfg = self.config.get("runtime", {}) or {}
|
||||
result.update({
|
||||
"runtime_name": runtime_cfg.get("name", result.get("runtime_name", "DTK")),
|
||||
"runtime_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")),
|
||||
"cuda_version": runtime_cfg.get("version", result.get("runtime_version", "N/A")),
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"detected_gpu_type": "bw1000",
|
||||
"gpu_label": get_gpu_label("bw1000"),
|
||||
})
|
||||
return result
|
||||
|
||||
if not shutil.which("nvidia-smi"):
|
||||
self.console.print("[bold red]nvidia-smi not found![/bold red]")
|
||||
return {"error": "nvidia-smi not found", "gpu_count": 0}
|
||||
@ -159,7 +174,9 @@ class GPUInfo:
|
||||
|
||||
c.print(f"\n[bold cyan]GPU Information[/bold cyan]")
|
||||
c.print(f" Driver Version : {results.get('driver_version', 'N/A')}")
|
||||
c.print(f" CUDA Version : {results.get('cuda_version', 'N/A')}")
|
||||
runtime_name = results.get("runtime_name", "CUDA")
|
||||
runtime_version = results.get("runtime_version", results.get("cuda_version", "N/A"))
|
||||
c.print(f" {runtime_name} Version{' ' * max(0, 4-len(runtime_name))}: {runtime_version}")
|
||||
c.print(f" GPU Count : {results.get('gpu_count', 0)}")
|
||||
c.print(f" Detected GPU : {results.get('gpu_label', 'Unknown')} ({results.get('detected_gpu_type', 'unknown')})")
|
||||
c.print(f" Timestamp : {results.get('timestamp', 'N/A')}")
|
||||
@ -181,7 +198,7 @@ class GPUInfo:
|
||||
|
||||
for g in gpus:
|
||||
name = g["name"]
|
||||
if any(k in name for k in ("A100", "A800", "H100", "H200", "B200", "B300")):
|
||||
if any(k in name for k in ("A100", "A800", "H100", "H200", "B200", "B300", "BW1000")):
|
||||
name = f"[bold green]{name}[/bold green]"
|
||||
vram = f"{g['vram_used_mb']}/{g['vram_total_mb']} MB"
|
||||
temp = f"{g['temperature']}°C"
|
||||
@ -204,7 +221,8 @@ class GPUInfo:
|
||||
|
||||
topology = results.get("topology", "")
|
||||
if topology and topology != "Unavailable":
|
||||
c.print(Panel(topology, title="[bold]NVLink/NVSwitch Topology[/bold]", border_style="cyan"))
|
||||
title = "HSW/XHCL Topology" if results.get("vendor") == "hygon" else "NVLink/NVSwitch Topology"
|
||||
c.print(Panel(topology, title=f"[bold]{title}[/bold]", border_style="cyan"))
|
||||
|
||||
warnings = []
|
||||
for g in gpus:
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
"""GPU specifications database for NVIDIA datacenter GPUs."""
|
||||
"""Accelerator specifications for NVIDIA GPUs and Hygon DCUs."""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
@ -8,6 +8,7 @@ from typing import List, Optional
|
||||
# GPU name patterns -> internal key mapping
|
||||
# Order matters: longer/more-specific patterns must come before shorter ones.
|
||||
GPU_NAME_PATTERNS = {
|
||||
"BW1000": "bw1000",
|
||||
"A100": "a100",
|
||||
"A800": "a800",
|
||||
"H100": "h100",
|
||||
@ -20,6 +21,31 @@ GPU_NAME_PATTERNS = {
|
||||
|
||||
# Specs database — ALL values are DENSE (non-sparse) TFLOPS
|
||||
GPU_SPECS = {
|
||||
"bw1000": {
|
||||
# Public Hygon material identifies the supported precisions but does not
|
||||
# publish stable dense peak figures for BW1000_H. Keep unknown peaks at
|
||||
# zero so reports show measured throughput without inventing efficiency.
|
||||
"full_name": "Hygon BW1000_H DCU",
|
||||
"vendor": "hygon",
|
||||
"architecture": "C-3000 GPGPU",
|
||||
"compute_capability": 0.0,
|
||||
"hbm_capacity_gb": 64,
|
||||
"hbm_type": "HBM",
|
||||
"memory_bandwidth_gbps": 0,
|
||||
"fp32_tflops": 0,
|
||||
"tf32_tflops": 0,
|
||||
"fp16_tflops": 0,
|
||||
"bf16_tflops": 0,
|
||||
"fp8_tflops": 0,
|
||||
"compute_pass_thresholds_tflops": {},
|
||||
"tdp_watts": 800,
|
||||
"nvlink_gen": 0,
|
||||
"nvlink_bandwidth_gbps": 0,
|
||||
"interconnect": "HSW/XHCL",
|
||||
"pcie_gen": 5,
|
||||
"min_driver_version": "",
|
||||
"min_cuda_version": "",
|
||||
},
|
||||
"h100": {
|
||||
# Peaks below are NVIDIA marketing dense peaks (theoretical Tensor Core max).
|
||||
# `compute_pass_thresholds_tflops` carries the absolute PASS thresholds used
|
||||
@ -240,30 +266,41 @@ _UNKNOWN_SPECS = {
|
||||
|
||||
|
||||
def detect_gpu_type() -> str:
|
||||
"""Detect GPU type via nvidia-smi and return the internal key (e.g. 'h200').
|
||||
"""Detect accelerator type via nvidia-smi or hy-smi.
|
||||
|
||||
Returns 'unknown' if nvidia-smi is unavailable or the GPU is not recognized.
|
||||
"""
|
||||
nvidia_smi = shutil.which("nvidia-smi")
|
||||
if not nvidia_smi:
|
||||
return "unknown"
|
||||
if nvidia_smi:
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[nvidia_smi, "--query-gpu=name", "--format=csv,noheader"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
if r.returncode == 0 and r.stdout.strip():
|
||||
first_line = r.stdout.strip().splitlines()[0].strip().upper()
|
||||
# Iterate longest-pattern-first so "H200" doesn't match "H20".
|
||||
for pattern, key in sorted(GPU_NAME_PATTERNS.items(), key=lambda kv: -len(kv[0])):
|
||||
if pattern in first_line:
|
||||
return key
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
|
||||
pass
|
||||
|
||||
# Hygon DTK hosts expose hy-smi on PATH or under /opt/hyhal.
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[nvidia_smi, "--query-gpu=name", "--format=csv,noheader"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return "unknown"
|
||||
|
||||
first_line = r.stdout.strip().splitlines()[0].strip().upper()
|
||||
# Iterate longest-pattern-first so "H200" doesn't get matched by "H20".
|
||||
for pattern, key in sorted(GPU_NAME_PATTERNS.items(), key=lambda kv: -len(kv[0])):
|
||||
if pattern in first_line:
|
||||
return key
|
||||
return "unknown"
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
|
||||
return "unknown"
|
||||
from modules.hygon_smi import find_hy_smi
|
||||
hy_smi = find_hy_smi()
|
||||
if hy_smi:
|
||||
r = subprocess.run(
|
||||
[hy_smi, "--showproductname"], capture_output=True,
|
||||
text=True, timeout=10,
|
||||
)
|
||||
product = (r.stdout or "").upper()
|
||||
if r.returncode == 0 and "BW1000" in product:
|
||||
return "bw1000"
|
||||
except (ImportError, subprocess.TimeoutExpired, FileNotFoundError, OSError):
|
||||
pass
|
||||
return "unknown"
|
||||
|
||||
|
||||
def get_gpu_specs(gpu_type: str = None) -> dict:
|
||||
@ -286,8 +323,7 @@ def get_gpu_label(gpu_type: str) -> str:
|
||||
specs = GPU_SPECS.get(gpu_type)
|
||||
if specs:
|
||||
full = specs["full_name"]
|
||||
# Strip the "NVIDIA " prefix for display
|
||||
return full.replace("NVIDIA ", "")
|
||||
return full.replace("NVIDIA ", "").replace("Hygon ", "")
|
||||
return "Unknown GPU"
|
||||
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
"""Hardware health monitoring module for NVIDIA datacenter GPUs."""
|
||||
"""Hardware health monitoring for NVIDIA GPUs and Hygon DCUs."""
|
||||
|
||||
import subprocess
|
||||
import shutil
|
||||
@ -12,6 +12,7 @@ from rich.panel import Panel
|
||||
from rich.text import Text
|
||||
|
||||
from modules.gpu_specs import detect_gpu_type, get_gpu_specs
|
||||
from modules.hygon_smi import collect_hygon_info, find_hy_smi
|
||||
|
||||
|
||||
class HealthCheck:
|
||||
@ -20,7 +21,8 @@ class HealthCheck:
|
||||
self.config = config
|
||||
self.console = Console()
|
||||
self.health_cfg = config.get("health", {})
|
||||
self.gpu_type = detect_gpu_type()
|
||||
configured = config.get("gpu_type", "auto")
|
||||
self.gpu_type = configured if configured != "auto" else detect_gpu_type()
|
||||
self.specs = get_gpu_specs(self.gpu_type)
|
||||
|
||||
def _run_smi(self, query: str) -> Optional[str]:
|
||||
@ -55,6 +57,9 @@ class HealthCheck:
|
||||
return default
|
||||
|
||||
def run(self) -> dict:
|
||||
if self.gpu_type == "bw1000" or (not shutil.which("nvidia-smi") and find_hy_smi()):
|
||||
return self._run_hygon()
|
||||
|
||||
if not shutil.which("nvidia-smi"):
|
||||
return {"error": "nvidia-smi not found", "passed": False}
|
||||
|
||||
@ -180,6 +185,102 @@ class HealthCheck:
|
||||
"detected_gpu_type": self.gpu_type,
|
||||
}
|
||||
|
||||
def _run_hygon(self) -> dict:
|
||||
info = collect_hygon_info(include_topology=False)
|
||||
if info.get("error"):
|
||||
return {"error": info["error"], "passed": False}
|
||||
|
||||
temp_warn = self.health_cfg.get("temp_warning", 80)
|
||||
temp_crit = self.health_cfg.get("temp_critical", 90)
|
||||
configured_power = self.health_cfg.get("power_limit")
|
||||
gpu_health = []
|
||||
overall_pass = True
|
||||
|
||||
for gpu in info.get("gpus", []):
|
||||
checks = {}
|
||||
temp = gpu.get("temperature", 0)
|
||||
if temp >= temp_crit:
|
||||
temp_status = "FAIL"
|
||||
elif temp >= temp_warn:
|
||||
temp_status = "WARN"
|
||||
else:
|
||||
temp_status = "PASS"
|
||||
checks["temperature"] = {
|
||||
"value": temp, "status": temp_status,
|
||||
"threshold": temp_crit if temp_status == "FAIL" else temp_warn,
|
||||
}
|
||||
|
||||
draw = gpu.get("power_draw", 0.0)
|
||||
limit = configured_power or gpu.get("power_limit") or self.specs.get("tdp_watts", 800)
|
||||
checks["power"] = {
|
||||
"value": draw, "limit": limit,
|
||||
"status": "PASS" if draw <= limit * 1.05 else "WARN",
|
||||
}
|
||||
|
||||
ecc_enabled = bool(gpu.get("ecc_enabled"))
|
||||
checks["ecc_errors"] = {
|
||||
"single": gpu.get("ecc_errors_single", 0),
|
||||
"double": gpu.get("ecc_errors_double", 0),
|
||||
"enabled": ecc_enabled,
|
||||
"status": "PASS" if ecc_enabled else "FAIL",
|
||||
}
|
||||
xgmi = int(gpu.get("xgmi_errors", 0))
|
||||
replay = int(gpu.get("pcie_replay_count", 0))
|
||||
healthy = bool(gpu.get("healthy"))
|
||||
checks["memory_errors"] = {
|
||||
"xgmi_errors": xgmi,
|
||||
"pcie_replay_count": replay,
|
||||
"device_healthy": healthy,
|
||||
"status": "PASS" if healthy and xgmi == 0 and replay == 0 else "FAIL",
|
||||
}
|
||||
|
||||
generation = int(gpu.get("pcie_link_gen", 0))
|
||||
width = int(gpu.get("pcie_link_width", 0))
|
||||
expected = self.specs.get("pcie_gen", 0)
|
||||
pcie_ok = generation >= expected and width >= 16
|
||||
checks["pcie_link"] = {
|
||||
"gen": generation, "width": width,
|
||||
"status": "PASS" if pcie_ok else "WARN",
|
||||
}
|
||||
|
||||
sm = int(gpu.get("clock_sm", 0))
|
||||
mem = int(gpu.get("clock_mem", 0))
|
||||
checks["clock_speed"] = {
|
||||
"sm": sm, "mem": mem,
|
||||
"status": "PASS" if sm > 0 and mem > 0 else "WARN",
|
||||
}
|
||||
throttling = bool(gpu.get("throttling_active"))
|
||||
checks["throttling"] = {
|
||||
"status": "FAIL" if throttling else "PASS",
|
||||
"reasons": [gpu.get("throttling_text")] if throttling else [],
|
||||
}
|
||||
# Hygon has no NVIDIA-style persistence mode; a loaded /dev/kfd
|
||||
# driver and successful hy-smi health query are the equivalent check.
|
||||
checks["persistence_mode"] = {
|
||||
"enabled": healthy, "status": "PASS" if healthy else "FAIL"
|
||||
}
|
||||
|
||||
worst = "PASS"
|
||||
for check in checks.values():
|
||||
if check["status"] == "FAIL":
|
||||
worst = "FAIL"
|
||||
break
|
||||
if check["status"] == "WARN":
|
||||
worst = "WARN"
|
||||
if worst == "FAIL":
|
||||
overall_pass = False
|
||||
gpu_health.append({"index": gpu["index"], "status": worst, "checks": checks})
|
||||
|
||||
return {
|
||||
"passed": overall_pass,
|
||||
"gpu_health": gpu_health,
|
||||
"system_health": self._check_system(),
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"detected_gpu_type": "bw1000",
|
||||
"driver_version": info.get("driver_version", "N/A"),
|
||||
"vendor": "hygon",
|
||||
}
|
||||
|
||||
def _check_system(self) -> dict:
|
||||
persistd = shutil.which("nvidia-persistenced") is not None
|
||||
persistd_running = False
|
||||
@ -227,7 +328,10 @@ class HealthCheck:
|
||||
if os.path.isdir("/sys/class/infiniband_verbs"):
|
||||
rdma_devs = os.listdir("/sys/class/infiniband_verbs")
|
||||
|
||||
nccl_env = {k: v for k, v in os.environ.items() if k.startswith("NCCL_")}
|
||||
nccl_env = {
|
||||
k: v for k, v in os.environ.items()
|
||||
if k.startswith(("NCCL_", "RCCL_", "HCCL_"))
|
||||
}
|
||||
|
||||
return {
|
||||
"nvidia_persistenced": {"installed": persistd, "running": persistd_running},
|
||||
|
||||
190
modules/hygon_smi.py
Normal file
190
modules/hygon_smi.py
Normal file
@ -0,0 +1,190 @@
|
||||
"""Helpers for querying Hygon DCU devices through ``hy-smi``."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
|
||||
def find_hy_smi() -> Optional[str]:
|
||||
"""Return the hy-smi executable path when available."""
|
||||
path = shutil.which("hy-smi")
|
||||
if path:
|
||||
return path
|
||||
path = "/opt/hyhal/bin/hy-smi"
|
||||
return path if os.path.isfile(path) and os.access(path, os.X_OK) else None
|
||||
|
||||
|
||||
def run_hy_smi(*args: str, timeout: int = 30) -> Optional[str]:
|
||||
"""Run hy-smi and return stdout, or ``None`` on failure."""
|
||||
binary = find_hy_smi()
|
||||
if not binary:
|
||||
return None
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[binary, *args], capture_output=True, text=True, timeout=timeout
|
||||
)
|
||||
except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
return result.stdout.strip() if result.returncode == 0 else None
|
||||
|
||||
|
||||
def _indexed_values(text: Optional[str], pattern: str) -> Dict[int, str]:
|
||||
if not text:
|
||||
return {}
|
||||
values: Dict[int, str] = {}
|
||||
for match in re.finditer(pattern, text, re.MULTILINE | re.IGNORECASE):
|
||||
values[int(match.group(1))] = match.group(2).strip()
|
||||
return values
|
||||
|
||||
|
||||
def _number(value, default=0.0):
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _pcie_gen(gt_per_second: float) -> int:
|
||||
if gt_per_second >= 32:
|
||||
return 5
|
||||
if gt_per_second >= 16:
|
||||
return 4
|
||||
if gt_per_second >= 8:
|
||||
return 3
|
||||
if gt_per_second >= 5:
|
||||
return 2
|
||||
return 1 if gt_per_second > 0 else 0
|
||||
|
||||
|
||||
def collect_hygon_info(include_topology: bool = True) -> dict:
|
||||
"""Collect normalized per-device information for GPUInfo and HealthCheck."""
|
||||
summary = run_hy_smi()
|
||||
if not summary:
|
||||
return {"error": "hy-smi query failed", "gpu_count": 0, "gpus": []}
|
||||
|
||||
rows: Dict[int, dict] = {}
|
||||
row_re = re.compile(
|
||||
r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W\s+(\S+)\s+"
|
||||
r"([\d.]+)W\s+(\d+)%\s+([\d.]+)%",
|
||||
re.MULTILINE,
|
||||
)
|
||||
for match in row_re.finditer(summary):
|
||||
index = int(match.group(1))
|
||||
rows[index] = {
|
||||
"index": index,
|
||||
"temperature": int(round(float(match.group(2)))),
|
||||
"power_draw": float(match.group(3)),
|
||||
"performance_level": match.group(4),
|
||||
"power_limit": float(match.group(5)),
|
||||
"vram_used_pct": int(match.group(6)),
|
||||
"utilization_pct": float(match.group(7)),
|
||||
}
|
||||
|
||||
product = run_hy_smi("--showproductname")
|
||||
driver = run_hy_smi("--showdriverversion")
|
||||
memory = run_hy_smi("--showmemavailable")
|
||||
bus = run_hy_smi("--showbus")
|
||||
unique = run_hy_smi("--showuniqueid")
|
||||
serial = run_hy_smi("--showserial")
|
||||
clocks = run_hy_smi("--showclocks")
|
||||
ecc = run_hy_smi("--showmemeccinfo")
|
||||
xgmi = run_hy_smi("--showxgmierr")
|
||||
replay = run_hy_smi("--showreplaycount")
|
||||
health = run_hy_smi("--healthcheck")
|
||||
throttle = run_hy_smi("--showfdr")
|
||||
|
||||
names = _indexed_values(product, r"HCU\[(\d+)\].*Card Series:\s*([^\s]+)")
|
||||
memories = _indexed_values(memory, r"HCU\[(\d+)\].*Available memory size \(MiB\):\s*(\d+)")
|
||||
buses = _indexed_values(bus, r"HCU\[(\d+)\].*PCI Bus:\s*([0-9a-fA-F:.]+)")
|
||||
unique_ids = _indexed_values(unique, r"HCU\[(\d+)\].*Unique ID:\s*(\S+)")
|
||||
serials = _indexed_values(serial, r"HCU\[(\d+)\].*Serial Number:\s*(\S+)")
|
||||
sclk = _indexed_values(clocks, r"HCU\[(\d+)\].*sclk clock level:.*\((\d+)Mhz\)")
|
||||
mclk = _indexed_values(clocks, r"HCU\[(\d+)\].*mclk clock level:.*\((\d+)Mhz\)")
|
||||
pcie = {}
|
||||
if clocks:
|
||||
for match in re.finditer(
|
||||
r"HCU\[(\d+)\].*pcie clock level:.*\(([\d.]+)GT/s,\s*x(\d+)", clocks
|
||||
):
|
||||
pcie[int(match.group(1))] = (float(match.group(2)), int(match.group(3)))
|
||||
ecc_enabled = {
|
||||
idx: value.upper() == "ON"
|
||||
for idx, value in _indexed_values(
|
||||
ecc, r"HCU\[(\d+)\].*Vram memory ECC status:\s*(\S+)"
|
||||
).items()
|
||||
}
|
||||
xgmi_errors = {
|
||||
idx: int(value)
|
||||
for idx, value in _indexed_values(
|
||||
xgmi, r"HCU\[(\d+)\].*XGMI Error count\s+(\d+)"
|
||||
).items()
|
||||
}
|
||||
replay_counts = {
|
||||
idx: int(value)
|
||||
for idx, value in _indexed_values(
|
||||
replay, r"HCU\[(\d+)\].*PCIe Replay Count:\s*(\d+)"
|
||||
).items()
|
||||
}
|
||||
healthy = {
|
||||
idx: value.lower() == "healthy"
|
||||
for idx, value in _indexed_values(
|
||||
health, r"HCU\[(\d+)\].*:\s*(Healthy|Unhealthy)\s*$"
|
||||
).items()
|
||||
}
|
||||
throttle_status = _indexed_values(
|
||||
throttle, r"HCU\[(\d+)\].*Throttler Status:\s*(.+?)\s*$"
|
||||
)
|
||||
|
||||
driver_match = re.search(r"Driver Version:\s*(\S+)", driver or "")
|
||||
driver_version = driver_match.group(1) if driver_match else "N/A"
|
||||
|
||||
gpu_count = max(rows.keys(), default=-1) + 1
|
||||
gpus: List[dict] = []
|
||||
for index in range(gpu_count):
|
||||
row = rows.get(index, {"index": index})
|
||||
total_mb = int(memories.get(index, 0))
|
||||
used_mb = int(total_mb * int(row.get("vram_used_pct", 0)) / 100)
|
||||
gt_s, width = pcie.get(index, (0.0, 0))
|
||||
throttle_text = throttle_status.get(index, "")
|
||||
real_throttle = bool(
|
||||
throttle_text
|
||||
and any(token not in {"IDLE", "ACTIVE", "NONE"}
|
||||
for token in re.findall(r"[A-Z_]+", throttle_text.upper()))
|
||||
)
|
||||
gpus.append({
|
||||
**row,
|
||||
"name": names.get(index, "Hygon DCU"),
|
||||
"uuid": unique_ids.get(index, "N/A"),
|
||||
"pci_bus_id": buses.get(index, "N/A"),
|
||||
"pcie_link_gen": _pcie_gen(gt_s),
|
||||
"pcie_link_width": width,
|
||||
"pcie_gt_s": gt_s,
|
||||
"vram_total_mb": total_mb,
|
||||
"vram_used_mb": used_mb,
|
||||
"vram_free_mb": max(total_mb - used_mb, 0),
|
||||
"clock_sm": int(_number(sclk.get(index))),
|
||||
"clock_mem": int(_number(mclk.get(index))),
|
||||
"fan_speed": 0,
|
||||
"persistence_mode": True,
|
||||
"compute_mode": row.get("performance_level", "auto"),
|
||||
"serial_number": serials.get(index, "N/A"),
|
||||
"ecc_enabled": ecc_enabled.get(index, False),
|
||||
"ecc_errors_single": 0,
|
||||
"ecc_errors_double": 0,
|
||||
"xgmi_errors": xgmi_errors.get(index, 0),
|
||||
"pcie_replay_count": replay_counts.get(index, 0),
|
||||
"healthy": healthy.get(index, False),
|
||||
"throttling_active": real_throttle,
|
||||
"throttling_text": throttle_text,
|
||||
})
|
||||
|
||||
return {
|
||||
"driver_version": driver_version,
|
||||
"runtime_name": "DTK",
|
||||
"runtime_version": os.environ.get("DTK_VERSION", "N/A"),
|
||||
"gpu_count": gpu_count,
|
||||
"gpus": gpus,
|
||||
"topology": run_hy_smi("--showtopo") if include_topology else "",
|
||||
"vendor": "hygon",
|
||||
}
|
||||
@ -286,10 +286,18 @@ class MultiNodeNCCLTest:
|
||||
"--mca", "btl_openib_warn_no_device_params_found", "0",
|
||||
"--mca", "btl_tcp_if_include", str(self.cfg.get("socket_ifname", "bond0")),
|
||||
"--mca", "oob_tcp_if_include", str(self.cfg.get("oob_tcp_ifname", self.cfg.get("socket_ifname", "bond0"))),
|
||||
]
|
||||
# HCOLL does an eager UCX IB wireup during MPI_Init that deadlocks on nodes
|
||||
# where UCX only sees mlx5_2/mlx5_3 instead of mlx5_0/mlx5_1. HCOLL
|
||||
# collective optimisations are irrelevant for NCCL tests (NCCL handles its
|
||||
# own collectives), so disable it unconditionally unless the caller opts in.
|
||||
if self.cfg.get("disable_hcoll", True):
|
||||
cmd.extend(["--mca", "coll", "^hcoll"])
|
||||
cmd.extend([
|
||||
"-H", host_arg,
|
||||
"--map-by", f"ppr:{gpus_per_node}:node",
|
||||
"-np", str(ranks),
|
||||
]
|
||||
])
|
||||
plm_rsh_args = self.cfg.get("plm_rsh_args")
|
||||
if plm_rsh_args:
|
||||
cmd.extend(["--mca", "plm_rsh_args", str(plm_rsh_args)])
|
||||
|
||||
@ -19,6 +19,10 @@ from modules.gpu_specs import detect_gpu_type, get_gpu_specs, resolve_tools_dir
|
||||
TORCH_AVAILABLE = False
|
||||
try:
|
||||
import torch
|
||||
try:
|
||||
import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim
|
||||
except ImportError:
|
||||
pass
|
||||
if torch.cuda.is_available():
|
||||
TORCH_AVAILABLE = True
|
||||
except ImportError:
|
||||
@ -46,7 +50,8 @@ class NCCLTest:
|
||||
self.console = Console()
|
||||
self.nccl_cfg = config.get("nccl", {})
|
||||
self.tools_dir = resolve_tools_dir(config)
|
||||
self.gpu_type = detect_gpu_type()
|
||||
configured = config.get("gpu_type", "auto")
|
||||
self.gpu_type = configured if configured != "auto" else detect_gpu_type()
|
||||
self.specs = get_gpu_specs(self.gpu_type)
|
||||
|
||||
def _find_nccl_test(self, name: str) -> Optional[str]:
|
||||
@ -295,60 +300,78 @@ class NCCLTest:
|
||||
}
|
||||
|
||||
def _run_torchrun_fallback(self, gpu_count: int) -> dict:
|
||||
"""Basic NCCL connectivity test via torchrun — verifies NCCL works but does not benchmark performance."""
|
||||
self.console.print("[yellow]nccl-tests not available, running basic NCCL connectivity check[/yellow]")
|
||||
"""Benchmark collectives through PyTorch when nccl-tests is unavailable."""
|
||||
self.console.print("[yellow]nccl-tests not available, benchmarking collectives via torchrun[/yellow]")
|
||||
size_mb = int(self.nccl_cfg.get("torchrun_size_mb", 64))
|
||||
warmup = int(self.nccl_cfg.get("torchrun_warmup", 5))
|
||||
iterations = int(self.nccl_cfg.get("torchrun_iterations", 20))
|
||||
backend = self.nccl_cfg.get("torch_backend", "nccl")
|
||||
enabled_ops = []
|
||||
for key, enabled in [
|
||||
("allreduce", self.nccl_cfg.get("test_allreduce", True)),
|
||||
("alltoall", self.nccl_cfg.get("test_alltoall", True)),
|
||||
("broadcast", self.nccl_cfg.get("test_broadcast", True)),
|
||||
("reducescatter", self.nccl_cfg.get("test_reduce_scatter", False)),
|
||||
("allgather", self.nccl_cfg.get("test_allgather", False)),
|
||||
]:
|
||||
if enabled:
|
||||
enabled_ops.append(key)
|
||||
|
||||
code = f"""
|
||||
import torch, torch.distributed as dist, os
|
||||
import os, time
|
||||
import torch
|
||||
try:
|
||||
import torch_dcu
|
||||
except ImportError:
|
||||
pass
|
||||
import torch.distributed as dist
|
||||
os.environ.setdefault("MASTER_ADDR","127.0.0.1")
|
||||
os.environ.setdefault("MASTER_PORT","29500")
|
||||
rank=int(os.environ.get("LOCAL_RANK",0))
|
||||
ws={gpu_count}
|
||||
dist.init_process_group("nccl",rank=rank,world_size=ws)
|
||||
dist.init_process_group({backend!r},rank=rank,world_size=ws)
|
||||
torch.cuda.set_device(rank)
|
||||
size_bytes={size_mb}*1024*1024
|
||||
numel=size_bytes//4
|
||||
x=torch.ones(numel,device=f"cuda:{{rank}}",dtype=torch.float32)
|
||||
allgather_out=torch.empty(numel*ws,device=f"cuda:{{rank}}",dtype=torch.float32)
|
||||
rs_out=torch.empty(numel//ws,device=f"cuda:{{rank}}",dtype=torch.float32)
|
||||
a2a_out=torch.empty_like(x)
|
||||
|
||||
x=torch.randn(1024*1024,device=f"cuda:{{rank}}",dtype=torch.float32)
|
||||
|
||||
# Test AllReduce
|
||||
try:
|
||||
dist.all_reduce(x.clone())
|
||||
if rank==0: print("allreduce:ok")
|
||||
except Exception as e:
|
||||
if rank==0: print(f"allreduce:fail:{{e}}")
|
||||
|
||||
# Test Broadcast
|
||||
try:
|
||||
dist.broadcast(x.clone(),src=0)
|
||||
if rank==0: print("broadcast:ok")
|
||||
except Exception as e:
|
||||
if rank==0: print(f"broadcast:fail:{{e}}")
|
||||
|
||||
# Test AllGather
|
||||
try:
|
||||
tensor_list=[torch.empty_like(x) for _ in range(ws)]
|
||||
dist.all_gather(tensor_list,x.clone())
|
||||
if rank==0: print("allgather:ok")
|
||||
except Exception as e:
|
||||
if rank==0: print(f"allgather:fail:{{e}}")
|
||||
|
||||
# Test ReduceScatter
|
||||
try:
|
||||
chunks=list(x.chunk(ws))
|
||||
output=torch.empty_like(chunks[0])
|
||||
dist.reduce_scatter(output,chunks)
|
||||
if rank==0: print("reducescatter:ok")
|
||||
except Exception as e:
|
||||
if rank==0: print(f"reducescatter:fail:{{e}}")
|
||||
|
||||
# Test AllToAll
|
||||
try:
|
||||
chunks=list(x.chunk(ws))
|
||||
output_list=[torch.empty_like(c) for c in chunks]
|
||||
dist.all_to_all(output_list,chunks)
|
||||
if rank==0: print("alltoall:ok")
|
||||
except Exception as e:
|
||||
if rank==0: print(f"alltoall:fail:{{e}}")
|
||||
def run_op(name):
|
||||
if name == "allreduce":
|
||||
return dist.all_reduce(x)
|
||||
if name == "broadcast":
|
||||
return dist.broadcast(x, src=0)
|
||||
if name == "allgather":
|
||||
return dist.all_gather_into_tensor(allgather_out, x)
|
||||
if name == "reducescatter":
|
||||
return dist.reduce_scatter_tensor(rs_out, x)
|
||||
if name == "alltoall":
|
||||
return dist.all_to_all_single(a2a_out, x)
|
||||
raise ValueError(name)
|
||||
|
||||
for name in {enabled_ops!r}:
|
||||
try:
|
||||
for _ in range({warmup}):
|
||||
run_op(name)
|
||||
torch.cuda.synchronize()
|
||||
dist.barrier()
|
||||
start=time.perf_counter()
|
||||
for _ in range({iterations}):
|
||||
run_op(name)
|
||||
torch.cuda.synchronize()
|
||||
elapsed=(time.perf_counter()-start)/{iterations}
|
||||
algbw=size_bytes/elapsed/1e9
|
||||
factor={{"allreduce":2*(ws-1)/ws,"allgather":(ws-1)/ws,
|
||||
"reducescatter":(ws-1)/ws,"alltoall":(ws-1)/ws,
|
||||
"broadcast":1.0}}[name]
|
||||
busbw=algbw*factor
|
||||
if rank==0:
|
||||
print(f"METRIC|{{name}}|{{elapsed*1e6:.3f}}|{{algbw:.3f}}|{{busbw:.3f}}", flush=True)
|
||||
except Exception as exc:
|
||||
if rank==0:
|
||||
print(f"ERROR|{{name}}|{{exc}}", flush=True)
|
||||
dist.destroy_process_group()
|
||||
"""
|
||||
import tempfile
|
||||
@ -364,41 +387,50 @@ dist.destroy_process_group()
|
||||
|
||||
r = subprocess.run(
|
||||
[torchrun_cmd, f"--nproc_per_node={gpu_count}", tmp.name],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
env={**os.environ, "NCCL_DEBUG": "WARN"},
|
||||
capture_output=True, text=True, timeout=300,
|
||||
env={**os.environ, "NCCL_DEBUG": "WARN", "RCCL_DEBUG": "WARN"},
|
||||
)
|
||||
os.unlink(tmp.name)
|
||||
|
||||
# Parse connectivity results — format: op_name:ok or op_name:fail:error
|
||||
|
||||
tests = {}
|
||||
all_passed = True
|
||||
for line in r.stdout.split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split(":")
|
||||
op_name = parts[0]
|
||||
result = parts[1] if len(parts) > 1 else "unknown"
|
||||
|
||||
if result == "ok":
|
||||
status = "PASS"
|
||||
else:
|
||||
status = "FAIL"
|
||||
all_passed = False
|
||||
|
||||
tests[op_name] = {
|
||||
"status": status,
|
||||
"error": ":".join(parts[2:]) if len(parts) > 2 and result == "fail" else None,
|
||||
}
|
||||
|
||||
if line.startswith("METRIC|"):
|
||||
_, name, time_us, algbw, busbw = line.split("|", 4)
|
||||
threshold = self.nccl_cfg.get("min_bandwidth_gbps") or 10.0
|
||||
busbw_f = float(busbw)
|
||||
tests[name] = {
|
||||
"status": "PASS" if busbw_f >= float(threshold) else "WARN",
|
||||
"best_algbw_gbps": round(float(algbw), 2),
|
||||
"best_busbw_gbps": round(busbw_f, 2),
|
||||
"min_required_gbps": float(threshold),
|
||||
"by_size": [{
|
||||
"size": size_mb * 1024 * 1024,
|
||||
"time_us": round(float(time_us), 2),
|
||||
"algbw_gbps": round(float(algbw), 2),
|
||||
"busbw_gbps": round(busbw_f, 2),
|
||||
}],
|
||||
}
|
||||
elif line.startswith("ERROR|"):
|
||||
_, name, error = line.split("|", 2)
|
||||
tests[name] = {"status": "FAIL", "error": error}
|
||||
if not tests:
|
||||
error = (r.stderr or r.stdout or "torchrun produced no metrics")[-1000:]
|
||||
return {"passed": False, "source": "torchrun_collective", "error": error}
|
||||
all_passed = all(value.get("status") == "PASS" for value in tests.values())
|
||||
return {
|
||||
"passed": all_passed,
|
||||
"source": "torchrun_fallback",
|
||||
"source": "torchrun_collective",
|
||||
"tests": tests,
|
||||
"gpu_count": gpu_count,
|
||||
"message_size_mb": size_mb,
|
||||
"warmup": warmup,
|
||||
"iterations": iterations,
|
||||
"backend": backend,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
except Exception as e:
|
||||
return {"passed": False, "source": "torchrun_fallback", "error": str(e)}
|
||||
return {"passed": False, "source": "torchrun_collective", "error": str(e)}
|
||||
|
||||
@staticmethod
|
||||
def print_results(results: dict, console: Console = None):
|
||||
|
||||
@ -76,6 +76,21 @@ class RDMATest:
|
||||
active_ib_port = True
|
||||
|
||||
device_info = self._collect_device_info(devices)
|
||||
cross_enabled = (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled")
|
||||
if not ib_devices and cross_enabled:
|
||||
# RoCE ports report link_layer=Ethernet. They are valid for configured
|
||||
# cross-node perftest even though local InfiniBand loopback is skipped.
|
||||
cross = self.run_cross_node()
|
||||
return {
|
||||
"passed": cross.get("status") == "PASS",
|
||||
"status": cross.get("status", "FAIL"),
|
||||
"reason": "RoCE cross-node mode",
|
||||
"devices": device_info,
|
||||
"bandwidth_tests": [],
|
||||
"latency_tests": [],
|
||||
"cross_node": cross,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
if not ib_devices:
|
||||
self.console.print(
|
||||
"[yellow]No InfiniBand-link_layer ports present — "
|
||||
@ -266,13 +281,14 @@ class RDMATest:
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _active_ib_devices(self) -> List[str]:
|
||||
"""IB devices whose port 1 is InfiniBand link_layer and ACTIVE."""
|
||||
"""Active IB devices, optionally including Ethernet/RoCE devices."""
|
||||
out = []
|
||||
allow_roce = bool((self.rdma_cfg.get("cross_node", {}) or {}).get("allow_roce"))
|
||||
for dev in self._get_ib_devices():
|
||||
for port in self._get_ib_ports(dev):
|
||||
ll = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/link_layer")
|
||||
st = self._read_sys(f"/sys/class/infiniband/{dev}/ports/{port}/state")
|
||||
if ll == "InfiniBand" and "ACTIVE" in st.upper():
|
||||
if (ll == "InfiniBand" or allow_roce) and "ACTIVE" in st.upper():
|
||||
out.append(dev)
|
||||
break
|
||||
return out
|
||||
@ -296,6 +312,8 @@ class RDMATest:
|
||||
gid_index = cn.get("gid_index")
|
||||
msg_size = cn.get("msg_size", 1048576)
|
||||
iters = cn.get("iters", 5000)
|
||||
duration_sec = cn.get("duration_sec")
|
||||
queue_pairs = int(cn.get("queue_pairs", 1))
|
||||
base_port = cn.get("base_oob_port", 18515)
|
||||
warmup = cn.get("server_warmup_sec", 2.0)
|
||||
min_bw = cn.get("min_bandwidth_gbps", 350)
|
||||
@ -322,10 +340,16 @@ class RDMATest:
|
||||
entry = {"device": dev}
|
||||
|
||||
if has_bw:
|
||||
bw_extra = ["--report_gbits", "-s", str(msg_size),
|
||||
"-q", str(queue_pairs)]
|
||||
if duration_sec:
|
||||
bw_extra += ["-D", str(duration_sec)]
|
||||
else:
|
||||
bw_extra += ["-n", str(iters)]
|
||||
bw = self._cross_node_perftest(
|
||||
"ib_write_bw", dev, server_target, server_addr, ib_port,
|
||||
oob, gid_index, warmup,
|
||||
extra=["--report_gbits", "-s", str(msg_size), "-n", str(iters)],
|
||||
extra=bw_extra,
|
||||
parse="bw")
|
||||
entry["bandwidth_gbps"] = bw
|
||||
if isinstance(bw, (int, float)):
|
||||
|
||||
@ -100,6 +100,10 @@ class ReportGenerator:
|
||||
|
||||
if "gpu_info" in results:
|
||||
gpus = results["gpu_info"].get("gpus", [])
|
||||
runtime_name = results["gpu_info"].get("runtime_name", "CUDA")
|
||||
runtime_version = results["gpu_info"].get(
|
||||
"runtime_version", results["gpu_info"].get("cuda_version", "N/A")
|
||||
)
|
||||
rows = ""
|
||||
for g in gpus:
|
||||
rows += f"<tr><td>GPU {g['index']}</td><td>{g['name']}</td>"
|
||||
@ -109,7 +113,7 @@ class ReportGenerator:
|
||||
sections.append(
|
||||
f'<div class="section"><h2>GPU Information</h2>'
|
||||
f'<p>Driver: {results["gpu_info"].get("driver_version", "N/A")} | '
|
||||
f'CUDA: {results["gpu_info"].get("cuda_version", "N/A")} | '
|
||||
f'{runtime_name}: {runtime_version} | '
|
||||
f'Count: {len(gpus)}</p>'
|
||||
f'<table><tr><th>GPU</th><th>Model</th><th>VRAM</th><th>Temp</th><th>SM Clock</th></tr>'
|
||||
f'{rows}</table></div>'
|
||||
@ -194,8 +198,10 @@ class ReportGenerator:
|
||||
gpus = gpu_info.get("gpus", [])
|
||||
gpu_name = gpus[0]["name"] if gpus else "Unknown"
|
||||
lines.append(f"- **GPU:** {gpu_name} x{gpu_info.get('gpu_count', len(gpus))}")
|
||||
runtime_name = gpu_info.get("runtime_name", "CUDA")
|
||||
runtime_version = gpu_info.get("runtime_version", gpu_info.get("cuda_version", "N/A"))
|
||||
lines.append(f"- **Driver:** {gpu_info.get('driver_version', 'N/A')} | "
|
||||
f"**CUDA:** {gpu_info.get('cuda_version', 'N/A')}")
|
||||
f"**{runtime_name}:** {runtime_version}")
|
||||
lines.append("")
|
||||
|
||||
# --- Summary table ---
|
||||
@ -270,9 +276,16 @@ class ReportGenerator:
|
||||
if not d2d_peak:
|
||||
d2d_peak = mem_data.get("peak_bandwidth_gbps") or 0
|
||||
d2d_eff = mem_data.get("efficiency_pct") or 0
|
||||
lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {h2d_peak:.0f} GB/s | {h2d_eff:.1f}% |")
|
||||
lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {d2h_peak:.0f} GB/s | {d2h_eff:.1f}% |")
|
||||
lines.append(f"| D2D (NVLink) | {d2d:.1f} GB/s | {d2d_peak:.0f} GB/s | {d2d_eff:.1f}% |")
|
||||
def peak_text(value):
|
||||
return f"{value:.0f} GB/s" if value else "N/A"
|
||||
|
||||
def efficiency_text(value):
|
||||
return f"{value:.1f}%" if value else "N/A"
|
||||
|
||||
lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {peak_text(h2d_peak)} | {efficiency_text(h2d_eff)} |")
|
||||
lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {peak_text(d2h_peak)} | {efficiency_text(d2h_eff)} |")
|
||||
d2d_label = "D2D (HBM copy)" if mem_data.get("accelerator_vendor") == "hygon" else "D2D (NVLink)"
|
||||
lines.append(f"| {d2d_label} | {d2d:.1f} GB/s | {peak_text(d2d_peak)} | {efficiency_text(d2d_eff)} |")
|
||||
lines.append("")
|
||||
# PyTorch fallback can't accurately measure HBM peak (intra-GPU copy_()
|
||||
# only reaches ~20% of HBM bandwidth). When fallback is used, report
|
||||
@ -298,6 +311,10 @@ class ReportGenerator:
|
||||
# When present, override the legacy 80%-of-peak rule on a per-dtype basis.
|
||||
pass_thresholds = comp_data.get("pass_thresholds_tflops", {}) or {}
|
||||
use_abs = bool(pass_thresholds)
|
||||
has_peak_reference = any(
|
||||
isinstance(value, (int, float)) and value > 0
|
||||
for value in peak_tflops.values()
|
||||
)
|
||||
if use_abs:
|
||||
lines.append("| DType | Achieved (TFLOPS) | Peak | Threshold | Status |")
|
||||
else:
|
||||
@ -305,7 +322,7 @@ class ReportGenerator:
|
||||
lines.append("|-------|-------------------|------|------------|--------|")
|
||||
worst_eff = 100.0
|
||||
overall_status = "PASS"
|
||||
rank = {"PASS": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
|
||||
rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
|
||||
for dt, val in per_dtype.items():
|
||||
if isinstance(val, str):
|
||||
# skipped or error
|
||||
@ -324,17 +341,22 @@ class ReportGenerator:
|
||||
else:
|
||||
status = "FAIL"
|
||||
lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | >= {thr} | {status} |")
|
||||
else:
|
||||
elif has_peak_reference:
|
||||
status = "PASS" if ef >= 80 else ("WARN" if ef >= 50 else "FAIL")
|
||||
lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | {ef:.1f}% | {status} |")
|
||||
else:
|
||||
status = "MEASURED"
|
||||
lines.append(f"| {dt.upper()} | {val:.1f} | N/A | N/A | {status} |")
|
||||
if rank.get(status, 0) > rank.get(overall_status, 0):
|
||||
overall_status = status
|
||||
lines.append("")
|
||||
if use_abs:
|
||||
lines.append(f"**Verdict: {overall_status}** (absolute TFLOPS thresholds; worst efficiency {worst_eff:.1f}%)\n")
|
||||
else:
|
||||
elif has_peak_reference:
|
||||
overall_status = "PASS" if worst_eff >= 80 else ("WARN" if worst_eff >= 50 else "FAIL")
|
||||
lines.append(f"**Verdict: {overall_status}** (worst efficiency {worst_eff:.1f}%)\n")
|
||||
else:
|
||||
lines.append("**Verdict: MEASURED** (no published peak/acceptance threshold configured)\n")
|
||||
|
||||
# --- NCCL ---
|
||||
nccl = results.get("nccl")
|
||||
@ -369,6 +391,17 @@ class ReportGenerator:
|
||||
lines.append(f"- **Source:** {source}")
|
||||
lines.append(f"- **Duration:** {elapsed:.0f}s (requested {duration}s)")
|
||||
lines.append(f"- **Result: {'PASS' if passed else 'FAIL'}**")
|
||||
max_temp = stress.get("max_temperature_c", {}) or {}
|
||||
max_power = stress.get("max_power_w", {}) or {}
|
||||
if max_temp or max_power:
|
||||
lines.append("")
|
||||
lines.append("| GPU | Peak Temp | Peak Power |")
|
||||
lines.append("|-----|-----------|------------|")
|
||||
for gpu in sorted(set(max_temp) | set(max_power), key=lambda value: int(value)):
|
||||
lines.append(
|
||||
f"| {gpu} | {max_temp.get(gpu, 'N/A')} C | "
|
||||
f"{max_power.get(gpu, 'N/A')} W |"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# --- RDMA ---
|
||||
@ -519,7 +552,8 @@ class ReportGenerator:
|
||||
verdict = "PASS" if worst >= 80 else ("WARN" if worst >= 50 else "FAIL")
|
||||
items.append(("Compute Throughput", f"{verdict} (worst {worst:.1f}%)"))
|
||||
else:
|
||||
items.append(("Compute Throughput", "N/A"))
|
||||
measured = [v for v in per_dtype.values() if isinstance(v, (int, float))]
|
||||
items.append(("Compute Throughput", "MEASURED (no published threshold)" if measured else "N/A"))
|
||||
|
||||
# NCCL
|
||||
if "nccl" in results:
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
@ -13,6 +14,7 @@ from rich.live import Live
|
||||
from rich.text import Text
|
||||
|
||||
from modules.gpu_specs import resolve_tools_dir
|
||||
from modules.hygon_smi import run_hy_smi
|
||||
|
||||
|
||||
class StressTest:
|
||||
@ -56,12 +58,12 @@ class StressTest:
|
||||
if not result.get("passed") and result.get("elapsed_sec", 0) < duration_sec * 0.5:
|
||||
self.console.print("\n[yellow]gpu-burn exited early (possible OOM), switching to PyTorch stress test[/yellow]")
|
||||
self.console.print("[dim]PyTorch mode dynamically adapts to available memory[/dim]\n")
|
||||
return self._run_pytorch_stress(duration_sec, memory_pct)
|
||||
return self._run_pytorch_stress(duration_sec, memory_pct, use_tensor_cores)
|
||||
|
||||
return result
|
||||
|
||||
self.console.print("[yellow]gpu_burn not found, using PyTorch stress test[/yellow]")
|
||||
return self._run_pytorch_stress(duration_sec, memory_pct)
|
||||
return self._run_pytorch_stress(duration_sec, memory_pct, use_tensor_cores)
|
||||
|
||||
def _run_gpu_burn(self, gpu_burn: str, duration: int,
|
||||
doubles: bool, tensor_cores: bool, target_gpus: str) -> dict:
|
||||
@ -116,9 +118,26 @@ class StressTest:
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
def _run_pytorch_stress(self, duration: int, memory_pct: int = 90) -> dict:
|
||||
@staticmethod
|
||||
def _hygon_snapshot() -> dict:
|
||||
output = run_hy_smi()
|
||||
temps, powers = {}, {}
|
||||
if output:
|
||||
pattern = re.compile(r"^\s*(\d+)\s+([\d.]+)C\s+([\d.]+)W", re.MULTILINE)
|
||||
for match in pattern.finditer(output):
|
||||
idx = int(match.group(1))
|
||||
temps[idx] = float(match.group(2))
|
||||
powers[idx] = float(match.group(3))
|
||||
return {"temperatures_c": temps, "power_w": powers}
|
||||
|
||||
def _run_pytorch_stress(self, duration: int, memory_pct: int = 90,
|
||||
tensor_cores: bool = True) -> dict:
|
||||
try:
|
||||
import torch
|
||||
try:
|
||||
import torch_dcu # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
if not torch.cuda.is_available():
|
||||
return {"error": "pytorch_not_available"}
|
||||
except ImportError:
|
||||
@ -129,9 +148,10 @@ class StressTest:
|
||||
|
||||
gpu_status = {}
|
||||
t0 = time.time()
|
||||
snapshots = [self._hygon_snapshot()]
|
||||
|
||||
try:
|
||||
tensors = {}
|
||||
work_a, work_b, outputs, ballast = {}, {}, {}, {}
|
||||
for i in range(gpu_count):
|
||||
with torch.cuda.device(i):
|
||||
# Get actual free memory (accounting for other processes)
|
||||
@ -143,16 +163,23 @@ class StressTest:
|
||||
# Cap at actual free memory with 5% safety margin
|
||||
alloc_bytes = min(target_mem, int(free_mem * 0.95))
|
||||
|
||||
# matmul(A, A.T) needs 2x input memory (input + output)
|
||||
mem_side = int((alloc_bytes / 4 / 2) ** 0.5)
|
||||
# Cap compute matrix so a single matmul completes in ~2s on H100/H200
|
||||
# (FP32 ≈ 67 TFLOPS → 2*4096³/67e12 ≈ 2s). Without this cap, a 141GB
|
||||
# HBM yields side ≈ 131K → single matmul ~68s × 8 GPUs serial → loop
|
||||
# overshoots a 60s duration request by 10×+.
|
||||
MAX_COMPUTE_SIDE = 4096
|
||||
side = min(mem_side, MAX_COMPUTE_SIDE)
|
||||
dtype = torch.bfloat16 if tensor_cores else torch.float32
|
||||
elem_size = torch.tensor([], dtype=dtype).element_size()
|
||||
mem_side = int((alloc_bytes / elem_size / 3) ** 0.5)
|
||||
max_compute_side = 8192 if tensor_cores else 4096
|
||||
side = min(mem_side, max_compute_side)
|
||||
compute_bytes = side * side * elem_size * 3
|
||||
ballast_target = max(0, alloc_bytes - compute_bytes - 1024**3)
|
||||
ballast[i] = []
|
||||
chunk_size = 1024**3
|
||||
while ballast_target > 0:
|
||||
this_chunk = min(chunk_size, ballast_target)
|
||||
tensor = torch.empty(this_chunk, device=f"cuda:{i}", dtype=torch.uint8)
|
||||
tensor.fill_(0x5A)
|
||||
ballast[i].append(tensor)
|
||||
ballast_target -= this_chunk
|
||||
|
||||
actual_mem_mb = side * side * 4 / 1024 / 1024
|
||||
actual_mem_mb = (alloc_bytes - max(0, ballast_target)) / 1024 / 1024
|
||||
total_mem_mb = total_mem / 1024 / 1024
|
||||
free_mem_mb = free_mem / 1024 / 1024
|
||||
|
||||
@ -161,7 +188,9 @@ class StressTest:
|
||||
f"alloc {actual_mem_mb:.0f}MB ({actual_mem_mb/total_mem_mb*100:.0f}%) - "
|
||||
f"matrix {side}x{side}[/dim]"
|
||||
)
|
||||
tensors[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=torch.float32)
|
||||
work_a[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=dtype)
|
||||
work_b[i] = torch.randn(side, side, device=f"cuda:{i}", dtype=dtype)
|
||||
outputs[i] = torch.empty(side, side, device=f"cuda:{i}", dtype=dtype)
|
||||
|
||||
self.console.print(f"\n[cyan]Starting stress test for {duration} seconds...[/cyan]")
|
||||
|
||||
@ -171,7 +200,7 @@ class StressTest:
|
||||
# GPUs, otherwise the 8 GPUs run serially and overshoot the duration.
|
||||
for i in range(gpu_count):
|
||||
with torch.cuda.device(i):
|
||||
tensors[i] = torch.matmul(tensors[i], tensors[i].T)
|
||||
outputs[i] = torch.matmul(work_a[i], work_b[i])
|
||||
# Single sync per pass — waits for all 8 streams concurrently
|
||||
for i in range(gpu_count):
|
||||
with torch.cuda.device(i):
|
||||
@ -181,6 +210,7 @@ class StressTest:
|
||||
current_elapsed = time.time() - t0
|
||||
if int(current_elapsed) != int(elapsed_check) and int(current_elapsed) % 10 == 0:
|
||||
self.console.print(f" [dim]Running {int(current_elapsed)}s / {duration}s[/dim]")
|
||||
snapshots.append(self._hygon_snapshot())
|
||||
elapsed_check = current_elapsed
|
||||
|
||||
for i in range(gpu_count):
|
||||
@ -200,16 +230,28 @@ class StressTest:
|
||||
"gpu_status": gpu_status,
|
||||
}
|
||||
finally:
|
||||
tensors.clear()
|
||||
for mapping in (work_a, work_b, outputs, ballast):
|
||||
mapping.clear()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
elapsed = round(time.time() - t0, 1)
|
||||
snapshots.append(self._hygon_snapshot())
|
||||
max_temp = {}
|
||||
max_power = {}
|
||||
for snapshot in snapshots:
|
||||
for idx, value in snapshot.get("temperatures_c", {}).items():
|
||||
max_temp[idx] = max(max_temp.get(idx, value), value)
|
||||
for idx, value in snapshot.get("power_w", {}).items():
|
||||
max_power[idx] = max(max_power.get(idx, value), value)
|
||||
return {
|
||||
"source": "pytorch",
|
||||
"passed": True,
|
||||
"duration_sec": duration,
|
||||
"elapsed_sec": elapsed,
|
||||
"gpu_status": gpu_status,
|
||||
"memory_pct": memory_pct,
|
||||
"max_temperature_c": max_temp,
|
||||
"max_power_w": max_power,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
@ -245,5 +287,15 @@ class StressTest:
|
||||
color = "green" if status == "PASS" else "red"
|
||||
c.print(f" GPU {gid}: [{color}]{status}[/{color}]")
|
||||
|
||||
max_temp = results.get("max_temperature_c", {})
|
||||
max_power = results.get("max_power_w", {})
|
||||
if max_temp or max_power:
|
||||
c.print("\n Peak telemetry:")
|
||||
for gid in sorted(set(max_temp) | set(max_power)):
|
||||
c.print(
|
||||
f" GPU {gid}: {max_temp.get(gid, 'N/A')} C, "
|
||||
f"{max_power.get(gid, 'N/A')} W"
|
||||
)
|
||||
|
||||
if results.get("error"):
|
||||
c.print(f" [red]Error: {results['error']}[/red]")
|
||||
|
||||
@ -13,6 +13,10 @@ from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeEl
|
||||
TORCH_AVAILABLE = False
|
||||
try:
|
||||
import torch
|
||||
try:
|
||||
import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim
|
||||
except ImportError:
|
||||
pass
|
||||
if torch.cuda.is_available():
|
||||
TORCH_AVAILABLE = True
|
||||
except ImportError:
|
||||
@ -49,6 +53,9 @@ class TrainingSim:
|
||||
self.console.print(f" Model: {model_name} | Batch: {batch_size} | Seq: {seq_length} | "
|
||||
f"DType: {dtype_str} | Steps: {num_steps} | GPUs: {gpu_count}")
|
||||
|
||||
if model_name == "synthetic":
|
||||
return self._run_synthetic(gpu_count, batch_size, seq_length, num_steps, dtype)
|
||||
|
||||
try:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
except ImportError:
|
||||
@ -145,10 +152,10 @@ class TrainingSim:
|
||||
def _run_synthetic(self, gpu_count, batch_size, seq_length, num_steps, dtype) -> dict:
|
||||
self.console.print(" Running synthetic training benchmark...")
|
||||
|
||||
hidden_size = 4096
|
||||
num_layers = 6
|
||||
num_heads = 32
|
||||
vocab_size = 32000
|
||||
hidden_size = int(self.train_cfg.get("hidden_size", 4096))
|
||||
num_layers = int(self.train_cfg.get("num_layers", 6))
|
||||
num_heads = int(self.train_cfg.get("num_heads", 32))
|
||||
vocab_size = int(self.train_cfg.get("vocab_size", 32000))
|
||||
|
||||
class SyntheticTransformer(torch.nn.Module):
|
||||
def __init__(self):
|
||||
|
||||
46
scripts/bw1000_smoke.py
Normal file
46
scripts/bw1000_smoke.py
Normal file
@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Minimal DTK/PyTorch smoke test used before the full BW1000 suite."""
|
||||
|
||||
import json
|
||||
|
||||
import torch
|
||||
|
||||
try:
|
||||
import torch_dcu # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def main() -> None:
|
||||
count = torch.cuda.device_count()
|
||||
if not torch.cuda.is_available() or count < 1:
|
||||
raise SystemExit("DTK PyTorch reports no available accelerator")
|
||||
|
||||
devices = []
|
||||
for index in range(count):
|
||||
props = torch.cuda.get_device_properties(index)
|
||||
devices.append({
|
||||
"index": index,
|
||||
"name": torch.cuda.get_device_name(index),
|
||||
"memory_gib": round(props.total_memory / 1024**3, 2),
|
||||
})
|
||||
|
||||
a = torch.randn(2048, 2048, device="cuda:0", dtype=torch.bfloat16)
|
||||
b = torch.randn(2048, 2048, device="cuda:0", dtype=torch.bfloat16)
|
||||
c = a @ b
|
||||
torch.cuda.synchronize()
|
||||
if not torch.isfinite(c).all().item():
|
||||
raise SystemExit("BF16 matmul produced non-finite output")
|
||||
|
||||
print(json.dumps({
|
||||
"torch_version": torch.__version__,
|
||||
"cuda_api_available": torch.cuda.is_available(),
|
||||
"device_count": count,
|
||||
"devices": devices,
|
||||
"bf16_matmul": "PASS",
|
||||
"checksum": round(float(c.float().mean().item()), 6),
|
||||
}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user