fix: improve B300 acceptance test logic

This commit is contained in:
zulifeng 2026-08-28 23:38:31 +08:00
parent 6ea2acf2d6
commit 6caeeb4400
12 changed files with 683 additions and 149 deletions

View File

@ -0,0 +1,62 @@
gpu_type: b300
tools:
install_dir: /root/test_gpu_scripts/.gpu-test-env/tools
benchmark:
memory:
size_mb: 4096
iterations: 10
nvbandwidth_buffer_mb: 512
nvbandwidth_samples: 3
compute:
dtypes: [fp32, tf32, fp16, bf16, fp8]
sweep_sizes:
- 3584
- 4608
- 5376
- 8192
- 11520
- [2048, 2048, 13312]
- [2048, 2048, 16384]
matrix_size: 8192
warmup: 20
iterations: 80
health:
temp_warning: 75
temp_critical: 85
power_limit: null
nccl:
min_bandwidth_gbps: null
test_allreduce: true
test_alltoall: true
test_broadcast: true
test_reduce_scatter: false
test_allgather: false
test_sendrecv: false
stress:
duration_sec: 600
use_doubles: false
use_tensor_cores: true
memory_pct: 90
gpus: all
rdma:
min_bandwidth_gbps: 50
max_latency_us: 10
cross_node:
enabled: false
training:
model: synthetic
batch_size: 8
seq_length: 2048
num_steps: 50
dtype: bf16
report:
output_dir: /root/test_gpu_scripts/reports/b300_single_node_20260827
format: json

View File

@ -21,7 +21,8 @@ from rich import box
from modules.gpu_info import GPUInfo
from modules.health_check import HealthCheck
from modules.benchmark import Benchmark
from modules.nccl_test import NCCLTest, MultiNodeNCCLTest
from modules.nccl_test import NCCLTest
from modules.multinode_nccl_test import MultiNodeNCCLTest
from modules.training_sim import TrainingSim
from modules.stress_test import StressTest
from modules.rdma_test import RDMATest
@ -333,14 +334,39 @@ def _run_full_suite(config: dict, console: Console) -> dict:
console.print(f"[bold red]{name} FAILED: {e}[/bold red]")
all_results[key] = {"error": str(e)}
# Summary
# Summary: classify actual verdicts. The old implementation counted every
# result without an exception as PASS, even when passed=False or status=FAIL.
console.print("\n" + "=" * 60)
# Only count test results, exclude metadata like timestamp
test_results = {k: v for k, v in all_results.items() if k != "timestamp"}
passed = sum(1 for v in test_results.values() if not isinstance(v, dict) or "error" not in v)
total = len(test_results)
color = "green" if passed == total else ("yellow" if passed > 0 else "red")
console.print(f"[bold {color}]Suite complete: {passed}/{total} tests passed[/bold {color}]")
statuses = {}
for key, value in test_results.items():
status = "PASS"
if isinstance(value, dict) and value.get("error"):
status = "FAIL"
elif key == "health":
status = "PASS" if value.get("passed") else "FAIL"
elif key == "memory_bench":
mem = value.get("memory", value)
if mem.get("error"):
status = "FAIL"
else:
eff = mem.get("d2d_efficiency_pct") or mem.get("efficiency_pct") or 0
status = "PASS" if eff >= 80 else ("WARN" if eff >= 60 else "FAIL")
elif key == "compute_bench":
status = Benchmark.judge_compute(value)["verdict"]
elif key in ("nccl", "rdma"):
status = value.get("status") or ("PASS" if value.get("passed") else "FAIL")
elif key == "stress":
status = "PASS" if value.get("passed") else "FAIL"
elif key == "training":
status = value.get("performance_verdict", "MEASURED")
statuses[key] = status
counts = {name: list(statuses.values()).count(name) for name in ("PASS", "WARN", "FAIL", "SKIP", "MEASURED")}
color = "red" if counts["FAIL"] else ("yellow" if counts["WARN"] else "green")
summary = ", ".join(f"{name} {count}" for name, count in counts.items() if count)
console.print(f"[bold {color}]Suite complete: {summary}[/bold {color}]")
all_results["suite_status"] = statuses
return all_results
@ -446,11 +472,14 @@ Examples:
if args.report:
ReportGenerator(config).generate(results, fmt=args.format, output=args.output)
has_errors = any("error" in v for v in results.values() if isinstance(v, dict))
sys.exit(1 if has_errors else 0)
has_failures = any(
status == "FAIL" for status in results.get("suite_status", {}).values()
)
sys.exit(1 if has_errors or has_failures else 0)
else:
result = _run_test(test_map[args.test], config, console)
if args.report and result:
ReportGenerator(config).generate({args.test: result, "timestamp": datetime.now().isoformat()},
ReportGenerator(config).generate({test_map[args.test]: result, "timestamp": datetime.now().isoformat()},
fmt=args.format, output=args.output)

View File

@ -150,11 +150,14 @@ class Benchmark:
progress.advance(task)
# Compare like with like: the 900 GB/s reference is per direction, so do
# not select the bidirectional SM result as the acceptance value. Keep it
# as a separate informational metric.
d2d_bw = max(
results_by_test.get("d2d_write", 0),
results_by_test.get("d2d_read", 0),
results_by_test.get("d2d_bidir", 0),
)
d2d_bidir_bw = results_by_test.get("d2d_bidir", 0)
h2d_bw = results_by_test.get("h2d", 0)
d2h_bw = results_by_test.get("d2h", 0)
@ -185,6 +188,7 @@ class Benchmark:
"h2d_bandwidth_gbps": round(h2d_bw, 1),
"d2h_bandwidth_gbps": round(d2h_bw, 1),
"d2d_bandwidth_gbps": round(d2d_bw, 1),
"d2d_bidirectional_gbps": round(d2d_bidir_bw, 1),
"h2d_peak_gbps": pcie_peak if pcie_peak else None,
"d2h_peak_gbps": pcie_peak if pcie_peak else None,
"d2d_peak_gbps": round(d2d_peak, 1) if d2d_peak else None,
@ -195,6 +199,7 @@ class Benchmark:
"efficiency_pct": d2d_efficiency,
"results_by_test": results_by_test,
"per_gpu": [],
"scope_note": "GPU peer/PCIe transfer bandwidth; local HBM bandwidth not measured",
"accelerator_vendor": self.specs.get("vendor", "nvidia"),
}
}
@ -385,11 +390,13 @@ class Benchmark:
BarColumn(), TextColumn("{task.completed}/{task.total}"),
TimeElapsedColumn(), console=self.console,
) as progress:
task = progress.add_task("Testing dtypes...", total=len(configured_dtypes))
task = progress.add_task(
"Testing dtypes on every GPU...", total=len(configured_dtypes) * gpu_count
)
for dtype_name in configured_dtypes:
if dtype_name not in dtype_map:
progress.advance(task)
progress.advance(task, advance=gpu_count)
continue
# Skip FP8 if GPU architecture doesn't support it
@ -399,7 +406,7 @@ class Benchmark:
arch = self.specs.get("architecture", "unknown")
results_by_dtype["fp8"] = f"skipped ({arch} does not support FP8)"
self.console.print(f"[dim] fp8: skipped - {arch} architecture has no FP8 support[/dim]")
progress.advance(task)
progress.advance(task, advance=gpu_count)
continue
dtype_val, peak_tflops = dtype_map[dtype_name]
@ -413,32 +420,48 @@ class Benchmark:
elif dtype_name == "fp32":
torch.backends.cuda.matmul.allow_tf32 = False
best_tflops, best_shape, last_err = 0.0, None, None
for (M, N, K) in shapes:
try:
t = self._bench_matmul_once(dtype_name, dtype_val, M, N, K, warmup, iterations)
if t > best_tflops:
best_tflops, best_shape = t, (M, N, K)
except Exception as e: # noqa: BLE001 - record and try the next shape
last_err = e
gpu_values = []
gpu_shapes = {}
last_err = None
fastest = (0.0, None)
for gpu_idx in range(gpu_count):
best_tflops, best_shape = 0.0, None
for (M, N, K) in shapes:
try:
t = self._bench_matmul_once(
dtype_name, dtype_val, M, N, K,
warmup, iterations, gpu_idx,
)
if t > best_tflops:
best_tflops, best_shape = t, (M, N, K)
except Exception as e: # noqa: BLE001 - record and try next shape
last_err = e
if best_shape is not None:
value = round(best_tflops, 1)
per_gpu_results[gpu_idx][dtype_name] = value
gpu_values.append(value)
gpu_shapes[str(gpu_idx)] = "x".join(str(d) for d in best_shape)
if best_tflops > fastest[0]:
fastest = (best_tflops, best_shape)
else:
per_gpu_results[gpu_idx][dtype_name] = f"error: {last_err}"
progress.advance(task)
torch.backends.cuda.matmul.allow_tf32 = old_tf32
if best_shape is None:
if not gpu_values:
results_by_dtype[dtype_name] = f"error: {last_err}"
self.console.print(f"[yellow] {dtype_name}: {last_err}[/yellow]")
else:
shape_str = "x".join(str(d) for d in best_shape)
results_by_dtype[dtype_name] = round(best_tflops, 1)
best_shapes[dtype_name] = shape_str
for pg in per_gpu_results:
pg[dtype_name] = round(best_tflops, 1)
avg_value = round(sum(gpu_values) / len(gpu_values), 1)
results_by_dtype[dtype_name] = avg_value
best_shapes[dtype_name] = "x".join(str(d) for d in fastest[1])
if len(shapes) > 1:
self.console.print(
f"[dim] {dtype_name}: {best_tflops:.1f} TFLOPS @ {shape_str}[/dim]"
f"[dim] {dtype_name}: avg {avg_value:.1f}, "
f"min {min(gpu_values):.1f}, max {max(gpu_values):.1f} TFLOPS[/dim]"
)
progress.advance(task)
best_shapes.setdefault("per_gpu", {})[dtype_name] = gpu_shapes
efficiency = {}
for dt, achieved in results_by_dtype.items():
@ -447,6 +470,18 @@ class Benchmark:
if peak_tp:
efficiency[dt] = round((achieved / peak_tp) * 100, 1)
worst_by_dtype = {}
best_by_dtype = {}
spread_by_dtype = {}
for dt in results_by_dtype:
values = [pg.get(dt) for pg in per_gpu_results]
values = [v for v in values if isinstance(v, (int, float))]
if values:
worst_by_dtype[dt] = min(values)
best_by_dtype[dt] = max(values)
avg = sum(values) / len(values)
spread_by_dtype[dt] = round((max(values) - min(values)) / avg * 100, 2) if avg else 0
return {
"compute": {
"per_dtype_tflops": results_by_dtype,
@ -459,6 +494,9 @@ class Benchmark:
self.specs.get("compute_pass_thresholds_tflops") or {}
),
"per_gpu": per_gpu_results,
"worst_per_dtype_tflops": worst_by_dtype,
"best_per_dtype_tflops": best_by_dtype,
"spread_pct": spread_by_dtype,
"sweep_sizes": list(sweep_sizes),
"best_shapes": best_shapes,
"matrix_size": matrix_size,
@ -469,7 +507,7 @@ class Benchmark:
}
def _bench_matmul_once(self, dtype_name: str, dtype_val, M: int, N: int, K: int,
warmup: int, iterations: int) -> float:
warmup: int, iterations: int, gpu_index: int = 0) -> float:
"""Time one (M×K)·(K×N) matmul for a dtype and return achieved TFLOPS.
Uses an L2-cache-busting pool of matrix pairs (total > 256 MB) so operands
@ -478,34 +516,37 @@ class Benchmark:
cuBLAS, which on H100 beats torch.compile/Triton for plain GEMM and avoids the
per-shape recompile cost that would make a sweep pathologically slow.
"""
elem_bytes = 1 if dtype_name == "fp8" else torch.tensor([], dtype=dtype_val).element_size()
device = f"cuda:{gpu_index}"
with torch.cuda.device(gpu_index):
elem_bytes = 1 if dtype_name == "fp8" else torch.tensor([], dtype=dtype_val).element_size()
pair_bytes = (M * K + K * N) * elem_bytes
num_pools = max(4, -(-256 * 1024 * 1024 // pair_bytes)) # ceil(256MB / pair)
if dtype_name == "fp8":
if not hasattr(torch, "_scaled_mm"):
raise RuntimeError("torch._scaled_mm unavailable — upgrade to PyTorch >= 2.1")
pools_a = [torch.randn(M, K, device="cuda", dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)]
pools_b = [torch.randn(N, K, device="cuda", dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)]
scale_a = torch.tensor(1.0, device="cuda")
scale_b = torch.tensor(1.0, device="cuda")
pools_a = [torch.randn(M, K, device=device, dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)]
pools_b = [torch.randn(N, K, device=device, dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)]
scale_a = torch.tensor(1.0, device=device)
scale_b = torch.tensor(1.0, device=device)
def op(i):
return torch._scaled_mm(pools_a[i], pools_b[i].T, scale_a=scale_a, scale_b=scale_b, out_dtype=torch.bfloat16)
else:
pools_a = [torch.randn(M, K, device="cuda", dtype=dtype_val) for _ in range(num_pools)]
pools_b = [torch.randn(K, N, device="cuda", dtype=dtype_val) for _ in range(num_pools)]
pools_a = [torch.randn(M, K, device=device, dtype=dtype_val) for _ in range(num_pools)]
pools_b = [torch.randn(K, N, device=device, dtype=dtype_val) for _ in range(num_pools)]
def op(i):
return torch.matmul(pools_a[i], pools_b[i])
try:
torch.cuda.set_device(gpu_index)
# Probe once so a broken/unsupported kernel raises before the timed loop.
_probe = op(0)
torch.cuda.synchronize()
torch.cuda.synchronize(gpu_index)
del _probe
for i in range(warmup):
op(i % num_pools)
torch.cuda.synchronize()
torch.cuda.synchronize(gpu_index)
start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True)
@ -513,7 +554,7 @@ class Benchmark:
for i in range(iterations):
op(i % num_pools)
end_event.record()
torch.cuda.synchronize()
torch.cuda.synchronize(gpu_index)
elapsed_ms = start_event.elapsed_time(end_event)
finally:
del pools_a, pools_b
@ -584,18 +625,24 @@ class Benchmark:
table = Table(box=None, padding=(0, 1))
table.add_column("DType", style="bold")
table.add_column("Achieved (TFLOPS)", justify="right")
table.add_column("Avg", justify="right")
table.add_column("Min", justify="right")
table.add_column("Max", justify="right")
table.add_column("Spread", justify="right")
table.add_column("Peak", justify="right")
table.add_column("Efficiency", justify="right")
peak = comp.get("peak_tflops", {})
per_dtype = comp.get("per_dtype_tflops", {})
eff = comp.get("efficiency_pct", {})
worst = comp.get("worst_per_dtype_tflops", {})
best = comp.get("best_per_dtype_tflops", {})
spread = comp.get("spread_pct", {})
for dt in per_dtype:
achieved = per_dtype[dt]
if isinstance(achieved, str):
table.add_row(dt, f"[red]{achieved}[/red]", str(peak.get(dt, "N/A")), "N/A")
table.add_row(dt, f"[red]{achieved}[/red]", "-", "-", "-", str(peak.get(dt, "N/A")), "N/A")
continue
pk = peak.get(dt, 0)
ef = eff.get(dt, 0)
@ -606,7 +653,11 @@ class Benchmark:
else:
peak_text = "N/A"
eff_text = "[cyan]MEASURED[/cyan]"
table.add_row(dt.upper(), f"{achieved:.1f}", peak_text, eff_text)
table.add_row(
dt.upper(), f"{achieved:.1f}", f"{worst.get(dt, achieved):.1f}",
f"{best.get(dt, achieved):.1f}", f"{spread.get(dt, 0):.2f}%",
peak_text, eff_text,
)
c.print(table)
@staticmethod
@ -623,6 +674,7 @@ class Benchmark:
"""
comp = results.get("compute", results)
per_dtype = comp.get("per_dtype_tflops", {})
worst_per_dtype = comp.get("worst_per_dtype_tflops", {}) or {}
thresholds = comp.get("pass_thresholds_tflops", {}) or {}
eff = comp.get("efficiency_pct", {})
rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
@ -630,11 +682,12 @@ class Benchmark:
evaluated = False
measured = False
for dt, val in per_dtype.items():
judged_val = worst_per_dtype.get(dt, val)
thr = thresholds.get(dt)
if isinstance(val, str):
status = "SKIP"
elif thr:
status = "PASS" if val >= thr else ("WARN" if val >= thr * 0.9 else "FAIL")
status = "PASS" if judged_val >= thr else ("WARN" if judged_val >= thr * 0.9 else "FAIL")
evaluated = True
elif dt not in eff:
status = "MEASURED"
@ -643,7 +696,7 @@ class Benchmark:
e = eff.get(dt, 0)
status = "PASS" if e >= 80 else ("WARN" if e >= 50 else "FAIL")
evaluated = True
rows.append((dt, val, thr, status))
rows.append((dt, judged_val, thr, status))
if rank[status] > rank[verdict]:
verdict = status
if measured and not evaluated:

View File

@ -16,6 +16,7 @@ GPU_NAME_PATTERNS = {
"H200": "h200",
"H20": "h20", # H20 / H20-3e is the China-compliance export variant, REDUCED peaks
"B200": "b200",
"B30-E": "b300", # nvidia-smi product name used by the B300 systems under test
"B300": "b300",
}
@ -228,15 +229,33 @@ GPU_SPECS = {
"hbm_capacity_gb": 288,
"hbm_type": "HBM3e",
"memory_bandwidth_gbps": 8000, # GB/s (8 TB/s)
"fp32_tflops": 125,
"tf32_tflops": 1750, # dense (estimated)
"fp16_tflops": 3500, # dense
"bf16_tflops": 3500, # dense
"fp8_tflops": 7000, # dense
# Dense, per-GPU reference peaks. NVIDIA publishes DGX B300 FP8 as
# 72 PFLOPS sparse for eight GPUs and explicitly states dense is half,
# i.e. 4.5 PFLOPS/GPU. The previous 7 PFLOPS dense value mixed product
# variants/modes and made healthy B300 cards fail by construction.
"fp32_tflops": 75,
"tf32_tflops": 1100,
"fp16_tflops": 2250,
"bf16_tflops": 2250,
"fp8_tflops": 4500,
# Acceptance floors are achievable eager-PyTorch GEMM values rather than
# a blanket percentage of marketing peak. BF16/FP8 are calibrated against
# NVIDIA Transformer Engine's public B300 GEMM examples; the remaining
# modes use conservative floors below sustained measurements on this path.
"compute_pass_thresholds_tflops": {
"fp32": 60, "tf32": 750, "fp16": 1450, "bf16": 1450, "fp8": 2800,
},
# Single-node 8-GPU floors derived from public B300 nccl-tests results.
# These are operation-specific because collectives do not reach the same
# fraction of raw NVLink bandwidth.
"nccl_pass_thresholds_gbps": {
"allreduce": 750, "alltoall": 620, "broadcast": 600,
"allgather": 700, "reducescatter": 700, "sendrecv": 600,
},
"tdp_watts": 1200,
"nvlink_gen": 5,
"nvlink_bandwidth_gbps": 1800,
"pcie_gen": 5,
"pcie_gen": 6,
"min_driver_version": "550",
"min_cuda_version": "12.4",
},

View File

@ -3,6 +3,7 @@
import json
import os
import re
import shlex
import shutil
import subprocess
from datetime import datetime
@ -210,9 +211,11 @@ class MultiNodeNCCLTest:
"-o", "BatchMode=yes",
"-o", "ConnectTimeout=5",
"-o", "StrictHostKeyChecking=accept-new",
target,
"hostname",
]
plm_rsh_args = self.cfg.get("plm_rsh_args")
if plm_rsh_args:
cmd.extend(shlex.split(str(plm_rsh_args)))
cmd.extend([target, "hostname"])
try:
r = subprocess.run(cmd, capture_output=True, text=True, timeout=8, env=self._runtime_env())
detail = r.stdout.strip() or r.stderr.strip()[:120]
@ -287,6 +290,15 @@ class MultiNodeNCCLTest:
"--mca", "btl_tcp_if_include", str(self.cfg.get("socket_ifname", "bond0")),
"--mca", "oob_tcp_if_include", str(self.cfg.get("oob_tcp_ifname", self.cfg.get("socket_ifname", "bond0"))),
]
# Keep the MPI launcher/control plane separately configurable from the
# NCCL data plane. For example, PML/UCX can be restricted to TCP/eth0
# while NCCL uses the configured RoCE HCAs for benchmark traffic.
mpi_pml = self.cfg.get("mpi_pml", "ob1")
if mpi_pml:
cmd.extend(["--mca", "pml", str(mpi_pml)])
mpi_btl = self.cfg.get("mpi_btl", "self,tcp,vader")
if mpi_btl:
cmd.extend(["--mca", "btl", str(mpi_btl)])
# HCOLL does an eager UCX IB wireup during MPI_Init that deadlocks on nodes
# where UCX only sees mlx5_2/mlx5_3 instead of mlx5_0/mlx5_1. HCOLL
# collective optimisations are irrelevant for NCCL tests (NCCL handles its

View File

@ -5,6 +5,7 @@ import os
import re
import shutil
import subprocess
import sysconfig
import tempfile
from datetime import datetime
from typing import Optional
@ -77,6 +78,30 @@ class NCCLTest:
return p
return None
@staticmethod
def _runtime_env() -> dict:
"""Return an environment that can load NCCL bundled in the active venv.
CUDA PyTorch wheels install libnccl under site-packages/nvidia/nccl/lib,
which is intentionally isolated from the system linker path. nccl-tests
is an external executable, so it needs that directory explicitly even
though importing torch works in the same virtual environment.
"""
env = os.environ.copy()
purelib = sysconfig.get_paths().get("purelib", "")
candidates = [
os.path.join(purelib, "nvidia", "nccl", "lib"),
os.path.join(purelib, "torch", "lib"),
]
library_dirs = [path for path in candidates if os.path.isdir(path)]
existing = env.get("LD_LIBRARY_PATH", "")
if existing:
library_dirs.append(existing)
if library_dirs:
env["LD_LIBRARY_PATH"] = os.pathsep.join(library_dirs)
env["NCCL_DEBUG"] = "WARN"
return env
def run(self) -> dict:
gpu_count = 0
if TORCH_AVAILABLE:
@ -101,6 +126,7 @@ class NCCLTest:
tests.append(("sendrecv_perf", "SendRecv"))
nvlink_bw = self.specs.get("nvlink_bandwidth_gbps", 0)
calibrated_thresholds = self.specs.get("nccl_pass_thresholds_gbps", {}) or {}
# User-provided override applies uniformly across all ops; otherwise
# each op gets its own threshold from _OP_BW_FRACTIONS.
user_override = self.nccl_cfg.get("min_bandwidth_gbps")
@ -108,6 +134,9 @@ class NCCLTest:
def threshold_for(label: str) -> float:
if user_override:
return float(user_override)
calibrated = calibrated_thresholds.get(label.lower())
if calibrated:
return float(calibrated)
if nvlink_bw <= 0:
return 10.0 # conservative floor
frac = _OP_BW_FRACTIONS.get(label.lower(), 0.45)
@ -152,14 +181,19 @@ class NCCLTest:
self.console.print("[yellow]nccl-tests binaries failed, falling back to torchrun[/yellow]")
return self._run_torchrun_fallback(gpu_count)
all_passed = all(
r.get("status") == "PASS"
for r in results.values()
if isinstance(r, dict) and "status" in r
statuses = [
r.get("status") for r in results.values()
if isinstance(r, dict) and r.get("status")
]
overall_status = (
"FAIL" if "FAIL" in statuses else
"WARN" if "WARN" in statuses else
"PASS" if statuses else "SKIP"
)
return {
"passed": all_passed,
"passed": overall_status in ("PASS", "WARN"),
"status": overall_status,
"source": "nccl-tests",
"min_bandwidth_gbps": {
lbl.lower(): threshold_for(lbl) for _, lbl in tests
@ -188,8 +222,7 @@ class NCCLTest:
]
try:
env = os.environ.copy()
env["NCCL_DEBUG"] = "WARN"
env = self._runtime_env()
r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env)
combined = r.stdout + r.stderr
@ -234,8 +267,7 @@ class NCCLTest:
]
try:
env = os.environ.copy()
env["NCCL_DEBUG"] = "WARN"
env = self._runtime_env()
r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env)
combined = r.stdout + r.stderr
@ -440,6 +472,7 @@ dist.destroy_process_group()
return
passed = results.get("passed", False)
overall_status = results.get("status") or ("PASS" if passed else "FAIL")
source = results.get("source", "unknown")
if source == "torchrun_fallback":
@ -464,7 +497,12 @@ dist.destroy_process_group()
c.print("\n[yellow]Note: functional connectivity test only (no performance data)[/yellow]")
else:
# nccl-tests mode
verdict = "[bold green]✓ NCCL tests PASSED[/bold green]" if passed else "[bold yellow]⚠ NCCL tests WARNING[/bold yellow]"
if overall_status == "PASS":
verdict = "[bold green]✓ NCCL tests PASSED[/bold green]"
elif overall_status == "WARN":
verdict = "[bold yellow]⚠ NCCL tests WARNING[/bold yellow]"
else:
verdict = "[bold red]✗ NCCL tests FAILED[/bold red]"
c.print(f"{verdict} [dim](via {source})[/dim]")
tests = results.get("tests", {})

View File

@ -62,6 +62,7 @@ class RDMATest:
# bond/management interfaces (e.g. mlx5_bond_0) can show ACTIVE state
# without actually providing IB fabric connectivity.
ib_devices = []
active_ib_devices = []
active_ib_port = False
for dev in devices:
for port in self._get_ib_ports(dev):
@ -74,6 +75,7 @@ class RDMATest:
f"/sys/class/infiniband/{dev}/ports/{port}/state")
if "ACTIVE" in state.upper():
active_ib_port = True
active_ib_devices.append(dev)
device_info = self._collect_device_info(devices)
cross_enabled = (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled")
@ -114,28 +116,35 @@ class RDMATest:
"timestamp": datetime.now().isoformat(),
}
self.console.print(f"[cyan]RDMA Test - Devices: {', '.join(devices)}[/cyan]")
# perftest is a two-endpoint benchmark. Running both endpoints on
# localhost can select a DOWN HCA, bypass the fabric, or return a zero
# result that looks like hardware failure. Inventory-only is objective
# until a peer is explicitly configured.
if not cross_enabled:
self.console.print(
"[yellow]Active IB ports detected, but no cross-node peer is configured — "
"skipping bandwidth/latency benchmarks[/yellow]"
)
return {
"status": "SKIP", "skipped": True,
"reason": "cross-node RDMA peer not configured; local loopback is not an acceptance test",
"devices": device_info,
"active_ib_devices": sorted(set(active_ib_devices)),
"timestamp": datetime.now().isoformat(),
}
bw_results = self._run_bandwidth_tests(devices)
latency_results = self._run_latency_tests(devices)
all_passed = all(
r.get("status") == "PASS"
for r in bw_results + latency_results
if isinstance(r, dict)
)
result = {
"passed": all_passed,
cross = self.run_cross_node()
return {
"passed": cross.get("status") == "PASS",
"status": cross.get("status", "FAIL"),
"reason": "cross-node RDMA benchmark",
"devices": device_info,
"bandwidth_tests": bw_results,
"latency_tests": latency_results,
"active_ib_devices": sorted(set(active_ib_devices)),
"bandwidth_tests": [],
"latency_tests": [],
"cross_node": cross,
"timestamp": datetime.now().isoformat(),
}
# Cross-node (two-host) RDMA, run only when a peer is configured.
if (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled"):
result["cross_node"] = self.run_cross_node()
return result
def _collect_device_info(self, devices: List[str]) -> List[dict]:
info = []

View File

@ -2,6 +2,7 @@
import json
import os
import re
from datetime import datetime
from pathlib import Path
from typing import Optional
@ -258,8 +259,10 @@ class ReportGenerator:
# --- Memory Bandwidth ---
mem_data = self._extract_memory_results(results)
if mem_data and not mem_data.get("error"):
lines.append("## Memory Bandwidth\n")
lines.append("## GPU Interconnect and PCIe Bandwidth\n")
lines.append(f"Source: {mem_data.get('source', 'unknown')}\n")
if mem_data.get("scope_note"):
lines.append(f"> Scope: {mem_data['scope_note']}.\n")
lines.append("| Metric | Value | Peak | Efficiency |")
lines.append("|--------|-------|------|------------|")
d2d = mem_data.get("d2d_bandwidth_gbps") or 0
@ -284,8 +287,13 @@ class ReportGenerator:
lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {peak_text(h2d_peak)} | {efficiency_text(h2d_eff)} |")
lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {peak_text(d2h_peak)} | {efficiency_text(d2h_eff)} |")
d2d_label = "D2D (HBM copy)" if mem_data.get("accelerator_vendor") == "hygon" else "D2D (NVLink)"
d2d_label = "D2D (HBM copy)" if mem_data.get("accelerator_vendor") == "hygon" else "D2D (NVLink, unidirectional)"
lines.append(f"| {d2d_label} | {d2d:.1f} GB/s | {peak_text(d2d_peak)} | {efficiency_text(d2d_eff)} |")
if mem_data.get("d2d_bidirectional_gbps"):
lines.append(
f"| D2D (NVLink, bidirectional informational) | "
f"{mem_data['d2d_bidirectional_gbps']:.1f} GB/s | N/A | N/A |"
)
lines.append("")
# PyTorch fallback can't accurately measure HBM peak (intra-GPU copy_()
# only reaches ~20% of HBM bandwidth). When fallback is used, report
@ -315,43 +323,58 @@ class ReportGenerator:
isinstance(value, (int, float)) and value > 0
for value in peak_tflops.values()
)
worst_values = comp_data.get("worst_per_dtype_tflops", {}) or {}
best_values = comp_data.get("best_per_dtype_tflops", {}) or {}
spreads = comp_data.get("spread_pct", {}) or {}
if use_abs:
lines.append("| DType | Achieved (TFLOPS) | Peak | Threshold | Status |")
lines.append("| DType | Avg | Min | Max | Spread | Dense Peak | Pass Floor | Status |")
else:
lines.append("| DType | Achieved (TFLOPS) | Peak | Efficiency | Status |")
lines.append("|-------|-------------------|------|------------|--------|")
lines.append("| DType | Avg | Min | Max | Spread | Dense Peak | Efficiency | Status |")
lines.append("|-------|-----|-----|-----|--------|------------|------------|--------|")
worst_eff = 100.0
overall_status = "PASS"
rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
for dt, val in per_dtype.items():
if isinstance(val, str):
# skipped or error
lines.append(f"| {dt.upper()} | {val} | - | N/A | SKIP |")
lines.append(f"| {dt.upper()} | {val} | - | - | - | - | N/A | SKIP |")
else:
pk = peak_tflops.get(dt, 0)
ef = eff_pct.get(dt, 0)
worst = worst_values.get(dt, val)
best = best_values.get(dt, val)
spread = spreads.get(dt, 0)
if isinstance(ef, (int, float)) and ef > 0:
worst_eff = min(worst_eff, ef)
thr = pass_thresholds.get(dt)
if use_abs and thr:
if val >= thr:
if worst >= thr:
status = "PASS"
elif val >= thr * 0.9:
elif worst >= thr * 0.9:
status = "WARN"
else:
status = "FAIL"
lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | >= {thr} | {status} |")
lines.append(
f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | "
f"{spread:.2f}% | {pk:.0f} | >= {thr} | {status} |"
)
elif has_peak_reference:
status = "PASS" if ef >= 80 else ("WARN" if ef >= 50 else "FAIL")
lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | {ef:.1f}% | {status} |")
lines.append(
f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | "
f"{spread:.2f}% | {pk:.0f} | {ef:.1f}% | {status} |"
)
else:
status = "MEASURED"
lines.append(f"| {dt.upper()} | {val:.1f} | N/A | N/A | {status} |")
lines.append(
f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | "
f"{spread:.2f}% | N/A | N/A | {status} |"
)
if rank.get(status, 0) > rank.get(overall_status, 0):
overall_status = status
lines.append("")
if use_abs:
lines.append(f"**Verdict: {overall_status}** (absolute TFLOPS thresholds; worst efficiency {worst_eff:.1f}%)\n")
lines.append(f"**Verdict: {overall_status}** (each dtype is judged by the slowest GPU against an achievable B300 floor)\n")
elif has_peak_reference:
overall_status = "PASS" if worst_eff >= 80 else ("WARN" if worst_eff >= 50 else "FAIL")
lines.append(f"**Verdict: {overall_status}** (worst efficiency {worst_eff:.1f}%)\n")
@ -377,8 +400,62 @@ class ReportGenerator:
elif isinstance(data, dict) and data.get("error"):
lines.append(f"| {op} | - | - | ERROR: {data['error']} |")
lines.append("")
passed = nccl.get("passed", False)
lines.append(f"**Overall: {'PASS' if passed else 'FAIL'}**\n")
nccl_status = nccl.get("status") or ("PASS" if nccl.get("passed") else "FAIL")
lines.append(f"**Overall: {nccl_status}**\n")
# --- Multi-node NCCL ---
multinode = results.get("multinode_nccl")
if multinode:
lines.append("## Multi-node NCCL\n")
lines.append(f"Source: {multinode.get('source', 'unknown')} | "
f"Mode: {multinode.get('mode', 'unknown')}\n")
hosts = multinode.get("hosts", []) or []
if hosts:
host_text = ", ".join(
f"{host.get('name', host.get('addr', '?'))} "
f"({host.get('addr', '?')}, {host.get('slots', '?')} GPUs)"
for host in hosts
)
lines.append(f"- **Hosts:** {host_text}")
lines.append(f"- **Overall: {'PASS' if multinode.get('passed') else 'FAIL'}**")
if multinode.get("artifact_dir"):
lines.append(f"- **Artifacts:** `{multinode['artifact_dir']}`")
lines.append("")
tests = multinode.get("tests", {}) or {}
if tests:
lines.append("| Operation | Topology | Peak Bus BW | Avg Bus BW | Peak Size | Threshold | Wrong | Network | GDR | Status |")
lines.append("|-----------|----------|-------------|------------|-----------|-----------|-------|---------|-----|--------|")
for op, test_data in tests.items():
for topo in test_data.get("topologies", []):
network = topo.get("network", {}) or {}
networks = ", ".join(network.get("networks", [])) or "N/A"
ib_using = " ".join(network.get("ib_using", []))
roce_hcas = sorted(set(re.findall(r"mlx5_bond_\d+", ib_using)))
if roce_hcas:
networks = f"{networks} ({len(roce_hcas)}x RoCE)"
threshold = topo.get("min_required_gbps", 0)
lines.append(
f"| {op} | {topo.get('label', '')} | "
f"{topo.get('peak_busbw_gbps', 0):.2f} GB/s | "
f"{topo.get('avg_busbw_gbps', 0):.2f} GB/s | "
f"{topo.get('peak_size', '')} | >= {threshold:g} GB/s | "
f"{topo.get('wrong_count', 0)} | {networks} | "
f"{network.get('gpu_direct_rdma', 'UNKNOWN')} | {topo.get('status', '?')} |"
)
lines.append("")
preflight = multinode.get("preflight", {}) or {}
checks = preflight.get("checks", []) or []
if checks:
lines.append("### Preflight\n")
lines.append("| Check | Status | Detail |")
lines.append("|-------|--------|--------|")
for check in checks:
detail = str(check.get("detail", "")).replace("|", "\\|").replace("\n", " ")
lines.append(f"| {check.get('name', '')} | {check.get('status', '?')} | {detail} |")
lines.append("")
# --- Stress Test ---
stress = results.get("stress")
@ -391,6 +468,12 @@ class ReportGenerator:
lines.append(f"- **Source:** {source}")
lines.append(f"- **Duration:** {elapsed:.0f}s (requested {duration}s)")
lines.append(f"- **Result: {'PASS' if passed else 'FAIL'}**")
lines.append(f"- **Validation scope:** {stress.get('validation_scope', 'runtime completion')}")
if stress.get("numerical_validation") is False:
lines.append("- **Numerical result validation:** Not performed by PyTorch fallback")
ecc_delta = stress.get("uncorrected_ecc_delta", {}) or {}
if ecc_delta:
lines.append(f"- **Uncorrected ECC delta:** {sum(ecc_delta.values())}")
max_temp = stress.get("max_temperature_c", {}) or {}
max_power = stress.get("max_power_w", {}) or {}
if max_temp or max_power:
@ -440,7 +523,10 @@ class ReportGenerator:
lines.append(f"| Avg Step Time | {training.get('avg_step_time_ms', 0):.1f} ms |")
lines.append(f"| Peak Memory | {training.get('peak_memory_gb', 0):.1f} GB |")
lines.append(f"| Final Loss | {training.get('final_loss', 'N/A')} |")
lines.append(f"| GPU Scope | {training.get('gpu_scope', 'framework-defined')} |")
lines.append(f"| SDPA Policy | {training.get('sdpa_backend_policy', 'framework_default')} |")
lines.append("")
lines.append("**Verdict: MEASURED** (synthetic workload; not an 8-GPU scaling acceptance test)\n")
# --- Footer ---
lines.append("---")
@ -500,15 +586,15 @@ class ReportGenerator:
mem = self._extract_memory_results(results)
if mem:
if mem.get("error"):
items.append(("Memory Bandwidth", f"ERROR: {mem['error']}"))
items.append(("GPU Interconnect / PCIe", f"ERROR: {mem['error']}"))
elif mem.get("source") == "pytorch":
# PyTorch fallback can't reach HBM peak — report as WARN, not FAIL.
d2d = mem.get("d2d_bandwidth_gbps") or 0
items.append(("Memory Bandwidth", f"WARN ({d2d:.0f} GB/s via PyTorch fallback)"))
items.append(("GPU Interconnect / PCIe", f"WARN ({d2d:.0f} GB/s via PyTorch fallback)"))
else:
eff = mem.get("efficiency_pct") or 0
verdict = "PASS" if eff >= 80 else ("WARN" if eff >= 60 else "FAIL")
items.append(("Memory Bandwidth", f"{verdict} ({eff:.1f}%)"))
items.append(("GPU Interconnect / PCIe", f"{verdict} ({eff:.1f}% unidirectional D2D)"))
# Compute
comp = self._extract_compute_results(results)
@ -517,6 +603,7 @@ class ReportGenerator:
items.append(("Compute Throughput", f"ERROR: {comp['error']}"))
else:
per_dtype = comp.get("per_dtype_tflops", {})
worst_per_dtype = comp.get("worst_per_dtype_tflops", {}) or {}
eff_pct = comp.get("efficiency_pct", {})
pass_thresholds = comp.get("pass_thresholds_tflops", {}) or {}
if pass_thresholds:
@ -525,7 +612,8 @@ class ReportGenerator:
worst_status = "PASS"
worst_dt = None
for dt, thr in pass_thresholds.items():
val = per_dtype.get(dt)
avg_val = per_dtype.get(dt)
val = worst_per_dtype.get(dt, avg_val)
if not isinstance(val, (int, float)):
continue
if val >= thr:
@ -541,7 +629,8 @@ class ReportGenerator:
items.append((
"Compute Throughput",
f"{worst_status} (worst {worst_dt.upper()} "
f"{per_dtype[worst_dt]:.0f} vs >= {pass_thresholds[worst_dt]})"
f"min {worst_per_dtype.get(worst_dt, per_dtype[worst_dt]):.0f} "
f"vs >= {pass_thresholds[worst_dt]})"
))
else:
items.append(("Compute Throughput", f"{worst_status}"))
@ -560,10 +649,19 @@ class ReportGenerator:
n = results["nccl"]
if n.get("error"):
items.append(("NCCL", f"ERROR: {n['error']}"))
elif n.get("passed"):
items.append(("NCCL", "PASS"))
else:
items.append(("NCCL", "FAIL"))
status = n.get("status") or ("PASS" if n.get("passed") else "FAIL")
items.append(("NCCL", status))
# Multi-node NCCL
if "multinode_nccl" in results:
n = results["multinode_nccl"]
if n.get("error"):
items.append(("Multi-node NCCL", f"ERROR: {n['error']}"))
elif n.get("passed"):
items.append(("Multi-node NCCL", "PASS"))
else:
items.append(("Multi-node NCCL", "FAIL"))
# Stress
if "stress" in results:
@ -594,6 +692,7 @@ class ReportGenerator:
items.append(("Training", f"ERROR: {t['error']}"))
else:
tps = t.get("throughput_tokens_per_sec", 0)
items.append(("Training", f"PASS ({tps:.0f} tokens/sec)"))
scope = t.get("gpu_scope", "framework-defined")
items.append(("Training", f"MEASURED ({tps:.0f} tokens/sec; {scope})"))
return items

View File

@ -119,7 +119,31 @@ class StressTest:
}
@staticmethod
def _hygon_snapshot() -> dict:
def _accelerator_snapshot() -> dict:
if shutil.which("nvidia-smi"):
try:
r = subprocess.run(
["nvidia-smi", "--query-gpu=index,temperature.gpu,power.draw,"
"ecc.errors.uncorrected.volatile.total", "--format=csv,noheader,nounits"],
capture_output=True, text=True, timeout=15,
)
temps, powers, ecc = {}, {}, {}
if r.returncode == 0:
for line in r.stdout.splitlines():
parts = [part.strip() for part in line.split(",")]
if len(parts) < 4:
continue
idx = int(parts[0])
temps[idx] = float(parts[1])
powers[idx] = float(parts[2])
ecc[idx] = int(parts[3]) if parts[3].isdigit() else 0
return {
"temperatures_c": temps, "power_w": powers,
"uncorrected_ecc": ecc,
}
except (OSError, ValueError, subprocess.SubprocessError):
pass
output = run_hy_smi()
temps, powers = {}, {}
if output:
@ -128,7 +152,7 @@ class StressTest:
idx = int(match.group(1))
temps[idx] = float(match.group(2))
powers[idx] = float(match.group(3))
return {"temperatures_c": temps, "power_w": powers}
return {"temperatures_c": temps, "power_w": powers, "uncorrected_ecc": {}}
def _run_pytorch_stress(self, duration: int, memory_pct: int = 90,
tensor_cores: bool = True) -> dict:
@ -148,7 +172,7 @@ class StressTest:
gpu_status = {}
t0 = time.time()
snapshots = [self._hygon_snapshot()]
snapshots = [self._accelerator_snapshot()]
try:
work_a, work_b, outputs, ballast = {}, {}, {}, {}
@ -210,7 +234,7 @@ class StressTest:
current_elapsed = time.time() - t0
if int(current_elapsed) != int(elapsed_check) and int(current_elapsed) % 10 == 0:
self.console.print(f" [dim]Running {int(current_elapsed)}s / {duration}s[/dim]")
snapshots.append(self._hygon_snapshot())
snapshots.append(self._accelerator_snapshot())
elapsed_check = current_elapsed
for i in range(gpu_count):
@ -235,7 +259,7 @@ class StressTest:
torch.cuda.empty_cache()
elapsed = round(time.time() - t0, 1)
snapshots.append(self._hygon_snapshot())
snapshots.append(self._accelerator_snapshot())
max_temp = {}
max_power = {}
for snapshot in snapshots:
@ -243,15 +267,29 @@ class StressTest:
max_temp[idx] = max(max_temp.get(idx, value), value)
for idx, value in snapshot.get("power_w", {}).items():
max_power[idx] = max(max_power.get(idx, value), value)
first_ecc = snapshots[0].get("uncorrected_ecc", {}) if snapshots else {}
last_ecc = snapshots[-1].get("uncorrected_ecc", {}) if snapshots else {}
ecc_delta = {
idx: max(0, last_ecc.get(idx, 0) - first_ecc.get(idx, 0))
for idx in set(first_ecc) | set(last_ecc)
}
passed = not any(ecc_delta.values())
if not passed:
for idx, delta in ecc_delta.items():
if delta:
gpu_status[idx] = "FAIL"
return {
"source": "pytorch",
"passed": True,
"passed": passed,
"duration_sec": duration,
"elapsed_sec": elapsed,
"gpu_status": gpu_status,
"memory_pct": memory_pct,
"max_temperature_c": max_temp,
"max_power_w": max_power,
"uncorrected_ecc_delta": ecc_delta,
"numerical_validation": False,
"validation_scope": "runtime stability, allocation, temperature, power, and ECC delta",
"timestamp": datetime.now().isoformat(),
}

View File

@ -3,6 +3,7 @@
import time
import subprocess
import shutil
from contextlib import nullcontext
from datetime import datetime
from typing import Optional
@ -30,6 +31,73 @@ class TrainingSim:
self.console = Console()
self.train_cfg = config.get("training", {})
@staticmethod
def _requires_b300_sdpa_workaround(
configured_type: str, capability, cudnn_version: int | None,
) -> bool:
is_b300 = str(configured_type).lower() == "b300" or capability == (10, 3)
# cuDNN 9.20 restores SM103 plan generation and PyTorch 2.13 correctly
# rejects unsupported cuDNN shapes (for example dropout=0.1) before
# dispatch, allowing another SDPA backend to run. Keep the workaround
# only for the affected older stack.
return is_b300 and (not cudnn_version or cudnn_version < 92000)
def _sdpa_context(self):
"""Select a reliable SDPA backend policy for B300/SM103.
PyTorch 2.11 + cuDNN 9.19 can dispatch BF16 attention on B300 to the
cuDNN frontend even though that backend cannot build an execution plan.
Keep the workaround scoped to the training operation and retain ordered
Flash/Efficient/Math fallbacks instead of changing global process state.
"""
configured_type = str(self.config.get("gpu_type", "")).lower()
capability = None
cudnn_version = None
if torch.cuda.is_available():
try:
capability = tuple(torch.cuda.get_device_capability(0))
except Exception: # noqa: BLE001 - backend discovery must not abort training
capability = None
try:
cudnn_version = torch.backends.cudnn.version()
except Exception: # noqa: BLE001 - backend discovery must not abort training
cudnn_version = None
requires_workaround = self._requires_b300_sdpa_workaround(
configured_type, capability, cudnn_version,
)
if not requires_workaround:
if (configured_type == "b300" or capability == (10, 3)) and cudnn_version:
return nullcontext(), (
f"framework_default (cuDNN {cudnn_version}; "
"unsupported SDPA shapes may fall back automatically)"
)
return nullcontext(), "framework_default"
try:
from torch.nn.attention import SDPBackend, sdpa_kernel
context = sdpa_kernel(
[
SDPBackend.FLASH_ATTENTION,
SDPBackend.EFFICIENT_ATTENTION,
SDPBackend.MATH,
],
set_priority=True,
)
return context, "flash>efficient>math (cuDNN SDPA excluded on B300)"
except (ImportError, AttributeError, TypeError):
# Compatibility path for older PyTorch versions that do not expose
# the context-manager API. Restore the global flag when leaving.
class _DisableCudnnSDPA:
def __enter__(self_inner):
self_inner.old = torch.backends.cuda.cudnn_sdp_enabled()
torch.backends.cuda.enable_cudnn_sdp(False)
def __exit__(self_inner, exc_type, exc, traceback):
torch.backends.cuda.enable_cudnn_sdp(self_inner.old)
return _DisableCudnnSDPA(), "framework fallback (cuDNN SDPA disabled on B300)"
def run(self) -> dict:
if not TORCH_AVAILABLE:
self.console.print("[yellow]PyTorch not available - skipping training simulation[/yellow]")
@ -89,6 +157,8 @@ class TrainingSim:
step_times = []
mem_usage = []
sdpa_context, sdpa_policy = self._sdpa_context()
self.console.print(f" SDPA policy: {sdpa_policy}")
with Progress(
SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
BarColumn(), TextColumn("{task.completed}/{task.total}"),
@ -96,35 +166,36 @@ class TrainingSim:
) as progress:
task = progress.add_task("Training steps...", total=num_steps)
for step in range(num_steps):
torch.cuda.synchronize()
t0 = time.perf_counter()
with sdpa_context:
for step in range(num_steps):
torch.cuda.synchronize()
t0 = time.perf_counter()
input_ids = input_ids.to(model.device)
attention_mask = attention_mask.to(model.device)
input_ids = input_ids.to(model.device)
attention_mask = attention_mask.to(model.device)
if dtype in (torch.float16, torch.bfloat16):
with torch.amp.autocast("cuda", dtype=dtype):
if dtype in (torch.float16, torch.bfloat16):
with torch.amp.autocast("cuda", dtype=dtype):
outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
loss = outputs.loss
else:
outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
loss = outputs.loss
else:
outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
loss = outputs.loss
loss.backward()
optimizer.step()
optimizer.zero_grad()
loss.backward()
optimizer.step()
optimizer.zero_grad()
torch.cuda.synchronize()
elapsed = time.perf_counter() - t0
step_times.append(elapsed)
torch.cuda.synchronize()
elapsed = time.perf_counter() - t0
step_times.append(elapsed)
if torch.cuda.is_available():
mem_used = torch.cuda.max_memory_allocated() / 1024**3
mem_usage.append(mem_used)
torch.cuda.reset_peak_memory_stats()
if torch.cuda.is_available():
mem_used = torch.cuda.max_memory_allocated() / 1024**3
mem_usage.append(mem_used)
torch.cuda.reset_peak_memory_stats()
progress.advance(task)
progress.advance(task)
avg_step_time = sum(step_times) / len(step_times)
throughput = batch_size * seq_length / avg_step_time
@ -142,6 +213,7 @@ class TrainingSim:
"throughput_samples_per_sec": round(batch_size / avg_step_time, 2),
"peak_memory_gb": round(max(mem_usage) if mem_usage else 0, 2),
"final_loss": round(loss.item(), 4) if hasattr(loss, 'item') else None,
"sdpa_backend_policy": sdpa_policy,
"timestamp": datetime.now().isoformat(),
}
@ -190,6 +262,8 @@ class TrainingSim:
step_times = []
mem_usage = []
sdpa_context, sdpa_policy = self._sdpa_context()
self.console.print(f" SDPA policy: {sdpa_policy}")
with Progress(
SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
BarColumn(), TextColumn("{task.completed}/{task.total}"),
@ -197,27 +271,28 @@ class TrainingSim:
) as progress:
task = progress.add_task("Synthetic training...", total=num_steps)
for step in range(num_steps):
torch.cuda.synchronize()
t0 = time.perf_counter()
with sdpa_context:
for step in range(num_steps):
torch.cuda.synchronize()
t0 = time.perf_counter()
logits = model(input_ids)
loss = torch.nn.functional.cross_entropy(
logits.view(-1, vocab_size), input_ids.view(-1)
)
loss.backward()
optimizer.step()
optimizer.zero_grad()
logits = model(input_ids)
loss = torch.nn.functional.cross_entropy(
logits.view(-1, vocab_size), input_ids.view(-1)
)
loss.backward()
optimizer.step()
optimizer.zero_grad()
torch.cuda.synchronize()
elapsed = time.perf_counter() - t0
step_times.append(elapsed)
torch.cuda.synchronize()
elapsed = time.perf_counter() - t0
step_times.append(elapsed)
mem_used = torch.cuda.max_memory_allocated() / 1024**3
mem_usage.append(mem_used)
torch.cuda.reset_peak_memory_stats()
mem_used = torch.cuda.max_memory_allocated() / 1024**3
mem_usage.append(mem_used)
torch.cuda.reset_peak_memory_stats()
progress.advance(task)
progress.advance(task)
avg_step_time = sum(step_times) / len(step_times)
throughput = batch_size * seq_length / avg_step_time
@ -228,6 +303,9 @@ class TrainingSim:
"num_layers": num_layers,
"hidden_size": hidden_size,
"gpu_count": gpu_count,
"gpu_scope": "single_gpu:0",
"performance_verdict": "MEASURED",
"sdpa_backend_policy": sdpa_policy,
"dtype": str(dtype).replace("torch.", ""),
"batch_size": batch_size,
"seq_length": seq_length,

View File

@ -17,10 +17,12 @@ dependencies = [
# uv sync --extra torch-cu121
# uv sync --extra torch-cu124 (recommended for CUDA 12.x + Driver 535+)
# uv sync --extra torch-cu128 (for CUDA 12.8+ / Driver 570+)
# uv sync --extra torch-cu130 (recommended for B300 / Driver 580+)
torch-cu118 = ["torch>=2.1.0"]
torch-cu121 = ["torch>=2.1.0"]
torch-cu124 = ["torch>=2.1.0"]
torch-cu128 = ["torch>=2.1.0"]
torch-cu130 = ["torch>=2.13.0"]
[build-system]
requires = ["hatchling"]
@ -47,6 +49,11 @@ name = "pytorch-cu128"
url = "https://download.pytorch.org/whl/cu128"
explicit = true
[[tool.uv.index]]
name = "pytorch-cu130"
url = "https://download.pytorch.org/whl/cu130"
explicit = true
# Default: resolve torch from cu124 index (most broadly compatible with 535+ drivers)
# To use a different CUDA version, override:
# uv pip install torch --index-url https://download.pytorch.org/whl/cu128

View File

@ -0,0 +1,90 @@
import os
import tempfile
import unittest
from unittest.mock import patch
from modules.benchmark import Benchmark
from modules.gpu_specs import get_gpu_specs
from modules.nccl_test import NCCLTest
from modules.rdma_test import RDMATest
from modules.report import ReportGenerator
from modules.training_sim import TrainingSim
class B300JudgementTests(unittest.TestCase):
def test_b300_training_excludes_cudnn_sdpa(self):
self.assertTrue(TrainingSim._requires_b300_sdpa_workaround("b300", None, 91900))
self.assertTrue(TrainingSim._requires_b300_sdpa_workaround("auto", (10, 3), 91900))
self.assertFalse(TrainingSim._requires_b300_sdpa_workaround("b300", (10, 3), 92000))
self.assertFalse(TrainingSim._requires_b300_sdpa_workaround("h200", (9, 0), 91900))
def test_b300_uses_dense_peaks_and_calibrated_floors(self):
specs = get_gpu_specs("b300")
self.assertEqual(specs["fp8_tflops"], 4500)
self.assertEqual(specs["bf16_tflops"], 2250)
self.assertEqual(specs["compute_pass_thresholds_tflops"]["fp8"], 2800)
self.assertEqual(specs["nccl_pass_thresholds_gbps"]["allreduce"], 750)
def test_compute_verdict_uses_slowest_gpu_not_average(self):
result = {
"compute": {
"per_dtype_tflops": {"fp8": 3000.0},
"worst_per_dtype_tflops": {"fp8": 2500.0},
"pass_thresholds_tflops": {"fp8": 2800.0},
"efficiency_pct": {"fp8": 66.7},
}
}
judged = Benchmark.judge_compute(result)
self.assertEqual(judged["verdict"], "FAIL")
self.assertEqual(judged["rows"][0][1], 2500.0)
def test_rdma_without_peer_is_skip_not_false_failure(self):
rdma = RDMATest({"rdma": {}})
with (
patch.object(rdma, "_get_ib_devices", return_value=["mlx5_0"]),
patch.object(rdma, "_get_ib_ports", return_value=["1"]),
patch.object(rdma, "_read_sys", side_effect=["InfiniBand", "4: ACTIVE"]),
patch.object(rdma, "_collect_device_info", return_value=[]),
):
result = rdma.run()
self.assertEqual(result["status"], "SKIP")
self.assertTrue(result["skipped"])
self.assertIn("peer not configured", result["reason"])
def test_nccl_runtime_env_preserves_and_prepends_venv_libraries(self):
with (
patch("modules.nccl_test.sysconfig.get_paths", return_value={"purelib": "/venv/site"}),
patch("modules.nccl_test.os.path.isdir", return_value=True),
patch.dict(os.environ, {"LD_LIBRARY_PATH": "/system/lib"}),
):
env = NCCLTest._runtime_env()
self.assertEqual(
env["LD_LIBRARY_PATH"],
"/venv/site/nvidia/nccl/lib:/venv/site/torch/lib:/system/lib",
)
self.assertEqual(env["NCCL_DEBUG"], "WARN")
def test_markdown_summary_keeps_warn_and_measured_semantics(self):
results = {
"nccl": {"status": "WARN", "passed": True, "tests": {}},
"training": {
"throughput_tokens_per_sec": 1234,
"gpu_scope": "single_gpu:0",
},
}
generator = ReportGenerator({"report": {"output_dir": "."}})
summary = dict(generator._build_summary(results))
self.assertEqual(summary["NCCL"], "WARN")
self.assertTrue(summary["Training"].startswith("MEASURED"))
with tempfile.TemporaryDirectory() as tmpdir:
path = os.path.join(tmpdir, "report.md")
generator.generate(results, fmt="md", output=path)
with open(path, encoding="utf-8") as report_file:
report = report_file.read()
self.assertIn("| NCCL | WARN |", report)
self.assertIn("synthetic workload; not an 8-GPU scaling acceptance test", report)
if __name__ == "__main__":
unittest.main()