diff --git a/configs/b300_single_node_acceptance_20260827.yaml b/configs/b300_single_node_acceptance_20260827.yaml new file mode 100644 index 0000000..ff9d6a7 --- /dev/null +++ b/configs/b300_single_node_acceptance_20260827.yaml @@ -0,0 +1,62 @@ +gpu_type: b300 + +tools: + install_dir: /root/test_gpu_scripts/.gpu-test-env/tools + +benchmark: + memory: + size_mb: 4096 + iterations: 10 + nvbandwidth_buffer_mb: 512 + nvbandwidth_samples: 3 + compute: + dtypes: [fp32, tf32, fp16, bf16, fp8] + sweep_sizes: + - 3584 + - 4608 + - 5376 + - 8192 + - 11520 + - [2048, 2048, 13312] + - [2048, 2048, 16384] + matrix_size: 8192 + warmup: 20 + iterations: 80 + +health: + temp_warning: 75 + temp_critical: 85 + power_limit: null + +nccl: + min_bandwidth_gbps: null + test_allreduce: true + test_alltoall: true + test_broadcast: true + test_reduce_scatter: false + test_allgather: false + test_sendrecv: false + +stress: + duration_sec: 600 + use_doubles: false + use_tensor_cores: true + memory_pct: 90 + gpus: all + +rdma: + min_bandwidth_gbps: 50 + max_latency_us: 10 + cross_node: + enabled: false + +training: + model: synthetic + batch_size: 8 + seq_length: 2048 + num_steps: 50 + dtype: bf16 + +report: + output_dir: /root/test_gpu_scripts/reports/b300_single_node_20260827 + format: json diff --git a/gpu_tester.py b/gpu_tester.py index 5de9833..7765bf6 100644 --- a/gpu_tester.py +++ b/gpu_tester.py @@ -21,7 +21,8 @@ from rich import box from modules.gpu_info import GPUInfo from modules.health_check import HealthCheck from modules.benchmark import Benchmark -from modules.nccl_test import NCCLTest, MultiNodeNCCLTest +from modules.nccl_test import NCCLTest +from modules.multinode_nccl_test import MultiNodeNCCLTest from modules.training_sim import TrainingSim from modules.stress_test import StressTest from modules.rdma_test import RDMATest @@ -333,14 +334,39 @@ def _run_full_suite(config: dict, console: Console) -> dict: console.print(f"[bold red]{name} FAILED: {e}[/bold red]") all_results[key] = {"error": str(e)} - # Summary + # Summary: classify actual verdicts. The old implementation counted every + # result without an exception as PASS, even when passed=False or status=FAIL. console.print("\n" + "=" * 60) - # Only count test results, exclude metadata like timestamp test_results = {k: v for k, v in all_results.items() if k != "timestamp"} - passed = sum(1 for v in test_results.values() if not isinstance(v, dict) or "error" not in v) - total = len(test_results) - color = "green" if passed == total else ("yellow" if passed > 0 else "red") - console.print(f"[bold {color}]Suite complete: {passed}/{total} tests passed[/bold {color}]") + statuses = {} + for key, value in test_results.items(): + status = "PASS" + if isinstance(value, dict) and value.get("error"): + status = "FAIL" + elif key == "health": + status = "PASS" if value.get("passed") else "FAIL" + elif key == "memory_bench": + mem = value.get("memory", value) + if mem.get("error"): + status = "FAIL" + else: + eff = mem.get("d2d_efficiency_pct") or mem.get("efficiency_pct") or 0 + status = "PASS" if eff >= 80 else ("WARN" if eff >= 60 else "FAIL") + elif key == "compute_bench": + status = Benchmark.judge_compute(value)["verdict"] + elif key in ("nccl", "rdma"): + status = value.get("status") or ("PASS" if value.get("passed") else "FAIL") + elif key == "stress": + status = "PASS" if value.get("passed") else "FAIL" + elif key == "training": + status = value.get("performance_verdict", "MEASURED") + statuses[key] = status + + counts = {name: list(statuses.values()).count(name) for name in ("PASS", "WARN", "FAIL", "SKIP", "MEASURED")} + color = "red" if counts["FAIL"] else ("yellow" if counts["WARN"] else "green") + summary = ", ".join(f"{name} {count}" for name, count in counts.items() if count) + console.print(f"[bold {color}]Suite complete: {summary}[/bold {color}]") + all_results["suite_status"] = statuses return all_results @@ -446,11 +472,14 @@ Examples: if args.report: ReportGenerator(config).generate(results, fmt=args.format, output=args.output) has_errors = any("error" in v for v in results.values() if isinstance(v, dict)) - sys.exit(1 if has_errors else 0) + has_failures = any( + status == "FAIL" for status in results.get("suite_status", {}).values() + ) + sys.exit(1 if has_errors or has_failures else 0) else: result = _run_test(test_map[args.test], config, console) if args.report and result: - ReportGenerator(config).generate({args.test: result, "timestamp": datetime.now().isoformat()}, + ReportGenerator(config).generate({test_map[args.test]: result, "timestamp": datetime.now().isoformat()}, fmt=args.format, output=args.output) diff --git a/modules/benchmark.py b/modules/benchmark.py index 38b185b..c75abb6 100644 --- a/modules/benchmark.py +++ b/modules/benchmark.py @@ -150,11 +150,14 @@ class Benchmark: progress.advance(task) + # Compare like with like: the 900 GB/s reference is per direction, so do + # not select the bidirectional SM result as the acceptance value. Keep it + # as a separate informational metric. d2d_bw = max( results_by_test.get("d2d_write", 0), results_by_test.get("d2d_read", 0), - results_by_test.get("d2d_bidir", 0), ) + d2d_bidir_bw = results_by_test.get("d2d_bidir", 0) h2d_bw = results_by_test.get("h2d", 0) d2h_bw = results_by_test.get("d2h", 0) @@ -185,6 +188,7 @@ class Benchmark: "h2d_bandwidth_gbps": round(h2d_bw, 1), "d2h_bandwidth_gbps": round(d2h_bw, 1), "d2d_bandwidth_gbps": round(d2d_bw, 1), + "d2d_bidirectional_gbps": round(d2d_bidir_bw, 1), "h2d_peak_gbps": pcie_peak if pcie_peak else None, "d2h_peak_gbps": pcie_peak if pcie_peak else None, "d2d_peak_gbps": round(d2d_peak, 1) if d2d_peak else None, @@ -195,6 +199,7 @@ class Benchmark: "efficiency_pct": d2d_efficiency, "results_by_test": results_by_test, "per_gpu": [], + "scope_note": "GPU peer/PCIe transfer bandwidth; local HBM bandwidth not measured", "accelerator_vendor": self.specs.get("vendor", "nvidia"), } } @@ -385,11 +390,13 @@ class Benchmark: BarColumn(), TextColumn("{task.completed}/{task.total}"), TimeElapsedColumn(), console=self.console, ) as progress: - task = progress.add_task("Testing dtypes...", total=len(configured_dtypes)) + task = progress.add_task( + "Testing dtypes on every GPU...", total=len(configured_dtypes) * gpu_count + ) for dtype_name in configured_dtypes: if dtype_name not in dtype_map: - progress.advance(task) + progress.advance(task, advance=gpu_count) continue # Skip FP8 if GPU architecture doesn't support it @@ -399,7 +406,7 @@ class Benchmark: arch = self.specs.get("architecture", "unknown") results_by_dtype["fp8"] = f"skipped ({arch} does not support FP8)" self.console.print(f"[dim] fp8: skipped - {arch} architecture has no FP8 support[/dim]") - progress.advance(task) + progress.advance(task, advance=gpu_count) continue dtype_val, peak_tflops = dtype_map[dtype_name] @@ -413,32 +420,48 @@ class Benchmark: elif dtype_name == "fp32": torch.backends.cuda.matmul.allow_tf32 = False - best_tflops, best_shape, last_err = 0.0, None, None - for (M, N, K) in shapes: - try: - t = self._bench_matmul_once(dtype_name, dtype_val, M, N, K, warmup, iterations) - if t > best_tflops: - best_tflops, best_shape = t, (M, N, K) - except Exception as e: # noqa: BLE001 - record and try the next shape - last_err = e + gpu_values = [] + gpu_shapes = {} + last_err = None + fastest = (0.0, None) + for gpu_idx in range(gpu_count): + best_tflops, best_shape = 0.0, None + for (M, N, K) in shapes: + try: + t = self._bench_matmul_once( + dtype_name, dtype_val, M, N, K, + warmup, iterations, gpu_idx, + ) + if t > best_tflops: + best_tflops, best_shape = t, (M, N, K) + except Exception as e: # noqa: BLE001 - record and try next shape + last_err = e + if best_shape is not None: + value = round(best_tflops, 1) + per_gpu_results[gpu_idx][dtype_name] = value + gpu_values.append(value) + gpu_shapes[str(gpu_idx)] = "x".join(str(d) for d in best_shape) + if best_tflops > fastest[0]: + fastest = (best_tflops, best_shape) + else: + per_gpu_results[gpu_idx][dtype_name] = f"error: {last_err}" + progress.advance(task) torch.backends.cuda.matmul.allow_tf32 = old_tf32 - if best_shape is None: + if not gpu_values: results_by_dtype[dtype_name] = f"error: {last_err}" self.console.print(f"[yellow] {dtype_name}: {last_err}[/yellow]") else: - shape_str = "x".join(str(d) for d in best_shape) - results_by_dtype[dtype_name] = round(best_tflops, 1) - best_shapes[dtype_name] = shape_str - for pg in per_gpu_results: - pg[dtype_name] = round(best_tflops, 1) + avg_value = round(sum(gpu_values) / len(gpu_values), 1) + results_by_dtype[dtype_name] = avg_value + best_shapes[dtype_name] = "x".join(str(d) for d in fastest[1]) if len(shapes) > 1: self.console.print( - f"[dim] {dtype_name}: {best_tflops:.1f} TFLOPS @ {shape_str}[/dim]" + f"[dim] {dtype_name}: avg {avg_value:.1f}, " + f"min {min(gpu_values):.1f}, max {max(gpu_values):.1f} TFLOPS[/dim]" ) - - progress.advance(task) + best_shapes.setdefault("per_gpu", {})[dtype_name] = gpu_shapes efficiency = {} for dt, achieved in results_by_dtype.items(): @@ -447,6 +470,18 @@ class Benchmark: if peak_tp: efficiency[dt] = round((achieved / peak_tp) * 100, 1) + worst_by_dtype = {} + best_by_dtype = {} + spread_by_dtype = {} + for dt in results_by_dtype: + values = [pg.get(dt) for pg in per_gpu_results] + values = [v for v in values if isinstance(v, (int, float))] + if values: + worst_by_dtype[dt] = min(values) + best_by_dtype[dt] = max(values) + avg = sum(values) / len(values) + spread_by_dtype[dt] = round((max(values) - min(values)) / avg * 100, 2) if avg else 0 + return { "compute": { "per_dtype_tflops": results_by_dtype, @@ -459,6 +494,9 @@ class Benchmark: self.specs.get("compute_pass_thresholds_tflops") or {} ), "per_gpu": per_gpu_results, + "worst_per_dtype_tflops": worst_by_dtype, + "best_per_dtype_tflops": best_by_dtype, + "spread_pct": spread_by_dtype, "sweep_sizes": list(sweep_sizes), "best_shapes": best_shapes, "matrix_size": matrix_size, @@ -469,7 +507,7 @@ class Benchmark: } def _bench_matmul_once(self, dtype_name: str, dtype_val, M: int, N: int, K: int, - warmup: int, iterations: int) -> float: + warmup: int, iterations: int, gpu_index: int = 0) -> float: """Time one (M×K)·(K×N) matmul for a dtype and return achieved TFLOPS. Uses an L2-cache-busting pool of matrix pairs (total > 256 MB) so operands @@ -478,34 +516,37 @@ class Benchmark: cuBLAS, which on H100 beats torch.compile/Triton for plain GEMM and avoids the per-shape recompile cost that would make a sweep pathologically slow. """ - elem_bytes = 1 if dtype_name == "fp8" else torch.tensor([], dtype=dtype_val).element_size() + device = f"cuda:{gpu_index}" + with torch.cuda.device(gpu_index): + elem_bytes = 1 if dtype_name == "fp8" else torch.tensor([], dtype=dtype_val).element_size() pair_bytes = (M * K + K * N) * elem_bytes num_pools = max(4, -(-256 * 1024 * 1024 // pair_bytes)) # ceil(256MB / pair) if dtype_name == "fp8": if not hasattr(torch, "_scaled_mm"): raise RuntimeError("torch._scaled_mm unavailable — upgrade to PyTorch >= 2.1") - pools_a = [torch.randn(M, K, device="cuda", dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)] - pools_b = [torch.randn(N, K, device="cuda", dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)] - scale_a = torch.tensor(1.0, device="cuda") - scale_b = torch.tensor(1.0, device="cuda") + pools_a = [torch.randn(M, K, device=device, dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)] + pools_b = [torch.randn(N, K, device=device, dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)] + scale_a = torch.tensor(1.0, device=device) + scale_b = torch.tensor(1.0, device=device) def op(i): return torch._scaled_mm(pools_a[i], pools_b[i].T, scale_a=scale_a, scale_b=scale_b, out_dtype=torch.bfloat16) else: - pools_a = [torch.randn(M, K, device="cuda", dtype=dtype_val) for _ in range(num_pools)] - pools_b = [torch.randn(K, N, device="cuda", dtype=dtype_val) for _ in range(num_pools)] + pools_a = [torch.randn(M, K, device=device, dtype=dtype_val) for _ in range(num_pools)] + pools_b = [torch.randn(K, N, device=device, dtype=dtype_val) for _ in range(num_pools)] def op(i): return torch.matmul(pools_a[i], pools_b[i]) try: + torch.cuda.set_device(gpu_index) # Probe once so a broken/unsupported kernel raises before the timed loop. _probe = op(0) - torch.cuda.synchronize() + torch.cuda.synchronize(gpu_index) del _probe for i in range(warmup): op(i % num_pools) - torch.cuda.synchronize() + torch.cuda.synchronize(gpu_index) start_event = torch.cuda.Event(enable_timing=True) end_event = torch.cuda.Event(enable_timing=True) @@ -513,7 +554,7 @@ class Benchmark: for i in range(iterations): op(i % num_pools) end_event.record() - torch.cuda.synchronize() + torch.cuda.synchronize(gpu_index) elapsed_ms = start_event.elapsed_time(end_event) finally: del pools_a, pools_b @@ -584,18 +625,24 @@ class Benchmark: table = Table(box=None, padding=(0, 1)) table.add_column("DType", style="bold") - table.add_column("Achieved (TFLOPS)", justify="right") + table.add_column("Avg", justify="right") + table.add_column("Min", justify="right") + table.add_column("Max", justify="right") + table.add_column("Spread", justify="right") table.add_column("Peak", justify="right") table.add_column("Efficiency", justify="right") peak = comp.get("peak_tflops", {}) per_dtype = comp.get("per_dtype_tflops", {}) eff = comp.get("efficiency_pct", {}) + worst = comp.get("worst_per_dtype_tflops", {}) + best = comp.get("best_per_dtype_tflops", {}) + spread = comp.get("spread_pct", {}) for dt in per_dtype: achieved = per_dtype[dt] if isinstance(achieved, str): - table.add_row(dt, f"[red]{achieved}[/red]", str(peak.get(dt, "N/A")), "N/A") + table.add_row(dt, f"[red]{achieved}[/red]", "-", "-", "-", str(peak.get(dt, "N/A")), "N/A") continue pk = peak.get(dt, 0) ef = eff.get(dt, 0) @@ -606,7 +653,11 @@ class Benchmark: else: peak_text = "N/A" eff_text = "[cyan]MEASURED[/cyan]" - table.add_row(dt.upper(), f"{achieved:.1f}", peak_text, eff_text) + table.add_row( + dt.upper(), f"{achieved:.1f}", f"{worst.get(dt, achieved):.1f}", + f"{best.get(dt, achieved):.1f}", f"{spread.get(dt, 0):.2f}%", + peak_text, eff_text, + ) c.print(table) @staticmethod @@ -623,6 +674,7 @@ class Benchmark: """ comp = results.get("compute", results) per_dtype = comp.get("per_dtype_tflops", {}) + worst_per_dtype = comp.get("worst_per_dtype_tflops", {}) or {} thresholds = comp.get("pass_thresholds_tflops", {}) or {} eff = comp.get("efficiency_pct", {}) rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0} @@ -630,11 +682,12 @@ class Benchmark: evaluated = False measured = False for dt, val in per_dtype.items(): + judged_val = worst_per_dtype.get(dt, val) thr = thresholds.get(dt) if isinstance(val, str): status = "SKIP" elif thr: - status = "PASS" if val >= thr else ("WARN" if val >= thr * 0.9 else "FAIL") + status = "PASS" if judged_val >= thr else ("WARN" if judged_val >= thr * 0.9 else "FAIL") evaluated = True elif dt not in eff: status = "MEASURED" @@ -643,7 +696,7 @@ class Benchmark: e = eff.get(dt, 0) status = "PASS" if e >= 80 else ("WARN" if e >= 50 else "FAIL") evaluated = True - rows.append((dt, val, thr, status)) + rows.append((dt, judged_val, thr, status)) if rank[status] > rank[verdict]: verdict = status if measured and not evaluated: diff --git a/modules/gpu_specs.py b/modules/gpu_specs.py index b36f112..40f4475 100644 --- a/modules/gpu_specs.py +++ b/modules/gpu_specs.py @@ -16,6 +16,7 @@ GPU_NAME_PATTERNS = { "H200": "h200", "H20": "h20", # H20 / H20-3e is the China-compliance export variant, REDUCED peaks "B200": "b200", + "B30-E": "b300", # nvidia-smi product name used by the B300 systems under test "B300": "b300", } @@ -228,15 +229,33 @@ GPU_SPECS = { "hbm_capacity_gb": 288, "hbm_type": "HBM3e", "memory_bandwidth_gbps": 8000, # GB/s (8 TB/s) - "fp32_tflops": 125, - "tf32_tflops": 1750, # dense (estimated) - "fp16_tflops": 3500, # dense - "bf16_tflops": 3500, # dense - "fp8_tflops": 7000, # dense + # Dense, per-GPU reference peaks. NVIDIA publishes DGX B300 FP8 as + # 72 PFLOPS sparse for eight GPUs and explicitly states dense is half, + # i.e. 4.5 PFLOPS/GPU. The previous 7 PFLOPS dense value mixed product + # variants/modes and made healthy B300 cards fail by construction. + "fp32_tflops": 75, + "tf32_tflops": 1100, + "fp16_tflops": 2250, + "bf16_tflops": 2250, + "fp8_tflops": 4500, + # Acceptance floors are achievable eager-PyTorch GEMM values rather than + # a blanket percentage of marketing peak. BF16/FP8 are calibrated against + # NVIDIA Transformer Engine's public B300 GEMM examples; the remaining + # modes use conservative floors below sustained measurements on this path. + "compute_pass_thresholds_tflops": { + "fp32": 60, "tf32": 750, "fp16": 1450, "bf16": 1450, "fp8": 2800, + }, + # Single-node 8-GPU floors derived from public B300 nccl-tests results. + # These are operation-specific because collectives do not reach the same + # fraction of raw NVLink bandwidth. + "nccl_pass_thresholds_gbps": { + "allreduce": 750, "alltoall": 620, "broadcast": 600, + "allgather": 700, "reducescatter": 700, "sendrecv": 600, + }, "tdp_watts": 1200, "nvlink_gen": 5, "nvlink_bandwidth_gbps": 1800, - "pcie_gen": 5, + "pcie_gen": 6, "min_driver_version": "550", "min_cuda_version": "12.4", }, diff --git a/modules/multinode_nccl_test.py b/modules/multinode_nccl_test.py index e00b30e..cf92e53 100644 --- a/modules/multinode_nccl_test.py +++ b/modules/multinode_nccl_test.py @@ -3,6 +3,7 @@ import json import os import re +import shlex import shutil import subprocess from datetime import datetime @@ -210,9 +211,11 @@ class MultiNodeNCCLTest: "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", "-o", "StrictHostKeyChecking=accept-new", - target, - "hostname", ] + plm_rsh_args = self.cfg.get("plm_rsh_args") + if plm_rsh_args: + cmd.extend(shlex.split(str(plm_rsh_args))) + cmd.extend([target, "hostname"]) try: r = subprocess.run(cmd, capture_output=True, text=True, timeout=8, env=self._runtime_env()) detail = r.stdout.strip() or r.stderr.strip()[:120] @@ -287,6 +290,15 @@ class MultiNodeNCCLTest: "--mca", "btl_tcp_if_include", str(self.cfg.get("socket_ifname", "bond0")), "--mca", "oob_tcp_if_include", str(self.cfg.get("oob_tcp_ifname", self.cfg.get("socket_ifname", "bond0"))), ] + # Keep the MPI launcher/control plane separately configurable from the + # NCCL data plane. For example, PML/UCX can be restricted to TCP/eth0 + # while NCCL uses the configured RoCE HCAs for benchmark traffic. + mpi_pml = self.cfg.get("mpi_pml", "ob1") + if mpi_pml: + cmd.extend(["--mca", "pml", str(mpi_pml)]) + mpi_btl = self.cfg.get("mpi_btl", "self,tcp,vader") + if mpi_btl: + cmd.extend(["--mca", "btl", str(mpi_btl)]) # HCOLL does an eager UCX IB wireup during MPI_Init that deadlocks on nodes # where UCX only sees mlx5_2/mlx5_3 instead of mlx5_0/mlx5_1. HCOLL # collective optimisations are irrelevant for NCCL tests (NCCL handles its diff --git a/modules/nccl_test.py b/modules/nccl_test.py index f67998d..f8f12ea 100644 --- a/modules/nccl_test.py +++ b/modules/nccl_test.py @@ -5,6 +5,7 @@ import os import re import shutil import subprocess +import sysconfig import tempfile from datetime import datetime from typing import Optional @@ -77,6 +78,30 @@ class NCCLTest: return p return None + @staticmethod + def _runtime_env() -> dict: + """Return an environment that can load NCCL bundled in the active venv. + + CUDA PyTorch wheels install libnccl under site-packages/nvidia/nccl/lib, + which is intentionally isolated from the system linker path. nccl-tests + is an external executable, so it needs that directory explicitly even + though importing torch works in the same virtual environment. + """ + env = os.environ.copy() + purelib = sysconfig.get_paths().get("purelib", "") + candidates = [ + os.path.join(purelib, "nvidia", "nccl", "lib"), + os.path.join(purelib, "torch", "lib"), + ] + library_dirs = [path for path in candidates if os.path.isdir(path)] + existing = env.get("LD_LIBRARY_PATH", "") + if existing: + library_dirs.append(existing) + if library_dirs: + env["LD_LIBRARY_PATH"] = os.pathsep.join(library_dirs) + env["NCCL_DEBUG"] = "WARN" + return env + def run(self) -> dict: gpu_count = 0 if TORCH_AVAILABLE: @@ -101,6 +126,7 @@ class NCCLTest: tests.append(("sendrecv_perf", "SendRecv")) nvlink_bw = self.specs.get("nvlink_bandwidth_gbps", 0) + calibrated_thresholds = self.specs.get("nccl_pass_thresholds_gbps", {}) or {} # User-provided override applies uniformly across all ops; otherwise # each op gets its own threshold from _OP_BW_FRACTIONS. user_override = self.nccl_cfg.get("min_bandwidth_gbps") @@ -108,6 +134,9 @@ class NCCLTest: def threshold_for(label: str) -> float: if user_override: return float(user_override) + calibrated = calibrated_thresholds.get(label.lower()) + if calibrated: + return float(calibrated) if nvlink_bw <= 0: return 10.0 # conservative floor frac = _OP_BW_FRACTIONS.get(label.lower(), 0.45) @@ -152,14 +181,19 @@ class NCCLTest: self.console.print("[yellow]nccl-tests binaries failed, falling back to torchrun[/yellow]") return self._run_torchrun_fallback(gpu_count) - all_passed = all( - r.get("status") == "PASS" - for r in results.values() - if isinstance(r, dict) and "status" in r + statuses = [ + r.get("status") for r in results.values() + if isinstance(r, dict) and r.get("status") + ] + overall_status = ( + "FAIL" if "FAIL" in statuses else + "WARN" if "WARN" in statuses else + "PASS" if statuses else "SKIP" ) return { - "passed": all_passed, + "passed": overall_status in ("PASS", "WARN"), + "status": overall_status, "source": "nccl-tests", "min_bandwidth_gbps": { lbl.lower(): threshold_for(lbl) for _, lbl in tests @@ -188,8 +222,7 @@ class NCCLTest: ] try: - env = os.environ.copy() - env["NCCL_DEBUG"] = "WARN" + env = self._runtime_env() r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env) combined = r.stdout + r.stderr @@ -234,8 +267,7 @@ class NCCLTest: ] try: - env = os.environ.copy() - env["NCCL_DEBUG"] = "WARN" + env = self._runtime_env() r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env) combined = r.stdout + r.stderr @@ -440,6 +472,7 @@ dist.destroy_process_group() return passed = results.get("passed", False) + overall_status = results.get("status") or ("PASS" if passed else "FAIL") source = results.get("source", "unknown") if source == "torchrun_fallback": @@ -464,7 +497,12 @@ dist.destroy_process_group() c.print("\n[yellow]Note: functional connectivity test only (no performance data)[/yellow]") else: # nccl-tests mode - verdict = "[bold green]✓ NCCL tests PASSED[/bold green]" if passed else "[bold yellow]⚠ NCCL tests WARNING[/bold yellow]" + if overall_status == "PASS": + verdict = "[bold green]✓ NCCL tests PASSED[/bold green]" + elif overall_status == "WARN": + verdict = "[bold yellow]⚠ NCCL tests WARNING[/bold yellow]" + else: + verdict = "[bold red]✗ NCCL tests FAILED[/bold red]" c.print(f"{verdict} [dim](via {source})[/dim]") tests = results.get("tests", {}) diff --git a/modules/rdma_test.py b/modules/rdma_test.py index c94a106..7952cf6 100644 --- a/modules/rdma_test.py +++ b/modules/rdma_test.py @@ -62,6 +62,7 @@ class RDMATest: # bond/management interfaces (e.g. mlx5_bond_0) can show ACTIVE state # without actually providing IB fabric connectivity. ib_devices = [] + active_ib_devices = [] active_ib_port = False for dev in devices: for port in self._get_ib_ports(dev): @@ -74,6 +75,7 @@ class RDMATest: f"/sys/class/infiniband/{dev}/ports/{port}/state") if "ACTIVE" in state.upper(): active_ib_port = True + active_ib_devices.append(dev) device_info = self._collect_device_info(devices) cross_enabled = (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled") @@ -114,28 +116,35 @@ class RDMATest: "timestamp": datetime.now().isoformat(), } - self.console.print(f"[cyan]RDMA Test - Devices: {', '.join(devices)}[/cyan]") + # perftest is a two-endpoint benchmark. Running both endpoints on + # localhost can select a DOWN HCA, bypass the fabric, or return a zero + # result that looks like hardware failure. Inventory-only is objective + # until a peer is explicitly configured. + if not cross_enabled: + self.console.print( + "[yellow]Active IB ports detected, but no cross-node peer is configured — " + "skipping bandwidth/latency benchmarks[/yellow]" + ) + return { + "status": "SKIP", "skipped": True, + "reason": "cross-node RDMA peer not configured; local loopback is not an acceptance test", + "devices": device_info, + "active_ib_devices": sorted(set(active_ib_devices)), + "timestamp": datetime.now().isoformat(), + } - bw_results = self._run_bandwidth_tests(devices) - latency_results = self._run_latency_tests(devices) - - all_passed = all( - r.get("status") == "PASS" - for r in bw_results + latency_results - if isinstance(r, dict) - ) - - result = { - "passed": all_passed, + cross = self.run_cross_node() + return { + "passed": cross.get("status") == "PASS", + "status": cross.get("status", "FAIL"), + "reason": "cross-node RDMA benchmark", "devices": device_info, - "bandwidth_tests": bw_results, - "latency_tests": latency_results, + "active_ib_devices": sorted(set(active_ib_devices)), + "bandwidth_tests": [], + "latency_tests": [], + "cross_node": cross, "timestamp": datetime.now().isoformat(), } - # Cross-node (two-host) RDMA, run only when a peer is configured. - if (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled"): - result["cross_node"] = self.run_cross_node() - return result def _collect_device_info(self, devices: List[str]) -> List[dict]: info = [] diff --git a/modules/report.py b/modules/report.py index 0d8f04f..597496b 100644 --- a/modules/report.py +++ b/modules/report.py @@ -2,6 +2,7 @@ import json import os +import re from datetime import datetime from pathlib import Path from typing import Optional @@ -258,8 +259,10 @@ class ReportGenerator: # --- Memory Bandwidth --- mem_data = self._extract_memory_results(results) if mem_data and not mem_data.get("error"): - lines.append("## Memory Bandwidth\n") + lines.append("## GPU Interconnect and PCIe Bandwidth\n") lines.append(f"Source: {mem_data.get('source', 'unknown')}\n") + if mem_data.get("scope_note"): + lines.append(f"> Scope: {mem_data['scope_note']}.\n") lines.append("| Metric | Value | Peak | Efficiency |") lines.append("|--------|-------|------|------------|") d2d = mem_data.get("d2d_bandwidth_gbps") or 0 @@ -284,8 +287,13 @@ class ReportGenerator: lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {peak_text(h2d_peak)} | {efficiency_text(h2d_eff)} |") lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {peak_text(d2h_peak)} | {efficiency_text(d2h_eff)} |") - d2d_label = "D2D (HBM copy)" if mem_data.get("accelerator_vendor") == "hygon" else "D2D (NVLink)" + d2d_label = "D2D (HBM copy)" if mem_data.get("accelerator_vendor") == "hygon" else "D2D (NVLink, unidirectional)" lines.append(f"| {d2d_label} | {d2d:.1f} GB/s | {peak_text(d2d_peak)} | {efficiency_text(d2d_eff)} |") + if mem_data.get("d2d_bidirectional_gbps"): + lines.append( + f"| D2D (NVLink, bidirectional informational) | " + f"{mem_data['d2d_bidirectional_gbps']:.1f} GB/s | N/A | N/A |" + ) lines.append("") # PyTorch fallback can't accurately measure HBM peak (intra-GPU copy_() # only reaches ~20% of HBM bandwidth). When fallback is used, report @@ -315,43 +323,58 @@ class ReportGenerator: isinstance(value, (int, float)) and value > 0 for value in peak_tflops.values() ) + worst_values = comp_data.get("worst_per_dtype_tflops", {}) or {} + best_values = comp_data.get("best_per_dtype_tflops", {}) or {} + spreads = comp_data.get("spread_pct", {}) or {} if use_abs: - lines.append("| DType | Achieved (TFLOPS) | Peak | Threshold | Status |") + lines.append("| DType | Avg | Min | Max | Spread | Dense Peak | Pass Floor | Status |") else: - lines.append("| DType | Achieved (TFLOPS) | Peak | Efficiency | Status |") - lines.append("|-------|-------------------|------|------------|--------|") + lines.append("| DType | Avg | Min | Max | Spread | Dense Peak | Efficiency | Status |") + lines.append("|-------|-----|-----|-----|--------|------------|------------|--------|") worst_eff = 100.0 overall_status = "PASS" rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0} for dt, val in per_dtype.items(): if isinstance(val, str): # skipped or error - lines.append(f"| {dt.upper()} | {val} | - | N/A | SKIP |") + lines.append(f"| {dt.upper()} | {val} | - | - | - | - | N/A | SKIP |") else: pk = peak_tflops.get(dt, 0) ef = eff_pct.get(dt, 0) + worst = worst_values.get(dt, val) + best = best_values.get(dt, val) + spread = spreads.get(dt, 0) if isinstance(ef, (int, float)) and ef > 0: worst_eff = min(worst_eff, ef) thr = pass_thresholds.get(dt) if use_abs and thr: - if val >= thr: + if worst >= thr: status = "PASS" - elif val >= thr * 0.9: + elif worst >= thr * 0.9: status = "WARN" else: status = "FAIL" - lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | >= {thr} | {status} |") + lines.append( + f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | " + f"{spread:.2f}% | {pk:.0f} | >= {thr} | {status} |" + ) elif has_peak_reference: status = "PASS" if ef >= 80 else ("WARN" if ef >= 50 else "FAIL") - lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | {ef:.1f}% | {status} |") + lines.append( + f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | " + f"{spread:.2f}% | {pk:.0f} | {ef:.1f}% | {status} |" + ) else: status = "MEASURED" - lines.append(f"| {dt.upper()} | {val:.1f} | N/A | N/A | {status} |") + lines.append( + f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | " + f"{spread:.2f}% | N/A | N/A | {status} |" + ) if rank.get(status, 0) > rank.get(overall_status, 0): overall_status = status lines.append("") if use_abs: - lines.append(f"**Verdict: {overall_status}** (absolute TFLOPS thresholds; worst efficiency {worst_eff:.1f}%)\n") + lines.append(f"**Verdict: {overall_status}** (each dtype is judged by the slowest GPU against an achievable B300 floor)\n") elif has_peak_reference: overall_status = "PASS" if worst_eff >= 80 else ("WARN" if worst_eff >= 50 else "FAIL") lines.append(f"**Verdict: {overall_status}** (worst efficiency {worst_eff:.1f}%)\n") @@ -377,8 +400,62 @@ class ReportGenerator: elif isinstance(data, dict) and data.get("error"): lines.append(f"| {op} | - | - | ERROR: {data['error']} |") lines.append("") - passed = nccl.get("passed", False) - lines.append(f"**Overall: {'PASS' if passed else 'FAIL'}**\n") + nccl_status = nccl.get("status") or ("PASS" if nccl.get("passed") else "FAIL") + lines.append(f"**Overall: {nccl_status}**\n") + + # --- Multi-node NCCL --- + multinode = results.get("multinode_nccl") + if multinode: + lines.append("## Multi-node NCCL\n") + lines.append(f"Source: {multinode.get('source', 'unknown')} | " + f"Mode: {multinode.get('mode', 'unknown')}\n") + + hosts = multinode.get("hosts", []) or [] + if hosts: + host_text = ", ".join( + f"{host.get('name', host.get('addr', '?'))} " + f"({host.get('addr', '?')}, {host.get('slots', '?')} GPUs)" + for host in hosts + ) + lines.append(f"- **Hosts:** {host_text}") + lines.append(f"- **Overall: {'PASS' if multinode.get('passed') else 'FAIL'}**") + if multinode.get("artifact_dir"): + lines.append(f"- **Artifacts:** `{multinode['artifact_dir']}`") + lines.append("") + + tests = multinode.get("tests", {}) or {} + if tests: + lines.append("| Operation | Topology | Peak Bus BW | Avg Bus BW | Peak Size | Threshold | Wrong | Network | GDR | Status |") + lines.append("|-----------|----------|-------------|------------|-----------|-----------|-------|---------|-----|--------|") + for op, test_data in tests.items(): + for topo in test_data.get("topologies", []): + network = topo.get("network", {}) or {} + networks = ", ".join(network.get("networks", [])) or "N/A" + ib_using = " ".join(network.get("ib_using", [])) + roce_hcas = sorted(set(re.findall(r"mlx5_bond_\d+", ib_using))) + if roce_hcas: + networks = f"{networks} ({len(roce_hcas)}x RoCE)" + threshold = topo.get("min_required_gbps", 0) + lines.append( + f"| {op} | {topo.get('label', '')} | " + f"{topo.get('peak_busbw_gbps', 0):.2f} GB/s | " + f"{topo.get('avg_busbw_gbps', 0):.2f} GB/s | " + f"{topo.get('peak_size', '')} | >= {threshold:g} GB/s | " + f"{topo.get('wrong_count', 0)} | {networks} | " + f"{network.get('gpu_direct_rdma', 'UNKNOWN')} | {topo.get('status', '?')} |" + ) + lines.append("") + + preflight = multinode.get("preflight", {}) or {} + checks = preflight.get("checks", []) or [] + if checks: + lines.append("### Preflight\n") + lines.append("| Check | Status | Detail |") + lines.append("|-------|--------|--------|") + for check in checks: + detail = str(check.get("detail", "")).replace("|", "\\|").replace("\n", " ") + lines.append(f"| {check.get('name', '')} | {check.get('status', '?')} | {detail} |") + lines.append("") # --- Stress Test --- stress = results.get("stress") @@ -391,6 +468,12 @@ class ReportGenerator: lines.append(f"- **Source:** {source}") lines.append(f"- **Duration:** {elapsed:.0f}s (requested {duration}s)") lines.append(f"- **Result: {'PASS' if passed else 'FAIL'}**") + lines.append(f"- **Validation scope:** {stress.get('validation_scope', 'runtime completion')}") + if stress.get("numerical_validation") is False: + lines.append("- **Numerical result validation:** Not performed by PyTorch fallback") + ecc_delta = stress.get("uncorrected_ecc_delta", {}) or {} + if ecc_delta: + lines.append(f"- **Uncorrected ECC delta:** {sum(ecc_delta.values())}") max_temp = stress.get("max_temperature_c", {}) or {} max_power = stress.get("max_power_w", {}) or {} if max_temp or max_power: @@ -440,7 +523,10 @@ class ReportGenerator: lines.append(f"| Avg Step Time | {training.get('avg_step_time_ms', 0):.1f} ms |") lines.append(f"| Peak Memory | {training.get('peak_memory_gb', 0):.1f} GB |") lines.append(f"| Final Loss | {training.get('final_loss', 'N/A')} |") + lines.append(f"| GPU Scope | {training.get('gpu_scope', 'framework-defined')} |") + lines.append(f"| SDPA Policy | {training.get('sdpa_backend_policy', 'framework_default')} |") lines.append("") + lines.append("**Verdict: MEASURED** (synthetic workload; not an 8-GPU scaling acceptance test)\n") # --- Footer --- lines.append("---") @@ -500,15 +586,15 @@ class ReportGenerator: mem = self._extract_memory_results(results) if mem: if mem.get("error"): - items.append(("Memory Bandwidth", f"ERROR: {mem['error']}")) + items.append(("GPU Interconnect / PCIe", f"ERROR: {mem['error']}")) elif mem.get("source") == "pytorch": # PyTorch fallback can't reach HBM peak — report as WARN, not FAIL. d2d = mem.get("d2d_bandwidth_gbps") or 0 - items.append(("Memory Bandwidth", f"WARN ({d2d:.0f} GB/s via PyTorch fallback)")) + items.append(("GPU Interconnect / PCIe", f"WARN ({d2d:.0f} GB/s via PyTorch fallback)")) else: eff = mem.get("efficiency_pct") or 0 verdict = "PASS" if eff >= 80 else ("WARN" if eff >= 60 else "FAIL") - items.append(("Memory Bandwidth", f"{verdict} ({eff:.1f}%)")) + items.append(("GPU Interconnect / PCIe", f"{verdict} ({eff:.1f}% unidirectional D2D)")) # Compute comp = self._extract_compute_results(results) @@ -517,6 +603,7 @@ class ReportGenerator: items.append(("Compute Throughput", f"ERROR: {comp['error']}")) else: per_dtype = comp.get("per_dtype_tflops", {}) + worst_per_dtype = comp.get("worst_per_dtype_tflops", {}) or {} eff_pct = comp.get("efficiency_pct", {}) pass_thresholds = comp.get("pass_thresholds_tflops", {}) or {} if pass_thresholds: @@ -525,7 +612,8 @@ class ReportGenerator: worst_status = "PASS" worst_dt = None for dt, thr in pass_thresholds.items(): - val = per_dtype.get(dt) + avg_val = per_dtype.get(dt) + val = worst_per_dtype.get(dt, avg_val) if not isinstance(val, (int, float)): continue if val >= thr: @@ -541,7 +629,8 @@ class ReportGenerator: items.append(( "Compute Throughput", f"{worst_status} (worst {worst_dt.upper()} " - f"{per_dtype[worst_dt]:.0f} vs >= {pass_thresholds[worst_dt]})" + f"min {worst_per_dtype.get(worst_dt, per_dtype[worst_dt]):.0f} " + f"vs >= {pass_thresholds[worst_dt]})" )) else: items.append(("Compute Throughput", f"{worst_status}")) @@ -560,10 +649,19 @@ class ReportGenerator: n = results["nccl"] if n.get("error"): items.append(("NCCL", f"ERROR: {n['error']}")) - elif n.get("passed"): - items.append(("NCCL", "PASS")) else: - items.append(("NCCL", "FAIL")) + status = n.get("status") or ("PASS" if n.get("passed") else "FAIL") + items.append(("NCCL", status)) + + # Multi-node NCCL + if "multinode_nccl" in results: + n = results["multinode_nccl"] + if n.get("error"): + items.append(("Multi-node NCCL", f"ERROR: {n['error']}")) + elif n.get("passed"): + items.append(("Multi-node NCCL", "PASS")) + else: + items.append(("Multi-node NCCL", "FAIL")) # Stress if "stress" in results: @@ -594,6 +692,7 @@ class ReportGenerator: items.append(("Training", f"ERROR: {t['error']}")) else: tps = t.get("throughput_tokens_per_sec", 0) - items.append(("Training", f"PASS ({tps:.0f} tokens/sec)")) + scope = t.get("gpu_scope", "framework-defined") + items.append(("Training", f"MEASURED ({tps:.0f} tokens/sec; {scope})")) return items diff --git a/modules/stress_test.py b/modules/stress_test.py index 583314b..ab72333 100644 --- a/modules/stress_test.py +++ b/modules/stress_test.py @@ -119,7 +119,31 @@ class StressTest: } @staticmethod - def _hygon_snapshot() -> dict: + def _accelerator_snapshot() -> dict: + if shutil.which("nvidia-smi"): + try: + r = subprocess.run( + ["nvidia-smi", "--query-gpu=index,temperature.gpu,power.draw," + "ecc.errors.uncorrected.volatile.total", "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=15, + ) + temps, powers, ecc = {}, {}, {} + if r.returncode == 0: + for line in r.stdout.splitlines(): + parts = [part.strip() for part in line.split(",")] + if len(parts) < 4: + continue + idx = int(parts[0]) + temps[idx] = float(parts[1]) + powers[idx] = float(parts[2]) + ecc[idx] = int(parts[3]) if parts[3].isdigit() else 0 + return { + "temperatures_c": temps, "power_w": powers, + "uncorrected_ecc": ecc, + } + except (OSError, ValueError, subprocess.SubprocessError): + pass + output = run_hy_smi() temps, powers = {}, {} if output: @@ -128,7 +152,7 @@ class StressTest: idx = int(match.group(1)) temps[idx] = float(match.group(2)) powers[idx] = float(match.group(3)) - return {"temperatures_c": temps, "power_w": powers} + return {"temperatures_c": temps, "power_w": powers, "uncorrected_ecc": {}} def _run_pytorch_stress(self, duration: int, memory_pct: int = 90, tensor_cores: bool = True) -> dict: @@ -148,7 +172,7 @@ class StressTest: gpu_status = {} t0 = time.time() - snapshots = [self._hygon_snapshot()] + snapshots = [self._accelerator_snapshot()] try: work_a, work_b, outputs, ballast = {}, {}, {}, {} @@ -210,7 +234,7 @@ class StressTest: current_elapsed = time.time() - t0 if int(current_elapsed) != int(elapsed_check) and int(current_elapsed) % 10 == 0: self.console.print(f" [dim]Running {int(current_elapsed)}s / {duration}s[/dim]") - snapshots.append(self._hygon_snapshot()) + snapshots.append(self._accelerator_snapshot()) elapsed_check = current_elapsed for i in range(gpu_count): @@ -235,7 +259,7 @@ class StressTest: torch.cuda.empty_cache() elapsed = round(time.time() - t0, 1) - snapshots.append(self._hygon_snapshot()) + snapshots.append(self._accelerator_snapshot()) max_temp = {} max_power = {} for snapshot in snapshots: @@ -243,15 +267,29 @@ class StressTest: max_temp[idx] = max(max_temp.get(idx, value), value) for idx, value in snapshot.get("power_w", {}).items(): max_power[idx] = max(max_power.get(idx, value), value) + first_ecc = snapshots[0].get("uncorrected_ecc", {}) if snapshots else {} + last_ecc = snapshots[-1].get("uncorrected_ecc", {}) if snapshots else {} + ecc_delta = { + idx: max(0, last_ecc.get(idx, 0) - first_ecc.get(idx, 0)) + for idx in set(first_ecc) | set(last_ecc) + } + passed = not any(ecc_delta.values()) + if not passed: + for idx, delta in ecc_delta.items(): + if delta: + gpu_status[idx] = "FAIL" return { "source": "pytorch", - "passed": True, + "passed": passed, "duration_sec": duration, "elapsed_sec": elapsed, "gpu_status": gpu_status, "memory_pct": memory_pct, "max_temperature_c": max_temp, "max_power_w": max_power, + "uncorrected_ecc_delta": ecc_delta, + "numerical_validation": False, + "validation_scope": "runtime stability, allocation, temperature, power, and ECC delta", "timestamp": datetime.now().isoformat(), } diff --git a/modules/training_sim.py b/modules/training_sim.py index bd36fab..3305504 100644 --- a/modules/training_sim.py +++ b/modules/training_sim.py @@ -3,6 +3,7 @@ import time import subprocess import shutil +from contextlib import nullcontext from datetime import datetime from typing import Optional @@ -30,6 +31,73 @@ class TrainingSim: self.console = Console() self.train_cfg = config.get("training", {}) + @staticmethod + def _requires_b300_sdpa_workaround( + configured_type: str, capability, cudnn_version: int | None, + ) -> bool: + is_b300 = str(configured_type).lower() == "b300" or capability == (10, 3) + # cuDNN 9.20 restores SM103 plan generation and PyTorch 2.13 correctly + # rejects unsupported cuDNN shapes (for example dropout=0.1) before + # dispatch, allowing another SDPA backend to run. Keep the workaround + # only for the affected older stack. + return is_b300 and (not cudnn_version or cudnn_version < 92000) + + def _sdpa_context(self): + """Select a reliable SDPA backend policy for B300/SM103. + + PyTorch 2.11 + cuDNN 9.19 can dispatch BF16 attention on B300 to the + cuDNN frontend even though that backend cannot build an execution plan. + Keep the workaround scoped to the training operation and retain ordered + Flash/Efficient/Math fallbacks instead of changing global process state. + """ + configured_type = str(self.config.get("gpu_type", "")).lower() + capability = None + cudnn_version = None + if torch.cuda.is_available(): + try: + capability = tuple(torch.cuda.get_device_capability(0)) + except Exception: # noqa: BLE001 - backend discovery must not abort training + capability = None + try: + cudnn_version = torch.backends.cudnn.version() + except Exception: # noqa: BLE001 - backend discovery must not abort training + cudnn_version = None + + requires_workaround = self._requires_b300_sdpa_workaround( + configured_type, capability, cudnn_version, + ) + if not requires_workaround: + if (configured_type == "b300" or capability == (10, 3)) and cudnn_version: + return nullcontext(), ( + f"framework_default (cuDNN {cudnn_version}; " + "unsupported SDPA shapes may fall back automatically)" + ) + return nullcontext(), "framework_default" + + try: + from torch.nn.attention import SDPBackend, sdpa_kernel + context = sdpa_kernel( + [ + SDPBackend.FLASH_ATTENTION, + SDPBackend.EFFICIENT_ATTENTION, + SDPBackend.MATH, + ], + set_priority=True, + ) + return context, "flash>efficient>math (cuDNN SDPA excluded on B300)" + except (ImportError, AttributeError, TypeError): + # Compatibility path for older PyTorch versions that do not expose + # the context-manager API. Restore the global flag when leaving. + class _DisableCudnnSDPA: + def __enter__(self_inner): + self_inner.old = torch.backends.cuda.cudnn_sdp_enabled() + torch.backends.cuda.enable_cudnn_sdp(False) + + def __exit__(self_inner, exc_type, exc, traceback): + torch.backends.cuda.enable_cudnn_sdp(self_inner.old) + + return _DisableCudnnSDPA(), "framework fallback (cuDNN SDPA disabled on B300)" + def run(self) -> dict: if not TORCH_AVAILABLE: self.console.print("[yellow]PyTorch not available - skipping training simulation[/yellow]") @@ -89,6 +157,8 @@ class TrainingSim: step_times = [] mem_usage = [] + sdpa_context, sdpa_policy = self._sdpa_context() + self.console.print(f" SDPA policy: {sdpa_policy}") with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), BarColumn(), TextColumn("{task.completed}/{task.total}"), @@ -96,35 +166,36 @@ class TrainingSim: ) as progress: task = progress.add_task("Training steps...", total=num_steps) - for step in range(num_steps): - torch.cuda.synchronize() - t0 = time.perf_counter() + with sdpa_context: + for step in range(num_steps): + torch.cuda.synchronize() + t0 = time.perf_counter() - input_ids = input_ids.to(model.device) - attention_mask = attention_mask.to(model.device) + input_ids = input_ids.to(model.device) + attention_mask = attention_mask.to(model.device) - if dtype in (torch.float16, torch.bfloat16): - with torch.amp.autocast("cuda", dtype=dtype): + if dtype in (torch.float16, torch.bfloat16): + with torch.amp.autocast("cuda", dtype=dtype): + outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids) + loss = outputs.loss + else: outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids) loss = outputs.loss - else: - outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids) - loss = outputs.loss - loss.backward() - optimizer.step() - optimizer.zero_grad() + loss.backward() + optimizer.step() + optimizer.zero_grad() - torch.cuda.synchronize() - elapsed = time.perf_counter() - t0 - step_times.append(elapsed) + torch.cuda.synchronize() + elapsed = time.perf_counter() - t0 + step_times.append(elapsed) - if torch.cuda.is_available(): - mem_used = torch.cuda.max_memory_allocated() / 1024**3 - mem_usage.append(mem_used) - torch.cuda.reset_peak_memory_stats() + if torch.cuda.is_available(): + mem_used = torch.cuda.max_memory_allocated() / 1024**3 + mem_usage.append(mem_used) + torch.cuda.reset_peak_memory_stats() - progress.advance(task) + progress.advance(task) avg_step_time = sum(step_times) / len(step_times) throughput = batch_size * seq_length / avg_step_time @@ -142,6 +213,7 @@ class TrainingSim: "throughput_samples_per_sec": round(batch_size / avg_step_time, 2), "peak_memory_gb": round(max(mem_usage) if mem_usage else 0, 2), "final_loss": round(loss.item(), 4) if hasattr(loss, 'item') else None, + "sdpa_backend_policy": sdpa_policy, "timestamp": datetime.now().isoformat(), } @@ -190,6 +262,8 @@ class TrainingSim: step_times = [] mem_usage = [] + sdpa_context, sdpa_policy = self._sdpa_context() + self.console.print(f" SDPA policy: {sdpa_policy}") with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), BarColumn(), TextColumn("{task.completed}/{task.total}"), @@ -197,27 +271,28 @@ class TrainingSim: ) as progress: task = progress.add_task("Synthetic training...", total=num_steps) - for step in range(num_steps): - torch.cuda.synchronize() - t0 = time.perf_counter() + with sdpa_context: + for step in range(num_steps): + torch.cuda.synchronize() + t0 = time.perf_counter() - logits = model(input_ids) - loss = torch.nn.functional.cross_entropy( - logits.view(-1, vocab_size), input_ids.view(-1) - ) - loss.backward() - optimizer.step() - optimizer.zero_grad() + logits = model(input_ids) + loss = torch.nn.functional.cross_entropy( + logits.view(-1, vocab_size), input_ids.view(-1) + ) + loss.backward() + optimizer.step() + optimizer.zero_grad() - torch.cuda.synchronize() - elapsed = time.perf_counter() - t0 - step_times.append(elapsed) + torch.cuda.synchronize() + elapsed = time.perf_counter() - t0 + step_times.append(elapsed) - mem_used = torch.cuda.max_memory_allocated() / 1024**3 - mem_usage.append(mem_used) - torch.cuda.reset_peak_memory_stats() + mem_used = torch.cuda.max_memory_allocated() / 1024**3 + mem_usage.append(mem_used) + torch.cuda.reset_peak_memory_stats() - progress.advance(task) + progress.advance(task) avg_step_time = sum(step_times) / len(step_times) throughput = batch_size * seq_length / avg_step_time @@ -228,6 +303,9 @@ class TrainingSim: "num_layers": num_layers, "hidden_size": hidden_size, "gpu_count": gpu_count, + "gpu_scope": "single_gpu:0", + "performance_verdict": "MEASURED", + "sdpa_backend_policy": sdpa_policy, "dtype": str(dtype).replace("torch.", ""), "batch_size": batch_size, "seq_length": seq_length, diff --git a/pyproject.toml b/pyproject.toml index 76422ce..788a3f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,10 +17,12 @@ dependencies = [ # uv sync --extra torch-cu121 # uv sync --extra torch-cu124 (recommended for CUDA 12.x + Driver 535+) # uv sync --extra torch-cu128 (for CUDA 12.8+ / Driver 570+) +# uv sync --extra torch-cu130 (recommended for B300 / Driver 580+) torch-cu118 = ["torch>=2.1.0"] torch-cu121 = ["torch>=2.1.0"] torch-cu124 = ["torch>=2.1.0"] torch-cu128 = ["torch>=2.1.0"] +torch-cu130 = ["torch>=2.13.0"] [build-system] requires = ["hatchling"] @@ -47,6 +49,11 @@ name = "pytorch-cu128" url = "https://download.pytorch.org/whl/cu128" explicit = true +[[tool.uv.index]] +name = "pytorch-cu130" +url = "https://download.pytorch.org/whl/cu130" +explicit = true + # Default: resolve torch from cu124 index (most broadly compatible with 535+ drivers) # To use a different CUDA version, override: # uv pip install torch --index-url https://download.pytorch.org/whl/cu128 diff --git a/tests/test_b300_judgement.py b/tests/test_b300_judgement.py new file mode 100644 index 0000000..86e8ede --- /dev/null +++ b/tests/test_b300_judgement.py @@ -0,0 +1,90 @@ +import os +import tempfile +import unittest +from unittest.mock import patch + +from modules.benchmark import Benchmark +from modules.gpu_specs import get_gpu_specs +from modules.nccl_test import NCCLTest +from modules.rdma_test import RDMATest +from modules.report import ReportGenerator +from modules.training_sim import TrainingSim + + +class B300JudgementTests(unittest.TestCase): + def test_b300_training_excludes_cudnn_sdpa(self): + self.assertTrue(TrainingSim._requires_b300_sdpa_workaround("b300", None, 91900)) + self.assertTrue(TrainingSim._requires_b300_sdpa_workaround("auto", (10, 3), 91900)) + self.assertFalse(TrainingSim._requires_b300_sdpa_workaround("b300", (10, 3), 92000)) + self.assertFalse(TrainingSim._requires_b300_sdpa_workaround("h200", (9, 0), 91900)) + + def test_b300_uses_dense_peaks_and_calibrated_floors(self): + specs = get_gpu_specs("b300") + self.assertEqual(specs["fp8_tflops"], 4500) + self.assertEqual(specs["bf16_tflops"], 2250) + self.assertEqual(specs["compute_pass_thresholds_tflops"]["fp8"], 2800) + self.assertEqual(specs["nccl_pass_thresholds_gbps"]["allreduce"], 750) + + def test_compute_verdict_uses_slowest_gpu_not_average(self): + result = { + "compute": { + "per_dtype_tflops": {"fp8": 3000.0}, + "worst_per_dtype_tflops": {"fp8": 2500.0}, + "pass_thresholds_tflops": {"fp8": 2800.0}, + "efficiency_pct": {"fp8": 66.7}, + } + } + judged = Benchmark.judge_compute(result) + self.assertEqual(judged["verdict"], "FAIL") + self.assertEqual(judged["rows"][0][1], 2500.0) + + def test_rdma_without_peer_is_skip_not_false_failure(self): + rdma = RDMATest({"rdma": {}}) + with ( + patch.object(rdma, "_get_ib_devices", return_value=["mlx5_0"]), + patch.object(rdma, "_get_ib_ports", return_value=["1"]), + patch.object(rdma, "_read_sys", side_effect=["InfiniBand", "4: ACTIVE"]), + patch.object(rdma, "_collect_device_info", return_value=[]), + ): + result = rdma.run() + self.assertEqual(result["status"], "SKIP") + self.assertTrue(result["skipped"]) + self.assertIn("peer not configured", result["reason"]) + + def test_nccl_runtime_env_preserves_and_prepends_venv_libraries(self): + with ( + patch("modules.nccl_test.sysconfig.get_paths", return_value={"purelib": "/venv/site"}), + patch("modules.nccl_test.os.path.isdir", return_value=True), + patch.dict(os.environ, {"LD_LIBRARY_PATH": "/system/lib"}), + ): + env = NCCLTest._runtime_env() + self.assertEqual( + env["LD_LIBRARY_PATH"], + "/venv/site/nvidia/nccl/lib:/venv/site/torch/lib:/system/lib", + ) + self.assertEqual(env["NCCL_DEBUG"], "WARN") + + def test_markdown_summary_keeps_warn_and_measured_semantics(self): + results = { + "nccl": {"status": "WARN", "passed": True, "tests": {}}, + "training": { + "throughput_tokens_per_sec": 1234, + "gpu_scope": "single_gpu:0", + }, + } + generator = ReportGenerator({"report": {"output_dir": "."}}) + summary = dict(generator._build_summary(results)) + self.assertEqual(summary["NCCL"], "WARN") + self.assertTrue(summary["Training"].startswith("MEASURED")) + + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, "report.md") + generator.generate(results, fmt="md", output=path) + with open(path, encoding="utf-8") as report_file: + report = report_file.read() + self.assertIn("| NCCL | WARN |", report) + self.assertIn("synthetic workload; not an 8-GPU scaling acceptance test", report) + + +if __name__ == "__main__": + unittest.main()