fix: improve B300 acceptance test logic
This commit is contained in:
parent
6ea2acf2d6
commit
6caeeb4400
62
configs/b300_single_node_acceptance_20260827.yaml
Normal file
62
configs/b300_single_node_acceptance_20260827.yaml
Normal file
@ -0,0 +1,62 @@
|
||||
gpu_type: b300
|
||||
|
||||
tools:
|
||||
install_dir: /root/test_gpu_scripts/.gpu-test-env/tools
|
||||
|
||||
benchmark:
|
||||
memory:
|
||||
size_mb: 4096
|
||||
iterations: 10
|
||||
nvbandwidth_buffer_mb: 512
|
||||
nvbandwidth_samples: 3
|
||||
compute:
|
||||
dtypes: [fp32, tf32, fp16, bf16, fp8]
|
||||
sweep_sizes:
|
||||
- 3584
|
||||
- 4608
|
||||
- 5376
|
||||
- 8192
|
||||
- 11520
|
||||
- [2048, 2048, 13312]
|
||||
- [2048, 2048, 16384]
|
||||
matrix_size: 8192
|
||||
warmup: 20
|
||||
iterations: 80
|
||||
|
||||
health:
|
||||
temp_warning: 75
|
||||
temp_critical: 85
|
||||
power_limit: null
|
||||
|
||||
nccl:
|
||||
min_bandwidth_gbps: null
|
||||
test_allreduce: true
|
||||
test_alltoall: true
|
||||
test_broadcast: true
|
||||
test_reduce_scatter: false
|
||||
test_allgather: false
|
||||
test_sendrecv: false
|
||||
|
||||
stress:
|
||||
duration_sec: 600
|
||||
use_doubles: false
|
||||
use_tensor_cores: true
|
||||
memory_pct: 90
|
||||
gpus: all
|
||||
|
||||
rdma:
|
||||
min_bandwidth_gbps: 50
|
||||
max_latency_us: 10
|
||||
cross_node:
|
||||
enabled: false
|
||||
|
||||
training:
|
||||
model: synthetic
|
||||
batch_size: 8
|
||||
seq_length: 2048
|
||||
num_steps: 50
|
||||
dtype: bf16
|
||||
|
||||
report:
|
||||
output_dir: /root/test_gpu_scripts/reports/b300_single_node_20260827
|
||||
format: json
|
||||
@ -21,7 +21,8 @@ from rich import box
|
||||
from modules.gpu_info import GPUInfo
|
||||
from modules.health_check import HealthCheck
|
||||
from modules.benchmark import Benchmark
|
||||
from modules.nccl_test import NCCLTest, MultiNodeNCCLTest
|
||||
from modules.nccl_test import NCCLTest
|
||||
from modules.multinode_nccl_test import MultiNodeNCCLTest
|
||||
from modules.training_sim import TrainingSim
|
||||
from modules.stress_test import StressTest
|
||||
from modules.rdma_test import RDMATest
|
||||
@ -333,14 +334,39 @@ def _run_full_suite(config: dict, console: Console) -> dict:
|
||||
console.print(f"[bold red]{name} FAILED: {e}[/bold red]")
|
||||
all_results[key] = {"error": str(e)}
|
||||
|
||||
# Summary
|
||||
# Summary: classify actual verdicts. The old implementation counted every
|
||||
# result without an exception as PASS, even when passed=False or status=FAIL.
|
||||
console.print("\n" + "=" * 60)
|
||||
# Only count test results, exclude metadata like timestamp
|
||||
test_results = {k: v for k, v in all_results.items() if k != "timestamp"}
|
||||
passed = sum(1 for v in test_results.values() if not isinstance(v, dict) or "error" not in v)
|
||||
total = len(test_results)
|
||||
color = "green" if passed == total else ("yellow" if passed > 0 else "red")
|
||||
console.print(f"[bold {color}]Suite complete: {passed}/{total} tests passed[/bold {color}]")
|
||||
statuses = {}
|
||||
for key, value in test_results.items():
|
||||
status = "PASS"
|
||||
if isinstance(value, dict) and value.get("error"):
|
||||
status = "FAIL"
|
||||
elif key == "health":
|
||||
status = "PASS" if value.get("passed") else "FAIL"
|
||||
elif key == "memory_bench":
|
||||
mem = value.get("memory", value)
|
||||
if mem.get("error"):
|
||||
status = "FAIL"
|
||||
else:
|
||||
eff = mem.get("d2d_efficiency_pct") or mem.get("efficiency_pct") or 0
|
||||
status = "PASS" if eff >= 80 else ("WARN" if eff >= 60 else "FAIL")
|
||||
elif key == "compute_bench":
|
||||
status = Benchmark.judge_compute(value)["verdict"]
|
||||
elif key in ("nccl", "rdma"):
|
||||
status = value.get("status") or ("PASS" if value.get("passed") else "FAIL")
|
||||
elif key == "stress":
|
||||
status = "PASS" if value.get("passed") else "FAIL"
|
||||
elif key == "training":
|
||||
status = value.get("performance_verdict", "MEASURED")
|
||||
statuses[key] = status
|
||||
|
||||
counts = {name: list(statuses.values()).count(name) for name in ("PASS", "WARN", "FAIL", "SKIP", "MEASURED")}
|
||||
color = "red" if counts["FAIL"] else ("yellow" if counts["WARN"] else "green")
|
||||
summary = ", ".join(f"{name} {count}" for name, count in counts.items() if count)
|
||||
console.print(f"[bold {color}]Suite complete: {summary}[/bold {color}]")
|
||||
all_results["suite_status"] = statuses
|
||||
return all_results
|
||||
|
||||
|
||||
@ -446,11 +472,14 @@ Examples:
|
||||
if args.report:
|
||||
ReportGenerator(config).generate(results, fmt=args.format, output=args.output)
|
||||
has_errors = any("error" in v for v in results.values() if isinstance(v, dict))
|
||||
sys.exit(1 if has_errors else 0)
|
||||
has_failures = any(
|
||||
status == "FAIL" for status in results.get("suite_status", {}).values()
|
||||
)
|
||||
sys.exit(1 if has_errors or has_failures else 0)
|
||||
else:
|
||||
result = _run_test(test_map[args.test], config, console)
|
||||
if args.report and result:
|
||||
ReportGenerator(config).generate({args.test: result, "timestamp": datetime.now().isoformat()},
|
||||
ReportGenerator(config).generate({test_map[args.test]: result, "timestamp": datetime.now().isoformat()},
|
||||
fmt=args.format, output=args.output)
|
||||
|
||||
|
||||
|
||||
@ -150,11 +150,14 @@ class Benchmark:
|
||||
|
||||
progress.advance(task)
|
||||
|
||||
# Compare like with like: the 900 GB/s reference is per direction, so do
|
||||
# not select the bidirectional SM result as the acceptance value. Keep it
|
||||
# as a separate informational metric.
|
||||
d2d_bw = max(
|
||||
results_by_test.get("d2d_write", 0),
|
||||
results_by_test.get("d2d_read", 0),
|
||||
results_by_test.get("d2d_bidir", 0),
|
||||
)
|
||||
d2d_bidir_bw = results_by_test.get("d2d_bidir", 0)
|
||||
h2d_bw = results_by_test.get("h2d", 0)
|
||||
d2h_bw = results_by_test.get("d2h", 0)
|
||||
|
||||
@ -185,6 +188,7 @@ class Benchmark:
|
||||
"h2d_bandwidth_gbps": round(h2d_bw, 1),
|
||||
"d2h_bandwidth_gbps": round(d2h_bw, 1),
|
||||
"d2d_bandwidth_gbps": round(d2d_bw, 1),
|
||||
"d2d_bidirectional_gbps": round(d2d_bidir_bw, 1),
|
||||
"h2d_peak_gbps": pcie_peak if pcie_peak else None,
|
||||
"d2h_peak_gbps": pcie_peak if pcie_peak else None,
|
||||
"d2d_peak_gbps": round(d2d_peak, 1) if d2d_peak else None,
|
||||
@ -195,6 +199,7 @@ class Benchmark:
|
||||
"efficiency_pct": d2d_efficiency,
|
||||
"results_by_test": results_by_test,
|
||||
"per_gpu": [],
|
||||
"scope_note": "GPU peer/PCIe transfer bandwidth; local HBM bandwidth not measured",
|
||||
"accelerator_vendor": self.specs.get("vendor", "nvidia"),
|
||||
}
|
||||
}
|
||||
@ -385,11 +390,13 @@ class Benchmark:
|
||||
BarColumn(), TextColumn("{task.completed}/{task.total}"),
|
||||
TimeElapsedColumn(), console=self.console,
|
||||
) as progress:
|
||||
task = progress.add_task("Testing dtypes...", total=len(configured_dtypes))
|
||||
task = progress.add_task(
|
||||
"Testing dtypes on every GPU...", total=len(configured_dtypes) * gpu_count
|
||||
)
|
||||
|
||||
for dtype_name in configured_dtypes:
|
||||
if dtype_name not in dtype_map:
|
||||
progress.advance(task)
|
||||
progress.advance(task, advance=gpu_count)
|
||||
continue
|
||||
|
||||
# Skip FP8 if GPU architecture doesn't support it
|
||||
@ -399,7 +406,7 @@ class Benchmark:
|
||||
arch = self.specs.get("architecture", "unknown")
|
||||
results_by_dtype["fp8"] = f"skipped ({arch} does not support FP8)"
|
||||
self.console.print(f"[dim] fp8: skipped - {arch} architecture has no FP8 support[/dim]")
|
||||
progress.advance(task)
|
||||
progress.advance(task, advance=gpu_count)
|
||||
continue
|
||||
|
||||
dtype_val, peak_tflops = dtype_map[dtype_name]
|
||||
@ -413,32 +420,48 @@ class Benchmark:
|
||||
elif dtype_name == "fp32":
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
|
||||
best_tflops, best_shape, last_err = 0.0, None, None
|
||||
for (M, N, K) in shapes:
|
||||
try:
|
||||
t = self._bench_matmul_once(dtype_name, dtype_val, M, N, K, warmup, iterations)
|
||||
if t > best_tflops:
|
||||
best_tflops, best_shape = t, (M, N, K)
|
||||
except Exception as e: # noqa: BLE001 - record and try the next shape
|
||||
last_err = e
|
||||
gpu_values = []
|
||||
gpu_shapes = {}
|
||||
last_err = None
|
||||
fastest = (0.0, None)
|
||||
for gpu_idx in range(gpu_count):
|
||||
best_tflops, best_shape = 0.0, None
|
||||
for (M, N, K) in shapes:
|
||||
try:
|
||||
t = self._bench_matmul_once(
|
||||
dtype_name, dtype_val, M, N, K,
|
||||
warmup, iterations, gpu_idx,
|
||||
)
|
||||
if t > best_tflops:
|
||||
best_tflops, best_shape = t, (M, N, K)
|
||||
except Exception as e: # noqa: BLE001 - record and try next shape
|
||||
last_err = e
|
||||
if best_shape is not None:
|
||||
value = round(best_tflops, 1)
|
||||
per_gpu_results[gpu_idx][dtype_name] = value
|
||||
gpu_values.append(value)
|
||||
gpu_shapes[str(gpu_idx)] = "x".join(str(d) for d in best_shape)
|
||||
if best_tflops > fastest[0]:
|
||||
fastest = (best_tflops, best_shape)
|
||||
else:
|
||||
per_gpu_results[gpu_idx][dtype_name] = f"error: {last_err}"
|
||||
progress.advance(task)
|
||||
|
||||
torch.backends.cuda.matmul.allow_tf32 = old_tf32
|
||||
|
||||
if best_shape is None:
|
||||
if not gpu_values:
|
||||
results_by_dtype[dtype_name] = f"error: {last_err}"
|
||||
self.console.print(f"[yellow] {dtype_name}: {last_err}[/yellow]")
|
||||
else:
|
||||
shape_str = "x".join(str(d) for d in best_shape)
|
||||
results_by_dtype[dtype_name] = round(best_tflops, 1)
|
||||
best_shapes[dtype_name] = shape_str
|
||||
for pg in per_gpu_results:
|
||||
pg[dtype_name] = round(best_tflops, 1)
|
||||
avg_value = round(sum(gpu_values) / len(gpu_values), 1)
|
||||
results_by_dtype[dtype_name] = avg_value
|
||||
best_shapes[dtype_name] = "x".join(str(d) for d in fastest[1])
|
||||
if len(shapes) > 1:
|
||||
self.console.print(
|
||||
f"[dim] {dtype_name}: {best_tflops:.1f} TFLOPS @ {shape_str}[/dim]"
|
||||
f"[dim] {dtype_name}: avg {avg_value:.1f}, "
|
||||
f"min {min(gpu_values):.1f}, max {max(gpu_values):.1f} TFLOPS[/dim]"
|
||||
)
|
||||
|
||||
progress.advance(task)
|
||||
best_shapes.setdefault("per_gpu", {})[dtype_name] = gpu_shapes
|
||||
|
||||
efficiency = {}
|
||||
for dt, achieved in results_by_dtype.items():
|
||||
@ -447,6 +470,18 @@ class Benchmark:
|
||||
if peak_tp:
|
||||
efficiency[dt] = round((achieved / peak_tp) * 100, 1)
|
||||
|
||||
worst_by_dtype = {}
|
||||
best_by_dtype = {}
|
||||
spread_by_dtype = {}
|
||||
for dt in results_by_dtype:
|
||||
values = [pg.get(dt) for pg in per_gpu_results]
|
||||
values = [v for v in values if isinstance(v, (int, float))]
|
||||
if values:
|
||||
worst_by_dtype[dt] = min(values)
|
||||
best_by_dtype[dt] = max(values)
|
||||
avg = sum(values) / len(values)
|
||||
spread_by_dtype[dt] = round((max(values) - min(values)) / avg * 100, 2) if avg else 0
|
||||
|
||||
return {
|
||||
"compute": {
|
||||
"per_dtype_tflops": results_by_dtype,
|
||||
@ -459,6 +494,9 @@ class Benchmark:
|
||||
self.specs.get("compute_pass_thresholds_tflops") or {}
|
||||
),
|
||||
"per_gpu": per_gpu_results,
|
||||
"worst_per_dtype_tflops": worst_by_dtype,
|
||||
"best_per_dtype_tflops": best_by_dtype,
|
||||
"spread_pct": spread_by_dtype,
|
||||
"sweep_sizes": list(sweep_sizes),
|
||||
"best_shapes": best_shapes,
|
||||
"matrix_size": matrix_size,
|
||||
@ -469,7 +507,7 @@ class Benchmark:
|
||||
}
|
||||
|
||||
def _bench_matmul_once(self, dtype_name: str, dtype_val, M: int, N: int, K: int,
|
||||
warmup: int, iterations: int) -> float:
|
||||
warmup: int, iterations: int, gpu_index: int = 0) -> float:
|
||||
"""Time one (M×K)·(K×N) matmul for a dtype and return achieved TFLOPS.
|
||||
|
||||
Uses an L2-cache-busting pool of matrix pairs (total > 256 MB) so operands
|
||||
@ -478,34 +516,37 @@ class Benchmark:
|
||||
cuBLAS, which on H100 beats torch.compile/Triton for plain GEMM and avoids the
|
||||
per-shape recompile cost that would make a sweep pathologically slow.
|
||||
"""
|
||||
elem_bytes = 1 if dtype_name == "fp8" else torch.tensor([], dtype=dtype_val).element_size()
|
||||
device = f"cuda:{gpu_index}"
|
||||
with torch.cuda.device(gpu_index):
|
||||
elem_bytes = 1 if dtype_name == "fp8" else torch.tensor([], dtype=dtype_val).element_size()
|
||||
pair_bytes = (M * K + K * N) * elem_bytes
|
||||
num_pools = max(4, -(-256 * 1024 * 1024 // pair_bytes)) # ceil(256MB / pair)
|
||||
|
||||
if dtype_name == "fp8":
|
||||
if not hasattr(torch, "_scaled_mm"):
|
||||
raise RuntimeError("torch._scaled_mm unavailable — upgrade to PyTorch >= 2.1")
|
||||
pools_a = [torch.randn(M, K, device="cuda", dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)]
|
||||
pools_b = [torch.randn(N, K, device="cuda", dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)]
|
||||
scale_a = torch.tensor(1.0, device="cuda")
|
||||
scale_b = torch.tensor(1.0, device="cuda")
|
||||
pools_a = [torch.randn(M, K, device=device, dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)]
|
||||
pools_b = [torch.randn(N, K, device=device, dtype=torch.float32).to(torch.float8_e4m3fn) for _ in range(num_pools)]
|
||||
scale_a = torch.tensor(1.0, device=device)
|
||||
scale_b = torch.tensor(1.0, device=device)
|
||||
def op(i):
|
||||
return torch._scaled_mm(pools_a[i], pools_b[i].T, scale_a=scale_a, scale_b=scale_b, out_dtype=torch.bfloat16)
|
||||
else:
|
||||
pools_a = [torch.randn(M, K, device="cuda", dtype=dtype_val) for _ in range(num_pools)]
|
||||
pools_b = [torch.randn(K, N, device="cuda", dtype=dtype_val) for _ in range(num_pools)]
|
||||
pools_a = [torch.randn(M, K, device=device, dtype=dtype_val) for _ in range(num_pools)]
|
||||
pools_b = [torch.randn(K, N, device=device, dtype=dtype_val) for _ in range(num_pools)]
|
||||
def op(i):
|
||||
return torch.matmul(pools_a[i], pools_b[i])
|
||||
|
||||
try:
|
||||
torch.cuda.set_device(gpu_index)
|
||||
# Probe once so a broken/unsupported kernel raises before the timed loop.
|
||||
_probe = op(0)
|
||||
torch.cuda.synchronize()
|
||||
torch.cuda.synchronize(gpu_index)
|
||||
del _probe
|
||||
|
||||
for i in range(warmup):
|
||||
op(i % num_pools)
|
||||
torch.cuda.synchronize()
|
||||
torch.cuda.synchronize(gpu_index)
|
||||
|
||||
start_event = torch.cuda.Event(enable_timing=True)
|
||||
end_event = torch.cuda.Event(enable_timing=True)
|
||||
@ -513,7 +554,7 @@ class Benchmark:
|
||||
for i in range(iterations):
|
||||
op(i % num_pools)
|
||||
end_event.record()
|
||||
torch.cuda.synchronize()
|
||||
torch.cuda.synchronize(gpu_index)
|
||||
elapsed_ms = start_event.elapsed_time(end_event)
|
||||
finally:
|
||||
del pools_a, pools_b
|
||||
@ -584,18 +625,24 @@ class Benchmark:
|
||||
|
||||
table = Table(box=None, padding=(0, 1))
|
||||
table.add_column("DType", style="bold")
|
||||
table.add_column("Achieved (TFLOPS)", justify="right")
|
||||
table.add_column("Avg", justify="right")
|
||||
table.add_column("Min", justify="right")
|
||||
table.add_column("Max", justify="right")
|
||||
table.add_column("Spread", justify="right")
|
||||
table.add_column("Peak", justify="right")
|
||||
table.add_column("Efficiency", justify="right")
|
||||
|
||||
peak = comp.get("peak_tflops", {})
|
||||
per_dtype = comp.get("per_dtype_tflops", {})
|
||||
eff = comp.get("efficiency_pct", {})
|
||||
worst = comp.get("worst_per_dtype_tflops", {})
|
||||
best = comp.get("best_per_dtype_tflops", {})
|
||||
spread = comp.get("spread_pct", {})
|
||||
|
||||
for dt in per_dtype:
|
||||
achieved = per_dtype[dt]
|
||||
if isinstance(achieved, str):
|
||||
table.add_row(dt, f"[red]{achieved}[/red]", str(peak.get(dt, "N/A")), "N/A")
|
||||
table.add_row(dt, f"[red]{achieved}[/red]", "-", "-", "-", str(peak.get(dt, "N/A")), "N/A")
|
||||
continue
|
||||
pk = peak.get(dt, 0)
|
||||
ef = eff.get(dt, 0)
|
||||
@ -606,7 +653,11 @@ class Benchmark:
|
||||
else:
|
||||
peak_text = "N/A"
|
||||
eff_text = "[cyan]MEASURED[/cyan]"
|
||||
table.add_row(dt.upper(), f"{achieved:.1f}", peak_text, eff_text)
|
||||
table.add_row(
|
||||
dt.upper(), f"{achieved:.1f}", f"{worst.get(dt, achieved):.1f}",
|
||||
f"{best.get(dt, achieved):.1f}", f"{spread.get(dt, 0):.2f}%",
|
||||
peak_text, eff_text,
|
||||
)
|
||||
c.print(table)
|
||||
|
||||
@staticmethod
|
||||
@ -623,6 +674,7 @@ class Benchmark:
|
||||
"""
|
||||
comp = results.get("compute", results)
|
||||
per_dtype = comp.get("per_dtype_tflops", {})
|
||||
worst_per_dtype = comp.get("worst_per_dtype_tflops", {}) or {}
|
||||
thresholds = comp.get("pass_thresholds_tflops", {}) or {}
|
||||
eff = comp.get("efficiency_pct", {})
|
||||
rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
|
||||
@ -630,11 +682,12 @@ class Benchmark:
|
||||
evaluated = False
|
||||
measured = False
|
||||
for dt, val in per_dtype.items():
|
||||
judged_val = worst_per_dtype.get(dt, val)
|
||||
thr = thresholds.get(dt)
|
||||
if isinstance(val, str):
|
||||
status = "SKIP"
|
||||
elif thr:
|
||||
status = "PASS" if val >= thr else ("WARN" if val >= thr * 0.9 else "FAIL")
|
||||
status = "PASS" if judged_val >= thr else ("WARN" if judged_val >= thr * 0.9 else "FAIL")
|
||||
evaluated = True
|
||||
elif dt not in eff:
|
||||
status = "MEASURED"
|
||||
@ -643,7 +696,7 @@ class Benchmark:
|
||||
e = eff.get(dt, 0)
|
||||
status = "PASS" if e >= 80 else ("WARN" if e >= 50 else "FAIL")
|
||||
evaluated = True
|
||||
rows.append((dt, val, thr, status))
|
||||
rows.append((dt, judged_val, thr, status))
|
||||
if rank[status] > rank[verdict]:
|
||||
verdict = status
|
||||
if measured and not evaluated:
|
||||
|
||||
@ -16,6 +16,7 @@ GPU_NAME_PATTERNS = {
|
||||
"H200": "h200",
|
||||
"H20": "h20", # H20 / H20-3e is the China-compliance export variant, REDUCED peaks
|
||||
"B200": "b200",
|
||||
"B30-E": "b300", # nvidia-smi product name used by the B300 systems under test
|
||||
"B300": "b300",
|
||||
}
|
||||
|
||||
@ -228,15 +229,33 @@ GPU_SPECS = {
|
||||
"hbm_capacity_gb": 288,
|
||||
"hbm_type": "HBM3e",
|
||||
"memory_bandwidth_gbps": 8000, # GB/s (8 TB/s)
|
||||
"fp32_tflops": 125,
|
||||
"tf32_tflops": 1750, # dense (estimated)
|
||||
"fp16_tflops": 3500, # dense
|
||||
"bf16_tflops": 3500, # dense
|
||||
"fp8_tflops": 7000, # dense
|
||||
# Dense, per-GPU reference peaks. NVIDIA publishes DGX B300 FP8 as
|
||||
# 72 PFLOPS sparse for eight GPUs and explicitly states dense is half,
|
||||
# i.e. 4.5 PFLOPS/GPU. The previous 7 PFLOPS dense value mixed product
|
||||
# variants/modes and made healthy B300 cards fail by construction.
|
||||
"fp32_tflops": 75,
|
||||
"tf32_tflops": 1100,
|
||||
"fp16_tflops": 2250,
|
||||
"bf16_tflops": 2250,
|
||||
"fp8_tflops": 4500,
|
||||
# Acceptance floors are achievable eager-PyTorch GEMM values rather than
|
||||
# a blanket percentage of marketing peak. BF16/FP8 are calibrated against
|
||||
# NVIDIA Transformer Engine's public B300 GEMM examples; the remaining
|
||||
# modes use conservative floors below sustained measurements on this path.
|
||||
"compute_pass_thresholds_tflops": {
|
||||
"fp32": 60, "tf32": 750, "fp16": 1450, "bf16": 1450, "fp8": 2800,
|
||||
},
|
||||
# Single-node 8-GPU floors derived from public B300 nccl-tests results.
|
||||
# These are operation-specific because collectives do not reach the same
|
||||
# fraction of raw NVLink bandwidth.
|
||||
"nccl_pass_thresholds_gbps": {
|
||||
"allreduce": 750, "alltoall": 620, "broadcast": 600,
|
||||
"allgather": 700, "reducescatter": 700, "sendrecv": 600,
|
||||
},
|
||||
"tdp_watts": 1200,
|
||||
"nvlink_gen": 5,
|
||||
"nvlink_bandwidth_gbps": 1800,
|
||||
"pcie_gen": 5,
|
||||
"pcie_gen": 6,
|
||||
"min_driver_version": "550",
|
||||
"min_cuda_version": "12.4",
|
||||
},
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
@ -210,9 +211,11 @@ class MultiNodeNCCLTest:
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "ConnectTimeout=5",
|
||||
"-o", "StrictHostKeyChecking=accept-new",
|
||||
target,
|
||||
"hostname",
|
||||
]
|
||||
plm_rsh_args = self.cfg.get("plm_rsh_args")
|
||||
if plm_rsh_args:
|
||||
cmd.extend(shlex.split(str(plm_rsh_args)))
|
||||
cmd.extend([target, "hostname"])
|
||||
try:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=8, env=self._runtime_env())
|
||||
detail = r.stdout.strip() or r.stderr.strip()[:120]
|
||||
@ -287,6 +290,15 @@ class MultiNodeNCCLTest:
|
||||
"--mca", "btl_tcp_if_include", str(self.cfg.get("socket_ifname", "bond0")),
|
||||
"--mca", "oob_tcp_if_include", str(self.cfg.get("oob_tcp_ifname", self.cfg.get("socket_ifname", "bond0"))),
|
||||
]
|
||||
# Keep the MPI launcher/control plane separately configurable from the
|
||||
# NCCL data plane. For example, PML/UCX can be restricted to TCP/eth0
|
||||
# while NCCL uses the configured RoCE HCAs for benchmark traffic.
|
||||
mpi_pml = self.cfg.get("mpi_pml", "ob1")
|
||||
if mpi_pml:
|
||||
cmd.extend(["--mca", "pml", str(mpi_pml)])
|
||||
mpi_btl = self.cfg.get("mpi_btl", "self,tcp,vader")
|
||||
if mpi_btl:
|
||||
cmd.extend(["--mca", "btl", str(mpi_btl)])
|
||||
# HCOLL does an eager UCX IB wireup during MPI_Init that deadlocks on nodes
|
||||
# where UCX only sees mlx5_2/mlx5_3 instead of mlx5_0/mlx5_1. HCOLL
|
||||
# collective optimisations are irrelevant for NCCL tests (NCCL handles its
|
||||
|
||||
@ -5,6 +5,7 @@ import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sysconfig
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
@ -77,6 +78,30 @@ class NCCLTest:
|
||||
return p
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _runtime_env() -> dict:
|
||||
"""Return an environment that can load NCCL bundled in the active venv.
|
||||
|
||||
CUDA PyTorch wheels install libnccl under site-packages/nvidia/nccl/lib,
|
||||
which is intentionally isolated from the system linker path. nccl-tests
|
||||
is an external executable, so it needs that directory explicitly even
|
||||
though importing torch works in the same virtual environment.
|
||||
"""
|
||||
env = os.environ.copy()
|
||||
purelib = sysconfig.get_paths().get("purelib", "")
|
||||
candidates = [
|
||||
os.path.join(purelib, "nvidia", "nccl", "lib"),
|
||||
os.path.join(purelib, "torch", "lib"),
|
||||
]
|
||||
library_dirs = [path for path in candidates if os.path.isdir(path)]
|
||||
existing = env.get("LD_LIBRARY_PATH", "")
|
||||
if existing:
|
||||
library_dirs.append(existing)
|
||||
if library_dirs:
|
||||
env["LD_LIBRARY_PATH"] = os.pathsep.join(library_dirs)
|
||||
env["NCCL_DEBUG"] = "WARN"
|
||||
return env
|
||||
|
||||
def run(self) -> dict:
|
||||
gpu_count = 0
|
||||
if TORCH_AVAILABLE:
|
||||
@ -101,6 +126,7 @@ class NCCLTest:
|
||||
tests.append(("sendrecv_perf", "SendRecv"))
|
||||
|
||||
nvlink_bw = self.specs.get("nvlink_bandwidth_gbps", 0)
|
||||
calibrated_thresholds = self.specs.get("nccl_pass_thresholds_gbps", {}) or {}
|
||||
# User-provided override applies uniformly across all ops; otherwise
|
||||
# each op gets its own threshold from _OP_BW_FRACTIONS.
|
||||
user_override = self.nccl_cfg.get("min_bandwidth_gbps")
|
||||
@ -108,6 +134,9 @@ class NCCLTest:
|
||||
def threshold_for(label: str) -> float:
|
||||
if user_override:
|
||||
return float(user_override)
|
||||
calibrated = calibrated_thresholds.get(label.lower())
|
||||
if calibrated:
|
||||
return float(calibrated)
|
||||
if nvlink_bw <= 0:
|
||||
return 10.0 # conservative floor
|
||||
frac = _OP_BW_FRACTIONS.get(label.lower(), 0.45)
|
||||
@ -152,14 +181,19 @@ class NCCLTest:
|
||||
self.console.print("[yellow]nccl-tests binaries failed, falling back to torchrun[/yellow]")
|
||||
return self._run_torchrun_fallback(gpu_count)
|
||||
|
||||
all_passed = all(
|
||||
r.get("status") == "PASS"
|
||||
for r in results.values()
|
||||
if isinstance(r, dict) and "status" in r
|
||||
statuses = [
|
||||
r.get("status") for r in results.values()
|
||||
if isinstance(r, dict) and r.get("status")
|
||||
]
|
||||
overall_status = (
|
||||
"FAIL" if "FAIL" in statuses else
|
||||
"WARN" if "WARN" in statuses else
|
||||
"PASS" if statuses else "SKIP"
|
||||
)
|
||||
|
||||
return {
|
||||
"passed": all_passed,
|
||||
"passed": overall_status in ("PASS", "WARN"),
|
||||
"status": overall_status,
|
||||
"source": "nccl-tests",
|
||||
"min_bandwidth_gbps": {
|
||||
lbl.lower(): threshold_for(lbl) for _, lbl in tests
|
||||
@ -188,8 +222,7 @@ class NCCLTest:
|
||||
]
|
||||
|
||||
try:
|
||||
env = os.environ.copy()
|
||||
env["NCCL_DEBUG"] = "WARN"
|
||||
env = self._runtime_env()
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env)
|
||||
|
||||
combined = r.stdout + r.stderr
|
||||
@ -234,8 +267,7 @@ class NCCLTest:
|
||||
]
|
||||
|
||||
try:
|
||||
env = os.environ.copy()
|
||||
env["NCCL_DEBUG"] = "WARN"
|
||||
env = self._runtime_env()
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env)
|
||||
|
||||
combined = r.stdout + r.stderr
|
||||
@ -440,6 +472,7 @@ dist.destroy_process_group()
|
||||
return
|
||||
|
||||
passed = results.get("passed", False)
|
||||
overall_status = results.get("status") or ("PASS" if passed else "FAIL")
|
||||
source = results.get("source", "unknown")
|
||||
|
||||
if source == "torchrun_fallback":
|
||||
@ -464,7 +497,12 @@ dist.destroy_process_group()
|
||||
c.print("\n[yellow]Note: functional connectivity test only (no performance data)[/yellow]")
|
||||
else:
|
||||
# nccl-tests mode
|
||||
verdict = "[bold green]✓ NCCL tests PASSED[/bold green]" if passed else "[bold yellow]⚠ NCCL tests WARNING[/bold yellow]"
|
||||
if overall_status == "PASS":
|
||||
verdict = "[bold green]✓ NCCL tests PASSED[/bold green]"
|
||||
elif overall_status == "WARN":
|
||||
verdict = "[bold yellow]⚠ NCCL tests WARNING[/bold yellow]"
|
||||
else:
|
||||
verdict = "[bold red]✗ NCCL tests FAILED[/bold red]"
|
||||
c.print(f"{verdict} [dim](via {source})[/dim]")
|
||||
|
||||
tests = results.get("tests", {})
|
||||
|
||||
@ -62,6 +62,7 @@ class RDMATest:
|
||||
# bond/management interfaces (e.g. mlx5_bond_0) can show ACTIVE state
|
||||
# without actually providing IB fabric connectivity.
|
||||
ib_devices = []
|
||||
active_ib_devices = []
|
||||
active_ib_port = False
|
||||
for dev in devices:
|
||||
for port in self._get_ib_ports(dev):
|
||||
@ -74,6 +75,7 @@ class RDMATest:
|
||||
f"/sys/class/infiniband/{dev}/ports/{port}/state")
|
||||
if "ACTIVE" in state.upper():
|
||||
active_ib_port = True
|
||||
active_ib_devices.append(dev)
|
||||
|
||||
device_info = self._collect_device_info(devices)
|
||||
cross_enabled = (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled")
|
||||
@ -114,28 +116,35 @@ class RDMATest:
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
self.console.print(f"[cyan]RDMA Test - Devices: {', '.join(devices)}[/cyan]")
|
||||
# perftest is a two-endpoint benchmark. Running both endpoints on
|
||||
# localhost can select a DOWN HCA, bypass the fabric, or return a zero
|
||||
# result that looks like hardware failure. Inventory-only is objective
|
||||
# until a peer is explicitly configured.
|
||||
if not cross_enabled:
|
||||
self.console.print(
|
||||
"[yellow]Active IB ports detected, but no cross-node peer is configured — "
|
||||
"skipping bandwidth/latency benchmarks[/yellow]"
|
||||
)
|
||||
return {
|
||||
"status": "SKIP", "skipped": True,
|
||||
"reason": "cross-node RDMA peer not configured; local loopback is not an acceptance test",
|
||||
"devices": device_info,
|
||||
"active_ib_devices": sorted(set(active_ib_devices)),
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
bw_results = self._run_bandwidth_tests(devices)
|
||||
latency_results = self._run_latency_tests(devices)
|
||||
|
||||
all_passed = all(
|
||||
r.get("status") == "PASS"
|
||||
for r in bw_results + latency_results
|
||||
if isinstance(r, dict)
|
||||
)
|
||||
|
||||
result = {
|
||||
"passed": all_passed,
|
||||
cross = self.run_cross_node()
|
||||
return {
|
||||
"passed": cross.get("status") == "PASS",
|
||||
"status": cross.get("status", "FAIL"),
|
||||
"reason": "cross-node RDMA benchmark",
|
||||
"devices": device_info,
|
||||
"bandwidth_tests": bw_results,
|
||||
"latency_tests": latency_results,
|
||||
"active_ib_devices": sorted(set(active_ib_devices)),
|
||||
"bandwidth_tests": [],
|
||||
"latency_tests": [],
|
||||
"cross_node": cross,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
# Cross-node (two-host) RDMA, run only when a peer is configured.
|
||||
if (self.rdma_cfg.get("cross_node", {}) or {}).get("enabled"):
|
||||
result["cross_node"] = self.run_cross_node()
|
||||
return result
|
||||
|
||||
def _collect_device_info(self, devices: List[str]) -> List[dict]:
|
||||
info = []
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
@ -258,8 +259,10 @@ class ReportGenerator:
|
||||
# --- Memory Bandwidth ---
|
||||
mem_data = self._extract_memory_results(results)
|
||||
if mem_data and not mem_data.get("error"):
|
||||
lines.append("## Memory Bandwidth\n")
|
||||
lines.append("## GPU Interconnect and PCIe Bandwidth\n")
|
||||
lines.append(f"Source: {mem_data.get('source', 'unknown')}\n")
|
||||
if mem_data.get("scope_note"):
|
||||
lines.append(f"> Scope: {mem_data['scope_note']}.\n")
|
||||
lines.append("| Metric | Value | Peak | Efficiency |")
|
||||
lines.append("|--------|-------|------|------------|")
|
||||
d2d = mem_data.get("d2d_bandwidth_gbps") or 0
|
||||
@ -284,8 +287,13 @@ class ReportGenerator:
|
||||
|
||||
lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {peak_text(h2d_peak)} | {efficiency_text(h2d_eff)} |")
|
||||
lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {peak_text(d2h_peak)} | {efficiency_text(d2h_eff)} |")
|
||||
d2d_label = "D2D (HBM copy)" if mem_data.get("accelerator_vendor") == "hygon" else "D2D (NVLink)"
|
||||
d2d_label = "D2D (HBM copy)" if mem_data.get("accelerator_vendor") == "hygon" else "D2D (NVLink, unidirectional)"
|
||||
lines.append(f"| {d2d_label} | {d2d:.1f} GB/s | {peak_text(d2d_peak)} | {efficiency_text(d2d_eff)} |")
|
||||
if mem_data.get("d2d_bidirectional_gbps"):
|
||||
lines.append(
|
||||
f"| D2D (NVLink, bidirectional informational) | "
|
||||
f"{mem_data['d2d_bidirectional_gbps']:.1f} GB/s | N/A | N/A |"
|
||||
)
|
||||
lines.append("")
|
||||
# PyTorch fallback can't accurately measure HBM peak (intra-GPU copy_()
|
||||
# only reaches ~20% of HBM bandwidth). When fallback is used, report
|
||||
@ -315,43 +323,58 @@ class ReportGenerator:
|
||||
isinstance(value, (int, float)) and value > 0
|
||||
for value in peak_tflops.values()
|
||||
)
|
||||
worst_values = comp_data.get("worst_per_dtype_tflops", {}) or {}
|
||||
best_values = comp_data.get("best_per_dtype_tflops", {}) or {}
|
||||
spreads = comp_data.get("spread_pct", {}) or {}
|
||||
if use_abs:
|
||||
lines.append("| DType | Achieved (TFLOPS) | Peak | Threshold | Status |")
|
||||
lines.append("| DType | Avg | Min | Max | Spread | Dense Peak | Pass Floor | Status |")
|
||||
else:
|
||||
lines.append("| DType | Achieved (TFLOPS) | Peak | Efficiency | Status |")
|
||||
lines.append("|-------|-------------------|------|------------|--------|")
|
||||
lines.append("| DType | Avg | Min | Max | Spread | Dense Peak | Efficiency | Status |")
|
||||
lines.append("|-------|-----|-----|-----|--------|------------|------------|--------|")
|
||||
worst_eff = 100.0
|
||||
overall_status = "PASS"
|
||||
rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0}
|
||||
for dt, val in per_dtype.items():
|
||||
if isinstance(val, str):
|
||||
# skipped or error
|
||||
lines.append(f"| {dt.upper()} | {val} | - | N/A | SKIP |")
|
||||
lines.append(f"| {dt.upper()} | {val} | - | - | - | - | N/A | SKIP |")
|
||||
else:
|
||||
pk = peak_tflops.get(dt, 0)
|
||||
ef = eff_pct.get(dt, 0)
|
||||
worst = worst_values.get(dt, val)
|
||||
best = best_values.get(dt, val)
|
||||
spread = spreads.get(dt, 0)
|
||||
if isinstance(ef, (int, float)) and ef > 0:
|
||||
worst_eff = min(worst_eff, ef)
|
||||
thr = pass_thresholds.get(dt)
|
||||
if use_abs and thr:
|
||||
if val >= thr:
|
||||
if worst >= thr:
|
||||
status = "PASS"
|
||||
elif val >= thr * 0.9:
|
||||
elif worst >= thr * 0.9:
|
||||
status = "WARN"
|
||||
else:
|
||||
status = "FAIL"
|
||||
lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | >= {thr} | {status} |")
|
||||
lines.append(
|
||||
f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | "
|
||||
f"{spread:.2f}% | {pk:.0f} | >= {thr} | {status} |"
|
||||
)
|
||||
elif has_peak_reference:
|
||||
status = "PASS" if ef >= 80 else ("WARN" if ef >= 50 else "FAIL")
|
||||
lines.append(f"| {dt.upper()} | {val:.1f} | {pk:.0f} | {ef:.1f}% | {status} |")
|
||||
lines.append(
|
||||
f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | "
|
||||
f"{spread:.2f}% | {pk:.0f} | {ef:.1f}% | {status} |"
|
||||
)
|
||||
else:
|
||||
status = "MEASURED"
|
||||
lines.append(f"| {dt.upper()} | {val:.1f} | N/A | N/A | {status} |")
|
||||
lines.append(
|
||||
f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | "
|
||||
f"{spread:.2f}% | N/A | N/A | {status} |"
|
||||
)
|
||||
if rank.get(status, 0) > rank.get(overall_status, 0):
|
||||
overall_status = status
|
||||
lines.append("")
|
||||
if use_abs:
|
||||
lines.append(f"**Verdict: {overall_status}** (absolute TFLOPS thresholds; worst efficiency {worst_eff:.1f}%)\n")
|
||||
lines.append(f"**Verdict: {overall_status}** (each dtype is judged by the slowest GPU against an achievable B300 floor)\n")
|
||||
elif has_peak_reference:
|
||||
overall_status = "PASS" if worst_eff >= 80 else ("WARN" if worst_eff >= 50 else "FAIL")
|
||||
lines.append(f"**Verdict: {overall_status}** (worst efficiency {worst_eff:.1f}%)\n")
|
||||
@ -377,8 +400,62 @@ class ReportGenerator:
|
||||
elif isinstance(data, dict) and data.get("error"):
|
||||
lines.append(f"| {op} | - | - | ERROR: {data['error']} |")
|
||||
lines.append("")
|
||||
passed = nccl.get("passed", False)
|
||||
lines.append(f"**Overall: {'PASS' if passed else 'FAIL'}**\n")
|
||||
nccl_status = nccl.get("status") or ("PASS" if nccl.get("passed") else "FAIL")
|
||||
lines.append(f"**Overall: {nccl_status}**\n")
|
||||
|
||||
# --- Multi-node NCCL ---
|
||||
multinode = results.get("multinode_nccl")
|
||||
if multinode:
|
||||
lines.append("## Multi-node NCCL\n")
|
||||
lines.append(f"Source: {multinode.get('source', 'unknown')} | "
|
||||
f"Mode: {multinode.get('mode', 'unknown')}\n")
|
||||
|
||||
hosts = multinode.get("hosts", []) or []
|
||||
if hosts:
|
||||
host_text = ", ".join(
|
||||
f"{host.get('name', host.get('addr', '?'))} "
|
||||
f"({host.get('addr', '?')}, {host.get('slots', '?')} GPUs)"
|
||||
for host in hosts
|
||||
)
|
||||
lines.append(f"- **Hosts:** {host_text}")
|
||||
lines.append(f"- **Overall: {'PASS' if multinode.get('passed') else 'FAIL'}**")
|
||||
if multinode.get("artifact_dir"):
|
||||
lines.append(f"- **Artifacts:** `{multinode['artifact_dir']}`")
|
||||
lines.append("")
|
||||
|
||||
tests = multinode.get("tests", {}) or {}
|
||||
if tests:
|
||||
lines.append("| Operation | Topology | Peak Bus BW | Avg Bus BW | Peak Size | Threshold | Wrong | Network | GDR | Status |")
|
||||
lines.append("|-----------|----------|-------------|------------|-----------|-----------|-------|---------|-----|--------|")
|
||||
for op, test_data in tests.items():
|
||||
for topo in test_data.get("topologies", []):
|
||||
network = topo.get("network", {}) or {}
|
||||
networks = ", ".join(network.get("networks", [])) or "N/A"
|
||||
ib_using = " ".join(network.get("ib_using", []))
|
||||
roce_hcas = sorted(set(re.findall(r"mlx5_bond_\d+", ib_using)))
|
||||
if roce_hcas:
|
||||
networks = f"{networks} ({len(roce_hcas)}x RoCE)"
|
||||
threshold = topo.get("min_required_gbps", 0)
|
||||
lines.append(
|
||||
f"| {op} | {topo.get('label', '')} | "
|
||||
f"{topo.get('peak_busbw_gbps', 0):.2f} GB/s | "
|
||||
f"{topo.get('avg_busbw_gbps', 0):.2f} GB/s | "
|
||||
f"{topo.get('peak_size', '')} | >= {threshold:g} GB/s | "
|
||||
f"{topo.get('wrong_count', 0)} | {networks} | "
|
||||
f"{network.get('gpu_direct_rdma', 'UNKNOWN')} | {topo.get('status', '?')} |"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
preflight = multinode.get("preflight", {}) or {}
|
||||
checks = preflight.get("checks", []) or []
|
||||
if checks:
|
||||
lines.append("### Preflight\n")
|
||||
lines.append("| Check | Status | Detail |")
|
||||
lines.append("|-------|--------|--------|")
|
||||
for check in checks:
|
||||
detail = str(check.get("detail", "")).replace("|", "\\|").replace("\n", " ")
|
||||
lines.append(f"| {check.get('name', '')} | {check.get('status', '?')} | {detail} |")
|
||||
lines.append("")
|
||||
|
||||
# --- Stress Test ---
|
||||
stress = results.get("stress")
|
||||
@ -391,6 +468,12 @@ class ReportGenerator:
|
||||
lines.append(f"- **Source:** {source}")
|
||||
lines.append(f"- **Duration:** {elapsed:.0f}s (requested {duration}s)")
|
||||
lines.append(f"- **Result: {'PASS' if passed else 'FAIL'}**")
|
||||
lines.append(f"- **Validation scope:** {stress.get('validation_scope', 'runtime completion')}")
|
||||
if stress.get("numerical_validation") is False:
|
||||
lines.append("- **Numerical result validation:** Not performed by PyTorch fallback")
|
||||
ecc_delta = stress.get("uncorrected_ecc_delta", {}) or {}
|
||||
if ecc_delta:
|
||||
lines.append(f"- **Uncorrected ECC delta:** {sum(ecc_delta.values())}")
|
||||
max_temp = stress.get("max_temperature_c", {}) or {}
|
||||
max_power = stress.get("max_power_w", {}) or {}
|
||||
if max_temp or max_power:
|
||||
@ -440,7 +523,10 @@ class ReportGenerator:
|
||||
lines.append(f"| Avg Step Time | {training.get('avg_step_time_ms', 0):.1f} ms |")
|
||||
lines.append(f"| Peak Memory | {training.get('peak_memory_gb', 0):.1f} GB |")
|
||||
lines.append(f"| Final Loss | {training.get('final_loss', 'N/A')} |")
|
||||
lines.append(f"| GPU Scope | {training.get('gpu_scope', 'framework-defined')} |")
|
||||
lines.append(f"| SDPA Policy | {training.get('sdpa_backend_policy', 'framework_default')} |")
|
||||
lines.append("")
|
||||
lines.append("**Verdict: MEASURED** (synthetic workload; not an 8-GPU scaling acceptance test)\n")
|
||||
|
||||
# --- Footer ---
|
||||
lines.append("---")
|
||||
@ -500,15 +586,15 @@ class ReportGenerator:
|
||||
mem = self._extract_memory_results(results)
|
||||
if mem:
|
||||
if mem.get("error"):
|
||||
items.append(("Memory Bandwidth", f"ERROR: {mem['error']}"))
|
||||
items.append(("GPU Interconnect / PCIe", f"ERROR: {mem['error']}"))
|
||||
elif mem.get("source") == "pytorch":
|
||||
# PyTorch fallback can't reach HBM peak — report as WARN, not FAIL.
|
||||
d2d = mem.get("d2d_bandwidth_gbps") or 0
|
||||
items.append(("Memory Bandwidth", f"WARN ({d2d:.0f} GB/s via PyTorch fallback)"))
|
||||
items.append(("GPU Interconnect / PCIe", f"WARN ({d2d:.0f} GB/s via PyTorch fallback)"))
|
||||
else:
|
||||
eff = mem.get("efficiency_pct") or 0
|
||||
verdict = "PASS" if eff >= 80 else ("WARN" if eff >= 60 else "FAIL")
|
||||
items.append(("Memory Bandwidth", f"{verdict} ({eff:.1f}%)"))
|
||||
items.append(("GPU Interconnect / PCIe", f"{verdict} ({eff:.1f}% unidirectional D2D)"))
|
||||
|
||||
# Compute
|
||||
comp = self._extract_compute_results(results)
|
||||
@ -517,6 +603,7 @@ class ReportGenerator:
|
||||
items.append(("Compute Throughput", f"ERROR: {comp['error']}"))
|
||||
else:
|
||||
per_dtype = comp.get("per_dtype_tflops", {})
|
||||
worst_per_dtype = comp.get("worst_per_dtype_tflops", {}) or {}
|
||||
eff_pct = comp.get("efficiency_pct", {})
|
||||
pass_thresholds = comp.get("pass_thresholds_tflops", {}) or {}
|
||||
if pass_thresholds:
|
||||
@ -525,7 +612,8 @@ class ReportGenerator:
|
||||
worst_status = "PASS"
|
||||
worst_dt = None
|
||||
for dt, thr in pass_thresholds.items():
|
||||
val = per_dtype.get(dt)
|
||||
avg_val = per_dtype.get(dt)
|
||||
val = worst_per_dtype.get(dt, avg_val)
|
||||
if not isinstance(val, (int, float)):
|
||||
continue
|
||||
if val >= thr:
|
||||
@ -541,7 +629,8 @@ class ReportGenerator:
|
||||
items.append((
|
||||
"Compute Throughput",
|
||||
f"{worst_status} (worst {worst_dt.upper()} "
|
||||
f"{per_dtype[worst_dt]:.0f} vs >= {pass_thresholds[worst_dt]})"
|
||||
f"min {worst_per_dtype.get(worst_dt, per_dtype[worst_dt]):.0f} "
|
||||
f"vs >= {pass_thresholds[worst_dt]})"
|
||||
))
|
||||
else:
|
||||
items.append(("Compute Throughput", f"{worst_status}"))
|
||||
@ -560,10 +649,19 @@ class ReportGenerator:
|
||||
n = results["nccl"]
|
||||
if n.get("error"):
|
||||
items.append(("NCCL", f"ERROR: {n['error']}"))
|
||||
elif n.get("passed"):
|
||||
items.append(("NCCL", "PASS"))
|
||||
else:
|
||||
items.append(("NCCL", "FAIL"))
|
||||
status = n.get("status") or ("PASS" if n.get("passed") else "FAIL")
|
||||
items.append(("NCCL", status))
|
||||
|
||||
# Multi-node NCCL
|
||||
if "multinode_nccl" in results:
|
||||
n = results["multinode_nccl"]
|
||||
if n.get("error"):
|
||||
items.append(("Multi-node NCCL", f"ERROR: {n['error']}"))
|
||||
elif n.get("passed"):
|
||||
items.append(("Multi-node NCCL", "PASS"))
|
||||
else:
|
||||
items.append(("Multi-node NCCL", "FAIL"))
|
||||
|
||||
# Stress
|
||||
if "stress" in results:
|
||||
@ -594,6 +692,7 @@ class ReportGenerator:
|
||||
items.append(("Training", f"ERROR: {t['error']}"))
|
||||
else:
|
||||
tps = t.get("throughput_tokens_per_sec", 0)
|
||||
items.append(("Training", f"PASS ({tps:.0f} tokens/sec)"))
|
||||
scope = t.get("gpu_scope", "framework-defined")
|
||||
items.append(("Training", f"MEASURED ({tps:.0f} tokens/sec; {scope})"))
|
||||
|
||||
return items
|
||||
|
||||
@ -119,7 +119,31 @@ class StressTest:
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _hygon_snapshot() -> dict:
|
||||
def _accelerator_snapshot() -> dict:
|
||||
if shutil.which("nvidia-smi"):
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["nvidia-smi", "--query-gpu=index,temperature.gpu,power.draw,"
|
||||
"ecc.errors.uncorrected.volatile.total", "--format=csv,noheader,nounits"],
|
||||
capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
temps, powers, ecc = {}, {}, {}
|
||||
if r.returncode == 0:
|
||||
for line in r.stdout.splitlines():
|
||||
parts = [part.strip() for part in line.split(",")]
|
||||
if len(parts) < 4:
|
||||
continue
|
||||
idx = int(parts[0])
|
||||
temps[idx] = float(parts[1])
|
||||
powers[idx] = float(parts[2])
|
||||
ecc[idx] = int(parts[3]) if parts[3].isdigit() else 0
|
||||
return {
|
||||
"temperatures_c": temps, "power_w": powers,
|
||||
"uncorrected_ecc": ecc,
|
||||
}
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
pass
|
||||
|
||||
output = run_hy_smi()
|
||||
temps, powers = {}, {}
|
||||
if output:
|
||||
@ -128,7 +152,7 @@ class StressTest:
|
||||
idx = int(match.group(1))
|
||||
temps[idx] = float(match.group(2))
|
||||
powers[idx] = float(match.group(3))
|
||||
return {"temperatures_c": temps, "power_w": powers}
|
||||
return {"temperatures_c": temps, "power_w": powers, "uncorrected_ecc": {}}
|
||||
|
||||
def _run_pytorch_stress(self, duration: int, memory_pct: int = 90,
|
||||
tensor_cores: bool = True) -> dict:
|
||||
@ -148,7 +172,7 @@ class StressTest:
|
||||
|
||||
gpu_status = {}
|
||||
t0 = time.time()
|
||||
snapshots = [self._hygon_snapshot()]
|
||||
snapshots = [self._accelerator_snapshot()]
|
||||
|
||||
try:
|
||||
work_a, work_b, outputs, ballast = {}, {}, {}, {}
|
||||
@ -210,7 +234,7 @@ class StressTest:
|
||||
current_elapsed = time.time() - t0
|
||||
if int(current_elapsed) != int(elapsed_check) and int(current_elapsed) % 10 == 0:
|
||||
self.console.print(f" [dim]Running {int(current_elapsed)}s / {duration}s[/dim]")
|
||||
snapshots.append(self._hygon_snapshot())
|
||||
snapshots.append(self._accelerator_snapshot())
|
||||
elapsed_check = current_elapsed
|
||||
|
||||
for i in range(gpu_count):
|
||||
@ -235,7 +259,7 @@ class StressTest:
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
elapsed = round(time.time() - t0, 1)
|
||||
snapshots.append(self._hygon_snapshot())
|
||||
snapshots.append(self._accelerator_snapshot())
|
||||
max_temp = {}
|
||||
max_power = {}
|
||||
for snapshot in snapshots:
|
||||
@ -243,15 +267,29 @@ class StressTest:
|
||||
max_temp[idx] = max(max_temp.get(idx, value), value)
|
||||
for idx, value in snapshot.get("power_w", {}).items():
|
||||
max_power[idx] = max(max_power.get(idx, value), value)
|
||||
first_ecc = snapshots[0].get("uncorrected_ecc", {}) if snapshots else {}
|
||||
last_ecc = snapshots[-1].get("uncorrected_ecc", {}) if snapshots else {}
|
||||
ecc_delta = {
|
||||
idx: max(0, last_ecc.get(idx, 0) - first_ecc.get(idx, 0))
|
||||
for idx in set(first_ecc) | set(last_ecc)
|
||||
}
|
||||
passed = not any(ecc_delta.values())
|
||||
if not passed:
|
||||
for idx, delta in ecc_delta.items():
|
||||
if delta:
|
||||
gpu_status[idx] = "FAIL"
|
||||
return {
|
||||
"source": "pytorch",
|
||||
"passed": True,
|
||||
"passed": passed,
|
||||
"duration_sec": duration,
|
||||
"elapsed_sec": elapsed,
|
||||
"gpu_status": gpu_status,
|
||||
"memory_pct": memory_pct,
|
||||
"max_temperature_c": max_temp,
|
||||
"max_power_w": max_power,
|
||||
"uncorrected_ecc_delta": ecc_delta,
|
||||
"numerical_validation": False,
|
||||
"validation_scope": "runtime stability, allocation, temperature, power, and ECC delta",
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
import time
|
||||
import subprocess
|
||||
import shutil
|
||||
from contextlib import nullcontext
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
@ -30,6 +31,73 @@ class TrainingSim:
|
||||
self.console = Console()
|
||||
self.train_cfg = config.get("training", {})
|
||||
|
||||
@staticmethod
|
||||
def _requires_b300_sdpa_workaround(
|
||||
configured_type: str, capability, cudnn_version: int | None,
|
||||
) -> bool:
|
||||
is_b300 = str(configured_type).lower() == "b300" or capability == (10, 3)
|
||||
# cuDNN 9.20 restores SM103 plan generation and PyTorch 2.13 correctly
|
||||
# rejects unsupported cuDNN shapes (for example dropout=0.1) before
|
||||
# dispatch, allowing another SDPA backend to run. Keep the workaround
|
||||
# only for the affected older stack.
|
||||
return is_b300 and (not cudnn_version or cudnn_version < 92000)
|
||||
|
||||
def _sdpa_context(self):
|
||||
"""Select a reliable SDPA backend policy for B300/SM103.
|
||||
|
||||
PyTorch 2.11 + cuDNN 9.19 can dispatch BF16 attention on B300 to the
|
||||
cuDNN frontend even though that backend cannot build an execution plan.
|
||||
Keep the workaround scoped to the training operation and retain ordered
|
||||
Flash/Efficient/Math fallbacks instead of changing global process state.
|
||||
"""
|
||||
configured_type = str(self.config.get("gpu_type", "")).lower()
|
||||
capability = None
|
||||
cudnn_version = None
|
||||
if torch.cuda.is_available():
|
||||
try:
|
||||
capability = tuple(torch.cuda.get_device_capability(0))
|
||||
except Exception: # noqa: BLE001 - backend discovery must not abort training
|
||||
capability = None
|
||||
try:
|
||||
cudnn_version = torch.backends.cudnn.version()
|
||||
except Exception: # noqa: BLE001 - backend discovery must not abort training
|
||||
cudnn_version = None
|
||||
|
||||
requires_workaround = self._requires_b300_sdpa_workaround(
|
||||
configured_type, capability, cudnn_version,
|
||||
)
|
||||
if not requires_workaround:
|
||||
if (configured_type == "b300" or capability == (10, 3)) and cudnn_version:
|
||||
return nullcontext(), (
|
||||
f"framework_default (cuDNN {cudnn_version}; "
|
||||
"unsupported SDPA shapes may fall back automatically)"
|
||||
)
|
||||
return nullcontext(), "framework_default"
|
||||
|
||||
try:
|
||||
from torch.nn.attention import SDPBackend, sdpa_kernel
|
||||
context = sdpa_kernel(
|
||||
[
|
||||
SDPBackend.FLASH_ATTENTION,
|
||||
SDPBackend.EFFICIENT_ATTENTION,
|
||||
SDPBackend.MATH,
|
||||
],
|
||||
set_priority=True,
|
||||
)
|
||||
return context, "flash>efficient>math (cuDNN SDPA excluded on B300)"
|
||||
except (ImportError, AttributeError, TypeError):
|
||||
# Compatibility path for older PyTorch versions that do not expose
|
||||
# the context-manager API. Restore the global flag when leaving.
|
||||
class _DisableCudnnSDPA:
|
||||
def __enter__(self_inner):
|
||||
self_inner.old = torch.backends.cuda.cudnn_sdp_enabled()
|
||||
torch.backends.cuda.enable_cudnn_sdp(False)
|
||||
|
||||
def __exit__(self_inner, exc_type, exc, traceback):
|
||||
torch.backends.cuda.enable_cudnn_sdp(self_inner.old)
|
||||
|
||||
return _DisableCudnnSDPA(), "framework fallback (cuDNN SDPA disabled on B300)"
|
||||
|
||||
def run(self) -> dict:
|
||||
if not TORCH_AVAILABLE:
|
||||
self.console.print("[yellow]PyTorch not available - skipping training simulation[/yellow]")
|
||||
@ -89,6 +157,8 @@ class TrainingSim:
|
||||
step_times = []
|
||||
mem_usage = []
|
||||
|
||||
sdpa_context, sdpa_policy = self._sdpa_context()
|
||||
self.console.print(f" SDPA policy: {sdpa_policy}")
|
||||
with Progress(
|
||||
SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(), TextColumn("{task.completed}/{task.total}"),
|
||||
@ -96,35 +166,36 @@ class TrainingSim:
|
||||
) as progress:
|
||||
task = progress.add_task("Training steps...", total=num_steps)
|
||||
|
||||
for step in range(num_steps):
|
||||
torch.cuda.synchronize()
|
||||
t0 = time.perf_counter()
|
||||
with sdpa_context:
|
||||
for step in range(num_steps):
|
||||
torch.cuda.synchronize()
|
||||
t0 = time.perf_counter()
|
||||
|
||||
input_ids = input_ids.to(model.device)
|
||||
attention_mask = attention_mask.to(model.device)
|
||||
input_ids = input_ids.to(model.device)
|
||||
attention_mask = attention_mask.to(model.device)
|
||||
|
||||
if dtype in (torch.float16, torch.bfloat16):
|
||||
with torch.amp.autocast("cuda", dtype=dtype):
|
||||
if dtype in (torch.float16, torch.bfloat16):
|
||||
with torch.amp.autocast("cuda", dtype=dtype):
|
||||
outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
|
||||
loss = outputs.loss
|
||||
else:
|
||||
outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
|
||||
loss = outputs.loss
|
||||
else:
|
||||
outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
|
||||
loss = outputs.loss
|
||||
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
optimizer.zero_grad()
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
optimizer.zero_grad()
|
||||
|
||||
torch.cuda.synchronize()
|
||||
elapsed = time.perf_counter() - t0
|
||||
step_times.append(elapsed)
|
||||
torch.cuda.synchronize()
|
||||
elapsed = time.perf_counter() - t0
|
||||
step_times.append(elapsed)
|
||||
|
||||
if torch.cuda.is_available():
|
||||
mem_used = torch.cuda.max_memory_allocated() / 1024**3
|
||||
mem_usage.append(mem_used)
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
if torch.cuda.is_available():
|
||||
mem_used = torch.cuda.max_memory_allocated() / 1024**3
|
||||
mem_usage.append(mem_used)
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
|
||||
progress.advance(task)
|
||||
progress.advance(task)
|
||||
|
||||
avg_step_time = sum(step_times) / len(step_times)
|
||||
throughput = batch_size * seq_length / avg_step_time
|
||||
@ -142,6 +213,7 @@ class TrainingSim:
|
||||
"throughput_samples_per_sec": round(batch_size / avg_step_time, 2),
|
||||
"peak_memory_gb": round(max(mem_usage) if mem_usage else 0, 2),
|
||||
"final_loss": round(loss.item(), 4) if hasattr(loss, 'item') else None,
|
||||
"sdpa_backend_policy": sdpa_policy,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
@ -190,6 +262,8 @@ class TrainingSim:
|
||||
step_times = []
|
||||
mem_usage = []
|
||||
|
||||
sdpa_context, sdpa_policy = self._sdpa_context()
|
||||
self.console.print(f" SDPA policy: {sdpa_policy}")
|
||||
with Progress(
|
||||
SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(), TextColumn("{task.completed}/{task.total}"),
|
||||
@ -197,27 +271,28 @@ class TrainingSim:
|
||||
) as progress:
|
||||
task = progress.add_task("Synthetic training...", total=num_steps)
|
||||
|
||||
for step in range(num_steps):
|
||||
torch.cuda.synchronize()
|
||||
t0 = time.perf_counter()
|
||||
with sdpa_context:
|
||||
for step in range(num_steps):
|
||||
torch.cuda.synchronize()
|
||||
t0 = time.perf_counter()
|
||||
|
||||
logits = model(input_ids)
|
||||
loss = torch.nn.functional.cross_entropy(
|
||||
logits.view(-1, vocab_size), input_ids.view(-1)
|
||||
)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
optimizer.zero_grad()
|
||||
logits = model(input_ids)
|
||||
loss = torch.nn.functional.cross_entropy(
|
||||
logits.view(-1, vocab_size), input_ids.view(-1)
|
||||
)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
optimizer.zero_grad()
|
||||
|
||||
torch.cuda.synchronize()
|
||||
elapsed = time.perf_counter() - t0
|
||||
step_times.append(elapsed)
|
||||
torch.cuda.synchronize()
|
||||
elapsed = time.perf_counter() - t0
|
||||
step_times.append(elapsed)
|
||||
|
||||
mem_used = torch.cuda.max_memory_allocated() / 1024**3
|
||||
mem_usage.append(mem_used)
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
mem_used = torch.cuda.max_memory_allocated() / 1024**3
|
||||
mem_usage.append(mem_used)
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
|
||||
progress.advance(task)
|
||||
progress.advance(task)
|
||||
|
||||
avg_step_time = sum(step_times) / len(step_times)
|
||||
throughput = batch_size * seq_length / avg_step_time
|
||||
@ -228,6 +303,9 @@ class TrainingSim:
|
||||
"num_layers": num_layers,
|
||||
"hidden_size": hidden_size,
|
||||
"gpu_count": gpu_count,
|
||||
"gpu_scope": "single_gpu:0",
|
||||
"performance_verdict": "MEASURED",
|
||||
"sdpa_backend_policy": sdpa_policy,
|
||||
"dtype": str(dtype).replace("torch.", ""),
|
||||
"batch_size": batch_size,
|
||||
"seq_length": seq_length,
|
||||
|
||||
@ -17,10 +17,12 @@ dependencies = [
|
||||
# uv sync --extra torch-cu121
|
||||
# uv sync --extra torch-cu124 (recommended for CUDA 12.x + Driver 535+)
|
||||
# uv sync --extra torch-cu128 (for CUDA 12.8+ / Driver 570+)
|
||||
# uv sync --extra torch-cu130 (recommended for B300 / Driver 580+)
|
||||
torch-cu118 = ["torch>=2.1.0"]
|
||||
torch-cu121 = ["torch>=2.1.0"]
|
||||
torch-cu124 = ["torch>=2.1.0"]
|
||||
torch-cu128 = ["torch>=2.1.0"]
|
||||
torch-cu130 = ["torch>=2.13.0"]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
@ -47,6 +49,11 @@ name = "pytorch-cu128"
|
||||
url = "https://download.pytorch.org/whl/cu128"
|
||||
explicit = true
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cu130"
|
||||
url = "https://download.pytorch.org/whl/cu130"
|
||||
explicit = true
|
||||
|
||||
# Default: resolve torch from cu124 index (most broadly compatible with 535+ drivers)
|
||||
# To use a different CUDA version, override:
|
||||
# uv pip install torch --index-url https://download.pytorch.org/whl/cu128
|
||||
|
||||
90
tests/test_b300_judgement.py
Normal file
90
tests/test_b300_judgement.py
Normal file
@ -0,0 +1,90 @@
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from modules.benchmark import Benchmark
|
||||
from modules.gpu_specs import get_gpu_specs
|
||||
from modules.nccl_test import NCCLTest
|
||||
from modules.rdma_test import RDMATest
|
||||
from modules.report import ReportGenerator
|
||||
from modules.training_sim import TrainingSim
|
||||
|
||||
|
||||
class B300JudgementTests(unittest.TestCase):
|
||||
def test_b300_training_excludes_cudnn_sdpa(self):
|
||||
self.assertTrue(TrainingSim._requires_b300_sdpa_workaround("b300", None, 91900))
|
||||
self.assertTrue(TrainingSim._requires_b300_sdpa_workaround("auto", (10, 3), 91900))
|
||||
self.assertFalse(TrainingSim._requires_b300_sdpa_workaround("b300", (10, 3), 92000))
|
||||
self.assertFalse(TrainingSim._requires_b300_sdpa_workaround("h200", (9, 0), 91900))
|
||||
|
||||
def test_b300_uses_dense_peaks_and_calibrated_floors(self):
|
||||
specs = get_gpu_specs("b300")
|
||||
self.assertEqual(specs["fp8_tflops"], 4500)
|
||||
self.assertEqual(specs["bf16_tflops"], 2250)
|
||||
self.assertEqual(specs["compute_pass_thresholds_tflops"]["fp8"], 2800)
|
||||
self.assertEqual(specs["nccl_pass_thresholds_gbps"]["allreduce"], 750)
|
||||
|
||||
def test_compute_verdict_uses_slowest_gpu_not_average(self):
|
||||
result = {
|
||||
"compute": {
|
||||
"per_dtype_tflops": {"fp8": 3000.0},
|
||||
"worst_per_dtype_tflops": {"fp8": 2500.0},
|
||||
"pass_thresholds_tflops": {"fp8": 2800.0},
|
||||
"efficiency_pct": {"fp8": 66.7},
|
||||
}
|
||||
}
|
||||
judged = Benchmark.judge_compute(result)
|
||||
self.assertEqual(judged["verdict"], "FAIL")
|
||||
self.assertEqual(judged["rows"][0][1], 2500.0)
|
||||
|
||||
def test_rdma_without_peer_is_skip_not_false_failure(self):
|
||||
rdma = RDMATest({"rdma": {}})
|
||||
with (
|
||||
patch.object(rdma, "_get_ib_devices", return_value=["mlx5_0"]),
|
||||
patch.object(rdma, "_get_ib_ports", return_value=["1"]),
|
||||
patch.object(rdma, "_read_sys", side_effect=["InfiniBand", "4: ACTIVE"]),
|
||||
patch.object(rdma, "_collect_device_info", return_value=[]),
|
||||
):
|
||||
result = rdma.run()
|
||||
self.assertEqual(result["status"], "SKIP")
|
||||
self.assertTrue(result["skipped"])
|
||||
self.assertIn("peer not configured", result["reason"])
|
||||
|
||||
def test_nccl_runtime_env_preserves_and_prepends_venv_libraries(self):
|
||||
with (
|
||||
patch("modules.nccl_test.sysconfig.get_paths", return_value={"purelib": "/venv/site"}),
|
||||
patch("modules.nccl_test.os.path.isdir", return_value=True),
|
||||
patch.dict(os.environ, {"LD_LIBRARY_PATH": "/system/lib"}),
|
||||
):
|
||||
env = NCCLTest._runtime_env()
|
||||
self.assertEqual(
|
||||
env["LD_LIBRARY_PATH"],
|
||||
"/venv/site/nvidia/nccl/lib:/venv/site/torch/lib:/system/lib",
|
||||
)
|
||||
self.assertEqual(env["NCCL_DEBUG"], "WARN")
|
||||
|
||||
def test_markdown_summary_keeps_warn_and_measured_semantics(self):
|
||||
results = {
|
||||
"nccl": {"status": "WARN", "passed": True, "tests": {}},
|
||||
"training": {
|
||||
"throughput_tokens_per_sec": 1234,
|
||||
"gpu_scope": "single_gpu:0",
|
||||
},
|
||||
}
|
||||
generator = ReportGenerator({"report": {"output_dir": "."}})
|
||||
summary = dict(generator._build_summary(results))
|
||||
self.assertEqual(summary["NCCL"], "WARN")
|
||||
self.assertTrue(summary["Training"].startswith("MEASURED"))
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = os.path.join(tmpdir, "report.md")
|
||||
generator.generate(results, fmt="md", output=path)
|
||||
with open(path, encoding="utf-8") as report_file:
|
||||
report = report_file.read()
|
||||
self.assertIn("| NCCL | WARN |", report)
|
||||
self.assertIn("synthetic workload; not an 8-GPU scaling acceptance test", report)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
x
Reference in New Issue
Block a user