"""Report generation module - export test results to JSON/HTML/Markdown.""" import json import os import re from datetime import datetime from pathlib import Path from typing import Optional try: from importlib.metadata import version as _pkg_version __version__ = _pkg_version("gpu-server-test-suite") except Exception: __version__ = "0.2.0" from rich.console import Console from rich.panel import Panel HTML_TEMPLATE = """ GPU Test Report - {timestamp}

GPU Training Server Test Report

Generated: {timestamp} | Server: {hostname}
{content} """ class ReportGenerator: def __init__(self, config: dict): self.config = config self.console = Console() self.report_cfg = config.get("report", {}) def generate(self, results: dict, fmt: str = None, output: str = None) -> str: fmt = fmt or self.report_cfg.get("format", "json") output_dir = self.report_cfg.get("output_dir", "./reports") os.makedirs(output_dir, exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") if not output: output = os.path.join(output_dir, f"gpu_report_{timestamp}.{fmt}") if fmt == "json": return self._generate_json(results, output) elif fmt == "html": return self._generate_html(results, output) elif fmt == "md": return self._generate_markdown(results, output) else: self.console.print(f"[red]Unsupported format: {fmt}[/red]") return "" def _generate_json(self, results: dict, output: str) -> str: with open(output, "w") as f: json.dump(results, f, indent=2, default=str) self.console.print(f"[green]JSON report saved to: {output}[/green]") return output def _generate_html(self, results: dict, output: str) -> str: import socket hostname = socket.gethostname() timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") sections = [] if "gpu_info" in results: gpus = results["gpu_info"].get("gpus", []) runtime_name = results["gpu_info"].get("runtime_name", "CUDA") runtime_version = results["gpu_info"].get( "runtime_version", results["gpu_info"].get("cuda_version", "N/A") ) rows = "" for g in gpus: rows += f"GPU {g['index']}{g['name']}" rows += f"{g['vram_total_mb']} MB" rows += f"{g['temperature']}°C" rows += f"{g['clock_sm']} MHz" sections.append( f'

GPU Information

' f'

Driver: {results["gpu_info"].get("driver_version", "N/A")} | ' f'{runtime_name}: {runtime_version} | ' f'Count: {len(gpus)}

' f'' f'{rows}
GPUModelVRAMTempSM Clock
' ) if "health" in results: h = results["health"] passed = h.get("passed", False) cls = "pass" if passed else "fail" txt = "ALL PASSED" if passed else "SOME CHECKS FAILED" sections.append(f'
{txt}
') if "benchmark" in results and "memory" in results["benchmark"]: mem = results["benchmark"]["memory"] sections.append( f'

Memory Bandwidth

' f'
{mem.get("d2d_bandwidth_gbps", "N/A")} GB/s
' f'
D2D (HBM)
' f'
{mem.get("efficiency_pct", "N/A")}%
' f'
Efficiency vs Peak ({mem.get("peak_bandwidth_gbps", "N/A")} GB/s)
' f'
' ) if "benchmark" in results and "compute" in results["benchmark"]: comp = results["benchmark"]["compute"] dtype_rows = "" per_dtype = comp.get("per_dtype_tflops", {}) eff = comp.get("efficiency_pct", {}) for dt, tflops in per_dtype.items(): ef = eff.get(dt, 0) cls = "pass" if ef >= 80 else ("warn" if ef >= 50 else "fail") if isinstance(tflops, (int, float)): dtype_rows += f'{dt.upper()}{tflops:.1f} TFLOPS' dtype_rows += f'{ef:.1f}%' if dtype_rows: sections.append( f'

Compute Throughput

' f'' f'{dtype_rows}
DTypeAchievedEfficiency
' ) if "training" in results: t = results["training"] sections.append( f'

Training Simulation

' f'
{t.get("throughput_tokens_per_sec", "N/A")}
' f'
Tokens/sec
' f'
{t.get("avg_step_time_ms", "N/A")} ms
' f'
Avg Step Time
' f'
{t.get("peak_memory_gb", "N/A")} GB
' f'
Peak Memory
' f'
' ) content = "\n".join(sections) html = HTML_TEMPLATE.format(timestamp=timestamp, hostname=hostname, content=content) with open(output, "w") as f: f.write(html) self.console.print(f"[green]HTML report saved to: {output}[/green]") return output # ------------------------------------------------------------------ # Markdown report # ------------------------------------------------------------------ def _generate_markdown(self, results: dict, output: str) -> str: import socket hostname = socket.gethostname() timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") lines: list[str] = [] # --- Header --- lines.append("# GPU Test Report\n") lines.append(f"- **Date:** {timestamp}") lines.append(f"- **Host:** {hostname}") # Extract GPU info for header gpu_info = results.get("gpu_info") if gpu_info and not gpu_info.get("error"): gpus = gpu_info.get("gpus", []) gpu_name = gpus[0]["name"] if gpus else "Unknown" lines.append(f"- **GPU:** {gpu_name} x{gpu_info.get('gpu_count', len(gpus))}") runtime_name = gpu_info.get("runtime_name", "CUDA") runtime_version = gpu_info.get("runtime_version", gpu_info.get("cuda_version", "N/A")) lines.append(f"- **Driver:** {gpu_info.get('driver_version', 'N/A')} | " f"**{runtime_name}:** {runtime_version}") lines.append("") # --- Summary table --- summary_items = self._build_summary(results) if summary_items: lines.append("## Summary\n") lines.append("| Test | Result |") lines.append("|------|--------|") for name, verdict in summary_items: lines.append(f"| {name} | {verdict} |") lines.append("") # --- GPU Information --- if gpu_info and not gpu_info.get("error"): lines.append("## GPU Information\n") gpus = gpu_info.get("gpus", []) lines.append("| GPU | Model | VRAM | Temp | Power | SM Clock |") lines.append("|-----|-------|------|------|-------|----------|") for g in gpus: vram = f"{g.get('vram_total_mb', 0)} MB" temp = f"{g.get('temperature', 'N/A')}C" power = f"{g.get('power_draw', 0):.0f}/{g.get('power_limit', 0):.0f}W" clock = f"{g.get('clock_sm', 0)} MHz" lines.append(f"| {g['index']} | {g['name']} | {vram} | {temp} | {power} | {clock} |") lines.append("") # --- Health Check --- health = results.get("health") if health and not health.get("error"): lines.append("## Health Check\n") passed = health.get("passed", False) lines.append(f"**Overall: {'PASS' if passed else 'FAIL'}**\n") gpu_health = health.get("gpu_health", []) if gpu_health: lines.append("| GPU | Temp | Power | ECC | PCIe | Throttle | Status |") lines.append("|-----|------|-------|-----|------|----------|--------|") for gh in gpu_health: checks = gh.get("checks", {}) temp_c = checks.get("temperature", {}) pwr = checks.get("power", {}) ecc = checks.get("ecc_errors", {}) pcie = checks.get("pcie_link", {}) throttle = checks.get("throttling", {}) temp_str = f"{temp_c.get('value', '?')}C {temp_c.get('status', '')}" pwr_str = f"{pwr.get('value', 0):.0f}W {pwr.get('status', '')}" ecc_str = f"S:{ecc.get('single', 0)} D:{ecc.get('double', 0)}" pcie_str = f"Gen{pcie.get('gen', '?')}x{pcie.get('width', '?')}" throt_str = throttle.get("status", "?") status = gh.get("status", "?") lines.append(f"| {gh['index']} | {temp_str} | {pwr_str} | " f"{ecc_str} | {pcie_str} | {throt_str} | **{status}** |") lines.append("") # --- Memory Bandwidth --- mem_data = self._extract_memory_results(results) if mem_data and not mem_data.get("error"): lines.append("## GPU Interconnect and PCIe Bandwidth\n") lines.append(f"Source: {mem_data.get('source', 'unknown')}\n") if mem_data.get("scope_note"): lines.append(f"> Scope: {mem_data['scope_note']}.\n") lines.append("| Metric | Value | Peak | Efficiency |") lines.append("|--------|-------|------|------------|") d2d = mem_data.get("d2d_bandwidth_gbps") or 0 h2d = mem_data.get("h2d_bandwidth_gbps") or 0 d2h = mem_data.get("d2h_bandwidth_gbps") or 0 # New format with per-metric peaks h2d_peak = mem_data.get("h2d_peak_gbps") or 0 d2h_peak = mem_data.get("d2h_peak_gbps") or 0 d2d_peak = mem_data.get("d2d_peak_gbps") or 0 h2d_eff = mem_data.get("h2d_efficiency_pct") or 0 d2h_eff = mem_data.get("d2h_efficiency_pct") or 0 d2d_eff = mem_data.get("d2d_efficiency_pct") or 0 # Fallback for old format if not d2d_peak: d2d_peak = mem_data.get("peak_bandwidth_gbps") or 0 d2d_eff = mem_data.get("efficiency_pct") or 0 def peak_text(value): return f"{value:.0f} GB/s" if value else "N/A" def efficiency_text(value): return f"{value:.1f}%" if value else "N/A" lines.append(f"| H2D (PCIe) | {h2d:.1f} GB/s | {peak_text(h2d_peak)} | {efficiency_text(h2d_eff)} |") lines.append(f"| D2H (PCIe) | {d2h:.1f} GB/s | {peak_text(d2h_peak)} | {efficiency_text(d2h_eff)} |") d2d_label = "D2D (HBM copy)" if mem_data.get("accelerator_vendor") == "hygon" else "D2D (NVLink, unidirectional)" lines.append(f"| {d2d_label} | {d2d:.1f} GB/s | {peak_text(d2d_peak)} | {efficiency_text(d2d_eff)} |") if mem_data.get("d2d_bidirectional_gbps"): lines.append( f"| D2D (NVLink, bidirectional informational) | " f"{mem_data['d2d_bidirectional_gbps']:.1f} GB/s | N/A | N/A |" ) lines.append("") # PyTorch fallback can't accurately measure HBM peak (intra-GPU copy_() # only reaches ~20% of HBM bandwidth). When fallback is used, report # the number but mark as WARN with a note instead of evaluating as FAIL. if mem_data.get("source") == "pytorch": lines.append( f"**Verdict: WARN** (D2D {d2d:.1f} GB/s via PyTorch fallback; " "nvbandwidth unavailable — figure is indicative only, not a true HBM peak)\n" ) else: # Tightened to match production acceptance: PASS >= 80%, WARN 60–80%, FAIL < 60%. verdict = "PASS" if d2d_eff >= 80 else ("WARN" if d2d_eff >= 60 else "FAIL") lines.append(f"**Verdict: {verdict}** (D2D efficiency {d2d_eff:.1f}%)\n") # --- Compute Throughput --- comp_data = self._extract_compute_results(results) if comp_data and not comp_data.get("error"): lines.append("## Compute Throughput\n") per_dtype = comp_data.get("per_dtype_tflops", {}) peak_tflops = comp_data.get("peak_tflops", {}) eff_pct = comp_data.get("efficiency_pct", {}) # Absolute PASS thresholds (TFLOPS) from gpu_specs.compute_pass_thresholds_tflops. # When present, override the legacy 80%-of-peak rule on a per-dtype basis. pass_thresholds = comp_data.get("pass_thresholds_tflops", {}) or {} use_abs = bool(pass_thresholds) has_peak_reference = any( isinstance(value, (int, float)) and value > 0 for value in peak_tflops.values() ) worst_values = comp_data.get("worst_per_dtype_tflops", {}) or {} best_values = comp_data.get("best_per_dtype_tflops", {}) or {} spreads = comp_data.get("spread_pct", {}) or {} if use_abs: lines.append("| DType | Avg | Min | Max | Spread | Dense Peak | Pass Floor | Status |") else: lines.append("| DType | Avg | Min | Max | Spread | Dense Peak | Efficiency | Status |") lines.append("|-------|-----|-----|-----|--------|------------|------------|--------|") worst_eff = 100.0 overall_status = "PASS" rank = {"PASS": 0, "MEASURED": 0, "WARN": 1, "FAIL": 2, "SKIP": 0} for dt, val in per_dtype.items(): if isinstance(val, str): # skipped or error lines.append(f"| {dt.upper()} | {val} | - | - | - | - | N/A | SKIP |") else: pk = peak_tflops.get(dt, 0) ef = eff_pct.get(dt, 0) worst = worst_values.get(dt, val) best = best_values.get(dt, val) spread = spreads.get(dt, 0) if isinstance(ef, (int, float)) and ef > 0: worst_eff = min(worst_eff, ef) thr = pass_thresholds.get(dt) if use_abs and thr: if worst >= thr: status = "PASS" elif worst >= thr * 0.9: status = "WARN" else: status = "FAIL" lines.append( f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | " f"{spread:.2f}% | {pk:.0f} | >= {thr} | {status} |" ) elif has_peak_reference: status = "PASS" if ef >= 80 else ("WARN" if ef >= 50 else "FAIL") lines.append( f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | " f"{spread:.2f}% | {pk:.0f} | {ef:.1f}% | {status} |" ) else: status = "MEASURED" lines.append( f"| {dt.upper()} | {val:.1f} | {worst:.1f} | {best:.1f} | " f"{spread:.2f}% | N/A | N/A | {status} |" ) if rank.get(status, 0) > rank.get(overall_status, 0): overall_status = status lines.append("") if use_abs: lines.append(f"**Verdict: {overall_status}** (each dtype is judged by the slowest GPU against an achievable B300 floor)\n") elif has_peak_reference: overall_status = "PASS" if worst_eff >= 80 else ("WARN" if worst_eff >= 50 else "FAIL") lines.append(f"**Verdict: {overall_status}** (worst efficiency {worst_eff:.1f}%)\n") else: lines.append("**Verdict: MEASURED** (no published peak/acceptance threshold configured)\n") # --- NCCL --- nccl = results.get("nccl") if nccl and not nccl.get("error"): lines.append("## NCCL Multi-GPU\n") lines.append(f"Source: {nccl.get('source', 'unknown')} | " f"GPUs: {nccl.get('gpu_count', '?')}\n") tests = nccl.get("tests", {}) if tests: lines.append("| Operation | Bus BW (GB/s) | Threshold | Status |") lines.append("|-----------|---------------|-----------|--------|") for op, data in tests.items(): if isinstance(data, dict) and not data.get("error"): bw = data.get("best_busbw_gbps", 0) req = data.get("min_required_gbps", 0) status = data.get("status", "?") lines.append(f"| {op} | {bw:.1f} | >= {req:.0f} | {status} |") elif isinstance(data, dict) and data.get("error"): lines.append(f"| {op} | - | - | ERROR: {data['error']} |") lines.append("") nccl_status = nccl.get("status") or ("PASS" if nccl.get("passed") else "FAIL") lines.append(f"**Overall: {nccl_status}**\n") # --- Multi-node NCCL --- multinode = results.get("multinode_nccl") if multinode: lines.append("## Multi-node NCCL\n") lines.append(f"Source: {multinode.get('source', 'unknown')} | " f"Mode: {multinode.get('mode', 'unknown')}\n") hosts = multinode.get("hosts", []) or [] if hosts: host_text = ", ".join( f"{host.get('name', host.get('addr', '?'))} " f"({host.get('addr', '?')}, {host.get('slots', '?')} GPUs)" for host in hosts ) lines.append(f"- **Hosts:** {host_text}") lines.append(f"- **Overall: {'PASS' if multinode.get('passed') else 'FAIL'}**") if multinode.get("artifact_dir"): lines.append(f"- **Artifacts:** `{multinode['artifact_dir']}`") lines.append("") tests = multinode.get("tests", {}) or {} if tests: lines.append("| Operation | Topology | Peak Bus BW | Avg Bus BW | Peak Size | Threshold | Wrong | Network | GDR | Status |") lines.append("|-----------|----------|-------------|------------|-----------|-----------|-------|---------|-----|--------|") for op, test_data in tests.items(): for topo in test_data.get("topologies", []): network = topo.get("network", {}) or {} networks = ", ".join(network.get("networks", [])) or "N/A" ib_using = " ".join(network.get("ib_using", [])) roce_hcas = sorted(set(re.findall(r"mlx5_bond_\d+", ib_using))) if roce_hcas: networks = f"{networks} ({len(roce_hcas)}x RoCE)" threshold = topo.get("min_required_gbps", 0) lines.append( f"| {op} | {topo.get('label', '')} | " f"{topo.get('peak_busbw_gbps', 0):.2f} GB/s | " f"{topo.get('avg_busbw_gbps', 0):.2f} GB/s | " f"{topo.get('peak_size', '')} | >= {threshold:g} GB/s | " f"{topo.get('wrong_count', 0)} | {networks} | " f"{network.get('gpu_direct_rdma', 'UNKNOWN')} | {topo.get('status', '?')} |" ) lines.append("") preflight = multinode.get("preflight", {}) or {} checks = preflight.get("checks", []) or [] if checks: lines.append("### Preflight\n") lines.append("| Check | Status | Detail |") lines.append("|-------|--------|--------|") for check in checks: detail = str(check.get("detail", "")).replace("|", "\\|").replace("\n", " ") lines.append(f"| {check.get('name', '')} | {check.get('status', '?')} | {detail} |") lines.append("") # --- Stress Test --- stress = results.get("stress") if stress and not stress.get("error"): lines.append("## Stress Test\n") passed = stress.get("passed", False) duration = stress.get("duration_sec") or 0 elapsed = stress.get("elapsed_sec") or 0 source = stress.get("source", "unknown") lines.append(f"- **Source:** {source}") lines.append(f"- **Duration:** {elapsed:.0f}s (requested {duration}s)") lines.append(f"- **Result: {'PASS' if passed else 'FAIL'}**") lines.append(f"- **Validation scope:** {stress.get('validation_scope', 'runtime completion')}") if stress.get("numerical_validation") is False: lines.append("- **Numerical result validation:** Not performed by PyTorch fallback") ecc_delta = stress.get("uncorrected_ecc_delta", {}) or {} if ecc_delta: lines.append(f"- **Uncorrected ECC delta:** {sum(ecc_delta.values())}") max_temp = stress.get("max_temperature_c", {}) or {} max_power = stress.get("max_power_w", {}) or {} if max_temp or max_power: lines.append("") lines.append("| GPU | Peak Temp | Peak Power |") lines.append("|-----|-----------|------------|") for gpu in sorted(set(max_temp) | set(max_power), key=lambda value: int(value)): lines.append( f"| {gpu} | {max_temp.get(gpu, 'N/A')} C | " f"{max_power.get(gpu, 'N/A')} W |" ) lines.append("") # --- RDMA --- rdma = results.get("rdma") if rdma and (rdma.get("skipped") or rdma.get("status") == "SKIP"): lines.append("## RDMA/InfiniBand\n") lines.append(f"**Overall: SKIP** [{rdma.get('reason', 'no IB hardware detected')}]\n") elif rdma and not rdma.get("error"): lines.append("## RDMA/InfiniBand\n") bw_tests = rdma.get("bandwidth_tests", []) lat_tests = rdma.get("latency_tests", []) if bw_tests or lat_tests: lines.append("| Test | Value | Threshold | Status |") lines.append("|------|-------|-----------|--------|") for bt in bw_tests: if not bt.get("error"): lines.append(f"| {bt['test']} | {bt.get('bandwidth_gbps', 0):.1f} GB/s | " f">= {bt.get('min_required_gbps', 0)} GB/s | {bt.get('status', '?')} |") for lt in lat_tests: if not lt.get("error"): lines.append(f"| {lt['test']} | {lt.get('latency_us', 0):.2f} us | " f"<= {lt.get('max_allowed_us', 0)} us | {lt.get('status', '?')} |") lines.append("") passed = rdma.get("passed", False) lines.append(f"**Overall: {'PASS' if passed else 'FAIL'}**\n") # --- Training --- training = results.get("training") if training and not training.get("error"): lines.append("## Training Simulation\n") lines.append("| Metric | Value |") lines.append("|--------|-------|") lines.append(f"| Model | {training.get('model', 'N/A')} |") lines.append(f"| Params | {training.get('total_params_m', 0):.1f}M |") lines.append(f"| Throughput | {training.get('throughput_tokens_per_sec', 0):.0f} tokens/sec |") lines.append(f"| Avg Step Time | {training.get('avg_step_time_ms', 0):.1f} ms |") lines.append(f"| Peak Memory | {training.get('peak_memory_gb', 0):.1f} GB |") lines.append(f"| Final Loss | {training.get('final_loss', 'N/A')} |") lines.append(f"| GPU Scope | {training.get('gpu_scope', 'framework-defined')} |") lines.append(f"| SDPA Policy | {training.get('sdpa_backend_policy', 'framework_default')} |") lines.append("") lines.append("**Verdict: MEASURED** (synthetic workload; not an 8-GPU scaling acceptance test)\n") # --- Footer --- lines.append("---") lines.append(f"*Generated by GPU Test Suite v{__version__}*") content = "\n".join(lines) with open(output, "w") as f: f.write(content) self.console.print(f"[green]Markdown report saved to: {output}[/green]") return output def _extract_memory_results(self, results: dict) -> dict: """Extract memory benchmark data from either full-suite or single-test format.""" if "memory_bench" in results: data = results["memory_bench"] return data.get("memory", data) if isinstance(data, dict) else {} if "benchmark" in results: bench = results["benchmark"] if isinstance(bench, dict) and "memory" in bench: return bench["memory"] return {} def _extract_compute_results(self, results: dict) -> dict: """Extract compute benchmark data from either full-suite or single-test format.""" if "compute_bench" in results: data = results["compute_bench"] return data.get("compute", data) if isinstance(data, dict) else {} if "benchmark" in results: bench = results["benchmark"] if isinstance(bench, dict) and "compute" in bench: return bench["compute"] return {} def _build_summary(self, results: dict) -> list[tuple[str, str]]: """Build summary verdict list from results.""" items = [] # GPU Info if "gpu_info" in results: gi = results["gpu_info"] if gi.get("error"): items.append(("GPU Info", f"ERROR: {gi['error']}")) else: items.append(("GPU Info", f"PASS ({gi.get('gpu_count', '?')} GPUs detected)")) # Health if "health" in results: h = results["health"] if h.get("error"): items.append(("Health Check", f"ERROR: {h['error']}")) elif h.get("passed"): items.append(("Health Check", "PASS")) else: items.append(("Health Check", "FAIL")) # Memory Bandwidth mem = self._extract_memory_results(results) if mem: if mem.get("error"): items.append(("GPU Interconnect / PCIe", f"ERROR: {mem['error']}")) elif mem.get("source") == "pytorch": # PyTorch fallback can't reach HBM peak — report as WARN, not FAIL. d2d = mem.get("d2d_bandwidth_gbps") or 0 items.append(("GPU Interconnect / PCIe", f"WARN ({d2d:.0f} GB/s via PyTorch fallback)")) else: eff = mem.get("efficiency_pct") or 0 verdict = "PASS" if eff >= 80 else ("WARN" if eff >= 60 else "FAIL") items.append(("GPU Interconnect / PCIe", f"{verdict} ({eff:.1f}% unidirectional D2D)")) # Compute comp = self._extract_compute_results(results) if comp: if comp.get("error"): items.append(("Compute Throughput", f"ERROR: {comp['error']}")) else: per_dtype = comp.get("per_dtype_tflops", {}) worst_per_dtype = comp.get("worst_per_dtype_tflops", {}) or {} eff_pct = comp.get("efficiency_pct", {}) pass_thresholds = comp.get("pass_thresholds_tflops", {}) or {} if pass_thresholds: # Absolute TFLOPS judgment, mirroring the per-dtype table above. rank = {"PASS": 0, "WARN": 1, "FAIL": 2} worst_status = "PASS" worst_dt = None for dt, thr in pass_thresholds.items(): avg_val = per_dtype.get(dt) val = worst_per_dtype.get(dt, avg_val) if not isinstance(val, (int, float)): continue if val >= thr: st = "PASS" elif val >= thr * 0.9: st = "WARN" else: st = "FAIL" if rank[st] > rank[worst_status]: worst_status = st worst_dt = dt if worst_dt: items.append(( "Compute Throughput", f"{worst_status} (worst {worst_dt.upper()} " f"min {worst_per_dtype.get(worst_dt, per_dtype[worst_dt]):.0f} " f"vs >= {pass_thresholds[worst_dt]})" )) else: items.append(("Compute Throughput", f"{worst_status}")) else: valid_effs = [v for v in eff_pct.values() if isinstance(v, (int, float)) and v > 0] if valid_effs: worst = min(valid_effs) verdict = "PASS" if worst >= 80 else ("WARN" if worst >= 50 else "FAIL") items.append(("Compute Throughput", f"{verdict} (worst {worst:.1f}%)")) else: measured = [v for v in per_dtype.values() if isinstance(v, (int, float))] items.append(("Compute Throughput", "MEASURED (no published threshold)" if measured else "N/A")) # NCCL if "nccl" in results: n = results["nccl"] if n.get("error"): items.append(("NCCL", f"ERROR: {n['error']}")) else: status = n.get("status") or ("PASS" if n.get("passed") else "FAIL") items.append(("NCCL", status)) # Multi-node NCCL if "multinode_nccl" in results: n = results["multinode_nccl"] if n.get("error"): items.append(("Multi-node NCCL", f"ERROR: {n['error']}")) elif n.get("passed"): items.append(("Multi-node NCCL", "PASS")) else: items.append(("Multi-node NCCL", "FAIL")) # Stress if "stress" in results: s = results["stress"] if s.get("error"): items.append(("Stress Test", f"ERROR: {s['error']}")) elif s.get("passed"): items.append(("Stress Test", "PASS")) else: items.append(("Stress Test", "FAIL")) # RDMA if "rdma" in results: r = results["rdma"] if r.get("skipped") or r.get("status") == "SKIP": items.append(("RDMA", f"SKIP ({r.get('reason', 'no IB hardware')})")) elif r.get("error"): items.append(("RDMA", f"ERROR: {r['error']}")) elif r.get("passed"): items.append(("RDMA", "PASS")) else: items.append(("RDMA", "FAIL")) # Training if "training" in results: t = results["training"] if t.get("error"): items.append(("Training", f"ERROR: {t['error']}")) else: tps = t.get("throughput_tokens_per_sec", 0) scope = t.get("gpu_scope", "framework-defined") items.append(("Training", f"MEASURED ({tps:.0f} tokens/sec; {scope})")) return items