test_gpu_scripts/modules/nccl_test.py
zulifeng cb5786c319 feat: 多机 NCCL Scaling 融入 Python 测试框架
- modules/nccl_test.py: 新增 MultiNodeNCCLTest 类,通过 mpirun + hostfile
  跑跨节点 NCCL Scaling 测试;支持 scale_levels 逐级测试,AllToAll 自动
  用 1G 消息避免大消息 hang,NCCL 环境变量通过 -x 传入各节点
- gpu_tester.py: 菜单加 [m] Multi-node NCCL Scaling,CLI 加 --test multinode-nccl
- configs/default.yaml: 新增 multinode_nccl 配置段(hostfile/mpi_home/HCA/网卡等)
- scripts/multinode_nccl/: 保留 shell 脚本版(nccl_env.sh/run_scaling.sh/
  run_cross_switch_test.sh/hosts_all,含 gpu-51/55 共 34 台)
- README: 新增 5m 多机 NCCL 章节,更新 SOP-3 和项目结构说明

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-06 18:58:08 +08:00

710 lines
28 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""NCCL multi-GPU communication test — wraps official nccl-tests."""
import glob
import os
import re
import shutil
import subprocess
import tempfile
from datetime import datetime
from typing import Optional
from rich import box
from rich.console import Console
from rich.table import Table
from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
from modules.gpu_specs import detect_gpu_type, get_gpu_specs, resolve_tools_dir
TORCH_AVAILABLE = False
try:
import torch
if torch.cuda.is_available():
TORCH_AVAILABLE = True
except ImportError:
pass
# Per-operation bandwidth thresholds, as a fraction of NVLink bidirectional BW.
# Values aligned with the H100 production acceptance criteria (acceptance doc §5).
# AllToAll runs ~10-20% lower than AllReduce on 8-GPU NVSwitch, so its fraction is
# set lower; broadcast/sendrecv sit between.
_OP_BW_FRACTIONS = {
"allreduce": 0.45,
"allgather": 0.45,
"reducescatter": 0.45,
"broadcast": 0.40,
"sendrecv": 0.40,
"alltoall": 0.35,
}
class NCCLTest:
def __init__(self, config: dict):
self.config = config
self.console = Console()
self.nccl_cfg = config.get("nccl", {})
self.tools_dir = resolve_tools_dir(config)
self.gpu_type = detect_gpu_type()
self.specs = get_gpu_specs(self.gpu_type)
def _find_nccl_test(self, name: str) -> Optional[str]:
p = shutil.which(name)
if p:
return p
build_dir = os.path.join(self.tools_dir, "nccl-tests", "build")
local = os.path.join(build_dir, name)
if os.path.isfile(local) and os.access(local, shutil.os.X_OK):
return local
matches = glob.glob(os.path.join(self.tools_dir, "nccl-tests", "**", name), recursive=True)
for m in matches:
if os.access(m, shutil.os.X_OK):
return m
return None
def _find_mpirun(self) -> Optional[str]:
for cmd in ["mpirun", "mpiexec", os.path.join(self.tools_dir, "mpi", "bin", "mpirun")]:
p = shutil.which(cmd)
if p:
return p
return None
def run(self) -> dict:
gpu_count = 0
if TORCH_AVAILABLE:
gpu_count = torch.cuda.device_count()
if gpu_count < 2:
self.console.print(f"[yellow]NCCL test requires at least 2 GPUs (found {gpu_count})[/yellow]")
return {"error": "need_at_least_2_gpus", "gpu_count": gpu_count}
tests = []
if self.nccl_cfg.get("test_allreduce", True):
tests.append(("all_reduce_perf", "AllReduce"))
if self.nccl_cfg.get("test_alltoall", True):
tests.append(("alltoall_perf", "AllToAll"))
if self.nccl_cfg.get("test_broadcast", True):
tests.append(("broadcast_perf", "Broadcast"))
if self.nccl_cfg.get("test_reduce_scatter", False):
tests.append(("reduce_scatter_perf", "ReduceScatter"))
if self.nccl_cfg.get("test_allgather", False):
tests.append(("all_gather_perf", "AllGather"))
if self.nccl_cfg.get("test_sendrecv", False):
tests.append(("sendrecv_perf", "SendRecv"))
nvlink_bw = self.specs.get("nvlink_bandwidth_gbps", 0)
# User-provided override applies uniformly across all ops; otherwise
# each op gets its own threshold from _OP_BW_FRACTIONS.
user_override = self.nccl_cfg.get("min_bandwidth_gbps")
def threshold_for(label: str) -> float:
if user_override:
return float(user_override)
if nvlink_bw <= 0:
return 10.0 # conservative floor
frac = _OP_BW_FRACTIONS.get(label.lower(), 0.45)
return round(nvlink_bw * frac)
if self.gpu_type == "unknown":
self.console.print("[yellow]Unknown GPU — using conservative bandwidth thresholds[/yellow]")
# Strategy: try nccl-tests binary directly (single-node, -g N),
# then mpirun, then torchrun fallback
results = {}
any_binary_worked = False
with Progress(
SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
TimeElapsedColumn(), console=self.console,
) as progress:
task = progress.add_task("NCCL tests...", total=len(tests))
for binary, label in tests:
progress.update(task, description=f"NCCL {label}...")
op_min_bw = threshold_for(label)
result = self._run_one_nccl_test_direct(
binary, label, gpu_count, op_min_bw
)
if result.get("status") not in ("SKIP", None) and "error" not in result:
any_binary_worked = True
results[label.lower()] = result
else:
# Try mpirun fallback
mpirun = self._find_mpirun()
if mpirun:
result = self._run_one_nccl_test_mpirun(
binary, label, gpu_count, mpirun, op_min_bw
)
if result.get("status") not in ("SKIP", None) and "error" not in result:
any_binary_worked = True
results[label.lower()] = result
progress.advance(task)
if not any_binary_worked:
self.console.print("[yellow]nccl-tests binaries failed, falling back to torchrun[/yellow]")
return self._run_torchrun_fallback(gpu_count)
all_passed = all(
r.get("status") == "PASS"
for r in results.values()
if isinstance(r, dict) and "status" in r
)
return {
"passed": all_passed,
"source": "nccl-tests",
"min_bandwidth_gbps": {
lbl.lower(): threshold_for(lbl) for _, lbl in tests
},
"tests": results,
"gpu_count": gpu_count,
"timestamp": datetime.now().isoformat(),
"detected_gpu_type": self.gpu_type,
}
def _run_one_nccl_test_direct(self, binary_name: str, label: str,
gpu_count: int, min_bw: float) -> dict:
"""Run nccl-tests binary directly with -g N (no mpirun needed for single-node)."""
binary = self._find_nccl_test(binary_name)
if not binary:
return {"status": "SKIP", "error": f"{binary_name} not found"}
cmd = [
binary,
"-b", "8M",
"-e", "8G",
"-f", "2",
"-g", str(gpu_count),
"-w", "5",
"-n", "20",
]
try:
env = os.environ.copy()
env["NCCL_DEBUG"] = "WARN"
r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env)
combined = r.stdout + r.stderr
# Check for NCCL/CUDA compatibility errors
if "CUDA driver version is insufficient" in combined or \
"Test NCCL failure" in combined:
error_msg = "NCCL/CUDA driver version mismatch" \
if "CUDA driver version" in combined \
else "NCCL test failure (library incompatibility)"
return {"status": "FAIL", "error": error_msg}
if r.returncode != 0:
return {"status": "FAIL", "error": r.stderr[:300]}
return self._parse_nccl_output(r.stdout, min_bw)
except subprocess.TimeoutExpired:
return {"status": "FAIL", "error": "timeout"}
except Exception as e:
return {"status": "FAIL", "error": str(e)}
def _run_one_nccl_test_mpirun(self, binary_name: str, label: str,
gpu_count: int, mpirun: str, min_bw: float) -> dict:
"""Run nccl-tests via mpirun (multi-node or per-GPU-process mode)."""
binary = self._find_nccl_test(binary_name)
if not binary:
return {"status": "SKIP", "error": f"{binary_name} not found"}
cmd = [
mpirun,
"-np", str(gpu_count),
"--allow-run-as-root",
"-x", "NCCL_DEBUG=WARN",
"-x", "CUDA_VISIBLE_DEVICES=" + ",".join(str(i) for i in range(gpu_count)),
binary,
"-b", "8",
"-e", "256M",
"-f", "2",
"-g", "1",
"-w", "5",
"-n", "20",
]
try:
env = os.environ.copy()
env["NCCL_DEBUG"] = "WARN"
r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env)
combined = r.stdout + r.stderr
if "CUDA driver version is insufficient" in combined or \
"Test NCCL failure" in combined:
error_msg = "NCCL/CUDA driver version mismatch" \
if "CUDA driver version" in combined \
else "NCCL test failure (library incompatibility)"
return {"status": "FAIL", "error": error_msg}
if r.returncode != 0:
return {"status": "FAIL", "error": r.stderr[:300]}
return self._parse_nccl_output(r.stdout, min_bw)
except subprocess.TimeoutExpired:
return {"status": "FAIL", "error": "timeout"}
except Exception as e:
return {"status": "FAIL", "error": str(e)}
@staticmethod
def _parse_nccl_output(stdout: str, min_bw: float) -> dict:
"""Parse nccl-tests tabular output and extract bandwidth results."""
best_algbw = 0.0
best_busbw = 0.0
size_results = []
for line in stdout.split("\n"):
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split()
# nccl-tests data lines: size count type redop root time algbw busbw #wrong [time algbw busbw #wrong]
if len(parts) >= 9:
try:
size = int(parts[0])
# parts[2] is dtype string ('float'/'int32'/etc.), not a number
# out-of-place columns: time=parts[5], algbw=parts[6], busbw=parts[7]
time_us = float(parts[5])
algbw = float(parts[6])
busbw = float(parts[7])
size_results.append({
"size": size,
"time_us": time_us,
"algbw_gbps": algbw,
"busbw_gbps": busbw,
})
if busbw > best_busbw:
best_busbw = busbw
if algbw > best_algbw:
best_algbw = algbw
except (ValueError, IndexError):
continue
status = "PASS" if best_busbw >= min_bw else "WARN"
return {
"status": status,
"best_algbw_gbps": round(best_algbw, 1),
"best_busbw_gbps": round(best_busbw, 1),
"min_required_gbps": min_bw,
"by_size": size_results[-5:] if size_results else [],
}
def _run_torchrun_fallback(self, gpu_count: int) -> dict:
"""Basic NCCL connectivity test via torchrun — verifies NCCL works but does not benchmark performance."""
self.console.print("[yellow]nccl-tests not available, running basic NCCL connectivity check[/yellow]")
code = f"""
import torch, torch.distributed as dist, os
os.environ.setdefault("MASTER_ADDR","127.0.0.1")
os.environ.setdefault("MASTER_PORT","29500")
rank=int(os.environ.get("LOCAL_RANK",0))
ws={gpu_count}
dist.init_process_group("nccl",rank=rank,world_size=ws)
torch.cuda.set_device(rank)
x=torch.randn(1024*1024,device=f"cuda:{{rank}}",dtype=torch.float32)
# Test AllReduce
try:
dist.all_reduce(x.clone())
if rank==0: print("allreduce:ok")
except Exception as e:
if rank==0: print(f"allreduce:fail:{{e}}")
# Test Broadcast
try:
dist.broadcast(x.clone(),src=0)
if rank==0: print("broadcast:ok")
except Exception as e:
if rank==0: print(f"broadcast:fail:{{e}}")
# Test AllGather
try:
tensor_list=[torch.empty_like(x) for _ in range(ws)]
dist.all_gather(tensor_list,x.clone())
if rank==0: print("allgather:ok")
except Exception as e:
if rank==0: print(f"allgather:fail:{{e}}")
# Test ReduceScatter
try:
chunks=list(x.chunk(ws))
output=torch.empty_like(chunks[0])
dist.reduce_scatter(output,chunks)
if rank==0: print("reducescatter:ok")
except Exception as e:
if rank==0: print(f"reducescatter:fail:{{e}}")
# Test AllToAll
try:
chunks=list(x.chunk(ws))
output_list=[torch.empty_like(c) for c in chunks]
dist.all_to_all(output_list,chunks)
if rank==0: print("alltoall:ok")
except Exception as e:
if rank==0: print(f"alltoall:fail:{{e}}")
dist.destroy_process_group()
"""
import tempfile
tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False)
tmp.write(code)
tmp.close()
try:
# Prefer torchrun from the same venv as the running Python
import sys
venv_torchrun = os.path.join(os.path.dirname(sys.executable), "torchrun")
torchrun_cmd = venv_torchrun if os.path.isfile(venv_torchrun) else "torchrun"
r = subprocess.run(
[torchrun_cmd, f"--nproc_per_node={gpu_count}", tmp.name],
capture_output=True, text=True, timeout=120,
env={**os.environ, "NCCL_DEBUG": "WARN"},
)
os.unlink(tmp.name)
# Parse connectivity results — format: op_name:ok or op_name:fail:error
tests = {}
all_passed = True
for line in r.stdout.split("\n"):
line = line.strip()
if not line:
continue
parts = line.split(":")
op_name = parts[0]
result = parts[1] if len(parts) > 1 else "unknown"
if result == "ok":
status = "PASS"
else:
status = "FAIL"
all_passed = False
tests[op_name] = {
"status": status,
"error": ":".join(parts[2:]) if len(parts) > 2 and result == "fail" else None,
}
return {
"passed": all_passed,
"source": "torchrun_fallback",
"tests": tests,
"gpu_count": gpu_count,
}
except Exception as e:
return {"passed": False, "source": "torchrun_fallback", "error": str(e)}
@staticmethod
def print_results(results: dict, console: Console = None):
c = console or Console()
if "error" in results:
c.print(f"[bold red]Error: {results['error']}[/bold red]")
return
passed = results.get("passed", False)
source = results.get("source", "unknown")
if source == "torchrun_fallback":
# Connectivity check mode
verdict = "[bold green]✓ NCCL Connectivity OK[/bold green]" if passed else "[bold red]✗ NCCL Connectivity FAILED[/bold red]"
c.print(f"{verdict} [dim](basic check via torchrun)[/dim]")
tests = results.get("tests", {})
if tests:
c.print("\n[dim]Operations tested:[/dim]")
for op_name, result in tests.items():
if not isinstance(result, dict):
continue
status = result.get("status", "FAIL")
s_color = "green" if status == "PASS" else "red"
error = result.get("error")
if error:
c.print(f" [{s_color}]{op_name}[/{s_color}] — {error}")
else:
c.print(f" [{s_color}]{op_name}[/{s_color}]")
c.print("\n[yellow]Note: functional connectivity test only (no performance data)[/yellow]")
else:
# nccl-tests mode
verdict = "[bold green]✓ NCCL tests PASSED[/bold green]" if passed else "[bold yellow]⚠ NCCL tests WARNING[/bold yellow]"
c.print(f"{verdict} [dim](via {source})[/dim]")
tests = results.get("tests", {})
for op_name, result in tests.items():
if not isinstance(result, dict):
continue
c.print(f"\n[bold cyan]{op_name.upper()}[/bold cyan]")
status = result.get("status", "FAIL")
s_color = "green" if status == "PASS" else ("yellow" if status == "WARN" else "red")
c.print(f" Status: [{s_color}]{status}[/{s_color}] "
f"Best bus BW: {result.get('best_busbw_gbps', 'N/A')} GB/s "
f"(min: {result.get('min_required_gbps', 'N/A')} GB/s)")
by_size = result.get("by_size", [])
if by_size:
t = Table(box=None, padding=(0, 1))
t.add_column("Size", style="bold", justify="right")
t.add_column("Time (us)", justify="right")
t.add_column("Alg BW (GB/s)", justify="right")
t.add_column("Bus BW (GB/s)", justify="right")
for r in by_size:
sz = r.get("size", 0)
sz_str = f"{sz/1024:.0f}K" if sz < 1048576 else f"{sz/1048576:.0f}M"
t.add_row(sz_str, f"{r.get('time_us',0):.1f}",
f"{r.get('algbw_gbps',0):.1f}", f"{r.get('busbw_gbps',0):.1f}")
c.print(t)
class MultiNodeNCCLTest:
"""Multi-node NCCL scaling test — runs nccl-tests via mpirun across a cluster hostfile.
Config section: multinode_nccl
hosts_file path to OpenMPI hostfile (one line: "<ip> slots=8")
mpi_home MPI installation prefix (non-default location)
nccl_ib_hca comma-separated IB HCAs (GPU-direct PIX cards only)
nccl_socket_ifname bootstrap NIC name (e.g. bond0)
scale_levels list of node counts to test
ops list of operations (all_reduce, alltoall, all_gather, reduce_scatter)
msg_size_bytes message size for most ops (default 8G)
msg_size_alltoall_bytes message size for alltoall (default 1G, avoids hang at 16+ nodes)
warmup / iterations nccl-tests -w / -n
timeout_sec per-op subprocess timeout
allreduce_min_busbw_gbps PASS threshold for AllReduce
"""
_OP_BINARY = {
"all_reduce": "all_reduce_perf",
"alltoall": "alltoall_perf",
"all_gather": "all_gather_perf",
"reduce_scatter": "reduce_scatter_perf",
}
_OP_LABEL = {
"all_reduce": "AllReduce",
"alltoall": "AllToAll",
"all_gather": "AllGather",
"reduce_scatter": "ReduceScatter",
}
def __init__(self, config: dict):
self.config = config
self.console = Console()
self.mn_cfg = config.get("multinode_nccl", {})
self.tools_dir = resolve_tools_dir(config)
def _find_mpirun(self) -> Optional[str]:
mpi_home = os.path.expanduser(self.mn_cfg.get("mpi_home", ""))
if mpi_home:
candidate = os.path.join(mpi_home, "bin", "mpirun")
if os.path.isfile(candidate):
return candidate
return shutil.which("mpirun") or shutil.which("mpiexec")
def _find_nccl_build_dir(self) -> Optional[str]:
candidates = [
"/opt/nccl-tests/build",
os.path.join(self.tools_dir, "nccl-tests", "build"),
]
for d in candidates:
probe = os.path.join(d, "all_reduce_perf")
if os.path.isfile(probe) and os.access(probe, os.X_OK):
return d
return None
def _build_env(self) -> dict:
env = os.environ.copy()
mpi_home = os.path.expanduser(self.mn_cfg.get("mpi_home", ""))
if mpi_home:
env["PATH"] = f"{mpi_home}/bin:{env.get('PATH', '')}"
env["LD_LIBRARY_PATH"] = f"{mpi_home}/lib:{env.get('LD_LIBRARY_PATH', '')}"
env["NCCL_IB_HCA"] = self.mn_cfg.get(
"nccl_ib_hca", "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9"
)
env["NCCL_IB_DISABLE"] = "0"
env["NCCL_SOCKET_IFNAME"] = self.mn_cfg.get("nccl_socket_ifname", "bond0")
env["NCCL_DEBUG"] = "WARN"
return env
def _load_hosts(self, hosts_file: str) -> list:
path = os.path.expanduser(hosts_file)
if not os.path.isfile(path):
return []
lines = []
with open(path) as f:
for line in f:
line = line.strip()
if line and not line.startswith("#"):
lines.append(line)
return lines
def _run_op(self, mpirun: str, nt_dir: str, op: str, n_nodes: int,
host_lines: list, env: dict) -> dict:
binary = os.path.join(nt_dir, self._OP_BINARY[op])
if not os.path.isfile(binary):
return {"busbw_gbps": None, "status": "SKIP", "error": f"{op} binary not found"}
if op == "alltoall":
msg_bytes = self.mn_cfg.get("msg_size_alltoall_bytes", 1073741824)
else:
msg_bytes = self.mn_cfg.get("msg_size_bytes", 8589934592)
with tempfile.NamedTemporaryFile(mode="w", suffix=".hosts", delete=False) as tf:
tf.write("\n".join(host_lines[:n_nodes]) + "\n")
hf_path = tf.name
cmd = [
mpirun,
"--allow-run-as-root",
"--mca", "plm_rsh_args", "-oStrictHostKeyChecking=no",
"-np", str(n_nodes * 8),
"-N", "8",
"--hostfile", hf_path,
"-x", "PATH",
"-x", "LD_LIBRARY_PATH",
"-x", "NCCL_IB_HCA",
"-x", "NCCL_IB_DISABLE",
"-x", "NCCL_SOCKET_IFNAME",
"-x", "NCCL_DEBUG",
binary,
"-b", "8",
"-e", str(msg_bytes),
"-f", "2",
"-g", "1",
"-w", str(self.mn_cfg.get("warmup", 5)),
"-n", str(self.mn_cfg.get("iterations", 30)),
]
timeout = self.mn_cfg.get("timeout_sec", 600)
try:
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env)
os.unlink(hf_path)
if r.returncode != 0:
return {"busbw_gbps": None, "status": "ERR", "error": r.stderr[:200]}
return self._parse_op_output(r.stdout, msg_bytes)
except subprocess.TimeoutExpired:
os.unlink(hf_path)
return {"busbw_gbps": None, "status": "ERR", "error": "timeout"}
except Exception as e:
os.unlink(hf_path)
return {"busbw_gbps": None, "status": "ERR", "error": str(e)}
@staticmethod
def _parse_op_output(stdout: str, target_bytes: int) -> dict:
for line in stdout.split("\n"):
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split()
if len(parts) < 9:
continue
try:
if int(parts[0]) == target_bytes:
return {"busbw_gbps": round(float(parts[7]), 2), "status": "OK"}
except (ValueError, IndexError):
continue
return {"busbw_gbps": None, "status": "ERR", "error": "size not found in output"}
def run(self) -> dict:
hosts_file = self.mn_cfg.get("hosts_file", "~/nccl_kit/hosts_all")
scale_levels = self.mn_cfg.get("scale_levels", [2, 4, 8])
ops = self.mn_cfg.get("ops", ["all_reduce", "alltoall", "all_gather", "reduce_scatter"])
min_busbw = self.mn_cfg.get("allreduce_min_busbw_gbps", 310)
mpirun = self._find_mpirun()
if not mpirun:
return {"error": "mpirun not found — set multinode_nccl.mpi_home in config"}
nt_dir = self._find_nccl_build_dir()
if not nt_dir:
return {"error": "nccl-tests not found (need MPI=1 build at /opt/nccl-tests/build)"}
host_lines = self._load_hosts(hosts_file)
if not host_lines:
return {"error": f"hostfile not found or empty: {hosts_file}"}
env = self._build_env()
results = {}
runnable = [n for n in scale_levels if n <= len(host_lines)]
skipped = [n for n in scale_levels if n > len(host_lines)]
if skipped:
self.console.print(f"[yellow]⚠ 跳过节点数 {skipped}:hostfile 只有 {len(host_lines)} 台[/yellow]")
with Progress(
SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
TimeElapsedColumn(), console=self.console,
) as progress:
task = progress.add_task("Multi-node NCCL...", total=len(runnable) * len(ops))
for n in runnable:
results[n] = {}
for op in ops:
progress.update(task, description=f"{n} nodes / {self._OP_LABEL.get(op, op)}...")
results[n][op] = self._run_op(mpirun, nt_dir, op, n, host_lines, env)
progress.advance(task)
passed = all(
(results.get(n, {}).get("all_reduce", {}).get("busbw_gbps") or 0) >= min_busbw
for n in runnable
)
return {
"passed": passed,
"hosts_file": hosts_file,
"available_nodes": len(host_lines),
"scale_levels": scale_levels,
"ops": ops,
"allreduce_min_busbw_gbps": min_busbw,
"results": results,
"timestamp": datetime.now().isoformat(),
}
def print_results(self, results: dict):
c = self.console
if "error" in results:
c.print(f"[bold red]Multi-node NCCL Error: {results['error']}[/bold red]")
return
passed = results.get("passed", False)
verdict = "[bold green]✓ PASS[/bold green]" if passed else "[bold red]✗ FAIL[/bold red]"
c.print(f"\n[bold cyan]Multi-node NCCL Scaling[/bold cyan] — {verdict}")
c.print(f" Hostfile: {results['hosts_file']} ({results['available_nodes']} nodes available)")
c.print(f" AllReduce PASS threshold: ≥ {results['allreduce_min_busbw_gbps']} GB/s")
data = results.get("results", {})
ops = results.get("ops", [])
scale_levels = [n for n in results.get("scale_levels", []) if n in data]
min_busbw = results.get("allreduce_min_busbw_gbps", 310)
t = Table(box=box.SIMPLE_HEAD, border_style="cyan")
t.add_column("Nodes / GPUs", style="bold", justify="center")
for op in ops:
t.add_column(self._OP_LABEL.get(op, op) + " (GB/s)", justify="right")
t.add_column("", justify="center")
for n in scale_levels:
row_data = data.get(n, {})
cells = []
ok = True
for op in ops:
r = row_data.get(op, {})
bw = r.get("busbw_gbps")
if bw is None:
err = r.get("error", "ERR")[:8]
cells.append(f"[red]{err}[/red]")
ok = False
else:
is_alltoall = op == "alltoall"
fail = (op == "all_reduce" and bw < min_busbw)
color = "red" if fail else "green"
suffix = "¹" if is_alltoall else ""
cells.append(f"[{color}]{bw:.1f}[/{color}]{suffix}")
status_str = "[green]PASS[/green]" if ok else "[red]FAIL[/red]"
t.add_row(f"{n} / {n * 8}", *cells, status_str)
c.print(t)
c.print("[dim]¹ AllToAll 使用 1G 消息(避免大消息 hang) | 其余操作使用 8G 消息[/dim]")