"""NCCL multi-GPU communication test — wraps official nccl-tests.""" import glob import os import re import shutil import subprocess import sysconfig import tempfile from datetime import datetime from typing import Optional from rich import box from rich.console import Console from rich.table import Table from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn from modules.gpu_specs import detect_gpu_type, get_gpu_specs, resolve_tools_dir TORCH_AVAILABLE = False try: import torch try: import torch_dcu # noqa: F401 - enables DTK's PyTorch device shim except ImportError: pass if torch.cuda.is_available(): TORCH_AVAILABLE = True except ImportError: pass # Per-operation bandwidth thresholds, as a fraction of NVLink bidirectional BW. # Values aligned with the H100 production acceptance criteria (acceptance doc §5). # AllToAll runs ~10-20% lower than AllReduce on 8-GPU NVSwitch, so its fraction is # set lower; broadcast/sendrecv sit between. _OP_BW_FRACTIONS = { "allreduce": 0.45, "allgather": 0.45, "reducescatter": 0.45, "broadcast": 0.40, "sendrecv": 0.40, "alltoall": 0.35, } class NCCLTest: def __init__(self, config: dict): self.config = config self.console = Console() self.nccl_cfg = config.get("nccl", {}) self.tools_dir = resolve_tools_dir(config) configured = config.get("gpu_type", "auto") self.gpu_type = configured if configured != "auto" else detect_gpu_type() self.specs = get_gpu_specs(self.gpu_type) def _find_nccl_test(self, name: str) -> Optional[str]: p = shutil.which(name) if p: return p build_dir = os.path.join(self.tools_dir, "nccl-tests", "build") local = os.path.join(build_dir, name) if os.path.isfile(local) and os.access(local, shutil.os.X_OK): return local matches = glob.glob(os.path.join(self.tools_dir, "nccl-tests", "**", name), recursive=True) for m in matches: if os.access(m, shutil.os.X_OK): return m return None def _find_mpirun(self) -> Optional[str]: for cmd in ["mpirun", "mpiexec", os.path.join(self.tools_dir, "mpi", "bin", "mpirun")]: p = shutil.which(cmd) if p: return p return None @staticmethod def _runtime_env() -> dict: """Return an environment that can load NCCL bundled in the active venv. CUDA PyTorch wheels install libnccl under site-packages/nvidia/nccl/lib, which is intentionally isolated from the system linker path. nccl-tests is an external executable, so it needs that directory explicitly even though importing torch works in the same virtual environment. """ env = os.environ.copy() purelib = sysconfig.get_paths().get("purelib", "") candidates = [ os.path.join(purelib, "nvidia", "nccl", "lib"), os.path.join(purelib, "torch", "lib"), ] library_dirs = [path for path in candidates if os.path.isdir(path)] existing = env.get("LD_LIBRARY_PATH", "") if existing: library_dirs.append(existing) if library_dirs: env["LD_LIBRARY_PATH"] = os.pathsep.join(library_dirs) env["NCCL_DEBUG"] = "WARN" return env def run(self) -> dict: gpu_count = 0 if TORCH_AVAILABLE: gpu_count = torch.cuda.device_count() if gpu_count < 2: self.console.print(f"[yellow]NCCL test requires at least 2 GPUs (found {gpu_count})[/yellow]") return {"error": "need_at_least_2_gpus", "gpu_count": gpu_count} tests = [] if self.nccl_cfg.get("test_allreduce", True): tests.append(("all_reduce_perf", "AllReduce")) if self.nccl_cfg.get("test_alltoall", True): tests.append(("alltoall_perf", "AllToAll")) if self.nccl_cfg.get("test_broadcast", True): tests.append(("broadcast_perf", "Broadcast")) if self.nccl_cfg.get("test_reduce_scatter", False): tests.append(("reduce_scatter_perf", "ReduceScatter")) if self.nccl_cfg.get("test_allgather", False): tests.append(("all_gather_perf", "AllGather")) if self.nccl_cfg.get("test_sendrecv", False): tests.append(("sendrecv_perf", "SendRecv")) nvlink_bw = self.specs.get("nvlink_bandwidth_gbps", 0) calibrated_thresholds = self.specs.get("nccl_pass_thresholds_gbps", {}) or {} # User-provided override applies uniformly across all ops; otherwise # each op gets its own threshold from _OP_BW_FRACTIONS. user_override = self.nccl_cfg.get("min_bandwidth_gbps") def threshold_for(label: str) -> float: if user_override: return float(user_override) calibrated = calibrated_thresholds.get(label.lower()) if calibrated: return float(calibrated) if nvlink_bw <= 0: return 10.0 # conservative floor frac = _OP_BW_FRACTIONS.get(label.lower(), 0.45) return round(nvlink_bw * frac) if self.gpu_type == "unknown": self.console.print("[yellow]Unknown GPU — using conservative bandwidth thresholds[/yellow]") # Strategy: try nccl-tests binary directly (single-node, -g N), # then mpirun, then torchrun fallback results = {} any_binary_worked = False with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), TimeElapsedColumn(), console=self.console, ) as progress: task = progress.add_task("NCCL tests...", total=len(tests)) for binary, label in tests: progress.update(task, description=f"NCCL {label}...") op_min_bw = threshold_for(label) result = self._run_one_nccl_test_direct( binary, label, gpu_count, op_min_bw ) if result.get("status") not in ("SKIP", None) and "error" not in result: any_binary_worked = True results[label.lower()] = result else: # Try mpirun fallback mpirun = self._find_mpirun() if mpirun: result = self._run_one_nccl_test_mpirun( binary, label, gpu_count, mpirun, op_min_bw ) if result.get("status") not in ("SKIP", None) and "error" not in result: any_binary_worked = True results[label.lower()] = result progress.advance(task) if not any_binary_worked: self.console.print("[yellow]nccl-tests binaries failed, falling back to torchrun[/yellow]") return self._run_torchrun_fallback(gpu_count) statuses = [ r.get("status") for r in results.values() if isinstance(r, dict) and r.get("status") ] overall_status = ( "FAIL" if "FAIL" in statuses else "WARN" if "WARN" in statuses else "PASS" if statuses else "SKIP" ) return { "passed": overall_status in ("PASS", "WARN"), "status": overall_status, "source": "nccl-tests", "min_bandwidth_gbps": { lbl.lower(): threshold_for(lbl) for _, lbl in tests }, "tests": results, "gpu_count": gpu_count, "timestamp": datetime.now().isoformat(), "detected_gpu_type": self.gpu_type, } def _run_one_nccl_test_direct(self, binary_name: str, label: str, gpu_count: int, min_bw: float) -> dict: """Run nccl-tests binary directly with -g N (no mpirun needed for single-node).""" binary = self._find_nccl_test(binary_name) if not binary: return {"status": "SKIP", "error": f"{binary_name} not found"} cmd = [ binary, "-b", "8M", "-e", "8G", "-f", "2", "-g", str(gpu_count), "-w", "5", "-n", "20", ] try: env = self._runtime_env() r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env) combined = r.stdout + r.stderr # Check for NCCL/CUDA compatibility errors if "CUDA driver version is insufficient" in combined or \ "Test NCCL failure" in combined: error_msg = "NCCL/CUDA driver version mismatch" \ if "CUDA driver version" in combined \ else "NCCL test failure (library incompatibility)" return {"status": "FAIL", "error": error_msg} if r.returncode != 0: return {"status": "FAIL", "error": r.stderr[:300]} return self._parse_nccl_output(r.stdout, min_bw) except subprocess.TimeoutExpired: return {"status": "FAIL", "error": "timeout"} except Exception as e: return {"status": "FAIL", "error": str(e)} def _run_one_nccl_test_mpirun(self, binary_name: str, label: str, gpu_count: int, mpirun: str, min_bw: float) -> dict: """Run nccl-tests via mpirun (multi-node or per-GPU-process mode).""" binary = self._find_nccl_test(binary_name) if not binary: return {"status": "SKIP", "error": f"{binary_name} not found"} cmd = [ mpirun, "-np", str(gpu_count), "--allow-run-as-root", "-x", "NCCL_DEBUG=WARN", "-x", "CUDA_VISIBLE_DEVICES=" + ",".join(str(i) for i in range(gpu_count)), binary, "-b", "8", "-e", "256M", "-f", "2", "-g", "1", "-w", "5", "-n", "20", ] try: env = self._runtime_env() r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env) combined = r.stdout + r.stderr if "CUDA driver version is insufficient" in combined or \ "Test NCCL failure" in combined: error_msg = "NCCL/CUDA driver version mismatch" \ if "CUDA driver version" in combined \ else "NCCL test failure (library incompatibility)" return {"status": "FAIL", "error": error_msg} if r.returncode != 0: return {"status": "FAIL", "error": r.stderr[:300]} return self._parse_nccl_output(r.stdout, min_bw) except subprocess.TimeoutExpired: return {"status": "FAIL", "error": "timeout"} except Exception as e: return {"status": "FAIL", "error": str(e)} @staticmethod def _parse_nccl_output(stdout: str, min_bw: float) -> dict: """Parse nccl-tests tabular output and extract bandwidth results.""" best_algbw = 0.0 best_busbw = 0.0 size_results = [] for line in stdout.split("\n"): line = line.strip() if not line or line.startswith("#"): continue parts = line.split() # nccl-tests data lines: size count type redop root time algbw busbw #wrong [time algbw busbw #wrong] if len(parts) >= 9: try: size = int(parts[0]) # parts[2] is dtype string ('float'/'int32'/etc.), not a number # out-of-place columns: time=parts[5], algbw=parts[6], busbw=parts[7] time_us = float(parts[5]) algbw = float(parts[6]) busbw = float(parts[7]) size_results.append({ "size": size, "time_us": time_us, "algbw_gbps": algbw, "busbw_gbps": busbw, }) if busbw > best_busbw: best_busbw = busbw if algbw > best_algbw: best_algbw = algbw except (ValueError, IndexError): continue status = "PASS" if best_busbw >= min_bw else "WARN" return { "status": status, "best_algbw_gbps": round(best_algbw, 1), "best_busbw_gbps": round(best_busbw, 1), "min_required_gbps": min_bw, "by_size": size_results[-5:] if size_results else [], } def _run_torchrun_fallback(self, gpu_count: int) -> dict: """Benchmark collectives through PyTorch when nccl-tests is unavailable.""" self.console.print("[yellow]nccl-tests not available, benchmarking collectives via torchrun[/yellow]") size_mb = int(self.nccl_cfg.get("torchrun_size_mb", 64)) warmup = int(self.nccl_cfg.get("torchrun_warmup", 5)) iterations = int(self.nccl_cfg.get("torchrun_iterations", 20)) backend = self.nccl_cfg.get("torch_backend", "nccl") enabled_ops = [] for key, enabled in [ ("allreduce", self.nccl_cfg.get("test_allreduce", True)), ("alltoall", self.nccl_cfg.get("test_alltoall", True)), ("broadcast", self.nccl_cfg.get("test_broadcast", True)), ("reducescatter", self.nccl_cfg.get("test_reduce_scatter", False)), ("allgather", self.nccl_cfg.get("test_allgather", False)), ]: if enabled: enabled_ops.append(key) code = f""" import os, time import torch try: import torch_dcu except ImportError: pass import torch.distributed as dist os.environ.setdefault("MASTER_ADDR","127.0.0.1") os.environ.setdefault("MASTER_PORT","29500") rank=int(os.environ.get("LOCAL_RANK",0)) ws={gpu_count} dist.init_process_group({backend!r},rank=rank,world_size=ws) torch.cuda.set_device(rank) size_bytes={size_mb}*1024*1024 numel=size_bytes//4 x=torch.ones(numel,device=f"cuda:{{rank}}",dtype=torch.float32) allgather_out=torch.empty(numel*ws,device=f"cuda:{{rank}}",dtype=torch.float32) rs_out=torch.empty(numel//ws,device=f"cuda:{{rank}}",dtype=torch.float32) a2a_out=torch.empty_like(x) def run_op(name): if name == "allreduce": return dist.all_reduce(x) if name == "broadcast": return dist.broadcast(x, src=0) if name == "allgather": return dist.all_gather_into_tensor(allgather_out, x) if name == "reducescatter": return dist.reduce_scatter_tensor(rs_out, x) if name == "alltoall": return dist.all_to_all_single(a2a_out, x) raise ValueError(name) for name in {enabled_ops!r}: try: for _ in range({warmup}): run_op(name) torch.cuda.synchronize() dist.barrier() start=time.perf_counter() for _ in range({iterations}): run_op(name) torch.cuda.synchronize() elapsed=(time.perf_counter()-start)/{iterations} algbw=size_bytes/elapsed/1e9 factor={{"allreduce":2*(ws-1)/ws,"allgather":(ws-1)/ws, "reducescatter":(ws-1)/ws,"alltoall":(ws-1)/ws, "broadcast":1.0}}[name] busbw=algbw*factor if rank==0: print(f"METRIC|{{name}}|{{elapsed*1e6:.3f}}|{{algbw:.3f}}|{{busbw:.3f}}", flush=True) except Exception as exc: if rank==0: print(f"ERROR|{{name}}|{{exc}}", flush=True) dist.destroy_process_group() """ import tempfile tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) tmp.write(code) tmp.close() try: # Prefer torchrun from the same venv as the running Python import sys venv_torchrun = os.path.join(os.path.dirname(sys.executable), "torchrun") torchrun_cmd = venv_torchrun if os.path.isfile(venv_torchrun) else "torchrun" r = subprocess.run( [torchrun_cmd, f"--nproc_per_node={gpu_count}", tmp.name], capture_output=True, text=True, timeout=300, env={**os.environ, "NCCL_DEBUG": "WARN", "RCCL_DEBUG": "WARN"}, ) os.unlink(tmp.name) tests = {} for line in r.stdout.split("\n"): line = line.strip() if line.startswith("METRIC|"): _, name, time_us, algbw, busbw = line.split("|", 4) threshold = self.nccl_cfg.get("min_bandwidth_gbps") or 10.0 busbw_f = float(busbw) tests[name] = { "status": "PASS" if busbw_f >= float(threshold) else "WARN", "best_algbw_gbps": round(float(algbw), 2), "best_busbw_gbps": round(busbw_f, 2), "min_required_gbps": float(threshold), "by_size": [{ "size": size_mb * 1024 * 1024, "time_us": round(float(time_us), 2), "algbw_gbps": round(float(algbw), 2), "busbw_gbps": round(busbw_f, 2), }], } elif line.startswith("ERROR|"): _, name, error = line.split("|", 2) tests[name] = {"status": "FAIL", "error": error} if not tests: error = (r.stderr or r.stdout or "torchrun produced no metrics")[-1000:] return {"passed": False, "source": "torchrun_collective", "error": error} all_passed = all(value.get("status") == "PASS" for value in tests.values()) return { "passed": all_passed, "source": "torchrun_collective", "tests": tests, "gpu_count": gpu_count, "message_size_mb": size_mb, "warmup": warmup, "iterations": iterations, "backend": backend, "timestamp": datetime.now().isoformat(), } except Exception as e: return {"passed": False, "source": "torchrun_collective", "error": str(e)} @staticmethod def print_results(results: dict, console: Console = None): c = console or Console() if "error" in results: c.print(f"[bold red]Error: {results['error']}[/bold red]") return passed = results.get("passed", False) overall_status = results.get("status") or ("PASS" if passed else "FAIL") source = results.get("source", "unknown") if source == "torchrun_fallback": # Connectivity check mode verdict = "[bold green]✓ NCCL Connectivity OK[/bold green]" if passed else "[bold red]✗ NCCL Connectivity FAILED[/bold red]" c.print(f"{verdict} [dim](basic check via torchrun)[/dim]") tests = results.get("tests", {}) if tests: c.print("\n[dim]Operations tested:[/dim]") for op_name, result in tests.items(): if not isinstance(result, dict): continue status = result.get("status", "FAIL") s_color = "green" if status == "PASS" else "red" error = result.get("error") if error: c.print(f" [{s_color}]{op_name}[/{s_color}] — {error}") else: c.print(f" [{s_color}]{op_name}[/{s_color}]") c.print("\n[yellow]Note: functional connectivity test only (no performance data)[/yellow]") else: # nccl-tests mode if overall_status == "PASS": verdict = "[bold green]✓ NCCL tests PASSED[/bold green]" elif overall_status == "WARN": verdict = "[bold yellow]⚠ NCCL tests WARNING[/bold yellow]" else: verdict = "[bold red]✗ NCCL tests FAILED[/bold red]" c.print(f"{verdict} [dim](via {source})[/dim]") tests = results.get("tests", {}) for op_name, result in tests.items(): if not isinstance(result, dict): continue c.print(f"\n[bold cyan]{op_name.upper()}[/bold cyan]") status = result.get("status", "FAIL") s_color = "green" if status == "PASS" else ("yellow" if status == "WARN" else "red") c.print(f" Status: [{s_color}]{status}[/{s_color}] " f"Best bus BW: {result.get('best_busbw_gbps', 'N/A')} GB/s " f"(min: {result.get('min_required_gbps', 'N/A')} GB/s)") by_size = result.get("by_size", []) if by_size: t = Table(box=None, padding=(0, 1)) t.add_column("Size", style="bold", justify="right") t.add_column("Time (us)", justify="right") t.add_column("Alg BW (GB/s)", justify="right") t.add_column("Bus BW (GB/s)", justify="right") for r in by_size: sz = r.get("size", 0) sz_str = f"{sz/1024:.0f}K" if sz < 1048576 else f"{sz/1048576:.0f}M" t.add_row(sz_str, f"{r.get('time_us',0):.1f}", f"{r.get('algbw_gbps',0):.1f}", f"{r.get('busbw_gbps',0):.1f}") c.print(t) class MultiNodeNCCLTest: """Multi-node NCCL scaling test — runs nccl-tests via mpirun across a cluster hostfile. Config section: multinode_nccl hosts_file path to OpenMPI hostfile (one line: " slots=8") mpi_home MPI installation prefix (non-default location) nccl_ib_hca comma-separated IB HCAs (GPU-direct PIX cards only) nccl_socket_ifname bootstrap NIC name (e.g. bond0) scale_levels list of node counts to test ops list of operations (all_reduce, alltoall, all_gather, reduce_scatter) msg_size_bytes message size for most ops (default 8G) msg_size_alltoall_bytes message size for alltoall (default 1G, avoids hang at 16+ nodes) warmup / iterations nccl-tests -w / -n timeout_sec per-op subprocess timeout allreduce_min_busbw_gbps PASS threshold for AllReduce """ _OP_BINARY = { "all_reduce": "all_reduce_perf", "alltoall": "alltoall_perf", "all_gather": "all_gather_perf", "reduce_scatter": "reduce_scatter_perf", } _OP_LABEL = { "all_reduce": "AllReduce", "alltoall": "AllToAll", "all_gather": "AllGather", "reduce_scatter": "ReduceScatter", } def __init__(self, config: dict): self.config = config self.console = Console() self.mn_cfg = config.get("multinode_nccl", {}) self.tools_dir = resolve_tools_dir(config) def _find_mpirun(self) -> Optional[str]: mpi_home = os.path.expanduser(self.mn_cfg.get("mpi_home", "")) if mpi_home: candidate = os.path.join(mpi_home, "bin", "mpirun") if os.path.isfile(candidate): return candidate return shutil.which("mpirun") or shutil.which("mpiexec") def _find_nccl_build_dir(self) -> Optional[str]: candidates = [ "/opt/nccl-tests/build", os.path.join(self.tools_dir, "nccl-tests", "build"), ] for d in candidates: probe = os.path.join(d, "all_reduce_perf") if os.path.isfile(probe) and os.access(probe, os.X_OK): return d return None def _build_env(self) -> dict: env = os.environ.copy() mpi_home = os.path.expanduser(self.mn_cfg.get("mpi_home", "")) if mpi_home: env["PATH"] = f"{mpi_home}/bin:{env.get('PATH', '')}" env["LD_LIBRARY_PATH"] = f"{mpi_home}/lib:{env.get('LD_LIBRARY_PATH', '')}" env["NCCL_IB_HCA"] = self.mn_cfg.get( "nccl_ib_hca", "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9" ) env["NCCL_IB_DISABLE"] = "0" env["NCCL_SOCKET_IFNAME"] = self.mn_cfg.get("nccl_socket_ifname", "bond0") env["NCCL_DEBUG"] = "WARN" return env def _load_hosts(self, hosts_file: str) -> list: path = os.path.expanduser(hosts_file) if not os.path.isfile(path): return [] lines = [] with open(path) as f: for line in f: line = line.strip() if line and not line.startswith("#"): lines.append(line) return lines def _run_op(self, mpirun: str, nt_dir: str, op: str, n_nodes: int, host_lines: list, env: dict) -> dict: binary = os.path.join(nt_dir, self._OP_BINARY[op]) if not os.path.isfile(binary): return {"busbw_gbps": None, "status": "SKIP", "error": f"{op} binary not found"} if op == "alltoall": msg_bytes = self.mn_cfg.get("msg_size_alltoall_bytes", 1073741824) else: msg_bytes = self.mn_cfg.get("msg_size_bytes", 8589934592) with tempfile.NamedTemporaryFile(mode="w", suffix=".hosts", delete=False) as tf: tf.write("\n".join(host_lines[:n_nodes]) + "\n") hf_path = tf.name cmd = [ mpirun, "--allow-run-as-root", "--mca", "plm_rsh_args", "-oStrictHostKeyChecking=no", "-np", str(n_nodes * 8), "-N", "8", "--hostfile", hf_path, "-x", "PATH", "-x", "LD_LIBRARY_PATH", "-x", "NCCL_IB_HCA", "-x", "NCCL_IB_DISABLE", "-x", "NCCL_SOCKET_IFNAME", "-x", "NCCL_DEBUG", binary, "-b", "8", "-e", str(msg_bytes), "-f", "2", "-g", "1", "-w", str(self.mn_cfg.get("warmup", 5)), "-n", str(self.mn_cfg.get("iterations", 30)), ] timeout = self.mn_cfg.get("timeout_sec", 600) try: r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env) os.unlink(hf_path) if r.returncode != 0: return {"busbw_gbps": None, "status": "ERR", "error": r.stderr[:200]} return self._parse_op_output(r.stdout, msg_bytes) except subprocess.TimeoutExpired: os.unlink(hf_path) return {"busbw_gbps": None, "status": "ERR", "error": "timeout"} except Exception as e: os.unlink(hf_path) return {"busbw_gbps": None, "status": "ERR", "error": str(e)} @staticmethod def _parse_op_output(stdout: str, target_bytes: int) -> dict: for line in stdout.split("\n"): line = line.strip() if not line or line.startswith("#"): continue parts = line.split() if len(parts) < 9: continue try: if int(parts[0]) == target_bytes: return {"busbw_gbps": round(float(parts[7]), 2), "status": "OK"} except (ValueError, IndexError): continue return {"busbw_gbps": None, "status": "ERR", "error": "size not found in output"} def run(self) -> dict: hosts_file = self.mn_cfg.get("hosts_file", "~/nccl_kit/hosts_all") scale_levels = self.mn_cfg.get("scale_levels", [2, 4, 8]) ops = self.mn_cfg.get("ops", ["all_reduce", "alltoall", "all_gather", "reduce_scatter"]) min_busbw = self.mn_cfg.get("allreduce_min_busbw_gbps", 310) mpirun = self._find_mpirun() if not mpirun: return {"error": "mpirun not found — set multinode_nccl.mpi_home in config"} nt_dir = self._find_nccl_build_dir() if not nt_dir: return {"error": "nccl-tests not found (need MPI=1 build at /opt/nccl-tests/build)"} host_lines = self._load_hosts(hosts_file) if not host_lines: return {"error": f"hostfile not found or empty: {hosts_file}"} env = self._build_env() results = {} runnable = [n for n in scale_levels if n <= len(host_lines)] skipped = [n for n in scale_levels if n > len(host_lines)] if skipped: self.console.print(f"[yellow]⚠ 跳过节点数 {skipped}:hostfile 只有 {len(host_lines)} 台[/yellow]") with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), TimeElapsedColumn(), console=self.console, ) as progress: task = progress.add_task("Multi-node NCCL...", total=len(runnable) * len(ops)) for n in runnable: results[n] = {} for op in ops: progress.update(task, description=f"{n} nodes / {self._OP_LABEL.get(op, op)}...") results[n][op] = self._run_op(mpirun, nt_dir, op, n, host_lines, env) progress.advance(task) passed = all( (results.get(n, {}).get("all_reduce", {}).get("busbw_gbps") or 0) >= min_busbw for n in runnable ) return { "passed": passed, "hosts_file": hosts_file, "available_nodes": len(host_lines), "scale_levels": scale_levels, "ops": ops, "allreduce_min_busbw_gbps": min_busbw, "results": results, "timestamp": datetime.now().isoformat(), } def print_results(self, results: dict): c = self.console if "error" in results: c.print(f"[bold red]Multi-node NCCL Error: {results['error']}[/bold red]") return passed = results.get("passed", False) verdict = "[bold green]✓ PASS[/bold green]" if passed else "[bold red]✗ FAIL[/bold red]" c.print(f"\n[bold cyan]Multi-node NCCL Scaling[/bold cyan] — {verdict}") c.print(f" Hostfile: {results['hosts_file']} ({results['available_nodes']} nodes available)") c.print(f" AllReduce PASS threshold: ≥ {results['allreduce_min_busbw_gbps']} GB/s") data = results.get("results", {}) ops = results.get("ops", []) scale_levels = [n for n in results.get("scale_levels", []) if n in data] min_busbw = results.get("allreduce_min_busbw_gbps", 310) t = Table(box=box.SIMPLE_HEAD, border_style="cyan") t.add_column("Nodes / GPUs", style="bold", justify="center") for op in ops: t.add_column(self._OP_LABEL.get(op, op) + " (GB/s)", justify="right") t.add_column("", justify="center") for n in scale_levels: row_data = data.get(n, {}) cells = [] ok = True for op in ops: r = row_data.get(op, {}) bw = r.get("busbw_gbps") if bw is None: err = r.get("error", "ERR")[:8] cells.append(f"[red]{err}[/red]") ok = False else: is_alltoall = op == "alltoall" fail = (op == "all_reduce" and bw < min_busbw) color = "red" if fail else "green" suffix = "¹" if is_alltoall else "" cells.append(f"[{color}]{bw:.1f}[/{color}]{suffix}") status_str = "[green]PASS[/green]" if ok else "[red]FAIL[/red]" t.add_row(f"{n} / {n * 8}", *cells, status_str) c.print(t) c.print("[dim]¹ AllToAll 使用 1G 消息(避免大消息 hang) | 其余操作使用 8G 消息[/dim]")