- modules/nccl_test.py: 新增 MultiNodeNCCLTest 类,通过 mpirun + hostfile 跑跨节点 NCCL Scaling 测试;支持 scale_levels 逐级测试,AllToAll 自动 用 1G 消息避免大消息 hang,NCCL 环境变量通过 -x 传入各节点 - gpu_tester.py: 菜单加 [m] Multi-node NCCL Scaling,CLI 加 --test multinode-nccl - configs/default.yaml: 新增 multinode_nccl 配置段(hostfile/mpi_home/HCA/网卡等) - scripts/multinode_nccl/: 保留 shell 脚本版(nccl_env.sh/run_scaling.sh/ run_cross_switch_test.sh/hosts_all,含 gpu-51/55 共 34 台) - README: 新增 5m 多机 NCCL 章节,更新 SOP-3 和项目结构说明 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
710 lines
28 KiB
Python
710 lines
28 KiB
Python
"""NCCL multi-GPU communication test — wraps official nccl-tests."""
|
||
|
||
import glob
|
||
import os
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import tempfile
|
||
from datetime import datetime
|
||
from typing import Optional
|
||
|
||
from rich import box
|
||
from rich.console import Console
|
||
from rich.table import Table
|
||
from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
|
||
|
||
from modules.gpu_specs import detect_gpu_type, get_gpu_specs, resolve_tools_dir
|
||
|
||
TORCH_AVAILABLE = False
|
||
try:
|
||
import torch
|
||
if torch.cuda.is_available():
|
||
TORCH_AVAILABLE = True
|
||
except ImportError:
|
||
pass
|
||
|
||
|
||
# Per-operation bandwidth thresholds, as a fraction of NVLink bidirectional BW.
|
||
# Values aligned with the H100 production acceptance criteria (acceptance doc §5).
|
||
# AllToAll runs ~10-20% lower than AllReduce on 8-GPU NVSwitch, so its fraction is
|
||
# set lower; broadcast/sendrecv sit between.
|
||
_OP_BW_FRACTIONS = {
|
||
"allreduce": 0.45,
|
||
"allgather": 0.45,
|
||
"reducescatter": 0.45,
|
||
"broadcast": 0.40,
|
||
"sendrecv": 0.40,
|
||
"alltoall": 0.35,
|
||
}
|
||
|
||
|
||
class NCCLTest:
|
||
|
||
def __init__(self, config: dict):
|
||
self.config = config
|
||
self.console = Console()
|
||
self.nccl_cfg = config.get("nccl", {})
|
||
self.tools_dir = resolve_tools_dir(config)
|
||
self.gpu_type = detect_gpu_type()
|
||
self.specs = get_gpu_specs(self.gpu_type)
|
||
|
||
def _find_nccl_test(self, name: str) -> Optional[str]:
|
||
p = shutil.which(name)
|
||
if p:
|
||
return p
|
||
|
||
build_dir = os.path.join(self.tools_dir, "nccl-tests", "build")
|
||
local = os.path.join(build_dir, name)
|
||
if os.path.isfile(local) and os.access(local, shutil.os.X_OK):
|
||
return local
|
||
|
||
matches = glob.glob(os.path.join(self.tools_dir, "nccl-tests", "**", name), recursive=True)
|
||
for m in matches:
|
||
if os.access(m, shutil.os.X_OK):
|
||
return m
|
||
return None
|
||
|
||
def _find_mpirun(self) -> Optional[str]:
|
||
for cmd in ["mpirun", "mpiexec", os.path.join(self.tools_dir, "mpi", "bin", "mpirun")]:
|
||
p = shutil.which(cmd)
|
||
if p:
|
||
return p
|
||
return None
|
||
|
||
def run(self) -> dict:
|
||
gpu_count = 0
|
||
if TORCH_AVAILABLE:
|
||
gpu_count = torch.cuda.device_count()
|
||
|
||
if gpu_count < 2:
|
||
self.console.print(f"[yellow]NCCL test requires at least 2 GPUs (found {gpu_count})[/yellow]")
|
||
return {"error": "need_at_least_2_gpus", "gpu_count": gpu_count}
|
||
|
||
tests = []
|
||
if self.nccl_cfg.get("test_allreduce", True):
|
||
tests.append(("all_reduce_perf", "AllReduce"))
|
||
if self.nccl_cfg.get("test_alltoall", True):
|
||
tests.append(("alltoall_perf", "AllToAll"))
|
||
if self.nccl_cfg.get("test_broadcast", True):
|
||
tests.append(("broadcast_perf", "Broadcast"))
|
||
if self.nccl_cfg.get("test_reduce_scatter", False):
|
||
tests.append(("reduce_scatter_perf", "ReduceScatter"))
|
||
if self.nccl_cfg.get("test_allgather", False):
|
||
tests.append(("all_gather_perf", "AllGather"))
|
||
if self.nccl_cfg.get("test_sendrecv", False):
|
||
tests.append(("sendrecv_perf", "SendRecv"))
|
||
|
||
nvlink_bw = self.specs.get("nvlink_bandwidth_gbps", 0)
|
||
# User-provided override applies uniformly across all ops; otherwise
|
||
# each op gets its own threshold from _OP_BW_FRACTIONS.
|
||
user_override = self.nccl_cfg.get("min_bandwidth_gbps")
|
||
|
||
def threshold_for(label: str) -> float:
|
||
if user_override:
|
||
return float(user_override)
|
||
if nvlink_bw <= 0:
|
||
return 10.0 # conservative floor
|
||
frac = _OP_BW_FRACTIONS.get(label.lower(), 0.45)
|
||
return round(nvlink_bw * frac)
|
||
|
||
if self.gpu_type == "unknown":
|
||
self.console.print("[yellow]Unknown GPU — using conservative bandwidth thresholds[/yellow]")
|
||
|
||
# Strategy: try nccl-tests binary directly (single-node, -g N),
|
||
# then mpirun, then torchrun fallback
|
||
results = {}
|
||
any_binary_worked = False
|
||
|
||
with Progress(
|
||
SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
|
||
TimeElapsedColumn(), console=self.console,
|
||
) as progress:
|
||
task = progress.add_task("NCCL tests...", total=len(tests))
|
||
|
||
for binary, label in tests:
|
||
progress.update(task, description=f"NCCL {label}...")
|
||
op_min_bw = threshold_for(label)
|
||
result = self._run_one_nccl_test_direct(
|
||
binary, label, gpu_count, op_min_bw
|
||
)
|
||
if result.get("status") not in ("SKIP", None) and "error" not in result:
|
||
any_binary_worked = True
|
||
results[label.lower()] = result
|
||
else:
|
||
# Try mpirun fallback
|
||
mpirun = self._find_mpirun()
|
||
if mpirun:
|
||
result = self._run_one_nccl_test_mpirun(
|
||
binary, label, gpu_count, mpirun, op_min_bw
|
||
)
|
||
if result.get("status") not in ("SKIP", None) and "error" not in result:
|
||
any_binary_worked = True
|
||
results[label.lower()] = result
|
||
progress.advance(task)
|
||
|
||
if not any_binary_worked:
|
||
self.console.print("[yellow]nccl-tests binaries failed, falling back to torchrun[/yellow]")
|
||
return self._run_torchrun_fallback(gpu_count)
|
||
|
||
all_passed = all(
|
||
r.get("status") == "PASS"
|
||
for r in results.values()
|
||
if isinstance(r, dict) and "status" in r
|
||
)
|
||
|
||
return {
|
||
"passed": all_passed,
|
||
"source": "nccl-tests",
|
||
"min_bandwidth_gbps": {
|
||
lbl.lower(): threshold_for(lbl) for _, lbl in tests
|
||
},
|
||
"tests": results,
|
||
"gpu_count": gpu_count,
|
||
"timestamp": datetime.now().isoformat(),
|
||
"detected_gpu_type": self.gpu_type,
|
||
}
|
||
|
||
def _run_one_nccl_test_direct(self, binary_name: str, label: str,
|
||
gpu_count: int, min_bw: float) -> dict:
|
||
"""Run nccl-tests binary directly with -g N (no mpirun needed for single-node)."""
|
||
binary = self._find_nccl_test(binary_name)
|
||
if not binary:
|
||
return {"status": "SKIP", "error": f"{binary_name} not found"}
|
||
|
||
cmd = [
|
||
binary,
|
||
"-b", "8M",
|
||
"-e", "8G",
|
||
"-f", "2",
|
||
"-g", str(gpu_count),
|
||
"-w", "5",
|
||
"-n", "20",
|
||
]
|
||
|
||
try:
|
||
env = os.environ.copy()
|
||
env["NCCL_DEBUG"] = "WARN"
|
||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env)
|
||
|
||
combined = r.stdout + r.stderr
|
||
# Check for NCCL/CUDA compatibility errors
|
||
if "CUDA driver version is insufficient" in combined or \
|
||
"Test NCCL failure" in combined:
|
||
error_msg = "NCCL/CUDA driver version mismatch" \
|
||
if "CUDA driver version" in combined \
|
||
else "NCCL test failure (library incompatibility)"
|
||
return {"status": "FAIL", "error": error_msg}
|
||
|
||
if r.returncode != 0:
|
||
return {"status": "FAIL", "error": r.stderr[:300]}
|
||
|
||
return self._parse_nccl_output(r.stdout, min_bw)
|
||
|
||
except subprocess.TimeoutExpired:
|
||
return {"status": "FAIL", "error": "timeout"}
|
||
except Exception as e:
|
||
return {"status": "FAIL", "error": str(e)}
|
||
|
||
def _run_one_nccl_test_mpirun(self, binary_name: str, label: str,
|
||
gpu_count: int, mpirun: str, min_bw: float) -> dict:
|
||
"""Run nccl-tests via mpirun (multi-node or per-GPU-process mode)."""
|
||
binary = self._find_nccl_test(binary_name)
|
||
if not binary:
|
||
return {"status": "SKIP", "error": f"{binary_name} not found"}
|
||
|
||
cmd = [
|
||
mpirun,
|
||
"-np", str(gpu_count),
|
||
"--allow-run-as-root",
|
||
"-x", "NCCL_DEBUG=WARN",
|
||
"-x", "CUDA_VISIBLE_DEVICES=" + ",".join(str(i) for i in range(gpu_count)),
|
||
binary,
|
||
"-b", "8",
|
||
"-e", "256M",
|
||
"-f", "2",
|
||
"-g", "1",
|
||
"-w", "5",
|
||
"-n", "20",
|
||
]
|
||
|
||
try:
|
||
env = os.environ.copy()
|
||
env["NCCL_DEBUG"] = "WARN"
|
||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=180, env=env)
|
||
|
||
combined = r.stdout + r.stderr
|
||
if "CUDA driver version is insufficient" in combined or \
|
||
"Test NCCL failure" in combined:
|
||
error_msg = "NCCL/CUDA driver version mismatch" \
|
||
if "CUDA driver version" in combined \
|
||
else "NCCL test failure (library incompatibility)"
|
||
return {"status": "FAIL", "error": error_msg}
|
||
|
||
if r.returncode != 0:
|
||
return {"status": "FAIL", "error": r.stderr[:300]}
|
||
|
||
return self._parse_nccl_output(r.stdout, min_bw)
|
||
|
||
except subprocess.TimeoutExpired:
|
||
return {"status": "FAIL", "error": "timeout"}
|
||
except Exception as e:
|
||
return {"status": "FAIL", "error": str(e)}
|
||
|
||
@staticmethod
|
||
def _parse_nccl_output(stdout: str, min_bw: float) -> dict:
|
||
"""Parse nccl-tests tabular output and extract bandwidth results."""
|
||
best_algbw = 0.0
|
||
best_busbw = 0.0
|
||
size_results = []
|
||
|
||
for line in stdout.split("\n"):
|
||
line = line.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
parts = line.split()
|
||
# nccl-tests data lines: size count type redop root time algbw busbw #wrong [time algbw busbw #wrong]
|
||
if len(parts) >= 9:
|
||
try:
|
||
size = int(parts[0])
|
||
# parts[2] is dtype string ('float'/'int32'/etc.), not a number
|
||
# out-of-place columns: time=parts[5], algbw=parts[6], busbw=parts[7]
|
||
time_us = float(parts[5])
|
||
algbw = float(parts[6])
|
||
busbw = float(parts[7])
|
||
size_results.append({
|
||
"size": size,
|
||
"time_us": time_us,
|
||
"algbw_gbps": algbw,
|
||
"busbw_gbps": busbw,
|
||
})
|
||
if busbw > best_busbw:
|
||
best_busbw = busbw
|
||
if algbw > best_algbw:
|
||
best_algbw = algbw
|
||
except (ValueError, IndexError):
|
||
continue
|
||
|
||
status = "PASS" if best_busbw >= min_bw else "WARN"
|
||
return {
|
||
"status": status,
|
||
"best_algbw_gbps": round(best_algbw, 1),
|
||
"best_busbw_gbps": round(best_busbw, 1),
|
||
"min_required_gbps": min_bw,
|
||
"by_size": size_results[-5:] if size_results else [],
|
||
}
|
||
|
||
def _run_torchrun_fallback(self, gpu_count: int) -> dict:
|
||
"""Basic NCCL connectivity test via torchrun — verifies NCCL works but does not benchmark performance."""
|
||
self.console.print("[yellow]nccl-tests not available, running basic NCCL connectivity check[/yellow]")
|
||
|
||
code = f"""
|
||
import torch, torch.distributed as dist, os
|
||
os.environ.setdefault("MASTER_ADDR","127.0.0.1")
|
||
os.environ.setdefault("MASTER_PORT","29500")
|
||
rank=int(os.environ.get("LOCAL_RANK",0))
|
||
ws={gpu_count}
|
||
dist.init_process_group("nccl",rank=rank,world_size=ws)
|
||
torch.cuda.set_device(rank)
|
||
|
||
x=torch.randn(1024*1024,device=f"cuda:{{rank}}",dtype=torch.float32)
|
||
|
||
# Test AllReduce
|
||
try:
|
||
dist.all_reduce(x.clone())
|
||
if rank==0: print("allreduce:ok")
|
||
except Exception as e:
|
||
if rank==0: print(f"allreduce:fail:{{e}}")
|
||
|
||
# Test Broadcast
|
||
try:
|
||
dist.broadcast(x.clone(),src=0)
|
||
if rank==0: print("broadcast:ok")
|
||
except Exception as e:
|
||
if rank==0: print(f"broadcast:fail:{{e}}")
|
||
|
||
# Test AllGather
|
||
try:
|
||
tensor_list=[torch.empty_like(x) for _ in range(ws)]
|
||
dist.all_gather(tensor_list,x.clone())
|
||
if rank==0: print("allgather:ok")
|
||
except Exception as e:
|
||
if rank==0: print(f"allgather:fail:{{e}}")
|
||
|
||
# Test ReduceScatter
|
||
try:
|
||
chunks=list(x.chunk(ws))
|
||
output=torch.empty_like(chunks[0])
|
||
dist.reduce_scatter(output,chunks)
|
||
if rank==0: print("reducescatter:ok")
|
||
except Exception as e:
|
||
if rank==0: print(f"reducescatter:fail:{{e}}")
|
||
|
||
# Test AllToAll
|
||
try:
|
||
chunks=list(x.chunk(ws))
|
||
output_list=[torch.empty_like(c) for c in chunks]
|
||
dist.all_to_all(output_list,chunks)
|
||
if rank==0: print("alltoall:ok")
|
||
except Exception as e:
|
||
if rank==0: print(f"alltoall:fail:{{e}}")
|
||
|
||
dist.destroy_process_group()
|
||
"""
|
||
import tempfile
|
||
tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False)
|
||
tmp.write(code)
|
||
tmp.close()
|
||
|
||
try:
|
||
# Prefer torchrun from the same venv as the running Python
|
||
import sys
|
||
venv_torchrun = os.path.join(os.path.dirname(sys.executable), "torchrun")
|
||
torchrun_cmd = venv_torchrun if os.path.isfile(venv_torchrun) else "torchrun"
|
||
|
||
r = subprocess.run(
|
||
[torchrun_cmd, f"--nproc_per_node={gpu_count}", tmp.name],
|
||
capture_output=True, text=True, timeout=120,
|
||
env={**os.environ, "NCCL_DEBUG": "WARN"},
|
||
)
|
||
os.unlink(tmp.name)
|
||
|
||
# Parse connectivity results — format: op_name:ok or op_name:fail:error
|
||
tests = {}
|
||
all_passed = True
|
||
for line in r.stdout.split("\n"):
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
parts = line.split(":")
|
||
op_name = parts[0]
|
||
result = parts[1] if len(parts) > 1 else "unknown"
|
||
|
||
if result == "ok":
|
||
status = "PASS"
|
||
else:
|
||
status = "FAIL"
|
||
all_passed = False
|
||
|
||
tests[op_name] = {
|
||
"status": status,
|
||
"error": ":".join(parts[2:]) if len(parts) > 2 and result == "fail" else None,
|
||
}
|
||
|
||
return {
|
||
"passed": all_passed,
|
||
"source": "torchrun_fallback",
|
||
"tests": tests,
|
||
"gpu_count": gpu_count,
|
||
}
|
||
except Exception as e:
|
||
return {"passed": False, "source": "torchrun_fallback", "error": str(e)}
|
||
|
||
@staticmethod
|
||
def print_results(results: dict, console: Console = None):
|
||
c = console or Console()
|
||
if "error" in results:
|
||
c.print(f"[bold red]Error: {results['error']}[/bold red]")
|
||
return
|
||
|
||
passed = results.get("passed", False)
|
||
source = results.get("source", "unknown")
|
||
|
||
if source == "torchrun_fallback":
|
||
# Connectivity check mode
|
||
verdict = "[bold green]✓ NCCL Connectivity OK[/bold green]" if passed else "[bold red]✗ NCCL Connectivity FAILED[/bold red]"
|
||
c.print(f"{verdict} [dim](basic check via torchrun)[/dim]")
|
||
|
||
tests = results.get("tests", {})
|
||
if tests:
|
||
c.print("\n[dim]Operations tested:[/dim]")
|
||
for op_name, result in tests.items():
|
||
if not isinstance(result, dict):
|
||
continue
|
||
status = result.get("status", "FAIL")
|
||
s_color = "green" if status == "PASS" else "red"
|
||
error = result.get("error")
|
||
if error:
|
||
c.print(f" [{s_color}]{op_name}[/{s_color}] — {error}")
|
||
else:
|
||
c.print(f" [{s_color}]{op_name}[/{s_color}]")
|
||
|
||
c.print("\n[yellow]Note: functional connectivity test only (no performance data)[/yellow]")
|
||
else:
|
||
# nccl-tests mode
|
||
verdict = "[bold green]✓ NCCL tests PASSED[/bold green]" if passed else "[bold yellow]⚠ NCCL tests WARNING[/bold yellow]"
|
||
c.print(f"{verdict} [dim](via {source})[/dim]")
|
||
|
||
tests = results.get("tests", {})
|
||
for op_name, result in tests.items():
|
||
if not isinstance(result, dict):
|
||
continue
|
||
c.print(f"\n[bold cyan]{op_name.upper()}[/bold cyan]")
|
||
status = result.get("status", "FAIL")
|
||
s_color = "green" if status == "PASS" else ("yellow" if status == "WARN" else "red")
|
||
c.print(f" Status: [{s_color}]{status}[/{s_color}] "
|
||
f"Best bus BW: {result.get('best_busbw_gbps', 'N/A')} GB/s "
|
||
f"(min: {result.get('min_required_gbps', 'N/A')} GB/s)")
|
||
|
||
by_size = result.get("by_size", [])
|
||
if by_size:
|
||
t = Table(box=None, padding=(0, 1))
|
||
t.add_column("Size", style="bold", justify="right")
|
||
t.add_column("Time (us)", justify="right")
|
||
t.add_column("Alg BW (GB/s)", justify="right")
|
||
t.add_column("Bus BW (GB/s)", justify="right")
|
||
for r in by_size:
|
||
sz = r.get("size", 0)
|
||
sz_str = f"{sz/1024:.0f}K" if sz < 1048576 else f"{sz/1048576:.0f}M"
|
||
t.add_row(sz_str, f"{r.get('time_us',0):.1f}",
|
||
f"{r.get('algbw_gbps',0):.1f}", f"{r.get('busbw_gbps',0):.1f}")
|
||
c.print(t)
|
||
|
||
|
||
class MultiNodeNCCLTest:
|
||
"""Multi-node NCCL scaling test — runs nccl-tests via mpirun across a cluster hostfile.
|
||
|
||
Config section: multinode_nccl
|
||
hosts_file path to OpenMPI hostfile (one line: "<ip> slots=8")
|
||
mpi_home MPI installation prefix (non-default location)
|
||
nccl_ib_hca comma-separated IB HCAs (GPU-direct PIX cards only)
|
||
nccl_socket_ifname bootstrap NIC name (e.g. bond0)
|
||
scale_levels list of node counts to test
|
||
ops list of operations (all_reduce, alltoall, all_gather, reduce_scatter)
|
||
msg_size_bytes message size for most ops (default 8G)
|
||
msg_size_alltoall_bytes message size for alltoall (default 1G, avoids hang at 16+ nodes)
|
||
warmup / iterations nccl-tests -w / -n
|
||
timeout_sec per-op subprocess timeout
|
||
allreduce_min_busbw_gbps PASS threshold for AllReduce
|
||
"""
|
||
|
||
_OP_BINARY = {
|
||
"all_reduce": "all_reduce_perf",
|
||
"alltoall": "alltoall_perf",
|
||
"all_gather": "all_gather_perf",
|
||
"reduce_scatter": "reduce_scatter_perf",
|
||
}
|
||
_OP_LABEL = {
|
||
"all_reduce": "AllReduce",
|
||
"alltoall": "AllToAll",
|
||
"all_gather": "AllGather",
|
||
"reduce_scatter": "ReduceScatter",
|
||
}
|
||
|
||
def __init__(self, config: dict):
|
||
self.config = config
|
||
self.console = Console()
|
||
self.mn_cfg = config.get("multinode_nccl", {})
|
||
self.tools_dir = resolve_tools_dir(config)
|
||
|
||
def _find_mpirun(self) -> Optional[str]:
|
||
mpi_home = os.path.expanduser(self.mn_cfg.get("mpi_home", ""))
|
||
if mpi_home:
|
||
candidate = os.path.join(mpi_home, "bin", "mpirun")
|
||
if os.path.isfile(candidate):
|
||
return candidate
|
||
return shutil.which("mpirun") or shutil.which("mpiexec")
|
||
|
||
def _find_nccl_build_dir(self) -> Optional[str]:
|
||
candidates = [
|
||
"/opt/nccl-tests/build",
|
||
os.path.join(self.tools_dir, "nccl-tests", "build"),
|
||
]
|
||
for d in candidates:
|
||
probe = os.path.join(d, "all_reduce_perf")
|
||
if os.path.isfile(probe) and os.access(probe, os.X_OK):
|
||
return d
|
||
return None
|
||
|
||
def _build_env(self) -> dict:
|
||
env = os.environ.copy()
|
||
mpi_home = os.path.expanduser(self.mn_cfg.get("mpi_home", ""))
|
||
if mpi_home:
|
||
env["PATH"] = f"{mpi_home}/bin:{env.get('PATH', '')}"
|
||
env["LD_LIBRARY_PATH"] = f"{mpi_home}/lib:{env.get('LD_LIBRARY_PATH', '')}"
|
||
env["NCCL_IB_HCA"] = self.mn_cfg.get(
|
||
"nccl_ib_hca", "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9"
|
||
)
|
||
env["NCCL_IB_DISABLE"] = "0"
|
||
env["NCCL_SOCKET_IFNAME"] = self.mn_cfg.get("nccl_socket_ifname", "bond0")
|
||
env["NCCL_DEBUG"] = "WARN"
|
||
return env
|
||
|
||
def _load_hosts(self, hosts_file: str) -> list:
|
||
path = os.path.expanduser(hosts_file)
|
||
if not os.path.isfile(path):
|
||
return []
|
||
lines = []
|
||
with open(path) as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if line and not line.startswith("#"):
|
||
lines.append(line)
|
||
return lines
|
||
|
||
def _run_op(self, mpirun: str, nt_dir: str, op: str, n_nodes: int,
|
||
host_lines: list, env: dict) -> dict:
|
||
binary = os.path.join(nt_dir, self._OP_BINARY[op])
|
||
if not os.path.isfile(binary):
|
||
return {"busbw_gbps": None, "status": "SKIP", "error": f"{op} binary not found"}
|
||
|
||
if op == "alltoall":
|
||
msg_bytes = self.mn_cfg.get("msg_size_alltoall_bytes", 1073741824)
|
||
else:
|
||
msg_bytes = self.mn_cfg.get("msg_size_bytes", 8589934592)
|
||
|
||
with tempfile.NamedTemporaryFile(mode="w", suffix=".hosts", delete=False) as tf:
|
||
tf.write("\n".join(host_lines[:n_nodes]) + "\n")
|
||
hf_path = tf.name
|
||
|
||
cmd = [
|
||
mpirun,
|
||
"--allow-run-as-root",
|
||
"--mca", "plm_rsh_args", "-oStrictHostKeyChecking=no",
|
||
"-np", str(n_nodes * 8),
|
||
"-N", "8",
|
||
"--hostfile", hf_path,
|
||
"-x", "PATH",
|
||
"-x", "LD_LIBRARY_PATH",
|
||
"-x", "NCCL_IB_HCA",
|
||
"-x", "NCCL_IB_DISABLE",
|
||
"-x", "NCCL_SOCKET_IFNAME",
|
||
"-x", "NCCL_DEBUG",
|
||
binary,
|
||
"-b", "8",
|
||
"-e", str(msg_bytes),
|
||
"-f", "2",
|
||
"-g", "1",
|
||
"-w", str(self.mn_cfg.get("warmup", 5)),
|
||
"-n", str(self.mn_cfg.get("iterations", 30)),
|
||
]
|
||
|
||
timeout = self.mn_cfg.get("timeout_sec", 600)
|
||
try:
|
||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env)
|
||
os.unlink(hf_path)
|
||
if r.returncode != 0:
|
||
return {"busbw_gbps": None, "status": "ERR", "error": r.stderr[:200]}
|
||
return self._parse_op_output(r.stdout, msg_bytes)
|
||
except subprocess.TimeoutExpired:
|
||
os.unlink(hf_path)
|
||
return {"busbw_gbps": None, "status": "ERR", "error": "timeout"}
|
||
except Exception as e:
|
||
os.unlink(hf_path)
|
||
return {"busbw_gbps": None, "status": "ERR", "error": str(e)}
|
||
|
||
@staticmethod
|
||
def _parse_op_output(stdout: str, target_bytes: int) -> dict:
|
||
for line in stdout.split("\n"):
|
||
line = line.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
parts = line.split()
|
||
if len(parts) < 9:
|
||
continue
|
||
try:
|
||
if int(parts[0]) == target_bytes:
|
||
return {"busbw_gbps": round(float(parts[7]), 2), "status": "OK"}
|
||
except (ValueError, IndexError):
|
||
continue
|
||
return {"busbw_gbps": None, "status": "ERR", "error": "size not found in output"}
|
||
|
||
def run(self) -> dict:
|
||
hosts_file = self.mn_cfg.get("hosts_file", "~/nccl_kit/hosts_all")
|
||
scale_levels = self.mn_cfg.get("scale_levels", [2, 4, 8])
|
||
ops = self.mn_cfg.get("ops", ["all_reduce", "alltoall", "all_gather", "reduce_scatter"])
|
||
min_busbw = self.mn_cfg.get("allreduce_min_busbw_gbps", 310)
|
||
|
||
mpirun = self._find_mpirun()
|
||
if not mpirun:
|
||
return {"error": "mpirun not found — set multinode_nccl.mpi_home in config"}
|
||
|
||
nt_dir = self._find_nccl_build_dir()
|
||
if not nt_dir:
|
||
return {"error": "nccl-tests not found (need MPI=1 build at /opt/nccl-tests/build)"}
|
||
|
||
host_lines = self._load_hosts(hosts_file)
|
||
if not host_lines:
|
||
return {"error": f"hostfile not found or empty: {hosts_file}"}
|
||
|
||
env = self._build_env()
|
||
results = {}
|
||
runnable = [n for n in scale_levels if n <= len(host_lines)]
|
||
skipped = [n for n in scale_levels if n > len(host_lines)]
|
||
if skipped:
|
||
self.console.print(f"[yellow]⚠ 跳过节点数 {skipped}:hostfile 只有 {len(host_lines)} 台[/yellow]")
|
||
|
||
with Progress(
|
||
SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
|
||
TimeElapsedColumn(), console=self.console,
|
||
) as progress:
|
||
task = progress.add_task("Multi-node NCCL...", total=len(runnable) * len(ops))
|
||
for n in runnable:
|
||
results[n] = {}
|
||
for op in ops:
|
||
progress.update(task, description=f"{n} nodes / {self._OP_LABEL.get(op, op)}...")
|
||
results[n][op] = self._run_op(mpirun, nt_dir, op, n, host_lines, env)
|
||
progress.advance(task)
|
||
|
||
passed = all(
|
||
(results.get(n, {}).get("all_reduce", {}).get("busbw_gbps") or 0) >= min_busbw
|
||
for n in runnable
|
||
)
|
||
|
||
return {
|
||
"passed": passed,
|
||
"hosts_file": hosts_file,
|
||
"available_nodes": len(host_lines),
|
||
"scale_levels": scale_levels,
|
||
"ops": ops,
|
||
"allreduce_min_busbw_gbps": min_busbw,
|
||
"results": results,
|
||
"timestamp": datetime.now().isoformat(),
|
||
}
|
||
|
||
def print_results(self, results: dict):
|
||
c = self.console
|
||
if "error" in results:
|
||
c.print(f"[bold red]Multi-node NCCL Error: {results['error']}[/bold red]")
|
||
return
|
||
|
||
passed = results.get("passed", False)
|
||
verdict = "[bold green]✓ PASS[/bold green]" if passed else "[bold red]✗ FAIL[/bold red]"
|
||
c.print(f"\n[bold cyan]Multi-node NCCL Scaling[/bold cyan] — {verdict}")
|
||
c.print(f" Hostfile: {results['hosts_file']} ({results['available_nodes']} nodes available)")
|
||
c.print(f" AllReduce PASS threshold: ≥ {results['allreduce_min_busbw_gbps']} GB/s")
|
||
|
||
data = results.get("results", {})
|
||
ops = results.get("ops", [])
|
||
scale_levels = [n for n in results.get("scale_levels", []) if n in data]
|
||
min_busbw = results.get("allreduce_min_busbw_gbps", 310)
|
||
|
||
t = Table(box=box.SIMPLE_HEAD, border_style="cyan")
|
||
t.add_column("Nodes / GPUs", style="bold", justify="center")
|
||
for op in ops:
|
||
t.add_column(self._OP_LABEL.get(op, op) + " (GB/s)", justify="right")
|
||
t.add_column("", justify="center")
|
||
|
||
for n in scale_levels:
|
||
row_data = data.get(n, {})
|
||
cells = []
|
||
ok = True
|
||
for op in ops:
|
||
r = row_data.get(op, {})
|
||
bw = r.get("busbw_gbps")
|
||
if bw is None:
|
||
err = r.get("error", "ERR")[:8]
|
||
cells.append(f"[red]{err}[/red]")
|
||
ok = False
|
||
else:
|
||
is_alltoall = op == "alltoall"
|
||
fail = (op == "all_reduce" and bw < min_busbw)
|
||
color = "red" if fail else "green"
|
||
suffix = "¹" if is_alltoall else ""
|
||
cells.append(f"[{color}]{bw:.1f}[/{color}]{suffix}")
|
||
status_str = "[green]PASS[/green]" if ok else "[red]FAIL[/red]"
|
||
t.add_row(f"{n} / {n * 8}", *cells, status_str)
|
||
|
||
c.print(t)
|
||
c.print("[dim]¹ AllToAll 使用 1G 消息(避免大消息 hang) | 其余操作使用 8G 消息[/dim]")
|