diff --git a/README.md b/README.md index ebe1ae6..9530f68 100644 --- a/README.md +++ b/README.md @@ -36,13 +36,22 @@ servertest/ │ ├── gpu_info.py # GPU 检测 & 信息 │ ├── health_check.py # 健康诊断 │ ├── benchmark.py # 内存带宽 + 计算吞吐 -│ ├── nccl_test.py # NCCL 多卡通信 +│ ├── nccl_test.py # NCCL 单节点多卡通信 (NCCLTest) + 多机 Scaling (MultiNodeNCCLTest) │ ├── stress_test.py # GPU 压力/稳定性 │ ├── rdma_test.py # RDMA/InfiniBand │ ├── training_sim.py # 训练模拟 │ └── report.py # 报告生成 +├── scripts/ +│ └── multinode_nccl/ # 独立 shell 脚本版 (历史备用) +│ ├── nccl_env.sh # 环境变量 (MPI路径/IB网卡/bond0) +│ ├── hosts_all # 全量 34 台节点列表 +│ ├── run_scaling.sh # Scaling 测试 (2→32节点, 同交换机) +│ └── run_cross_switch_test.sh # 跨交换机新机接入验证 +├── reports/ # 测试报告 +│ ├── multinode_nccl_scaling_20260602.md +│ └── multinode_nccl_cross_switch_20260606.md ├── requirements.txt -└── 调研.md # 行业框架调研 +└── doc/ # 验收标准文档 ``` --- @@ -282,7 +291,7 @@ python3 gpu_tester.py --config /path/to/config.yaml --test all 默认配置:4096×4096 矩阵,10 次 warmup,100 次迭代。 -### 5. NCCL Multi-GPU Test(多卡通信) +### 5. NCCL Multi-GPU Test(单节点多卡通信) 优先使用官方 nccl-tests(通过 mpirun 调用),不可用时 torchrun fallback。 @@ -299,6 +308,34 @@ python3 gpu_tester.py --config /path/to/config.yaml --test all **NVLink 参考带宽:** A100/A800 ≥ 240 GB/s | H100/H200 ≥ 360 GB/s | B200/B300 ≥ 720 GB/s(40% NVLink 峰值) +### 5m. Multi-node NCCL Scaling(多机跨节点通信) + +通过 mpirun + hostfile 在集群多台节点上运行 NCCL Scaling 测试,验证 IB fabric 健康和带宽随节点数的线性扩展。**菜单选 `m`,CLI 用 `--test multinode-nccl`。** + +```bash +# 交互式菜单选 m +python3 gpu_tester.py + +# 或 CLI 模式 +python3 gpu_tester.py --test multinode-nccl + +# 自定义节点数和 hostfile +python3 gpu_tester.py --test multinode-nccl --config configs/h20_cluster.yaml +``` + +配置(`configs/default.yaml` 的 `multinode_nccl` 段): + +| 参数 | 默认值 | 说明 | +|---|---|---| +| `hosts_file` | `~/nccl_kit/hosts_all` | OpenMPI hostfile 路径 | +| `mpi_home` | `/usr/mpi/gcc/openmpi-4.1.7rc1` | MPI 安装路径 | +| `nccl_ib_hca` | `mlx5_0~5,8,9` | GPU 直连 IB 卡(排除存储 NIC) | +| `nccl_socket_ifname` | `bond0` | NCCL bootstrap 网卡 | +| `scale_levels` | `[2, 4, 8]` | 要测试的节点数列表 | +| `allreduce_min_busbw_gbps` | `310` | AllReduce PASS 阈值(GB/s) | + +**验收标准:** AllReduce busbw ≥ 310 GB/s(4 节点起稳态 ~317 GB/s,32 节点无衰减) + ### 6. GPU Stress Test(压力测试) 使用 gpu-burn 进行长时满载测试,验证热稳定性和内存正确性。 @@ -479,9 +516,10 @@ report: --- -### SOP-3: 多节点集群验收 +### SOP-3: 多节点集群验收 (H20-3e, 34 台) -**适用场景:** 多台 GPU 服务器组成训练集群,验证节点间通信。 +**适用场景:** 多台 GPU 服务器组成训练集群,验证节点间 NCCL 通信及 IB fabric 健康。 +**工具脚本位置:** `scripts/multinode_nccl/`,主节点 `~/nccl_kit/`(gpu-20) ``` 前置条件: 每台单节点已通过 SOP-1 @@ -490,32 +528,61 @@ report: ├── 在每台节点上执行 SOP-1 └── 确保所有单节点测试通过 -步骤 2: RDMA 网络测试 -├── python3 gpu_tester.py --test rdma -├── 确认: IB 设备被识别 -├── 确认: 端口状态 Active -├── 确认: 写带宽 ≥ 50 GB/s -├── 确认: 延迟 ≤ 10 μs -└── 异常: 检查 IB 线缆、交换机配置、子网管理器 +步骤 2: 确认 IB 网卡拓扑 +├── nvidia-smi topo -m +├── 确认 GPU 直连 NIC (PIX): mlx5_0/1/2/3/4/5/8/9 (每节点 8 张) +├── mlx5_6/7 为存储 NIC (NODE 级别), 不参与 NCCL +└── 全集群 34 台拓扑一致: md5 均为 65a5f1bd43922dc879adadc5cce523b2 -步骤 3: 多节点 NCCL 测试 -├── 在每个节点上配置: -│ export MASTER_ADDR=<主节点IP> -│ export MASTER_PORT=29500 -│ export NCCL_SOCKET_IFNAME=ib0 # IB 网卡名 -│ export NCCL_DEBUG=INFO -├── 运行 nccl-tests 手动测试: -│ mpirun -np <总GPU数> -hostfile hosts \ -│ /opt/gpu-test-tools/nccl-tests/build/all_reduce_perf \ -│ -b 8 -e 256M -f 2 -g 1 -w 5 -n 20 -└── 确认: 多节点 AllReduce 带宽正常 +步骤 3: 准备主节点环境 +├── # 在 gpu-20 上: +├── mkdir -p ~/nccl_kit && cd ~/nccl_kit +├── # 上传 scripts/multinode_nccl/ 下的所有文件到 ~/nccl_kit/ +├── # 若 nccl-tests 为 MPI=0 编译, 需重编: +│ cd /opt/nccl-tests && make clean +│ make MPI=1 MPI_HOME=/usr/mpi/gcc/openmpi-4.1.7rc1 CUDA_HOME=/usr/local/cuda -j +│ ldd build/all_reduce_perf | grep -i mpi # 确认 libmpi 链接 +└── # 确认各节点免密 SSH 已打通 (集群预置共享 key) -步骤 4: 训练验证 -├── python3 gpu_tester.py --test training -├── 可选: 加载更大模型(如 llama 模型) -└── 确认: 训练 loss 正常下降 +步骤 4: Scaling 测试 (同交换机, 2→32 节点) +├── # 在 gpu-20 上: +├── source ~/nccl_kit/nccl_env.sh +├── bash ~/nccl_kit/run_scaling.sh 2 4 8 16 32 +├── 验收标准: +│ AllReduce busbw: 2节点 ~480 GB/s, 4节点起稳定 ~317 GB/s +│ AllGather / ReduceScatter: ~315-340 GB/s (随节点数平稳) +│ AllToAll (1G): ~48-70 GB/s (随节点数轻微下降属正常) +└── 结果: ~/nccl_kit/results_*/summary.csv + +步骤 5: 新机 / 跨交换机验证 (按需) +├── # 新机接入时需额外操作: +│ 1. SSH 互信: 在新机和现有节点间互相追加 authorized_keys +│ 2. 新机重编 nccl-tests (预装版通常为 MPI=0): +│ cd /opt/nccl-tests && make clean +│ make MPI=1 MPI_HOME=/usr/mpi/gcc/openmpi-4.1.7rc1 CUDA_HOME=/usr/local/cuda -j +│ 3. 确认新机 nvidia-smi topo -m 拓扑与集群一致 +├── bash ~/nccl_kit/run_cross_switch_test.sh +├── 验收标准: 跨交换机 AllReduce ~317 GB/s (与同交换机持平即正常) +└── AllToAll 跨交换机较同交换机下降 ≤20% 属正常 (多一跳 IB spine) ``` +**关键环境变量说明 (nccl_env.sh):** + +| 变量 | 值 | 原因 | +|---|---|---| +| `NCCL_IB_HCA` | `mlx5_0,1,2,3,4,5,8,9` | 仅 GPU 直连的 8 张 IB 卡;mlx5_6/7 为存储 NIC,不参与 NCCL | +| `NCCL_SOCKET_IFNAME` | `bond0` | 节点有 k8s 虚拟网卡,不指定会 bootstrap 选错导致 connection refused | +| `MPI_HOME` | `/usr/mpi/gcc/openmpi-4.1.7rc1` | 非系统默认路径,必须显式指定 | + +**常见问题:** + +| 现象 | 原因 | 解决 | +|---|---|---| +| busbw=0.00,每进程各打印一行结果 | nccl-tests 是 MPI=0 编译的 | `make clean && make MPI=1 ...` 重编 | +| `socketPollConnect: Connection refused` | NCCL 选了 k8s 虚拟网卡 | 确认 `NCCL_SOCKET_IFNAME=bond0` | +| AllToAll 在 8G 消息时 hang | 已知问题,16+ 节点大消息 hang | AllToAll 消息上限用 ≤2G | +| 新机 mpirun 连接失败 | SSH 互信未打通 | 互相追加 authorized_keys | + --- ### SOP-4: 故障诊断 diff --git a/configs/default.yaml b/configs/default.yaml index a432c11..c8674e1 100644 --- a/configs/default.yaml +++ b/configs/default.yaml @@ -81,6 +81,31 @@ rdma: min_bandwidth_gbps: 350 # per-port PASS floor (NDR400 ≈ 0.9 × 400) max_latency_us: 5 +multinode_nccl: + # Path to OpenMPI hostfile: one line per node, format " slots=8" + hosts_file: ~/nccl_kit/hosts_all + # MPI installation prefix (non-standard path, set empty to use system mpirun) + mpi_home: /usr/mpi/gcc/openmpi-4.1.7rc1 + # IB HCAs for NCCL — GPU-direct (PIX) cards only; mlx5_6/7 are storage NICs, exclude them + # GPU↔NIC map: GPU0↔mlx5_3, GPU1↔mlx5_2, GPU2↔mlx5_1, GPU3↔mlx5_0 + # GPU4↔mlx5_5, GPU5↔mlx5_4, GPU6↔mlx5_9, GPU7↔mlx5_8 + nccl_ib_hca: "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9" + # NCCL bootstrap NIC — must be explicit (k8s virtual NICs cause connection refused) + nccl_socket_ifname: bond0 + # Node counts to run scaling test at + scale_levels: [2, 4, 8] + # Operations to test + ops: [all_reduce, alltoall, all_gather, reduce_scatter] + # Message size for allreduce/allgather/reducescatter (bytes) + msg_size_bytes: 8589934592 # 8G + # Message size for alltoall — cap at 1G to avoid hang at 16+ nodes + msg_size_alltoall_bytes: 1073741824 # 1G + warmup: 5 + iterations: 30 + timeout_sec: 600 # per-op timeout (10 min) + # AllReduce busbw PASS floor (GB/s) — 4-node+ stable state on NDR400 cluster + allreduce_min_busbw_gbps: 310 + training: model: gpt2 batch_size: 8 diff --git a/gpu_tester.py b/gpu_tester.py index 4cfa47c..8c00282 100644 --- a/gpu_tester.py +++ b/gpu_tester.py @@ -21,7 +21,7 @@ from rich import box from modules.gpu_info import GPUInfo from modules.health_check import HealthCheck from modules.benchmark import Benchmark -from modules.nccl_test import NCCLTest +from modules.nccl_test import NCCLTest, MultiNodeNCCLTest from modules.training_sim import TrainingSim from modules.stress_test import StressTest from modules.rdma_test import RDMATest @@ -48,6 +48,20 @@ DEFAULT_CONFIG = { "test_allgather": False, "test_sendrecv": False, }, + "multinode_nccl": { + "hosts_file": "~/nccl_kit/hosts_all", + "mpi_home": "/usr/mpi/gcc/openmpi-4.1.7rc1", + "nccl_ib_hca": "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9", + "nccl_socket_ifname": "bond0", + "scale_levels": [2, 4, 8], + "ops": ["all_reduce", "alltoall", "all_gather", "reduce_scatter"], + "msg_size_bytes": 8589934592, + "msg_size_alltoall_bytes": 1073741824, + "warmup": 5, + "iterations": 30, + "timeout_sec": 600, + "allreduce_min_busbw_gbps": 310, + }, "stress": { "duration_sec": 60, "use_doubles": False, @@ -142,6 +156,7 @@ def interactive_menu(config: dict): ("6", "GPU Stress Test (gpu-burn)", "stress"), ("7", "RDMA/IB Test", "rdma"), ("8", "Training Simulation", "training"), + ("m", "Multi-node NCCL Scaling", "multinode_nccl"), ("9", "Full Test Suite (All Tests)", "all"), ("0", "Generate Report", "report"), ] @@ -164,6 +179,7 @@ def interactive_menu(config: dict): "memory_bench": "HBM bandwidth via nvbandwidth", "compute_bench": "GEMM TFLOPS across FP32/TF32/FP16/BF16/FP8", "nccl": "AllReduce, AllToAll, Broadcast via nccl-tests", + "multinode_nccl": "跨节点 NCCL Scaling via mpirun + hostfile", "stress": "Long-running GPU stress via gpu-burn", "rdma": "InfiniBand bandwidth & latency (ib_write_bw)", "training": "Simulate LLM training with PyTorch", @@ -245,6 +261,12 @@ def _run_test(test_name: str, config: dict, console: Console) -> dict: m.print_results(result) return result + elif test_name == "multinode_nccl": + m = MultiNodeNCCLTest(config) + result = m.run() + m.print_results(result) + return result + elif test_name == "stress": m = StressTest(config) result = m.run() @@ -340,7 +362,7 @@ Examples: python gpu_tester.py --report --format json --output report.json """, ) - parser.add_argument("--test", choices=["gpu-info", "health", "benchmark", "nccl", "stress", "rdma", "training", "all"], + parser.add_argument("--test", choices=["gpu-info", "health", "benchmark", "nccl", "multinode-nccl", "stress", "rdma", "training", "all"], help="Run a specific test") parser.add_argument("--type", choices=["memory", "compute"], help="Benchmark type (with --test benchmark)") parser.add_argument("--dtype", choices=["fp32", "tf32", "fp16", "bf16", "fp8"], @@ -397,6 +419,7 @@ Examples: "health": "health", "benchmark": None, "nccl": "nccl", + "multinode-nccl": "multinode_nccl", "stress": "stress", "rdma": "rdma", "training": "training", diff --git a/modules/nccl_test.py b/modules/nccl_test.py index fd9ab6a..5c04af5 100644 --- a/modules/nccl_test.py +++ b/modules/nccl_test.py @@ -5,9 +5,11 @@ import os import re import shutil import subprocess +import tempfile from datetime import datetime from typing import Optional +from rich import box from rich.console import Console from rich.table import Table from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn @@ -89,7 +91,7 @@ class NCCLTest: if self.nccl_cfg.get("test_reduce_scatter", False): tests.append(("reduce_scatter_perf", "ReduceScatter")) if self.nccl_cfg.get("test_allgather", False): - tests.append(("allgather_perf", "AllGather")) + tests.append(("all_gather_perf", "AllGather")) if self.nccl_cfg.get("test_sendrecv", False): tests.append(("sendrecv_perf", "SendRecv")) @@ -457,3 +459,251 @@ dist.destroy_process_group() t.add_row(sz_str, f"{r.get('time_us',0):.1f}", f"{r.get('algbw_gbps',0):.1f}", f"{r.get('busbw_gbps',0):.1f}") c.print(t) + + +class MultiNodeNCCLTest: + """Multi-node NCCL scaling test — runs nccl-tests via mpirun across a cluster hostfile. + + Config section: multinode_nccl + hosts_file path to OpenMPI hostfile (one line: " slots=8") + mpi_home MPI installation prefix (non-default location) + nccl_ib_hca comma-separated IB HCAs (GPU-direct PIX cards only) + nccl_socket_ifname bootstrap NIC name (e.g. bond0) + scale_levels list of node counts to test + ops list of operations (all_reduce, alltoall, all_gather, reduce_scatter) + msg_size_bytes message size for most ops (default 8G) + msg_size_alltoall_bytes message size for alltoall (default 1G, avoids hang at 16+ nodes) + warmup / iterations nccl-tests -w / -n + timeout_sec per-op subprocess timeout + allreduce_min_busbw_gbps PASS threshold for AllReduce + """ + + _OP_BINARY = { + "all_reduce": "all_reduce_perf", + "alltoall": "alltoall_perf", + "all_gather": "all_gather_perf", + "reduce_scatter": "reduce_scatter_perf", + } + _OP_LABEL = { + "all_reduce": "AllReduce", + "alltoall": "AllToAll", + "all_gather": "AllGather", + "reduce_scatter": "ReduceScatter", + } + + def __init__(self, config: dict): + self.config = config + self.console = Console() + self.mn_cfg = config.get("multinode_nccl", {}) + self.tools_dir = resolve_tools_dir(config) + + def _find_mpirun(self) -> Optional[str]: + mpi_home = os.path.expanduser(self.mn_cfg.get("mpi_home", "")) + if mpi_home: + candidate = os.path.join(mpi_home, "bin", "mpirun") + if os.path.isfile(candidate): + return candidate + return shutil.which("mpirun") or shutil.which("mpiexec") + + def _find_nccl_build_dir(self) -> Optional[str]: + candidates = [ + "/opt/nccl-tests/build", + os.path.join(self.tools_dir, "nccl-tests", "build"), + ] + for d in candidates: + probe = os.path.join(d, "all_reduce_perf") + if os.path.isfile(probe) and os.access(probe, os.X_OK): + return d + return None + + def _build_env(self) -> dict: + env = os.environ.copy() + mpi_home = os.path.expanduser(self.mn_cfg.get("mpi_home", "")) + if mpi_home: + env["PATH"] = f"{mpi_home}/bin:{env.get('PATH', '')}" + env["LD_LIBRARY_PATH"] = f"{mpi_home}/lib:{env.get('LD_LIBRARY_PATH', '')}" + env["NCCL_IB_HCA"] = self.mn_cfg.get( + "nccl_ib_hca", "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9" + ) + env["NCCL_IB_DISABLE"] = "0" + env["NCCL_SOCKET_IFNAME"] = self.mn_cfg.get("nccl_socket_ifname", "bond0") + env["NCCL_DEBUG"] = "WARN" + return env + + def _load_hosts(self, hosts_file: str) -> list: + path = os.path.expanduser(hosts_file) + if not os.path.isfile(path): + return [] + lines = [] + with open(path) as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + lines.append(line) + return lines + + def _run_op(self, mpirun: str, nt_dir: str, op: str, n_nodes: int, + host_lines: list, env: dict) -> dict: + binary = os.path.join(nt_dir, self._OP_BINARY[op]) + if not os.path.isfile(binary): + return {"busbw_gbps": None, "status": "SKIP", "error": f"{op} binary not found"} + + if op == "alltoall": + msg_bytes = self.mn_cfg.get("msg_size_alltoall_bytes", 1073741824) + else: + msg_bytes = self.mn_cfg.get("msg_size_bytes", 8589934592) + + with tempfile.NamedTemporaryFile(mode="w", suffix=".hosts", delete=False) as tf: + tf.write("\n".join(host_lines[:n_nodes]) + "\n") + hf_path = tf.name + + cmd = [ + mpirun, + "--allow-run-as-root", + "--mca", "plm_rsh_args", "-oStrictHostKeyChecking=no", + "-np", str(n_nodes * 8), + "-N", "8", + "--hostfile", hf_path, + "-x", "PATH", + "-x", "LD_LIBRARY_PATH", + "-x", "NCCL_IB_HCA", + "-x", "NCCL_IB_DISABLE", + "-x", "NCCL_SOCKET_IFNAME", + "-x", "NCCL_DEBUG", + binary, + "-b", "8", + "-e", str(msg_bytes), + "-f", "2", + "-g", "1", + "-w", str(self.mn_cfg.get("warmup", 5)), + "-n", str(self.mn_cfg.get("iterations", 30)), + ] + + timeout = self.mn_cfg.get("timeout_sec", 600) + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env) + os.unlink(hf_path) + if r.returncode != 0: + return {"busbw_gbps": None, "status": "ERR", "error": r.stderr[:200]} + return self._parse_op_output(r.stdout, msg_bytes) + except subprocess.TimeoutExpired: + os.unlink(hf_path) + return {"busbw_gbps": None, "status": "ERR", "error": "timeout"} + except Exception as e: + os.unlink(hf_path) + return {"busbw_gbps": None, "status": "ERR", "error": str(e)} + + @staticmethod + def _parse_op_output(stdout: str, target_bytes: int) -> dict: + for line in stdout.split("\n"): + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) < 9: + continue + try: + if int(parts[0]) == target_bytes: + return {"busbw_gbps": round(float(parts[7]), 2), "status": "OK"} + except (ValueError, IndexError): + continue + return {"busbw_gbps": None, "status": "ERR", "error": "size not found in output"} + + def run(self) -> dict: + hosts_file = self.mn_cfg.get("hosts_file", "~/nccl_kit/hosts_all") + scale_levels = self.mn_cfg.get("scale_levels", [2, 4, 8]) + ops = self.mn_cfg.get("ops", ["all_reduce", "alltoall", "all_gather", "reduce_scatter"]) + min_busbw = self.mn_cfg.get("allreduce_min_busbw_gbps", 310) + + mpirun = self._find_mpirun() + if not mpirun: + return {"error": "mpirun not found — set multinode_nccl.mpi_home in config"} + + nt_dir = self._find_nccl_build_dir() + if not nt_dir: + return {"error": "nccl-tests not found (need MPI=1 build at /opt/nccl-tests/build)"} + + host_lines = self._load_hosts(hosts_file) + if not host_lines: + return {"error": f"hostfile not found or empty: {hosts_file}"} + + env = self._build_env() + results = {} + runnable = [n for n in scale_levels if n <= len(host_lines)] + skipped = [n for n in scale_levels if n > len(host_lines)] + if skipped: + self.console.print(f"[yellow]⚠ 跳过节点数 {skipped}:hostfile 只有 {len(host_lines)} 台[/yellow]") + + with Progress( + SpinnerColumn(), TextColumn("[progress.description]{task.description}"), + TimeElapsedColumn(), console=self.console, + ) as progress: + task = progress.add_task("Multi-node NCCL...", total=len(runnable) * len(ops)) + for n in runnable: + results[n] = {} + for op in ops: + progress.update(task, description=f"{n} nodes / {self._OP_LABEL.get(op, op)}...") + results[n][op] = self._run_op(mpirun, nt_dir, op, n, host_lines, env) + progress.advance(task) + + passed = all( + (results.get(n, {}).get("all_reduce", {}).get("busbw_gbps") or 0) >= min_busbw + for n in runnable + ) + + return { + "passed": passed, + "hosts_file": hosts_file, + "available_nodes": len(host_lines), + "scale_levels": scale_levels, + "ops": ops, + "allreduce_min_busbw_gbps": min_busbw, + "results": results, + "timestamp": datetime.now().isoformat(), + } + + def print_results(self, results: dict): + c = self.console + if "error" in results: + c.print(f"[bold red]Multi-node NCCL Error: {results['error']}[/bold red]") + return + + passed = results.get("passed", False) + verdict = "[bold green]✓ PASS[/bold green]" if passed else "[bold red]✗ FAIL[/bold red]" + c.print(f"\n[bold cyan]Multi-node NCCL Scaling[/bold cyan] — {verdict}") + c.print(f" Hostfile: {results['hosts_file']} ({results['available_nodes']} nodes available)") + c.print(f" AllReduce PASS threshold: ≥ {results['allreduce_min_busbw_gbps']} GB/s") + + data = results.get("results", {}) + ops = results.get("ops", []) + scale_levels = [n for n in results.get("scale_levels", []) if n in data] + min_busbw = results.get("allreduce_min_busbw_gbps", 310) + + t = Table(box=box.SIMPLE_HEAD, border_style="cyan") + t.add_column("Nodes / GPUs", style="bold", justify="center") + for op in ops: + t.add_column(self._OP_LABEL.get(op, op) + " (GB/s)", justify="right") + t.add_column("", justify="center") + + for n in scale_levels: + row_data = data.get(n, {}) + cells = [] + ok = True + for op in ops: + r = row_data.get(op, {}) + bw = r.get("busbw_gbps") + if bw is None: + err = r.get("error", "ERR")[:8] + cells.append(f"[red]{err}[/red]") + ok = False + else: + is_alltoall = op == "alltoall" + fail = (op == "all_reduce" and bw < min_busbw) + color = "red" if fail else "green" + suffix = "¹" if is_alltoall else "" + cells.append(f"[{color}]{bw:.1f}[/{color}]{suffix}") + status_str = "[green]PASS[/green]" if ok else "[red]FAIL[/red]" + t.add_row(f"{n} / {n * 8}", *cells, status_str) + + c.print(t) + c.print("[dim]¹ AllToAll 使用 1G 消息(避免大消息 hang) | 其余操作使用 8G 消息[/dim]") diff --git a/scripts/multinode_nccl/hosts_all b/scripts/multinode_nccl/hosts_all new file mode 100644 index 0000000..313e3d5 --- /dev/null +++ b/scripts/multinode_nccl/hosts_all @@ -0,0 +1,34 @@ +10.184.0.20 slots=8 +10.184.0.21 slots=8 +10.184.0.22 slots=8 +10.184.0.23 slots=8 +10.184.0.24 slots=8 +10.184.0.25 slots=8 +10.184.0.26 slots=8 +10.184.0.27 slots=8 +10.184.0.28 slots=8 +10.184.0.29 slots=8 +10.184.0.30 slots=8 +10.184.0.31 slots=8 +10.184.0.32 slots=8 +10.184.0.33 slots=8 +10.184.0.34 slots=8 +10.184.0.35 slots=8 +10.184.0.36 slots=8 +10.184.0.37 slots=8 +10.184.0.38 slots=8 +10.184.0.39 slots=8 +10.184.0.40 slots=8 +10.184.0.41 slots=8 +10.184.0.42 slots=8 +10.184.0.43 slots=8 +10.184.0.44 slots=8 +10.184.0.45 slots=8 +10.184.0.46 slots=8 +10.184.0.47 slots=8 +10.184.0.48 slots=8 +10.184.0.49 slots=8 +10.184.0.50 slots=8 +10.184.0.51 slots=8 +10.184.0.52 slots=8 +10.184.0.55 slots=8 diff --git a/scripts/multinode_nccl/nccl_env.sh b/scripts/multinode_nccl/nccl_env.sh new file mode 100644 index 0000000..8d7b82a --- /dev/null +++ b/scripts/multinode_nccl/nccl_env.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# ===== 多机 NCCL 测试环境 — H20-3e 集群 (34台, 8xNDR400 per node) ===== +# 主节点: gpu-20 (10.184.0.20), 在此节点 source 后执行 mpirun +# 验证日期: 2026-06-06 + +# OpenMPI 不在默认 loader 路径,必须显式加 +export PATH=/usr/mpi/gcc/openmpi-4.1.7rc1/bin:$PATH +export LD_LIBRARY_PATH=/usr/mpi/gcc/openmpi-4.1.7rc1/lib:$LD_LIBRARY_PATH + +# NCCL: 使用 8 张 GPU 直连 IB 网卡 (PIX 级别直连) +# 排除 mlx5_6/7: 虽为 IB Active, 但无 GPU PIX 直连, 为存储 NIC +# 排除 mlx5_bond_0: 25G 以太管理口 +# GPU↔NIC 对应: GPU0↔mlx5_3, GPU1↔mlx5_2, GPU2↔mlx5_1, GPU3↔mlx5_0 +# GPU4↔mlx5_5, GPU5↔mlx5_4, GPU6↔mlx5_9, GPU7↔mlx5_8 +export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9 +export NCCL_IB_DISABLE=0 + +# NCCL bootstrap 网卡: 节点上有 k8s 虚拟网卡 (lvscare/kube-ipvs0), +# 不指定会选错导致 connection refused, 内网 IP 在 bond0 上 +# 注意: 不能用 ^bond (会把 bond0 也排掉), 必须明确写 bond0 +export NCCL_SOCKET_IFNAME=bond0 + +export NCCL_IB_DISABLE=0 +export NCCL_DEBUG=${NCCL_DEBUG:-WARN} + +# IB 为 InfiniBand link_layer (非 RoCE), GID 一般不用设 +# 如报 GID 问题再放开: +# export NCCL_IB_GID_INDEX=3 diff --git a/scripts/multinode_nccl/run_cross_switch_test.sh b/scripts/multinode_nccl/run_cross_switch_test.sh new file mode 100644 index 0000000..04a24d4 --- /dev/null +++ b/scripts/multinode_nccl/run_cross_switch_test.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# ===== 跨交换机多机 NCCL 通信测试 ===== +# 适用场景: 验证不同交换机的节点加入集群后跨交换机通信是否正常 +# 用法: bash run_cross_switch_test.sh +# 主节点运行: gpu-20 (需与目标节点已打通免密 SSH) +# +# 默认分三个阶段: +# old2node — 旧集群内 2 台基准 (gpu-20/21) +# new2node — 新机 2 台同交换机 (gpu-51/55) +# cross4node — 跨交换机 4 台联合 +# +# 新机接入前置步骤: +# 1. 四台节点互相打通 authorized_keys +# 2. 新机上重编 nccl-tests: cd /opt/nccl-tests && make clean && +# make MPI=1 MPI_HOME=/usr/mpi/gcc/openmpi-4.1.7rc1 CUDA_HOME=/usr/local/cuda -j + +set -euo pipefail + +source "$(dirname "$0")/nccl_env.sh" + +NT=/opt/nccl-tests/build +OUT=~/nccl_kit/results_cross_$(date +%Y%m%d_%H%M%S) +mkdir -p "$OUT" +SUMMARY="$OUT/summary.csv" +echo "test,nodes,op,size,busbw_GBps" > "$SUMMARY" + +MPI=(mpirun --allow-run-as-root + --mca plm_rsh_args "-oStrictHostKeyChecking=no" + -x PATH -x LD_LIBRARY_PATH + -x NCCL_IB_HCA -x NCCL_IB_DISABLE + -x NCCL_SOCKET_IFNAME -x NCCL_DEBUG) + +run_test() { + local tag="$1" hostfile="$2" + local np=$(( $(wc -l < "$hostfile") * 8 )) + echo "" + echo "======== $tag ($np GPUs) ========" + for op in all_reduce alltoall all_gather reduce_scatter; do + log="$OUT/${tag}_${op}.log" + "${MPI[@]}" -np "$np" -N 8 --hostfile "$hostfile" \ + "$NT/${op}_perf" -b 8 -e 4G -f 2 -g 1 -w 5 -n 30 > "$log" 2>&1 + row=$(awk '$1==4294967296{print}' "$log" | tail -1) + busbw=$(echo "$row" | awk '{print $(NF-1)}') + [ -z "$busbw" ] && busbw="ERR(见 $log)" + printf " %-14s 4G busbw=%s GB/s\n" "$op" "$busbw" + echo "$tag,$(wc -l < "$hostfile"),$op,4G,$busbw" >> "$SUMMARY" + done +} + +# === 阶段1: 旧集群内 2 台 (基准) === +cat > "$OUT/hosts_old2" << HEOF +10.184.0.20 slots=8 +10.184.0.21 slots=8 +HEOF +run_test "old2node" "$OUT/hosts_old2" + +# === 阶段2: 新机 2 台同交换机 === +cat > "$OUT/hosts_new2" << HEOF +10.184.0.51 slots=8 +10.184.0.55 slots=8 +HEOF +run_test "new2node" "$OUT/hosts_new2" + +# === 阶段3: 跨交换机 4 台 === +cat > "$OUT/hosts_cross4" << HEOF +10.184.0.20 slots=8 +10.184.0.21 slots=8 +10.184.0.51 slots=8 +10.184.0.55 slots=8 +HEOF +run_test "cross4node" "$OUT/hosts_cross4" + +echo "" +echo "==== 汇总 ====" +column -t -s, "$SUMMARY" +echo "结果目录: $OUT" diff --git a/scripts/multinode_nccl/run_scaling.sh b/scripts/multinode_nccl/run_scaling.sh new file mode 100644 index 0000000..ede9579 --- /dev/null +++ b/scripts/multinode_nccl/run_scaling.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# ===== 多机 NCCL Scaling 测试 ===== +# 用法: bash run_scaling.sh [节点级别...] +# 默认: "2 4 8 16 32" +# 示例: bash run_scaling.sh 2 4 8 +# 前置: source nccl_env.sh, hosts_all 文件与本脚本同目录或指定 HOSTS_ALL +# 主节点运行: gpu-20 + +set -euo pipefail + +source "$(dirname "$0")/nccl_env.sh" + +NT=/opt/nccl-tests/build +HOSTS_ALL="${HOSTS_ALL:-$(dirname "$0")/hosts_all}" +LEVELS="${*:-2 4 8 16 32}" +OUT=~/nccl_kit/results_$(date +%Y%m%d_%H%M%S) +mkdir -p "$OUT" +SUMMARY="$OUT/summary.csv" +echo "nodes,gpus,op,size,busbw_GBps" > "$SUMMARY" + +MPI=(mpirun --allow-run-as-root + --mca plm_rsh_args "-oStrictHostKeyChecking=no" + -x PATH -x LD_LIBRARY_PATH + -x NCCL_IB_HCA -x NCCL_IB_DISABLE + -x NCCL_SOCKET_IFNAME -x NCCL_DEBUG) + +echo "Scaling levels: $LEVELS" +echo "结果目录: $OUT" + +for N in $LEVELS; do + hf="$OUT/hosts_$N" + head -"$N" "$HOSTS_ALL" > "$hf" + avail=$(wc -l < "$hf") + if [ "$avail" -lt "$N" ]; then + echo "!! hosts_all 只有 $avail 台, 跳过 $N 节点" + continue + fi + np=$((N * 8)) + echo "" + echo "================ $N nodes / $np GPUs ================" + for op in all_reduce alltoall all_gather reduce_scatter; do + log="$OUT/${op}_${N}n.log" + "${MPI[@]}" -np "$np" -N 8 --hostfile "$hf" \ + "$NT/${op}_perf" -b 8 -e 8G -f 2 -g 1 -w 5 -n 50 > "$log" 2>&1 + # AllToAll 在大消息 (>2G) + 多节点会 hang, 优先取 1G 行 + if [[ "$op" == "alltoall" ]]; then + row=$(awk '$1==1073741824{print}' "$log" | tail -1) + sz="1G" + else + row=$(awk '$1==8589934592{print}' "$log" | tail -1) + sz="8G" + fi + busbw=$(echo "$row" | awk '{print $(NF-1)}') + [ -z "$busbw" ] && busbw="ERR(见 $log)" + printf " %-14s %s busbw=%s GB/s\n" "$op" "$sz" "$busbw" + echo "$N,$np,$op,$sz,$busbw" >> "$SUMMARY" + done +done + +echo "" +echo "==== 汇总 (AllReduce 应随节点数基本持平 ~317 GB/s) ====" +column -t -s, "$SUMMARY" +echo "结果目录: $OUT"