#!/usr/bin/env python3 """fleet — 全机进程清单:collector、活 rollout、孤儿、重负载 stage 子进程。 只读:扫 /proc + 只读打开 state/jobs.sqlite,不写任何状态。 背景:调度进程是各终端 setsid nohup 起的,bench status/watch/heartbeat 都是 per-run, 没有"这台机器现在跑着什么"的全局视图——本脚本补这个视图。 用法:python3 scripts/fleet.py [--json] """ import argparse import json import os import pathlib import sqlite3 import sys import time BENCH_ROOT = pathlib.Path(__file__).resolve().parent.parent DB = BENCH_ROOT / "state" / "jobs.sqlite" RUNS_DIR = BENCH_ROOT / "runs" HZ = os.sysconf("SC_CLK_TCK") def boot_time(): with open("/proc/stat") as f: for line in f: if line.startswith("btime"): return float(line.split()[1]) return 0.0 def proc_stats(pid): """返回 (etime_s, cpu_s, cmdline, cwd);进程消失返回 None。""" try: with open(f"/proc/{pid}/stat") as f: raw = f.read() # comm 字段可能含空格,从最后一个 ')' 后切 fields = raw[raw.rindex(")") + 2:].split() starttime = float(fields[19]) / HZ utime, stime = float(fields[11]), float(fields[12]) etime = time.time() - boot_time() - starttime with open(f"/proc/{pid}/cmdline", "rb") as f: cmdline = f.read().decode(errors="replace").split("\0") cmdline = [c for c in cmdline if c] try: cwd = os.readlink(f"/proc/{pid}/cwd") except (PermissionError, FileNotFoundError, ProcessLookupError): cwd = "" # unshare jail / zombie 下同 UID 也可能拒读 return etime, utime + stime, cmdline, cwd except (FileNotFoundError, ProcessLookupError, ValueError, IndexError, PermissionError): return None def scan_procs(): """按类别返回本用户的进程:collectors / pi / stage 子进程。""" me = os.getuid() collectors, pis, heavies = [], [], [] for entry in pathlib.Path("/proc").iterdir(): if not entry.name.isdigit(): continue pid = int(entry.name) try: if os.stat(f"/proc/{pid}").st_uid != me: continue except FileNotFoundError: continue st = proc_stats(pid) if st is None or not st[2]: continue etime, cpu_s, cmdline, cwd = st argv0 = os.path.basename(cmdline[0]) joined = " ".join(cmdline) if argv0 == "pi" or (argv0 == "node" and cmdline[1:2] and cmdline[1].endswith("/pi")): m = parse_rollout_cwd(cwd) if m: pis.append({"pid": pid, "age_s": etime, **m}) elif "harness.cli" in joined: run = arg_value(cmdline, "--run") kind = next((c for c in ("collect", "run", "export", "heartbeat") if f"-m harness.cli {c}" in joined or f"harness.cli {c}" in joined), "?") collectors.append({ "pid": pid, "age_s": etime, "cpu_s": cpu_s, "kind": kind, "run": run or "?", "parallel": arg_value(cmdline, "--parallel") or "1", "drain": bool((BENCH_ROOT / "state" / f"drain-collect-{run}").exists()) if run else False, }) elif any(k in argv0 for k in ("infer_hbm", "qemu-system-riscv64", "hbcm-module-entry", "hbrt4-run-model", "eval.run")) or "-m harness.eval.run" in joined: heavies.append({"pid": pid, "age_s": etime, "cpu_s": cpu_s, "argv0": argv0, "path": rollout_path_in(joined, cwd)}) return collectors, pis, heavies def arg_value(cmdline, flag): for i, a in enumerate(cmdline): if a == flag and i + 1 < len(cmdline): return cmdline[i + 1] return None def parse_rollout_cwd(cwd): """.../runs///rollouts//workspace → dict(run, job, kk)""" p = str(cwd) if "/rollouts/" not in p or not p.rstrip("/").endswith("/workspace"): return None try: head = p.split("/runs/", 1)[1].rstrip("/").split("/") run, job, _, kk = head[0], head[1], head[2], head[3] return {"run": run, "job": job, "kk": int(kk)} except (IndexError, ValueError): return None def rollout_path_in(joined, cwd): for hay in (joined, cwd): if "/runs/" in hay and "/rollouts/" in hay: seg = hay.split("/runs/", 1)[1].split("/rollouts/")[0].split("/") return f"{seg[0]}/{seg[1]}" return "?" def db_rollouts(): if not DB.exists(): return [] db = sqlite3.connect(f"file:{DB}?mode=ro", uri=True) db.row_factory = sqlite3.Row rows = db.execute( "SELECT run_id, job_id, kk, state, started_at FROM rollouts WHERE state='RUNNING'" ).fetchall() db.close() return [dict(r) for r in rows] def per_run_todo(): db = sqlite3.connect(f"file:{DB}?mode=ro", uri=True) db.row_factory = sqlite3.Row rows = db.execute( "SELECT run_id, state, COUNT(*) n FROM rollouts WHERE state IN ('QUEUED','RUNNING','PASS','FAIL','UNVERIFIED','TIMEOUT','BLOCKED') GROUP BY run_id, state" ).fetchall() db.close() agg = {} for r in rows: agg.setdefault(r["run_id"], {})[r["state"]] = r["n"] return agg def fmt_age(s): s = int(s) if s < 3600: return f"{s // 60}m" if s < 86400: return f"{s / 3600:.1f}h" return f"{s / 86400:.1f}d" def main(): ap = argparse.ArgumentParser() ap.add_argument("--json", action="store_true") args = ap.parse_args() collectors, pis, heavies = scan_procs() running = db_rollouts() now = time.time() live_keys = {(p["run"], p["job"], p["kk"]) for p in pis} orphans, live_rows = [], [] for r in running: key = (r["run_id"], r["job_id"], int(r["kk"] or 0)) age_s = now - (r["started_at"] or 0) (live_rows if key in live_keys else orphans).append({**r, "age_s": age_s}) if args.json: print(json.dumps({"collectors": collectors, "pi_live": pis, "orphans": orphans, "heavies": heavies}, ensure_ascii=False, indent=2, default=str)) return 0 print(f"fleet · {time.strftime('%Y-%m-%d %H:%M:%S')} · 只读快照") print(f"\n■ collectors ({len(collectors)})") for c in sorted(collectors, key=lambda x: x["run"]): flag = " [drain 中]" if c["drain"] else "" print(f" {c['pid']:>8} {fmt_age(c['age_s']):>6} {c['kind']:<8} {c['run']:<20} --parallel {c['parallel']}{flag}") print(f"\n■ rollouts RUNNING: {len(running)} 条 = 活 pi {len(live_rows)} + 孤儿 {len(orphans)}") if live_rows: print(" 活的:") for r in sorted(live_rows, key=lambda x: x["age_s"]): print(f" {r['run_id']:<18} {r['job_id'][:56]:<58} kk={r['kk']} {fmt_age(r['age_s'])}") if orphans: print(" 孤儿(pi 已死,行还占 agent 槽;涉及 run 的 collector 在收尾时会核验,否则需人工回收):") for r in sorted(orphans, key=lambda x: -x["age_s"]): over = " ⚠超12h硬顶" if r["age_s"] > 43200 else "" kids = [h for h in heavies if h["path"] == f"{r['run_id']}/{r['job_id']}"] kid = f",stage 子进程 {len(kids)} 个还在烧" if kids else "" print(f" {r['run_id']:<18} {r['job_id'][:56]:<58} kk={r['kk']} {fmt_age(r['age_s'])}{over}{kid}") big = sorted(heavies, key=lambda x: -x["cpu_s"])[:8] if big: print(f"\n■ 重负载 stage 子进程(top {len(big)},按累计 CPU)") for h in big: print(f" {h['pid']:>8} {fmt_age(h['age_s']):>6} cpu={h['cpu_s'] / 3600:5.1f}h {h['argv0']:<22} {h['path']}") todo = per_run_todo() active = {run: s for run, s in todo.items() if s.get("QUEUED") or s.get("RUNNING")} if active: print(f"\n■ 各 run 待办(QUEUED / RUNNING)") for run in sorted(active): s = active[run] print(f" {run:<20} QUEUED={s.get('QUEUED', 0):<5} RUNNING={s.get('RUNNING', 0)}") done_states = ["PASS", "FAIL", "UNVERIFIED", "TIMEOUT", "BLOCKED"] finished = {run: sum(s.get(k, 0) for k in done_states) for run, s in todo.items()} recent = sorted(finished.items(), key=lambda kv: -kv[1])[:5] if recent: print(f"\n■ 已结束 rollout 最多的 run:") for run, n in recent: print(f" {run:<20} {n}") return 0 if __name__ == "__main__": sys.exit(main())