218 lines
8.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""fleet — 全机进程清单:collector、活 rollout、孤儿、重负载 stage 子进程。
只读:扫 /proc + 只读打开 state/jobs.sqlite,不写任何状态。
背景:调度进程是各终端 setsid nohup 起的,bench status/watch/heartbeat 都是 per-run,
没有"这台机器现在跑着什么"的全局视图——本脚本补这个视图。
用法:python3 scripts/fleet.py [--json]
"""
import argparse
import json
import os
import pathlib
import sqlite3
import sys
import time
BENCH_ROOT = pathlib.Path(__file__).resolve().parent.parent
DB = BENCH_ROOT / "state" / "jobs.sqlite"
RUNS_DIR = BENCH_ROOT / "runs"
HZ = os.sysconf("SC_CLK_TCK")
def boot_time():
with open("/proc/stat") as f:
for line in f:
if line.startswith("btime"):
return float(line.split()[1])
return 0.0
def proc_stats(pid):
"""返回 (etime_s, cpu_s, cmdline, cwd);进程消失返回 None。"""
try:
with open(f"/proc/{pid}/stat") as f:
raw = f.read()
# comm 字段可能含空格,从最后一个 ')' 后切
fields = raw[raw.rindex(")") + 2:].split()
starttime = float(fields[19]) / HZ
utime, stime = float(fields[11]), float(fields[12])
etime = time.time() - boot_time() - starttime
with open(f"/proc/{pid}/cmdline", "rb") as f:
cmdline = f.read().decode(errors="replace").split("\0")
cmdline = [c for c in cmdline if c]
try:
cwd = os.readlink(f"/proc/{pid}/cwd")
except (PermissionError, FileNotFoundError, ProcessLookupError):
cwd = "" # unshare jail / zombie 下同 UID 也可能拒读
return etime, utime + stime, cmdline, cwd
except (FileNotFoundError, ProcessLookupError, ValueError, IndexError, PermissionError):
return None
def scan_procs():
"""按类别返回本用户的进程:collectors / pi / stage 子进程。"""
me = os.getuid()
collectors, pis, heavies = [], [], []
for entry in pathlib.Path("/proc").iterdir():
if not entry.name.isdigit():
continue
pid = int(entry.name)
try:
if os.stat(f"/proc/{pid}").st_uid != me:
continue
except FileNotFoundError:
continue
st = proc_stats(pid)
if st is None or not st[2]:
continue
etime, cpu_s, cmdline, cwd = st
argv0 = os.path.basename(cmdline[0])
joined = " ".join(cmdline)
if argv0 == "pi" or (argv0 == "node" and cmdline[1:2] and cmdline[1].endswith("/pi")):
m = parse_rollout_cwd(cwd)
if m:
pis.append({"pid": pid, "age_s": etime, **m})
elif "harness.cli" in joined:
run = arg_value(cmdline, "--run")
kind = next((c for c in ("collect", "run", "export", "heartbeat") if f"-m harness.cli {c}" in joined or f"harness.cli {c}" in joined), "?")
collectors.append({
"pid": pid, "age_s": etime, "cpu_s": cpu_s, "kind": kind,
"run": run or "?", "parallel": arg_value(cmdline, "--parallel") or "1",
"drain": bool((BENCH_ROOT / "state" / f"drain-collect-{run}").exists()) if run else False,
})
elif any(k in argv0 for k in ("infer_hbm", "qemu-system-riscv64", "hbcm-module-entry", "hbrt4-run-model", "eval.run")) or "-m harness.eval.run" in joined:
heavies.append({"pid": pid, "age_s": etime, "cpu_s": cpu_s, "argv0": argv0, "path": rollout_path_in(joined, cwd)})
return collectors, pis, heavies
def arg_value(cmdline, flag):
for i, a in enumerate(cmdline):
if a == flag and i + 1 < len(cmdline):
return cmdline[i + 1]
return None
def parse_rollout_cwd(cwd):
""".../runs/<run>/<job>/rollouts/<nn>/workspace → dict(run, job, kk)"""
p = str(cwd)
if "/rollouts/" not in p or not p.rstrip("/").endswith("/workspace"):
return None
try:
head = p.split("/runs/", 1)[1].rstrip("/").split("/")
run, job, _, kk = head[0], head[1], head[2], head[3]
return {"run": run, "job": job, "kk": int(kk)}
except (IndexError, ValueError):
return None
def rollout_path_in(joined, cwd):
for hay in (joined, cwd):
if "/runs/" in hay and "/rollouts/" in hay:
seg = hay.split("/runs/", 1)[1].split("/rollouts/")[0].split("/")
return f"{seg[0]}/{seg[1]}"
return "?"
def db_rollouts():
if not DB.exists():
return []
db = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
db.row_factory = sqlite3.Row
rows = db.execute(
"SELECT run_id, job_id, kk, state, started_at FROM rollouts WHERE state='RUNNING'"
).fetchall()
db.close()
return [dict(r) for r in rows]
def per_run_todo():
db = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
db.row_factory = sqlite3.Row
rows = db.execute(
"SELECT run_id, state, COUNT(*) n FROM rollouts WHERE state IN ('QUEUED','RUNNING','PASS','FAIL','UNVERIFIED','TIMEOUT','BLOCKED') GROUP BY run_id, state"
).fetchall()
db.close()
agg = {}
for r in rows:
agg.setdefault(r["run_id"], {})[r["state"]] = r["n"]
return agg
def fmt_age(s):
s = int(s)
if s < 3600:
return f"{s // 60}m"
if s < 86400:
return f"{s / 3600:.1f}h"
return f"{s / 86400:.1f}d"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--json", action="store_true")
args = ap.parse_args()
collectors, pis, heavies = scan_procs()
running = db_rollouts()
now = time.time()
live_keys = {(p["run"], p["job"], p["kk"]) for p in pis}
orphans, live_rows = [], []
for r in running:
key = (r["run_id"], r["job_id"], int(r["kk"] or 0))
age_s = now - (r["started_at"] or 0)
(live_rows if key in live_keys else orphans).append({**r, "age_s": age_s})
if args.json:
print(json.dumps({"collectors": collectors, "pi_live": pis,
"orphans": orphans, "heavies": heavies}, ensure_ascii=False, indent=2, default=str))
return 0
print(f"fleet · {time.strftime('%Y-%m-%d %H:%M:%S')} · 只读快照")
print(f"\n■ collectors ({len(collectors)})")
for c in sorted(collectors, key=lambda x: x["run"]):
flag = " [drain 中]" if c["drain"] else ""
print(f" {c['pid']:>8} {fmt_age(c['age_s']):>6} {c['kind']:<8} {c['run']:<20} --parallel {c['parallel']}{flag}")
print(f"\n■ rollouts RUNNING: {len(running)} 条 = 活 pi {len(live_rows)} + 孤儿 {len(orphans)}")
if live_rows:
print(" 活的:")
for r in sorted(live_rows, key=lambda x: x["age_s"]):
print(f" {r['run_id']:<18} {r['job_id'][:56]:<58} kk={r['kk']} {fmt_age(r['age_s'])}")
if orphans:
print(" 孤儿(pi 已死,行还占 agent 槽;涉及 run 的 collector 在收尾时会核验,否则需人工回收):")
for r in sorted(orphans, key=lambda x: -x["age_s"]):
over = " ⚠超12h硬顶" if r["age_s"] > 43200 else ""
kids = [h for h in heavies if h["path"] == f"{r['run_id']}/{r['job_id']}"]
kid = f",stage 子进程 {len(kids)} 个还在烧" if kids else ""
print(f" {r['run_id']:<18} {r['job_id'][:56]:<58} kk={r['kk']} {fmt_age(r['age_s'])}{over}{kid}")
big = sorted(heavies, key=lambda x: -x["cpu_s"])[:8]
if big:
print(f"\n■ 重负载 stage 子进程(top {len(big)},按累计 CPU)")
for h in big:
print(f" {h['pid']:>8} {fmt_age(h['age_s']):>6} cpu={h['cpu_s'] / 3600:5.1f}h {h['argv0']:<22} {h['path']}")
todo = per_run_todo()
active = {run: s for run, s in todo.items() if s.get("QUEUED") or s.get("RUNNING")}
if active:
print(f"\n■ 各 run 待办(QUEUED / RUNNING)")
for run in sorted(active):
s = active[run]
print(f" {run:<20} QUEUED={s.get('QUEUED', 0):<5} RUNNING={s.get('RUNNING', 0)}")
done_states = ["PASS", "FAIL", "UNVERIFIED", "TIMEOUT", "BLOCKED"]
finished = {run: sum(s.get(k, 0) for k in done_states) for run, s in todo.items()}
recent = sorted(finished.items(), key=lambda kv: -kv[1])[:5]
if recent:
print(f"\n■ 已结束 rollout 最多的 run:")
for run, n in recent:
print(f" {run:<20} {n}")
return 0
if __name__ == "__main__":
sys.exit(main())