218 lines
8.3 KiB
Python
218 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
||
"""fleet — 全机进程清单:collector、活 rollout、孤儿、重负载 stage 子进程。
|
||
|
||
只读:扫 /proc + 只读打开 state/jobs.sqlite,不写任何状态。
|
||
背景:调度进程是各终端 setsid nohup 起的,bench status/watch/heartbeat 都是 per-run,
|
||
没有"这台机器现在跑着什么"的全局视图——本脚本补这个视图。
|
||
|
||
用法:python3 scripts/fleet.py [--json]
|
||
"""
|
||
import argparse
|
||
import json
|
||
import os
|
||
import pathlib
|
||
import sqlite3
|
||
import sys
|
||
import time
|
||
|
||
BENCH_ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||
DB = BENCH_ROOT / "state" / "jobs.sqlite"
|
||
RUNS_DIR = BENCH_ROOT / "runs"
|
||
HZ = os.sysconf("SC_CLK_TCK")
|
||
|
||
|
||
def boot_time():
|
||
with open("/proc/stat") as f:
|
||
for line in f:
|
||
if line.startswith("btime"):
|
||
return float(line.split()[1])
|
||
return 0.0
|
||
|
||
|
||
def proc_stats(pid):
|
||
"""返回 (etime_s, cpu_s, cmdline, cwd);进程消失返回 None。"""
|
||
try:
|
||
with open(f"/proc/{pid}/stat") as f:
|
||
raw = f.read()
|
||
# comm 字段可能含空格,从最后一个 ')' 后切
|
||
fields = raw[raw.rindex(")") + 2:].split()
|
||
starttime = float(fields[19]) / HZ
|
||
utime, stime = float(fields[11]), float(fields[12])
|
||
etime = time.time() - boot_time() - starttime
|
||
with open(f"/proc/{pid}/cmdline", "rb") as f:
|
||
cmdline = f.read().decode(errors="replace").split("\0")
|
||
cmdline = [c for c in cmdline if c]
|
||
try:
|
||
cwd = os.readlink(f"/proc/{pid}/cwd")
|
||
except (PermissionError, FileNotFoundError, ProcessLookupError):
|
||
cwd = "" # unshare jail / zombie 下同 UID 也可能拒读
|
||
return etime, utime + stime, cmdline, cwd
|
||
except (FileNotFoundError, ProcessLookupError, ValueError, IndexError, PermissionError):
|
||
return None
|
||
|
||
|
||
def scan_procs():
|
||
"""按类别返回本用户的进程:collectors / pi / stage 子进程。"""
|
||
me = os.getuid()
|
||
collectors, pis, heavies = [], [], []
|
||
for entry in pathlib.Path("/proc").iterdir():
|
||
if not entry.name.isdigit():
|
||
continue
|
||
pid = int(entry.name)
|
||
try:
|
||
if os.stat(f"/proc/{pid}").st_uid != me:
|
||
continue
|
||
except FileNotFoundError:
|
||
continue
|
||
st = proc_stats(pid)
|
||
if st is None or not st[2]:
|
||
continue
|
||
etime, cpu_s, cmdline, cwd = st
|
||
argv0 = os.path.basename(cmdline[0])
|
||
joined = " ".join(cmdline)
|
||
if argv0 == "pi" or (argv0 == "node" and cmdline[1:2] and cmdline[1].endswith("/pi")):
|
||
m = parse_rollout_cwd(cwd)
|
||
if m:
|
||
pis.append({"pid": pid, "age_s": etime, **m})
|
||
elif "harness.cli" in joined:
|
||
run = arg_value(cmdline, "--run")
|
||
kind = next((c for c in ("collect", "run", "export", "heartbeat") if f"-m harness.cli {c}" in joined or f"harness.cli {c}" in joined), "?")
|
||
collectors.append({
|
||
"pid": pid, "age_s": etime, "cpu_s": cpu_s, "kind": kind,
|
||
"run": run or "?", "parallel": arg_value(cmdline, "--parallel") or "1",
|
||
"drain": bool((BENCH_ROOT / "state" / f"drain-collect-{run}").exists()) if run else False,
|
||
})
|
||
elif any(k in argv0 for k in ("infer_hbm", "qemu-system-riscv64", "hbcm-module-entry", "hbrt4-run-model", "eval.run")) or "-m harness.eval.run" in joined:
|
||
heavies.append({"pid": pid, "age_s": etime, "cpu_s": cpu_s, "argv0": argv0, "path": rollout_path_in(joined, cwd)})
|
||
return collectors, pis, heavies
|
||
|
||
|
||
def arg_value(cmdline, flag):
|
||
for i, a in enumerate(cmdline):
|
||
if a == flag and i + 1 < len(cmdline):
|
||
return cmdline[i + 1]
|
||
return None
|
||
|
||
|
||
def parse_rollout_cwd(cwd):
|
||
""".../runs/<run>/<job>/rollouts/<nn>/workspace → dict(run, job, kk)"""
|
||
p = str(cwd)
|
||
if "/rollouts/" not in p or not p.rstrip("/").endswith("/workspace"):
|
||
return None
|
||
try:
|
||
head = p.split("/runs/", 1)[1].rstrip("/").split("/")
|
||
run, job, _, kk = head[0], head[1], head[2], head[3]
|
||
return {"run": run, "job": job, "kk": int(kk)}
|
||
except (IndexError, ValueError):
|
||
return None
|
||
|
||
|
||
def rollout_path_in(joined, cwd):
|
||
for hay in (joined, cwd):
|
||
if "/runs/" in hay and "/rollouts/" in hay:
|
||
seg = hay.split("/runs/", 1)[1].split("/rollouts/")[0].split("/")
|
||
return f"{seg[0]}/{seg[1]}"
|
||
return "?"
|
||
|
||
|
||
def db_rollouts():
|
||
if not DB.exists():
|
||
return []
|
||
db = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
|
||
db.row_factory = sqlite3.Row
|
||
rows = db.execute(
|
||
"SELECT run_id, job_id, kk, state, started_at FROM rollouts WHERE state='RUNNING'"
|
||
).fetchall()
|
||
db.close()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def per_run_todo():
|
||
db = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
|
||
db.row_factory = sqlite3.Row
|
||
rows = db.execute(
|
||
"SELECT run_id, state, COUNT(*) n FROM rollouts WHERE state IN ('QUEUED','RUNNING','PASS','FAIL','UNVERIFIED','TIMEOUT','BLOCKED') GROUP BY run_id, state"
|
||
).fetchall()
|
||
db.close()
|
||
agg = {}
|
||
for r in rows:
|
||
agg.setdefault(r["run_id"], {})[r["state"]] = r["n"]
|
||
return agg
|
||
|
||
|
||
def fmt_age(s):
|
||
s = int(s)
|
||
if s < 3600:
|
||
return f"{s // 60}m"
|
||
if s < 86400:
|
||
return f"{s / 3600:.1f}h"
|
||
return f"{s / 86400:.1f}d"
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--json", action="store_true")
|
||
args = ap.parse_args()
|
||
|
||
collectors, pis, heavies = scan_procs()
|
||
running = db_rollouts()
|
||
now = time.time()
|
||
|
||
live_keys = {(p["run"], p["job"], p["kk"]) for p in pis}
|
||
orphans, live_rows = [], []
|
||
for r in running:
|
||
key = (r["run_id"], r["job_id"], int(r["kk"] or 0))
|
||
age_s = now - (r["started_at"] or 0)
|
||
(live_rows if key in live_keys else orphans).append({**r, "age_s": age_s})
|
||
|
||
if args.json:
|
||
print(json.dumps({"collectors": collectors, "pi_live": pis,
|
||
"orphans": orphans, "heavies": heavies}, ensure_ascii=False, indent=2, default=str))
|
||
return 0
|
||
|
||
print(f"fleet · {time.strftime('%Y-%m-%d %H:%M:%S')} · 只读快照")
|
||
print(f"\n■ collectors ({len(collectors)})")
|
||
for c in sorted(collectors, key=lambda x: x["run"]):
|
||
flag = " [drain 中]" if c["drain"] else ""
|
||
print(f" {c['pid']:>8} {fmt_age(c['age_s']):>6} {c['kind']:<8} {c['run']:<20} --parallel {c['parallel']}{flag}")
|
||
|
||
print(f"\n■ rollouts RUNNING: {len(running)} 条 = 活 pi {len(live_rows)} + 孤儿 {len(orphans)}")
|
||
if live_rows:
|
||
print(" 活的:")
|
||
for r in sorted(live_rows, key=lambda x: x["age_s"]):
|
||
print(f" {r['run_id']:<18} {r['job_id'][:56]:<58} kk={r['kk']} {fmt_age(r['age_s'])}")
|
||
if orphans:
|
||
print(" 孤儿(pi 已死,行还占 agent 槽;涉及 run 的 collector 在收尾时会核验,否则需人工回收):")
|
||
for r in sorted(orphans, key=lambda x: -x["age_s"]):
|
||
over = " ⚠超12h硬顶" if r["age_s"] > 43200 else ""
|
||
kids = [h for h in heavies if h["path"] == f"{r['run_id']}/{r['job_id']}"]
|
||
kid = f",stage 子进程 {len(kids)} 个还在烧" if kids else ""
|
||
print(f" {r['run_id']:<18} {r['job_id'][:56]:<58} kk={r['kk']} {fmt_age(r['age_s'])}{over}{kid}")
|
||
|
||
big = sorted(heavies, key=lambda x: -x["cpu_s"])[:8]
|
||
if big:
|
||
print(f"\n■ 重负载 stage 子进程(top {len(big)},按累计 CPU)")
|
||
for h in big:
|
||
print(f" {h['pid']:>8} {fmt_age(h['age_s']):>6} cpu={h['cpu_s'] / 3600:5.1f}h {h['argv0']:<22} {h['path']}")
|
||
|
||
todo = per_run_todo()
|
||
active = {run: s for run, s in todo.items() if s.get("QUEUED") or s.get("RUNNING")}
|
||
if active:
|
||
print(f"\n■ 各 run 待办(QUEUED / RUNNING)")
|
||
for run in sorted(active):
|
||
s = active[run]
|
||
print(f" {run:<20} QUEUED={s.get('QUEUED', 0):<5} RUNNING={s.get('RUNNING', 0)}")
|
||
|
||
done_states = ["PASS", "FAIL", "UNVERIFIED", "TIMEOUT", "BLOCKED"]
|
||
finished = {run: sum(s.get(k, 0) for k in done_states) for run, s in todo.items()}
|
||
recent = sorted(finished.items(), key=lambda kv: -kv[1])[:5]
|
||
if recent:
|
||
print(f"\n■ 已结束 rollout 最多的 run:")
|
||
for run, n in recent:
|
||
print(f" {run:<20} {n}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|