124 lines
5.8 KiB
Python
124 lines
5.8 KiB
Python
"""bench watch — 终端实时看板:在跑什么、跑了多久、日志多久没动、谁可能卡死、还剩多少。
|
||
|
||
bench watch [--run r…] [--interval 30] [--once]
|
||
|
||
"卡死"的判断口径与看门狗一致但更早预警:
|
||
日志 idle > 5 min → ⚠ 提示
|
||
日志 idle > stall_s × 2/3 → 🔥 看门狗即将介入(stall_s 默认 900 s,且要求 CPU 也空闲才真杀)
|
||
看门狗真正杀掉的会以 STALLED 出现在"最近完成"里,error_class=stalled。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
from . import paths, db
|
||
from .scheduler import pool_usage, pool_size, estimate_stage
|
||
|
||
|
||
def _fmt(s: float | None) -> str:
|
||
if s is None:
|
||
return "-"
|
||
s = int(s)
|
||
return f"{s // 3600}h{(s % 3600) // 60:02d}m" if s >= 3600 else (f"{s // 60}m{s % 60:02d}s" if s >= 60 else f"{s}s")
|
||
|
||
|
||
def _log_age(run_id: str, job_id: str, rollout: int, stage: str, attempt: int) -> float | None:
|
||
jd = paths.RUNS / run_id / job_id
|
||
root = jd / "oracle" / "stages" if rollout < 0 else jd / "rollouts" / f"{rollout:02d}" / "stages"
|
||
sd = root / stage / f"a{attempt:02d}"
|
||
latest = 0.0
|
||
for p in (sd / "stdout.log", sd / "stderr.log"):
|
||
try:
|
||
latest = max(latest, p.stat().st_mtime)
|
||
except OSError:
|
||
pass
|
||
return (time.time() - latest) if latest else None
|
||
|
||
|
||
def snapshot(run_id: str) -> str:
|
||
cfg = paths.config()
|
||
stall_s = int(cfg["watchdog"]["stall_s"])
|
||
lines = []
|
||
with db.readonly() as con:
|
||
run = db.get_run(con, run_id)
|
||
jobs = db.list_jobs(con, run_id, mode="oracle")
|
||
counts = {}
|
||
for j in jobs:
|
||
counts[j["state"]] = counts.get(j["state"], 0) + 1
|
||
running = con.execute("SELECT * FROM stages WHERE run_id=? AND state='RUNNING' ORDER BY started_at", (run_id,)).fetchall()
|
||
done = con.execute("SELECT * FROM stages WHERE run_id=? AND state!='RUNNING' AND finished_at IS NOT NULL ORDER BY finished_at DESC LIMIT 8", (run_id,)).fetchall()
|
||
ec = db.error_class_counts(con, run_id)
|
||
from .verify import summarize_results
|
||
agent_counts = summarize_results(r[0] for r in con.execute("SELECT verdict FROM rollouts WHERE run_id=?", (run_id,)))
|
||
pipeline_counts = summarize_results(j.get("result") for j in jobs)
|
||
# 完成的 job 平均耗时 → ETA
|
||
durs = [ (j["finished_at"] - j["started_at"]) for j in jobs if j.get("finished_at") and j.get("started_at") and j["state"] in ("PASS", "FAIL", "UNVERIFIED")]
|
||
from .registry import status_counts
|
||
try:
|
||
sc = status_counts()
|
||
except Exception:
|
||
sc = {}
|
||
now = time.strftime("%Y-%m-%d %H:%M:%S")
|
||
lines.append(f"bench watch · run {run_id} · {now} · {run.get('note','') if run else ''}")
|
||
lines.append(" jobs: " + " ".join(f"{k}={v}" for k, v in sorted(counts.items())) + f" (registry: {', '.join(f'{k}={v}' for k, v in sc.items())})")
|
||
for label, summary in (("pipeline", pipeline_counts), ("agent", agent_counts)):
|
||
lines.append(f" {label} 实际完成: " + " ".join(f"{k}={v}" for k, v in summary.items()))
|
||
lines.append(" pools: " + " ".join(f"{p}={n}/{pool_size(p)}" for p, n in pool_usage().items()))
|
||
pending = counts.get("PENDING", 0)
|
||
if durs and pending:
|
||
avg = sum(durs) / len(durs)
|
||
par = max(1, pool_usage().get("cpu", 1) or cfg["pools"]["cpu"]["workers"])
|
||
lines.append(f" ETA: 剩 {pending} 个 job × 平均 {_fmt(avg)} / 并行 {par} ≈ {_fmt(pending * avg / par)}(已完成 {len(durs)} 个)")
|
||
lines.append("")
|
||
lines.append(f" RUNNING ({len(running)})")
|
||
lines.append(f" {'job_id':<48} {'stage':<13} {'att':<4} {'elapsed':>8} {'est':>7} {'log idle':>9} flag")
|
||
for s in running:
|
||
el = time.time() - (s["started_at"] or time.time())
|
||
age = _log_age(run_id, s["job_id"], s["rollout"], s["stage"], s["attempt"])
|
||
try:
|
||
with db.readonly() as con:
|
||
jrow = db.get_job(con, run_id, s["job_id"])
|
||
est = estimate_stage(con, jrow["model"], jrow["task"], s["stage"]) if jrow else None
|
||
except Exception:
|
||
est = None
|
||
flag = ""
|
||
if s["lane"] == "queued":
|
||
# rollout 提交的阶段还在等池槽位:不是卡死,别报看门狗
|
||
lines.append(f" {s['job_id']:<48} {s['stage']:<13} a{s['attempt']:02d}r{s['rollout']:02d} {_fmt(el):>8} {'-':>7} {'-':>9} ⏸ 排队等槽位")
|
||
continue
|
||
if age is not None and age > stall_s * 2 / 3:
|
||
flag = "🔥 看门狗即将介入"
|
||
elif age is not None and age > 300:
|
||
flag = "⚠ 日志 5 min 没动"
|
||
elif est and el > est * 3:
|
||
flag = "⏳ 超历史 p95×3(只告警)"
|
||
tag = f"r{s['rollout']:02d}" if s["rollout"] >= 0 else " "
|
||
lines.append(f" {s['job_id']:<48} {s['stage']:<13} a{s['attempt']:02d}{tag} {_fmt(el):>8} {_fmt(est):>7} {_fmt(age):>9} {flag}")
|
||
lines.append("")
|
||
lines.append(" 最近完成")
|
||
for s in done:
|
||
lines.append(f" {s['job_id']:<48} {s['stage']:<13} {s['state']:<8} {s['error_class'] or '':<16} {_fmt(s['duration_s']):>8}")
|
||
if ec:
|
||
lines.append("")
|
||
lines.append(" FAIL by error_class: " + " ".join(f"{r['error_class']}@{r['stage_failed']}={r['n']}" for r in ec))
|
||
return "\n".join(lines)
|
||
|
||
|
||
def watch(run_id: str, interval: int = 30, once: bool = False) -> int:
|
||
once = once or not sys.stdout.isatty()
|
||
while True:
|
||
text = snapshot(run_id)
|
||
if not once:
|
||
os.system("clear" if os.name != "nt" else "cls")
|
||
print(text)
|
||
if once:
|
||
return 0
|
||
print(f"\n (每 {interval}s 刷新 · Ctrl-C 退出 · bench show <job_id> 看细节)")
|
||
try:
|
||
time.sleep(interval)
|
||
except KeyboardInterrupt:
|
||
return 0
|