124 lines
5.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""bench watch — 终端实时看板:在跑什么、跑了多久、日志多久没动、谁可能卡死、还剩多少。
bench watch [--run r…] [--interval 30] [--once]
"卡死"的判断口径与看门狗一致但更早预警:
日志 idle > 5 min → ⚠ 提示
日志 idle > stall_s × 2/3 → 🔥 看门狗即将介入(stall_s 默认 900 s,且要求 CPU 也空闲才真杀)
看门狗真正杀掉的会以 STALLED 出现在"最近完成"里,error_class=stalled。
"""
from __future__ import annotations
import os
import sys
import time
from pathlib import Path
from . import paths, db
from .scheduler import pool_usage, pool_size, estimate_stage
def _fmt(s: float | None) -> str:
if s is None:
return "-"
s = int(s)
return f"{s // 3600}h{(s % 3600) // 60:02d}m" if s >= 3600 else (f"{s // 60}m{s % 60:02d}s" if s >= 60 else f"{s}s")
def _log_age(run_id: str, job_id: str, rollout: int, stage: str, attempt: int) -> float | None:
jd = paths.RUNS / run_id / job_id
root = jd / "oracle" / "stages" if rollout < 0 else jd / "rollouts" / f"{rollout:02d}" / "stages"
sd = root / stage / f"a{attempt:02d}"
latest = 0.0
for p in (sd / "stdout.log", sd / "stderr.log"):
try:
latest = max(latest, p.stat().st_mtime)
except OSError:
pass
return (time.time() - latest) if latest else None
def snapshot(run_id: str) -> str:
cfg = paths.config()
stall_s = int(cfg["watchdog"]["stall_s"])
lines = []
with db.readonly() as con:
run = db.get_run(con, run_id)
jobs = db.list_jobs(con, run_id, mode="oracle")
counts = {}
for j in jobs:
counts[j["state"]] = counts.get(j["state"], 0) + 1
running = con.execute("SELECT * FROM stages WHERE run_id=? AND state='RUNNING' ORDER BY started_at", (run_id,)).fetchall()
done = con.execute("SELECT * FROM stages WHERE run_id=? AND state!='RUNNING' AND finished_at IS NOT NULL ORDER BY finished_at DESC LIMIT 8", (run_id,)).fetchall()
ec = db.error_class_counts(con, run_id)
from .verify import summarize_results
agent_counts = summarize_results(r[0] for r in con.execute("SELECT verdict FROM rollouts WHERE run_id=?", (run_id,)))
pipeline_counts = summarize_results(j.get("result") for j in jobs)
# 完成的 job 平均耗时 → ETA
durs = [ (j["finished_at"] - j["started_at"]) for j in jobs if j.get("finished_at") and j.get("started_at") and j["state"] in ("PASS", "FAIL", "UNVERIFIED")]
from .registry import status_counts
try:
sc = status_counts()
except Exception:
sc = {}
now = time.strftime("%Y-%m-%d %H:%M:%S")
lines.append(f"bench watch · run {run_id} · {now} · {run.get('note','') if run else ''}")
lines.append(" jobs: " + " ".join(f"{k}={v}" for k, v in sorted(counts.items())) + f" (registry: {', '.join(f'{k}={v}' for k, v in sc.items())})")
for label, summary in (("pipeline", pipeline_counts), ("agent", agent_counts)):
lines.append(f" {label} 实际完成: " + " ".join(f"{k}={v}" for k, v in summary.items()))
lines.append(" pools: " + " ".join(f"{p}={n}/{pool_size(p)}" for p, n in pool_usage().items()))
pending = counts.get("PENDING", 0)
if durs and pending:
avg = sum(durs) / len(durs)
par = max(1, pool_usage().get("cpu", 1) or cfg["pools"]["cpu"]["workers"])
lines.append(f" ETA: 剩 {pending} 个 job × 平均 {_fmt(avg)} / 并行 {par} ≈ {_fmt(pending * avg / par)}(已完成 {len(durs)} 个)")
lines.append("")
lines.append(f" RUNNING ({len(running)})")
lines.append(f" {'job_id':<48} {'stage':<13} {'att':<4} {'elapsed':>8} {'est':>7} {'log idle':>9} flag")
for s in running:
el = time.time() - (s["started_at"] or time.time())
age = _log_age(run_id, s["job_id"], s["rollout"], s["stage"], s["attempt"])
try:
with db.readonly() as con:
jrow = db.get_job(con, run_id, s["job_id"])
est = estimate_stage(con, jrow["model"], jrow["task"], s["stage"]) if jrow else None
except Exception:
est = None
flag = ""
if s["lane"] == "queued":
# rollout 提交的阶段还在等池槽位:不是卡死,别报看门狗
lines.append(f" {s['job_id']:<48} {s['stage']:<13} a{s['attempt']:02d}r{s['rollout']:02d} {_fmt(el):>8} {'-':>7} {'-':>9} ⏸ 排队等槽位")
continue
if age is not None and age > stall_s * 2 / 3:
flag = "🔥 看门狗即将介入"
elif age is not None and age > 300:
flag = "⚠ 日志 5 min 没动"
elif est and el > est * 3:
flag = "⏳ 超历史 p95×3(只告警)"
tag = f"r{s['rollout']:02d}" if s["rollout"] >= 0 else " "
lines.append(f" {s['job_id']:<48} {s['stage']:<13} a{s['attempt']:02d}{tag} {_fmt(el):>8} {_fmt(est):>7} {_fmt(age):>9} {flag}")
lines.append("")
lines.append(" 最近完成")
for s in done:
lines.append(f" {s['job_id']:<48} {s['stage']:<13} {s['state']:<8} {s['error_class'] or '':<16} {_fmt(s['duration_s']):>8}")
if ec:
lines.append("")
lines.append(" FAIL by error_class: " + " ".join(f"{r['error_class']}@{r['stage_failed']}={r['n']}" for r in ec))
return "\n".join(lines)
def watch(run_id: str, interval: int = 30, once: bool = False) -> int:
once = once or not sys.stdout.isatty()
while True:
text = snapshot(run_id)
if not once:
os.system("clear" if os.name != "nt" else "cls")
print(text)
if once:
return 0
print(f"\n (每 {interval}s 刷新 · Ctrl-C 退出 · bench show <job_id> 看细节)")
try:
time.sleep(interval)
except KeyboardInterrupt:
return 0