433 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""`bench` 命令。
bench preflight [--modes oracle,e2e] [--target s100p]
bench new-run [--note ...] [--target s100p]
bench expand [--run r…] [--modes …]
bench run [--run r…] [--job id …] [--dry-run] [--parallel N]
bench status [--run r…] [--fail]
bench show <job_id> [--run r…]
bench annotate <job_id> --note "…"
bench report [--run r…]
bench verify <rollout_dir>
bench stage run <stage> [--config path] (rollout 内,agent 用)
bench stage wait <handle> [--poll 60]
bench stage status <handle>
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from . import paths, db
def _run_id(con, given: str | None) -> str:
rid = given or db.latest_run_id(con)
if not rid:
raise SystemExit("没有 run:先 bench new-run")
return rid
def cmd_preflight(a) -> int:
from .pipeline import preflight
errs, warns = preflight(a.modes.split(",") if a.modes else None,
oracle_only=a.oracle_only, target=a.target)
for w in warns:
print(f"WARN {w}")
for e in errs:
print(f"ERROR {e}")
print(f"preflight: {len(errs)} errors, {len(warns)} warnings")
return 1 if errs else 0
def cmd_new_run(a) -> int:
from .pipeline import new_run
print(new_run(a.note or "", target=a.target))
return 0
def cmd_expand(a) -> int:
from .pipeline import expand
with db.connect() as con:
rid = _run_id(con, a.run)
print(json.dumps({"run_id": rid, **expand(rid, a.modes.split(",") if a.modes else None, a.marches.split(",") if a.marches else None,
model_ids=a.model or None, eval_manifests=a.t1 or None)}))
return 0
def cmd_run(a) -> int:
from .pipeline import run_oracle
with db.connect() as con:
rid = _run_id(con, a.run)
s = run_oracle(rid, a.job or None, dry_run=a.dry_run, parallel=a.parallel, prepare_only=not a.oracle)
print(json.dumps({"run_id": rid, **s}))
return 0
def cmd_status(a) -> int:
with db.connect() as con:
rid = _run_id(con, a.run)
run = db.get_run(con, rid)
target = json.loads(run["manifest"]).get("target") if run else None
rows = db.status_counts(con, rid)
target_text = f" target={target['board_type']}/{target['march']}" if target else " target=legacy/mixed"
print(f"run {rid}{target_text}")
if a.fail:
for r in db.error_class_counts(con, rid):
print(f" {r['n']:4d} {r['error_class']:<20} {r['stage_failed'] or ''}")
for j in db.list_jobs(con, rid, state="FAIL"):
print(f" FAIL {j['job_id']:<50} {j['error_class'] or '':<18} {j['stage_failed'] or ''}")
return 0
agg: dict = {}
for r in rows:
agg.setdefault(r["state"], {}).setdefault(r["task"], {})[r["march"]] = r["n"]
for st, tasks in agg.items():
total = sum(sum(m.values()) for m in tasks.values())
print(f" {st:<15} {total:5d}")
for t, m in tasks.items():
print(f" {t:<8} " + " ".join(f"{k}={v}" for k, v in m.items()))
ro = con.execute(
"SELECT j.march, r.state, COUNT(*) n FROM rollouts r"
" JOIN jobs j ON j.run_id=r.run_id AND j.job_id=r.job_id AND j.mode='oracle'"
" WHERE r.run_id=? GROUP BY j.march,r.state ORDER BY j.march,r.state", (rid,)).fetchall()
if ro:
by_march = {}
for r in ro:
by_march.setdefault(r["march"], []).append(f"{r['state']}={r['n']}")
for march, counts in by_march.items():
print(f" rollouts {march}: " + " ".join(counts))
from .verify import summarize_results
for label, records in (("pipeline", con.execute("SELECT result FROM jobs WHERE run_id=? AND mode='oracle'", (rid,))),
("agent", con.execute("SELECT verdict FROM rollouts WHERE run_id=?", (rid,)))):
counts = summarize_results(r[0] for r in records)
print(f" {label} 实际完成: " + " ".join(f"{k}={v}" for k, v in counts.items()))
manifest = paths.DATASETS / f"{rid}-training" / "manifest.json"
if manifest.exists():
m = json.loads(manifest.read_text())
print(f" dataset: {manifest.parent} stats={json.dumps(m.get('stats', {}), ensure_ascii=False)}")
# 导出没被触发是静默的(collector 被 database is locked 打死过,manifest 就停在那一刻)。
# 拿 manifest 里记的可导出条数和现在的比一比,落后就提示补导。复用上面的 ro,零新增查询。
skip = {"RUNNING", "QUEUED", "VERIFYING", "TRANSFERRED"} # 与 to_agentrl.scan() 的口径一致
now_n = sum(r["n"] for r in ro if r["state"] not in skip)
was = (m.get("coverage", {}).get(rid) or {}).get("agent") or {}
was_n = sum(n for s, n in was.items() if s not in skip)
if now_n > was_n:
# collector / export --follow 活着时导出已经在排队了(pipeline.py 的 export_dirty),
# 这时的落后是正常在途,不该催人补导——否则 --parallel 12 下会常态刷屏。
from .pipeline import scheduler_holders
busy = bool(scheduler_holders(rid))
tail = "(导出在途)" if busy else f":bench export --run {rid}"
print(f" {'' if busy else '⚠ '}dataset 落后 {now_n - was_n} 条(manifest {was_n} / 现在 {now_n}){tail}")
from .scheduler import pool_usage, pool_size
from .registry import status_counts
print(" pools: " + " ".join(f"{p}={n}/{pool_size(p)}" for p, n in pool_usage().items()))
print(" registry: " + " ".join(f"{k}={v}" for k, v in status_counts().items()) + " (原始标签;显式批次进度见 coverage --all-runs)")
return 0
def cmd_show(a) -> int:
with db.connect() as con:
rid = _run_id(con, a.run)
j = db.get_job(con, rid, a.job_id)
if not j:
raise SystemExit(f"{rid} 里没有 {a.job_id}")
print(f"{a.job_id} [{rid}] state={j['state']} ec={j['error_class']} stage={j['stage_failed']} est={j['estimate_s']}")
st = con.execute("SELECT stage, attempt, lane, state, exit_code, error_class, duration_s FROM stages"
" WHERE run_id=? AND job_id=? AND rollout=-1 ORDER BY stage, attempt", (rid, a.job_id)).fetchall()
for s in st:
print(f" {s['stage']:<14} a{s['attempt']:02d} {s['lane']:<6} {s['state']:<8} exit={s['exit_code']!s:<4} {s['error_class'] or '':<18} {s['duration_s'] or 0:8.1f}s")
jd = paths.RUNS / rid / a.job_id
if j["stage_failed"]:
for p in sorted((jd / "oracle" / "stages" / j["stage_failed"]).glob("a*/stderr.log")):
print(f"--- {p} (tail 50) ---")
print("\n".join(p.read_text(errors="replace").splitlines()[-50:]))
rs = db.list_rollouts(con, rid, a.job_id)
if rs:
print("rollouts:")
for r in rs:
print(f" {r['kk']:02d} {r['mode']:<6} {r['policy'] or '':<8} {r['state']:<10} claim={r['claim_terminal']} consistent={r['claim_consistent']} review={r['review']} turns={r['turns']} tokens={r['tokens']}")
notes = con.execute("SELECT ts, note FROM annotations WHERE run_id=? AND job_id=? ORDER BY ts", (rid, a.job_id)).fetchall()
for n in notes:
print(f" note: {n['note']}")
return 0
def cmd_annotate(a) -> int:
with db.connect() as con:
rid = _run_id(con, a.run)
db.annotate(con, rid, a.job_id, a.note)
print("ok")
return 0
def cmd_report(a) -> int:
from .report import render
with db.connect() as con:
rid = _run_id(con, a.run)
print(render(rid))
return 0
def cmd_coverage(a) -> int:
from .coverage import render, to_csv
if a.all_runs:
from .coverage import write_master
print(write_master(a.csv or str(paths.REPORTS / "model-coverage.csv")))
return 0
with db.connect() as con:
rid = _run_id(con, a.run)
if a.csv:
print(to_csv(rid, a.csv)); return 0
print(render(rid, a.task, a.missing, a.detail))
return 0
def cmd_drain(a) -> int:
from .pipeline import drain_file, collect_drain_file
with db.connect() as con:
rid = _run_id(con, a.run)
drain_file(rid).touch()
if a.all:
collect_drain_file(rid).touch()
print(f"drain 已置位:{rid},范围={'流水线和 Agent' if a.all else '流水线'}。等已领取的任务结束后再交接。")
print("注意:2026-09-15 之前启动的旧调度进程(代码在内存里)不认这个文件,只能等它跑完或 kill + reset-stale。")
return 0
def cmd_continue(a) -> int:
from . import continuation
try:
if a.op == "plan":
result = continuation.plan(a.run, a.t1, a.to)
elif a.op == "apply":
result = continuation.apply(json.loads(Path(a.plan).read_text()))
else:
result = continuation.check(a.run)
except (ValueError, OSError) as exc:
print(f"continue: {exc}", file=sys.stderr)
return 2
print(json.dumps(result, indent=2, ensure_ascii=False))
return 0
def cmd_heartbeat(a) -> int:
from .heartbeat import heartbeat
return heartbeat(a.run)
def cmd_watch(a) -> int:
from .watch import watch
with db.connect() as con:
rid = _run_id(con, a.run)
return watch(rid, a.interval, a.once)
def cmd_reset_stale(a) -> int:
from .pipeline import reset_stale
with db.connect() as con:
rid = _run_id(con, a.run)
print(json.dumps(reset_stale(rid, a.force)))
return 0
def cmd_board_backfill(a) -> int:
from .pipeline import board_backfill, board_backfill_rollout
with db.connect() as con:
rid = _run_id(con, a.run)
if a.rollout:
# --rollout JOB_ID[:KK],可重复;只动 rollout 树,不碰 oracle
res = []
for spec in a.rollout:
job_id, _, kk = spec.partition(":")
res.append(board_backfill_rollout(rid, job_id, int(kk or 0)))
print(json.dumps(res, ensure_ascii=False))
return 0
print(json.dumps(board_backfill(rid, a.job or None), ensure_ascii=False))
return 0
def cmd_rejudge(a) -> int:
from .pipeline import rejudge
with db.connect() as con:
rid = _run_id(con, a.run)
ids = [a.job_id] if a.job_id not in ("all", "--all") else [j["job_id"] for j in db.list_jobs(con, rid, mode="oracle") if j["state"] in ("PASS", "FAIL", "UNVERIFIED")]
from collections import Counter
c = Counter()
for jid in ids:
r = rejudge(rid, jid)
c[(r["terminal"], r.get("failed_gate"))] += 1
if a.job_id not in ("all", "--all"):
print(json.dumps(r, ensure_ascii=False))
if a.job_id in ("all", "--all"):
print(json.dumps({"rejudged": len(ids), "by_terminal_gate": {f"{k[0]}/{k[1]}": v for k, v in c.items()}}, ensure_ascii=False))
return 0
def cmd_verify(a) -> int:
from .verify import verify_rollout
v = verify_rollout(Path(a.rollout_dir))
print(json.dumps({k: v[k] for k in ("job_id", "rollout", "terminal", "failed_gate", "claim_consistent", "review")}))
return 0
def cmd_collect(a) -> int:
from .pipeline import collect_rollouts
with db.connect() as con:
rid = _run_id(con, a.run)
s = collect_rollouts(rid, a.job or None, a.modes.split(",") if a.modes else None,
parallel=a.parallel, dry_run=a.dry_run, limit=a.limit,
coverage=a.coverage, watch=a.watch, first_pass=a.first_pass)
print(json.dumps({"run_id": rid, **s}, ensure_ascii=False))
return 0
def cmd_export(a) -> int:
"""给已经在跑的批次补上持续物化;新调度器每个任务完成时也会调用同一导出器。"""
import hashlib
import time
import fcntl
import os
from .pipeline import refresh_run_dataset
with db.readonly() as con:
rid = _run_id(con, a.run)
lock = None
if a.follow:
if a.interval < 1:
raise SystemExit("interval 必须 >= 1")
(paths.STATE / "locks").mkdir(parents=True, exist_ok=True)
lock = open(paths.STATE / "locks" / f"export-follow-{rid}.lock", "a")
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise SystemExit(f"{rid} 已有 export --follow 进程")
previous = None
while True:
with db.readonly() as con:
jobs = con.execute("SELECT job_id,state,finished_at,result FROM jobs WHERE run_id=? AND mode='oracle' ORDER BY job_id", (rid,)).fetchall()
rollouts = con.execute("SELECT job_id,kk,state,finished_at,verdict FROM rollouts WHERE run_id=? ORDER BY job_id,kk", (rid,)).fetchall()
signature = hashlib.sha256(json.dumps([list(map(tuple, jobs)), list(map(tuple, rollouts))]).encode()).hexdigest()
if signature != previous:
try:
# 导出器只读重核旧结果;历史 verdict 和数据库保持原样。
manifest = refresh_run_dataset(rid, strict=True)
except Exception as exc:
if not a.follow:
raise
print(f"dataset export failed: {type(exc).__name__}: {exc}; {a.interval}s 后重试", file=sys.stderr, flush=True)
time.sleep(a.interval)
continue
print(json.dumps({"run_id": rid, "dataset": str(paths.DATASETS / f"{rid}-training"),
"stats": manifest["stats"]}, ensure_ascii=False), flush=True)
previous = signature
if not a.follow:
return 0
# 队列暂空不代表在途 Agent 已结束。
if a.until_pid:
try:
os.kill(a.until_pid, 0)
except ProcessLookupError:
refresh_run_dataset(rid, strict=True)
return 0
time.sleep(a.interval)
def cmd_rollout(a) -> int:
from .rollout import run_rollout
with db.connect() as con:
rid = _run_id(con, a.run)
print(json.dumps(run_rollout(rid, a.job_id, a.kk, a.mode, a.policy, dry_run=a.dry_run), ensure_ascii=False))
if not a.dry_run:
from .pipeline import refresh_run_dataset
refresh_run_dataset(rid)
return 0
def cmd_agent(a) -> int:
from .rollout import setup_models_json
if a.op == "setup":
print(setup_models_json(force=a.force))
return 0
raise SystemExit("agent op: setup")
def cmd_stage(a) -> int:
from . import stagerun
if a.op == "run":
stagerun.submit(a.stage, a.config)
return 0
if a.op == "wait":
return stagerun.wait(a.stage, a.poll)
if a.op == "status":
return stagerun.status(a.stage)
raise SystemExit("stage op: run | wait | status")
def main(argv=None) -> int:
p = argparse.ArgumentParser(prog="bench")
sp = p.add_subparsers(dest="cmd", required=True)
x = sp.add_parser("preflight"); x.add_argument("--modes"); x.add_argument("--target", help="板型目标,如 s100p / s600")
x.add_argument("--oracle-only", action="store_true", help="只跑 benchmark 那一半时放行 agent 运行时缺失"); x.set_defaults(f=cmd_preflight)
x = sp.add_parser("new-run"); x.add_argument("--note")
x.add_argument("--target", required=True, help="固定本 run 的唯一板型目标")
x.set_defaults(f=cmd_new_run)
x = sp.add_parser("expand"); x.add_argument("--run"); x.add_argument("--modes"); x.add_argument("--marches", help="只展开这些 march,逗号分隔")
x.add_argument("--model", action="append", help="显式分批接入;核查真实前置,保留原 march/backend/policy/k,不改 CSV 标签")
x.add_argument("--t1", action="append", help="新批次直接使用已验证的 T1 manifest;每个数据集一份,不改旧任务")
x.set_defaults(f=cmd_expand)
x = sp.add_parser("run"); x.add_argument("--run"); x.add_argument("--job", action="append")
x.add_argument("--oracle", action="store_true", help="显式运行完整01–08对照;默认只准备01–03输入")
x.add_argument("--dry-run", action="store_true"); x.add_argument("--parallel", type=int); x.set_defaults(f=cmd_run)
x = sp.add_parser("status"); x.add_argument("--run"); x.add_argument("--fail", action="store_true"); x.set_defaults(f=cmd_status)
x = sp.add_parser("show"); x.add_argument("job_id"); x.add_argument("--run"); x.set_defaults(f=cmd_show)
x = sp.add_parser("annotate"); x.add_argument("job_id"); x.add_argument("--note", required=True); x.add_argument("--run"); x.set_defaults(f=cmd_annotate)
x = sp.add_parser("report"); x.add_argument("--run"); x.set_defaults(f=cmd_report)
x = sp.add_parser("verify"); x.add_argument("rollout_dir"); x.set_defaults(f=cmd_verify)
x = sp.add_parser("rejudge"); x.add_argument("job_id", help="job_id 或 all(该 run 所有已结束的 job)"); x.add_argument("--run"); x.set_defaults(f=cmd_rejudge)
x = sp.add_parser("board-backfill", help="用现成 HBM 补跑 07/08:默认补 oracle job;--rollout JOB_ID[:KK] 补已结束的 rollout 并重新 verify")
x.add_argument("--run"); x.add_argument("--job", action="append")
x.add_argument("--rollout", action="append", metavar="JOB_ID[:KK]", help="补测 rollout(可重复),板卡故障导致 UNVERIFIED 的轨迹用这个回收")
x.set_defaults(f=cmd_board_backfill)
x = sp.add_parser("coverage"); x.add_argument("--run"); x.add_argument("--task"); x.add_argument("--missing", action="store_true")
x.add_argument("--all-runs", action="store_true", help="338 模型总表及逐任务/阶段/真实轨迹 JSON")
x.add_argument("--detail", action="store_true"); x.add_argument("--csv"); x.set_defaults(f=cmd_coverage)
x = sp.add_parser("heartbeat"); x.add_argument("--run", required=True); x.set_defaults(f=cmd_heartbeat)
x = sp.add_parser("watch"); x.add_argument("--run"); x.add_argument("--interval", type=int, default=30); x.add_argument("--once", action="store_true"); x.set_defaults(f=cmd_watch)
# --run 必填:默认取最新 run,而最新 run 往往正是有活 rollout 的那个,误打代价太大
x = sp.add_parser("reset-stale"); x.add_argument("--run", required=True); x.add_argument("--force", action="store_true"); x.set_defaults(f=cmd_reset_stale)
x = sp.add_parser("drain", help="让 oracle 调度进程跑完手里的 job 后不再认领新 job(换代码/配置前用,不用 kill)")
x.add_argument("--run"); x.add_argument("--all", action="store_true", help="同时停止 Agent 领取"); x.set_defaults(f=cmd_drain)
x = sp.add_parser("continue", help="预览、应用和核对剩余任务交接;不全量展开")
cs = x.add_subparsers(dest="op", required=True)
y = cs.add_parser("plan"); y.add_argument("--run", required=True); y.add_argument("--to")
y.add_argument("--t1", action="append", required=True, help="新 T1 的 manifest.json,每个数据集一份"); y.set_defaults(f=cmd_continue)
y = cs.add_parser("apply"); y.add_argument("plan", help="已核对的 JSON 清单"); y.set_defaults(f=cmd_continue)
y = cs.add_parser("check"); y.add_argument("--run", required=True); y.set_defaults(f=cmd_continue)
x = sp.add_parser("collect"); x.add_argument("--run"); x.add_argument("--job", action="append")
x.add_argument("--modes", help="e2e,repair"); x.add_argument("--parallel", type=int)
x.add_argument("--limit", type=int, help="先采 N 条试水"); x.add_argument("--dry-run", action="store_true")
x.add_argument("--coverage", choices=["matrix", "all"], default="matrix",
help="all 为本 run 的 oracle 模型补齐 e2e 计划,只放开 priority 限制")
x.add_argument("--watch", action="store_true", help="等待输入准备与在途任务,持续采集")
x.add_argument("--first-pass", action="store_true", help="每个 job 只领取首条,保留后续采样编号和原 k")
x.set_defaults(f=cmd_collect)
x = sp.add_parser("export", help="导出本 run 已完成的流程与 Agent 数据;--follow 跟随当前批次更新同一目录")
x.add_argument("--run"); x.add_argument("--follow", action="store_true")
x.add_argument("--until-pid", type=int, help="跟随到指定批次进程退出;避免队列暂空时提前结束")
x.add_argument("--interval", type=int, default=30); x.set_defaults(f=cmd_export)
x = sp.add_parser("rollout"); x.add_argument("job_id"); x.add_argument("--kk", type=int, default=0)
x.add_argument("--mode", choices=["e2e", "repair"], default="e2e"); x.add_argument("--policy", choices=["teacher", "student"], default="teacher")
x.add_argument("--run"); x.add_argument("--dry-run", action="store_true"); x.set_defaults(f=cmd_rollout)
x = sp.add_parser("agent"); x.add_argument("op", choices=["setup"]); x.add_argument("--force", action="store_true"); x.set_defaults(f=cmd_agent)
x = sp.add_parser("stage"); x.add_argument("op", choices=["run", "wait", "status"]); x.add_argument("stage")
x.add_argument("--config"); x.add_argument("--poll", type=int, default=60); x.set_defaults(f=cmd_stage)
a = p.parse_args(argv)
return a.f(a)
if __name__ == "__main__":
sys.exit(main())