Reuse execution, stage and review evidence to recover database registration and SFT publication without replaying completed work. Keep finished futures during SQLite contention, and hand durable results to startup recovery when draining. Release proven local leases independently of unresolved board isolation. Keep UNKNOWN review retries explicit and archived, publish completed receipts after transient export failures, and prevent a malformed sample from aborting the entire dataset. Scope routine reconciliation to newly finished attempts. Add failure-injection coverage and the production acceptance plan.
538 lines
27 KiB
Python
538 lines
27 KiB
Python
"""`bench` 命令。
|
||
|
||
bench preflight [--modes oracle,e2e] [--target s100p]
|
||
bench new-run [--note ...] [--target s100p]
|
||
bench expand [--run r…] [--modes …]
|
||
bench run [--run r…] [--job id …] [--dry-run] [--parallel N]
|
||
bench status [--run r…] [--fail]
|
||
bench show <job_id> [--run r…]
|
||
bench annotate <job_id> --note "…"
|
||
bench report [--run r…]
|
||
bench verify <rollout_dir>
|
||
bench stage run <stage> [--config path] (rollout 内,agent 用)
|
||
bench stage wait <handle> [--poll 60]
|
||
bench stage status <handle>
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
from . import paths, db
|
||
|
||
|
||
def _run_id(con, given: str | None) -> str:
|
||
rid = given or db.latest_run_id(con)
|
||
if not rid:
|
||
raise SystemExit("没有 run:先 bench new-run")
|
||
return rid
|
||
|
||
|
||
def cmd_preflight(a) -> int:
|
||
from .pipeline import preflight
|
||
errs, warns = preflight(a.modes.split(",") if a.modes else None,
|
||
oracle_only=a.oracle_only, target=a.target)
|
||
for w in warns:
|
||
print(f"WARN {w}")
|
||
for e in errs:
|
||
print(f"ERROR {e}")
|
||
print(f"preflight: {len(errs)} errors, {len(warns)} warnings")
|
||
return 1 if errs else 0
|
||
|
||
|
||
def cmd_new_run(a) -> int:
|
||
from .pipeline import new_run
|
||
print(new_run(a.note or "", target=a.target))
|
||
return 0
|
||
|
||
|
||
def cmd_expand(a) -> int:
|
||
from .pipeline import expand
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
print(json.dumps({"run_id": rid, **expand(rid, a.modes.split(",") if a.modes else None, a.marches.split(",") if a.marches else None,
|
||
model_ids=a.model or None, eval_manifests=a.t1 or None)}))
|
||
return 0
|
||
|
||
|
||
def cmd_run(a) -> int:
|
||
from .pipeline import run_oracle
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
s = run_oracle(rid, a.job or None, dry_run=a.dry_run, parallel=a.parallel, prepare_only=not a.oracle)
|
||
print(json.dumps({"run_id": rid, **s}))
|
||
return 0
|
||
|
||
|
||
def cmd_status(a) -> int:
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
run = db.get_run(con, rid)
|
||
target = json.loads(run["manifest"]).get("target") if run else None
|
||
rows = db.status_counts(con, rid)
|
||
target_text = f" target={target['board_type']}/{target['march']}" if target else " target=legacy/mixed"
|
||
print(f"run {rid}{target_text}")
|
||
if a.fail:
|
||
for r in db.error_class_counts(con, rid):
|
||
print(f" {r['n']:4d} {r['error_class']:<20} {r['stage_failed'] or ''}")
|
||
for j in db.list_jobs(con, rid, state="FAIL"):
|
||
print(f" FAIL {j['job_id']:<50} {j['error_class'] or '':<18} {j['stage_failed'] or ''}")
|
||
return 0
|
||
agg: dict = {}
|
||
for r in rows:
|
||
agg.setdefault(r["state"], {}).setdefault(r["task"], {})[r["march"]] = r["n"]
|
||
for st, tasks in agg.items():
|
||
total = sum(sum(m.values()) for m in tasks.values())
|
||
print(f" {st:<15} {total:5d}")
|
||
for t, m in tasks.items():
|
||
print(f" {t:<8} " + " ".join(f"{k}={v}" for k, v in m.items()))
|
||
ro = con.execute(
|
||
"SELECT j.march, r.state, COUNT(*) n FROM rollouts r"
|
||
" JOIN jobs j ON j.run_id=r.run_id AND j.job_id=r.job_id AND j.mode='oracle'"
|
||
" WHERE r.run_id=? GROUP BY j.march,r.state ORDER BY j.march,r.state", (rid,)).fetchall()
|
||
if ro:
|
||
by_march = {}
|
||
for r in ro:
|
||
by_march.setdefault(r["march"], []).append(f"{r['state']}={r['n']}")
|
||
for march, counts in by_march.items():
|
||
print(f" rollouts {march}: " + " ".join(counts))
|
||
from .verify import summarize_results
|
||
for label, records in (("pipeline", con.execute("SELECT result FROM jobs WHERE run_id=? AND mode='oracle'", (rid,))),
|
||
("agent", con.execute("SELECT verdict FROM rollouts WHERE run_id=?", (rid,)))):
|
||
counts = summarize_results(r[0] for r in records)
|
||
print(f" {label} 实际完成: " + " ".join(f"{k}={v}" for k, v in counts.items()))
|
||
manifest = paths.DATASETS / f"{rid}-training" / "manifest.json"
|
||
if manifest.exists():
|
||
m = json.loads(manifest.read_text())
|
||
print(f" dataset: {manifest.parent} stats={json.dumps(m.get('stats', {}), ensure_ascii=False)}")
|
||
# 导出没被触发是静默的(collector 被 database is locked 打死过,manifest 就停在那一刻)。
|
||
# 拿 manifest 里记的可导出条数和现在的比一比,落后就提示补导。复用上面的 ro,零新增查询。
|
||
skip = {"RUNNING", "QUEUED", "VERIFYING", "TRANSFERRED"} # 与 to_agentrl.scan() 的口径一致
|
||
now_n = sum(r["n"] for r in ro if r["state"] not in skip)
|
||
was = (m.get("coverage", {}).get(rid) or {}).get("agent") or {}
|
||
was_n = sum(n for s, n in was.items() if s not in skip)
|
||
if now_n > was_n:
|
||
# collector / export --follow 活着时导出已经在排队了(pipeline.py 的 export_dirty),
|
||
# 这时的落后是正常在途,不该催人补导——否则 --parallel 12 下会常态刷屏。
|
||
from .pipeline import scheduler_holders
|
||
busy = bool(scheduler_holders(rid))
|
||
tail = "(导出在途)" if busy else f":bench export --run {rid}"
|
||
print(f" {'' if busy else '⚠ '}dataset 落后 {now_n - was_n} 条(manifest {was_n} / 现在 {now_n}){tail}")
|
||
from .scheduler import pool_usage, pool_size
|
||
from .registry import status_counts
|
||
print(" pools: " + " ".join(f"{p}={n}/{pool_size(p)}" for p, n in pool_usage().items()))
|
||
print(" registry: " + " ".join(f"{k}={v}" for k, v in status_counts().items()) + " (原始标签;显式批次进度见 coverage --all-runs)")
|
||
return 0
|
||
|
||
|
||
def cmd_show(a) -> int:
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
j = db.get_job(con, rid, a.job_id)
|
||
if not j:
|
||
raise SystemExit(f"{rid} 里没有 {a.job_id}")
|
||
print(f"{a.job_id} [{rid}] state={j['state']} ec={j['error_class']} stage={j['stage_failed']} est={j['estimate_s']}")
|
||
st = con.execute("SELECT stage, attempt, lane, state, exit_code, error_class, duration_s FROM stages"
|
||
" WHERE run_id=? AND job_id=? AND rollout=-1 ORDER BY stage, attempt", (rid, a.job_id)).fetchall()
|
||
for s in st:
|
||
print(f" {s['stage']:<14} a{s['attempt']:02d} {s['lane']:<6} {s['state']:<8} exit={s['exit_code']!s:<4} {s['error_class'] or '':<18} {s['duration_s'] or 0:8.1f}s")
|
||
jd = paths.RUNS / rid / a.job_id
|
||
if j["stage_failed"]:
|
||
for p in sorted((jd / "oracle" / "stages" / j["stage_failed"]).glob("a*/stderr.log")):
|
||
print(f"--- {p} (tail 50) ---")
|
||
print("\n".join(p.read_text(errors="replace").splitlines()[-50:]))
|
||
rs = db.list_rollouts(con, rid, a.job_id)
|
||
if rs:
|
||
print("rollouts:")
|
||
for r in rs:
|
||
print(f" {r['kk']:02d} {r['mode']:<6} {r['policy'] or '':<8} {r['state']:<10} claim={r['claim_terminal']} consistent={r['claim_consistent']} review={r['review']} turns={r['turns']} tokens={r['tokens']}")
|
||
notes = con.execute("SELECT ts, note FROM annotations WHERE run_id=? AND job_id=? ORDER BY ts", (rid, a.job_id)).fetchall()
|
||
for n in notes:
|
||
print(f" note: {n['note']}")
|
||
return 0
|
||
|
||
|
||
def cmd_annotate(a) -> int:
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
db.annotate(con, rid, a.job_id, a.note)
|
||
print("ok")
|
||
return 0
|
||
|
||
|
||
def cmd_report(a) -> int:
|
||
from .report import render
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
print(render(rid))
|
||
return 0
|
||
|
||
|
||
def cmd_coverage(a) -> int:
|
||
from .coverage import render, to_csv
|
||
if a.all_runs:
|
||
from .coverage import write_master
|
||
print(write_master(a.csv or str(paths.REPORTS / "model-coverage.csv")))
|
||
return 0
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
if a.csv:
|
||
print(to_csv(rid, a.csv)); return 0
|
||
print(render(rid, a.task, a.missing, a.detail))
|
||
return 0
|
||
|
||
|
||
def cmd_drain(a) -> int:
|
||
from .pipeline import drain_file, collect_drain_file
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
drain_file(rid).touch()
|
||
if a.all:
|
||
collect_drain_file(rid).touch()
|
||
print(f"drain 已置位:{rid},范围={'流水线和 Agent' if a.all else '流水线'}。等已领取的任务结束后再交接。")
|
||
print("注意:2026-09-15 之前启动的旧调度进程(代码在内存里)不认这个文件,只能等它跑完或 kill + reset-stale。")
|
||
return 0
|
||
|
||
|
||
def cmd_continue(a) -> int:
|
||
from . import continuation
|
||
try:
|
||
if a.op == "plan":
|
||
result = continuation.plan(a.run, a.t1, a.to)
|
||
elif a.op == "apply":
|
||
result = continuation.apply(json.loads(Path(a.plan).read_text()))
|
||
else:
|
||
result = continuation.check(a.run)
|
||
except (ValueError, OSError) as exc:
|
||
print(f"continue: {exc}", file=sys.stderr)
|
||
return 2
|
||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
def cmd_heartbeat(a) -> int:
|
||
from .heartbeat import heartbeat
|
||
return heartbeat(a.run)
|
||
|
||
|
||
def cmd_watch(a) -> int:
|
||
from .watch import watch
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
return watch(rid, a.interval, a.once)
|
||
|
||
|
||
def cmd_reset_stale(a) -> int:
|
||
from .pipeline import reset_stale
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
print(json.dumps(reset_stale(rid, a.force)))
|
||
return 0
|
||
|
||
|
||
def cmd_recover_rollout(a) -> int:
|
||
from . import recovery
|
||
if not a.apply:
|
||
result = recovery.diagnose(a.run, a.job, a.kk)
|
||
else:
|
||
if not a.job or a.kk is None or not a.reason:
|
||
raise SystemExit("--apply requires --job, --kk and --reason")
|
||
result = recovery.recover(a.run, a.job, a.kk, reason=a.reason,
|
||
confirm_owner_dead=a.confirm_owner_dead,
|
||
confirm_work_stopped=a.confirm_work_stopped)
|
||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
def cmd_capabilities(a) -> int:
|
||
from .registry import boards, target_spec, matrix, board_resource_summary
|
||
from .eval import TASK_MODULES, task_contract
|
||
from .runners import MODULES
|
||
from .toolchains import resolve
|
||
result = {"targets": boards().get("march_by_board", {}), "runners": list(MODULES),
|
||
"tasks": {name: task_contract(name) for name in sorted(TASK_MODULES)},
|
||
"toolchains": list(paths.config().get("toolchains", {})),
|
||
"unsupported_targets": ["x5"], "opencode_status": "prototype-awaiting-live-acceptance"}
|
||
if a.target:
|
||
target = target_spec(a.target)
|
||
qcfg = matrix().get("qcfg_by_march", {}).get(target["march"], "int8")
|
||
result["selection"] = {"target": target, "qcfg": qcfg,
|
||
"toolchain": resolve(a.backend, target["march"], qcfg, a.toolchain),
|
||
"board_resources": board_resource_summary(target['march'], target['board_type'])}
|
||
else:
|
||
result['board_resources'] = {name: board_resource_summary(target_spec(name)['march'], name)
|
||
for name in result['targets']}
|
||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
def cmd_board_backfill(a) -> int:
|
||
from .pipeline import board_backfill, board_backfill_rollout
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
if a.rollout:
|
||
# --rollout JOB_ID[:KK],可重复;只动 rollout 树,不碰 oracle
|
||
res = []
|
||
for spec in a.rollout:
|
||
job_id, _, kk = spec.partition(":")
|
||
res.append(board_backfill_rollout(rid, job_id, int(kk or 0)))
|
||
print(json.dumps(res, ensure_ascii=False))
|
||
return 0
|
||
print(json.dumps(board_backfill(rid, a.job or None), ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
def cmd_rejudge(a) -> int:
|
||
from .pipeline import rejudge
|
||
with db.connect() as con:
|
||
rid = _run_id(con, a.run)
|
||
ids = [a.job_id] if a.job_id not in ("all", "--all") else [j["job_id"] for j in db.list_jobs(con, rid, mode="oracle") if j["state"] in ("PASS", "FAIL", "UNVERIFIED")]
|
||
from collections import Counter
|
||
c = Counter()
|
||
for jid in ids:
|
||
r = rejudge(rid, jid)
|
||
c[(r["terminal"], r.get("failed_gate"))] += 1
|
||
if a.job_id not in ("all", "--all"):
|
||
print(json.dumps(r, ensure_ascii=False))
|
||
if a.job_id in ("all", "--all"):
|
||
print(json.dumps({"rejudged": len(ids), "by_terminal_gate": {f"{k[0]}/{k[1]}": v for k, v in c.items()}}, ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
def cmd_verify(a) -> int:
|
||
from .verify import verify_rollout
|
||
v = verify_rollout(Path(a.rollout_dir))
|
||
print(json.dumps({k: v[k] for k in ("job_id", "rollout", "terminal", "failed_gate", "claim_consistent", "review")}))
|
||
return 0
|
||
|
||
|
||
def cmd_collect(a) -> int:
|
||
from .pipeline import collect_rollouts
|
||
with (db.readonly() if a.dry_run else db.connect()) as con:
|
||
rid = _run_id(con, a.run)
|
||
s = collect_rollouts(rid, a.job or None, a.modes.split(",") if a.modes else None,
|
||
parallel=a.parallel, dry_run=a.dry_run, limit=a.limit,
|
||
coverage=a.coverage, watch=a.watch, first_pass=a.first_pass)
|
||
print(json.dumps({"run_id": rid, **s}, ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
def cmd_export(a) -> int:
|
||
"""给已经在跑的批次补上持续物化;新调度器每个任务完成时也会调用同一导出器。"""
|
||
import hashlib
|
||
import time
|
||
import fcntl
|
||
import os
|
||
from .pipeline import refresh_run_dataset
|
||
retry_unknown = getattr(a, "retry_unknown", False)
|
||
if retry_unknown and (not getattr(a, "review", False) or a.follow):
|
||
raise SystemExit("--retry-unknown requires --review and cannot be used with --follow")
|
||
if retry_unknown and not paths.config().get("content_review"):
|
||
raise SystemExit("--retry-unknown requires configured managed content_review")
|
||
with db.readonly() as con:
|
||
rid = _run_id(con, a.run)
|
||
lock = None
|
||
if a.follow:
|
||
if a.interval < 1:
|
||
raise SystemExit("interval 必须 >= 1")
|
||
(paths.STATE / "locks").mkdir(parents=True, exist_ok=True)
|
||
lock = open(paths.STATE / "locks" / f"export-follow-{rid}.lock", "a")
|
||
try:
|
||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
except BlockingIOError:
|
||
raise SystemExit(f"{rid} 已有 export --follow 进程")
|
||
previous = None
|
||
while True:
|
||
with db.readonly() as con:
|
||
jobs = con.execute("SELECT job_id,state,finished_at,result FROM jobs WHERE run_id=? AND mode='oracle' ORDER BY job_id", (rid,)).fetchall()
|
||
rollouts = con.execute("SELECT job_id,kk,state,finished_at,verdict FROM rollouts WHERE run_id=? ORDER BY job_id,kk", (rid,)).fetchall()
|
||
signature = hashlib.sha256(json.dumps([list(map(tuple, jobs)), list(map(tuple, rollouts))]).encode()).hexdigest()
|
||
pending_only = signature == previous
|
||
if not pending_only or (paths.config().get("content_review") or {}).get("enabled") or getattr(a, "review", False):
|
||
try:
|
||
# 导出器只读重核旧结果;历史 verdict 和数据库保持原样。
|
||
manifest = refresh_run_dataset(rid, strict=True, review=getattr(a, "review", False),
|
||
**({"retry_unknown": True} if retry_unknown else {}),
|
||
**({"pending_only": True} if pending_only else {}))
|
||
except Exception as exc:
|
||
if not a.follow:
|
||
raise
|
||
print(f"dataset export failed: {type(exc).__name__}: {exc}; {a.interval}s 后重试", file=sys.stderr, flush=True)
|
||
time.sleep(a.interval)
|
||
continue
|
||
if manifest is not None:
|
||
print(json.dumps({"run_id": rid, "dataset": str(paths.DATASETS / f"{rid}-training"),
|
||
"stats": manifest["stats"]}, ensure_ascii=False), flush=True)
|
||
previous = signature
|
||
if not a.follow:
|
||
return 0
|
||
# 队列暂空不代表在途 Agent 已结束。
|
||
if a.until_pid:
|
||
try:
|
||
os.kill(a.until_pid, 0)
|
||
except ProcessLookupError:
|
||
refresh_run_dataset(rid, strict=True)
|
||
return 0
|
||
time.sleep(a.interval)
|
||
|
||
|
||
def cmd_rollout(a) -> int:
|
||
from .rollout import run_rollout
|
||
with (db.readonly() if a.dry_run else db.connect()) as con:
|
||
rid = _run_id(con, a.run)
|
||
print(json.dumps(run_rollout(rid, a.job_id, a.kk, a.mode, a.policy, dry_run=a.dry_run, runner=a.runner), ensure_ascii=False))
|
||
if not a.dry_run:
|
||
from .pipeline import refresh_run_dataset
|
||
refresh_run_dataset(rid)
|
||
return 0
|
||
|
||
|
||
def cmd_agent(a) -> int:
|
||
from .rollout import setup_models_json
|
||
if a.op == "setup":
|
||
print(setup_models_json(force=a.force))
|
||
return 0
|
||
raise SystemExit("agent op: setup")
|
||
|
||
|
||
def cmd_stage(a) -> int:
|
||
from . import stagerun
|
||
if a.op == "run":
|
||
stagerun.submit(a.stage, a.config, recipe=a.recipe, submission=a.submission)
|
||
return 0
|
||
if a.op == "wait":
|
||
return stagerun.wait(a.stage, a.poll)
|
||
if a.op == "status":
|
||
return stagerun.status(a.stage)
|
||
raise SystemExit("stage op: run | wait | status")
|
||
|
||
|
||
def main(argv=None) -> int:
|
||
from .runners import MODULES
|
||
p = argparse.ArgumentParser(prog="bench")
|
||
sp = p.add_subparsers(dest="cmd", required=True)
|
||
|
||
x = sp.add_parser("monitor", help="只读实时看板;共享采样,不修改任务或数据")
|
||
x.add_argument("--host", default="127.0.0.1")
|
||
x.add_argument("--port", type=int, default=8787)
|
||
x.add_argument("--interval", type=float, default=5)
|
||
def monitor(a):
|
||
from .monitor import serve
|
||
if a.interval <= 0 or not 0 < a.port < 65536:
|
||
p.error("monitor requires a positive interval and port 1..65535")
|
||
serve(paths.BENCH_ROOT, state_dir=paths.STATE, host=a.host, port=a.port, interval=a.interval)
|
||
return 0
|
||
x.set_defaults(f=monitor)
|
||
|
||
x = sp.add_parser("execution", help="核对、取消或恢复受保护的采集执行")
|
||
x.add_argument("op", choices=['status', 'cancel', 'resume', 'reconcile'])
|
||
x.add_argument("--run", required=True)
|
||
x.add_argument("--job")
|
||
x.add_argument("--kk", type=int)
|
||
x.add_argument("--reason")
|
||
def execution_command(a):
|
||
from .execution import Execution
|
||
from .rollout import rollout_dir, resume_rollout, settle_execution
|
||
if a.op == 'reconcile':
|
||
from .recovery import reconcile_run
|
||
result = reconcile_run(a.run)
|
||
for row in result.get('executions', []):
|
||
row.get('execution', {}).pop('token', None)
|
||
else:
|
||
if not a.job or a.kk is None:
|
||
p.error('execution requires --job and --kk')
|
||
if any('/' in value or value in ('.', '..') for value in (a.run, a.job)) or a.kk < 0:
|
||
p.error('invalid execution identity')
|
||
execution = Execution(rollout_dir(a.run, a.job, a.kk)).attach()
|
||
if a.op == 'status':
|
||
result = execution.snapshot()
|
||
result.pop('token', None)
|
||
elif a.op == 'resume':
|
||
result = resume_rollout(a.run, a.job, a.kk)
|
||
else:
|
||
if not a.reason:
|
||
p.error('cancel requires --reason')
|
||
result = settle_execution(a.run, a.job, a.kk, execution, a.reason, cancel=True)
|
||
print(json.dumps(result, ensure_ascii=False))
|
||
return 0
|
||
print(json.dumps(result, ensure_ascii=False))
|
||
return 0 if result.get('ready', True) else 2
|
||
x.set_defaults(f=execution_command)
|
||
|
||
x = sp.add_parser("capabilities", help="只读预览扩展入口与目标量化条件,不创建任务")
|
||
x.add_argument("--target"); x.add_argument("--backend", default="ptq"); x.add_argument("--toolchain"); x.set_defaults(f=cmd_capabilities)
|
||
x = sp.add_parser("recover-rollout", help="默认只诊断;显式封存旧尝试并分配新 kk")
|
||
x.add_argument("--run", required=True); x.add_argument("--job"); x.add_argument("--kk", type=int)
|
||
x.add_argument("--apply", action="store_true"); x.add_argument("--reason")
|
||
x.add_argument("--confirm-owner-dead", action="store_true"); x.add_argument("--confirm-work-stopped", action="store_true")
|
||
x.set_defaults(f=cmd_recover_rollout)
|
||
|
||
x = sp.add_parser("preflight"); x.add_argument("--modes"); x.add_argument("--target", help="板型目标,如 s100p / s600")
|
||
x.add_argument("--oracle-only", action="store_true", help="只跑 benchmark 那一半时放行 agent 运行时缺失"); x.set_defaults(f=cmd_preflight)
|
||
x = sp.add_parser("new-run"); x.add_argument("--note")
|
||
x.add_argument("--target", required=True, help="固定本 run 的唯一板型目标")
|
||
x.set_defaults(f=cmd_new_run)
|
||
x = sp.add_parser("expand"); x.add_argument("--run"); x.add_argument("--modes"); x.add_argument("--marches", help="只展开这些 march,逗号分隔")
|
||
x.add_argument("--model", action="append", help="显式分批接入;核查真实前置,保留原 march/backend/policy/k,不改 CSV 标签")
|
||
x.add_argument("--t1", action="append", help="新批次直接使用已验证的 T1 manifest;每个数据集一份,不改旧任务")
|
||
x.set_defaults(f=cmd_expand)
|
||
x = sp.add_parser("run"); x.add_argument("--run"); x.add_argument("--job", action="append")
|
||
x.add_argument("--oracle", action="store_true", help="显式运行完整01–08对照;默认只准备01–03输入")
|
||
x.add_argument("--dry-run", action="store_true"); x.add_argument("--parallel", type=int); x.set_defaults(f=cmd_run)
|
||
x = sp.add_parser("status"); x.add_argument("--run"); x.add_argument("--fail", action="store_true"); x.set_defaults(f=cmd_status)
|
||
x = sp.add_parser("show"); x.add_argument("job_id"); x.add_argument("--run"); x.set_defaults(f=cmd_show)
|
||
x = sp.add_parser("annotate"); x.add_argument("job_id"); x.add_argument("--note", required=True); x.add_argument("--run"); x.set_defaults(f=cmd_annotate)
|
||
x = sp.add_parser("report"); x.add_argument("--run"); x.set_defaults(f=cmd_report)
|
||
x = sp.add_parser("verify"); x.add_argument("rollout_dir"); x.set_defaults(f=cmd_verify)
|
||
x = sp.add_parser("rejudge"); x.add_argument("job_id", help="job_id 或 all(该 run 所有已结束的 job)"); x.add_argument("--run"); x.set_defaults(f=cmd_rejudge)
|
||
x = sp.add_parser("board-backfill", help="用现成 HBM 补跑 07/08:默认补 oracle job;--rollout JOB_ID[:KK] 补已结束的 rollout 并重新 verify")
|
||
x.add_argument("--run"); x.add_argument("--job", action="append")
|
||
x.add_argument("--rollout", action="append", metavar="JOB_ID[:KK]", help="补测 rollout(可重复),板卡故障导致 UNVERIFIED 的轨迹用这个回收")
|
||
x.set_defaults(f=cmd_board_backfill)
|
||
x = sp.add_parser("coverage"); x.add_argument("--run"); x.add_argument("--task"); x.add_argument("--missing", action="store_true")
|
||
x.add_argument("--all-runs", action="store_true", help="338 模型总表及逐任务/阶段/真实轨迹 JSON")
|
||
x.add_argument("--detail", action="store_true"); x.add_argument("--csv"); x.set_defaults(f=cmd_coverage)
|
||
x = sp.add_parser("heartbeat"); x.add_argument("--run", required=True); x.set_defaults(f=cmd_heartbeat)
|
||
x = sp.add_parser("watch"); x.add_argument("--run"); x.add_argument("--interval", type=int, default=30); x.add_argument("--once", action="store_true"); x.set_defaults(f=cmd_watch)
|
||
# --run 必填:默认取最新 run,而最新 run 往往正是有活 rollout 的那个,误打代价太大
|
||
x = sp.add_parser("reset-stale"); x.add_argument("--run", required=True); x.add_argument("--force", action="store_true"); x.set_defaults(f=cmd_reset_stale)
|
||
x = sp.add_parser("drain", help="让 oracle 调度进程跑完手里的 job 后不再认领新 job(换代码/配置前用,不用 kill)")
|
||
x.add_argument("--run"); x.add_argument("--all", action="store_true", help="同时停止 Agent 领取"); x.set_defaults(f=cmd_drain)
|
||
x = sp.add_parser("continue", help="预览、应用和核对剩余任务交接;不全量展开")
|
||
cs = x.add_subparsers(dest="op", required=True)
|
||
y = cs.add_parser("plan"); y.add_argument("--run", required=True); y.add_argument("--to")
|
||
y.add_argument("--t1", action="append", required=True, help="新 T1 的 manifest.json,每个数据集一份"); y.set_defaults(f=cmd_continue)
|
||
y = cs.add_parser("apply"); y.add_argument("plan", help="已核对的 JSON 清单"); y.set_defaults(f=cmd_continue)
|
||
y = cs.add_parser("check"); y.add_argument("--run", required=True); y.set_defaults(f=cmd_continue)
|
||
x = sp.add_parser("collect"); x.add_argument("--run"); x.add_argument("--job", action="append")
|
||
x.add_argument("--modes", help="e2e,repair"); x.add_argument("--parallel", type=int)
|
||
x.add_argument("--limit", type=int, help="先采 N 条试水"); x.add_argument("--dry-run", action="store_true")
|
||
x.add_argument("--coverage", choices=["matrix", "all"], default="matrix",
|
||
help="all 为本 run 的 oracle 模型补齐 e2e 计划,只放开 priority 限制")
|
||
x.add_argument("--watch", action="store_true", help="等待输入准备与在途任务,持续采集")
|
||
x.add_argument("--first-pass", action="store_true", help="每个 job 只领取首条,保留后续采样编号和原 k")
|
||
x.set_defaults(f=cmd_collect)
|
||
x = sp.add_parser("export", help="导出本 run 已完成的流程与 Agent 数据;--follow 跟随当前批次更新同一目录")
|
||
x.add_argument("--run"); x.add_argument("--follow", action="store_true")
|
||
x.add_argument("--review", action="store_true", help="对脚本合格候选执行一次内容审核,须配置 reviewer 环境")
|
||
x.add_argument("--retry-unknown", action="store_true", help="显式补审已完成 UNKNOWN 并保留旧回执;须 --review,不能与 --follow 同用")
|
||
x.add_argument("--until-pid", type=int, help="跟随到指定批次进程退出;避免队列暂空时提前结束")
|
||
x.add_argument("--interval", type=int, default=30); x.set_defaults(f=cmd_export)
|
||
x = sp.add_parser("rollout"); x.add_argument("job_id"); x.add_argument("--kk", type=int, default=0)
|
||
x.add_argument("--mode", choices=["e2e", "repair"], default="e2e"); x.add_argument("--policy", choices=["teacher", "student"], default="teacher")
|
||
x.add_argument("--runner", choices=list(MODULES), default="pi")
|
||
x.add_argument("--run"); x.add_argument("--dry-run", action="store_true"); x.set_defaults(f=cmd_rollout)
|
||
x = sp.add_parser("agent"); x.add_argument("op", choices=["setup"]); x.add_argument("--force", action="store_true"); x.set_defaults(f=cmd_agent)
|
||
x = sp.add_parser("stage"); x.add_argument("op", choices=["run", "wait", "status"]); x.add_argument("stage")
|
||
x.add_argument("--config"); x.add_argument("--poll", type=int, default=60); x.set_defaults(f=cmd_stage)
|
||
x.add_argument("--recipe"); x.add_argument("--submission")
|
||
|
||
a = p.parse_args(argv)
|
||
return a.f(a)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|