data-gen-bpu/bin/stageclient.py
ming01.wang 498187569c fix: close rollout finalization and review publication gaps
Reuse execution, stage and review evidence to recover database registration and
SFT publication without replaying completed work. Keep finished futures during
SQLite contention, and hand durable results to startup recovery when draining.
Release proven local leases independently of unresolved board isolation.

Keep UNKNOWN review retries explicit and archived, publish completed receipts
after transient export failures, and prevent a malformed sample from aborting
the entire dataset. Scope routine reconciliation to newly finished attempts.
Add failure-injection coverage and the production acceptance plan.
2026-09-29 16:09:01 +08:00

131 lines
5.3 KiB
Python

#!/usr/bin/env python3
"""Untrusted, stdlib-only stage ticket client. Authority lives in the outer worker."""
import argparse
import json
import os
import re
import sys
import time
import uuid
from pathlib import Path
NAMES = {n.split('-', 1)[1]: n for n in ('01-fetch', '02-baseline', '03-export', '04-calibrate', '05-compile', '06-hbm', '07-board', '08-eval')}
def rollout_dir():
return Path(os.environ['BENCH_ROLLOUT_DIR'])
def submit(stage, config, poll_hint=60, *, recipe=None, submission=None):
name = NAMES.get(stage, stage)
if name not in NAMES.values() and name != 'noop':
raise SystemExit('阶段不允许: ' + name)
rd = rollout_dir()
if recipe and submission:
raise ValueError('--recipe and --submission are mutually exclusive')
if submission and not re.fullmatch(r'[a-zA-Z0-9_-]+', submission):
raise ValueError('submission must be a handle from this rollout')
if recipe:
p = Path(recipe)
if p.is_absolute():
p = p.relative_to(rd / 'workspace')
elif p.parts and p.parts[0] == 'workspace':
p = Path(*p.parts[1:])
recipe = str(p)
cfg = None
if config:
ws = rd / 'workspace'
cfg = Path(config)
if not cfg.is_absolute():
if cfg.parts[0] == 'workspace':
cfg = Path(*cfg.parts[1:])
cfg = ws / cfg
cfg = cfg.resolve()
if not cfg.is_relative_to(ws.resolve()) or not cfg.is_file():
raise SystemExit('--config 必须是 workspace 内文件')
hid = name + '-' + uuid.uuid4().hex
hd = rd / 'handles' / hid
hd.mkdir()
(hd / 'status.json').write_text(json.dumps({'state': 'QUEUED', 'queued_at': time.time()}))
tmp = hd / 'request.tmp'
tmp.write_text(json.dumps({'stage': name, 'config': str(cfg) if cfg else None,
'recipe': recipe, 'submission': submission, 'submitted_at': time.time()}))
tmp.rename(hd / 'request.json')
print(hid)
print('# Pi wait: terminal_exec(' + json.dumps({'wait_handle': hid}) + ')')
print(f'# wait: bench stage wait {hid} --poll {poll_hint}')
return hid
def wait(handle: str, poll: int = 60, quiet: bool = False, max_wait_s: float = 60) -> int:
"""单次最多等 60 秒;返回 2 表示仍在排队/运行,阶段继续执行。"""
if not 0 < max_wait_s <= 60:
raise ValueError("stage wait budget must be in (0, 60] seconds")
hd = rollout_dir() / "handles" / handle
if not hd.exists():
raise SystemExit(f"handle 不存在:{handle}")
t0 = time.time()
deadline = time.monotonic() + max_wait_s
while True:
st = json.loads((hd / "status.json").read_text())
if st.get("state") == "DONE":
break
if not quiet:
since = st.get("started_at") or st.get("queued_at") or t0
print(f"[{time.strftime('%H:%M:%S')}] {handle}: {st.get('state')}"
f"{'(等 ' + st.get('pool', '?') + ' 池槽位)' if st.get('state') == 'QUEUED' else ''}"
f"{'(阶段计算已结束,等待数据库登记;无需重跑)' if st.get('registration') == 'PENDING' else ''}"
f" 已 {int(time.time() - since)}s", flush=True)
remaining = deadline - time.monotonic()
if remaining <= 0:
print(f"(已等 {int(max_wait_s - remaining)}s 仍未结束;这不是失败,再调一次 bench stage wait {handle} 继续等)")
return 2
time.sleep(min(max(1, poll), remaining))
_print_status(st)
return 0 if st["result"] in ("OK", "SKIPPED") else 1
def status(handle: str) -> int:
hd = rollout_dir() / "handles" / handle
st = json.loads((hd / "status.json").read_text())
_print_status(st)
return 0
def _print_status(st: dict) -> None:
if st.get("state") != "DONE":
print(json.dumps(st)); return
print(f"result: {st['result']} exit_code: {st['exit_code']} error_class: {st['error_class']} {st['duration_s']}s")
print(f"stdout: {st['stdout']}\nstderr: {st['stderr']}\nmeta: {st['meta']}")
if st.get("measurements"):
print("measurements: " + json.dumps(st["measurements"], ensure_ascii=False))
if st["result"] != "OK":
try:
tail = Path(st["stderr"]).read_text(errors="replace").splitlines()[-30:]
print("--- stderr tail ---"); print("\n".join(tail))
except OSError:
pass
def main():
p = argparse.ArgumentParser()
p.add_argument('command', choices=['stage'])
p.add_argument('op', choices=['run', 'wait', 'status'])
p.add_argument('stage')
p.add_argument('--config')
p.add_argument('--recipe')
p.add_argument('--submission')
p.add_argument('--poll', type=int, default=60)
a = p.parse_args()
if not re.fullmatch(r'[a-zA-Z0-9_-]+', a.stage):
p.error('invalid handle')
try:
if a.op == 'run':
submit(a.stage, a.config, recipe=a.recipe, submission=a.submission); return 0
if a.op == 'wait':
return wait(a.stage, max(1, a.poll))
return status(a.stage)
except (OSError, ValueError) as e:
print(f'handle 不存在或尚未就绪: {a.stage} ({e.strerror if isinstance(e, OSError) else e})', file=sys.stderr)
return 1
if __name__ == '__main__':
sys.exit(main())