Reuse execution, stage and review evidence to recover database registration and SFT publication without replaying completed work. Keep finished futures during SQLite contention, and hand durable results to startup recovery when draining. Release proven local leases independently of unresolved board isolation. Keep UNKNOWN review retries explicit and archived, publish completed receipts after transient export failures, and prevent a malformed sample from aborting the entire dataset. Scope routine reconciliation to newly finished attempts. Add failure-injection coverage and the production acceptance plan.
131 lines
5.3 KiB
Python
131 lines
5.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Untrusted, stdlib-only stage ticket client. Authority lives in the outer worker."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
NAMES = {n.split('-', 1)[1]: n for n in ('01-fetch', '02-baseline', '03-export', '04-calibrate', '05-compile', '06-hbm', '07-board', '08-eval')}
|
|
|
|
def rollout_dir():
|
|
return Path(os.environ['BENCH_ROLLOUT_DIR'])
|
|
|
|
def submit(stage, config, poll_hint=60, *, recipe=None, submission=None):
|
|
name = NAMES.get(stage, stage)
|
|
if name not in NAMES.values() and name != 'noop':
|
|
raise SystemExit('阶段不允许: ' + name)
|
|
rd = rollout_dir()
|
|
if recipe and submission:
|
|
raise ValueError('--recipe and --submission are mutually exclusive')
|
|
if submission and not re.fullmatch(r'[a-zA-Z0-9_-]+', submission):
|
|
raise ValueError('submission must be a handle from this rollout')
|
|
if recipe:
|
|
p = Path(recipe)
|
|
if p.is_absolute():
|
|
p = p.relative_to(rd / 'workspace')
|
|
elif p.parts and p.parts[0] == 'workspace':
|
|
p = Path(*p.parts[1:])
|
|
recipe = str(p)
|
|
cfg = None
|
|
if config:
|
|
ws = rd / 'workspace'
|
|
cfg = Path(config)
|
|
if not cfg.is_absolute():
|
|
if cfg.parts[0] == 'workspace':
|
|
cfg = Path(*cfg.parts[1:])
|
|
cfg = ws / cfg
|
|
cfg = cfg.resolve()
|
|
if not cfg.is_relative_to(ws.resolve()) or not cfg.is_file():
|
|
raise SystemExit('--config 必须是 workspace 内文件')
|
|
hid = name + '-' + uuid.uuid4().hex
|
|
hd = rd / 'handles' / hid
|
|
hd.mkdir()
|
|
(hd / 'status.json').write_text(json.dumps({'state': 'QUEUED', 'queued_at': time.time()}))
|
|
tmp = hd / 'request.tmp'
|
|
tmp.write_text(json.dumps({'stage': name, 'config': str(cfg) if cfg else None,
|
|
'recipe': recipe, 'submission': submission, 'submitted_at': time.time()}))
|
|
tmp.rename(hd / 'request.json')
|
|
print(hid)
|
|
print('# Pi wait: terminal_exec(' + json.dumps({'wait_handle': hid}) + ')')
|
|
print(f'# wait: bench stage wait {hid} --poll {poll_hint}')
|
|
return hid
|
|
|
|
def wait(handle: str, poll: int = 60, quiet: bool = False, max_wait_s: float = 60) -> int:
|
|
"""单次最多等 60 秒;返回 2 表示仍在排队/运行,阶段继续执行。"""
|
|
if not 0 < max_wait_s <= 60:
|
|
raise ValueError("stage wait budget must be in (0, 60] seconds")
|
|
hd = rollout_dir() / "handles" / handle
|
|
if not hd.exists():
|
|
raise SystemExit(f"handle 不存在:{handle}")
|
|
t0 = time.time()
|
|
deadline = time.monotonic() + max_wait_s
|
|
while True:
|
|
st = json.loads((hd / "status.json").read_text())
|
|
if st.get("state") == "DONE":
|
|
break
|
|
if not quiet:
|
|
since = st.get("started_at") or st.get("queued_at") or t0
|
|
print(f"[{time.strftime('%H:%M:%S')}] {handle}: {st.get('state')}"
|
|
f"{'(等 ' + st.get('pool', '?') + ' 池槽位)' if st.get('state') == 'QUEUED' else ''}"
|
|
f"{'(阶段计算已结束,等待数据库登记;无需重跑)' if st.get('registration') == 'PENDING' else ''}"
|
|
f" 已 {int(time.time() - since)}s", flush=True)
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
print(f"(已等 {int(max_wait_s - remaining)}s 仍未结束;这不是失败,再调一次 bench stage wait {handle} 继续等)")
|
|
return 2
|
|
time.sleep(min(max(1, poll), remaining))
|
|
_print_status(st)
|
|
return 0 if st["result"] in ("OK", "SKIPPED") else 1
|
|
|
|
|
|
def status(handle: str) -> int:
|
|
hd = rollout_dir() / "handles" / handle
|
|
st = json.loads((hd / "status.json").read_text())
|
|
_print_status(st)
|
|
return 0
|
|
|
|
|
|
def _print_status(st: dict) -> None:
|
|
if st.get("state") != "DONE":
|
|
print(json.dumps(st)); return
|
|
print(f"result: {st['result']} exit_code: {st['exit_code']} error_class: {st['error_class']} {st['duration_s']}s")
|
|
print(f"stdout: {st['stdout']}\nstderr: {st['stderr']}\nmeta: {st['meta']}")
|
|
if st.get("measurements"):
|
|
print("measurements: " + json.dumps(st["measurements"], ensure_ascii=False))
|
|
if st["result"] != "OK":
|
|
try:
|
|
tail = Path(st["stderr"]).read_text(errors="replace").splitlines()[-30:]
|
|
print("--- stderr tail ---"); print("\n".join(tail))
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('command', choices=['stage'])
|
|
p.add_argument('op', choices=['run', 'wait', 'status'])
|
|
p.add_argument('stage')
|
|
p.add_argument('--config')
|
|
p.add_argument('--recipe')
|
|
p.add_argument('--submission')
|
|
p.add_argument('--poll', type=int, default=60)
|
|
a = p.parse_args()
|
|
if not re.fullmatch(r'[a-zA-Z0-9_-]+', a.stage):
|
|
p.error('invalid handle')
|
|
try:
|
|
if a.op == 'run':
|
|
submit(a.stage, a.config, recipe=a.recipe, submission=a.submission); return 0
|
|
if a.op == 'wait':
|
|
return wait(a.stage, max(1, a.poll))
|
|
return status(a.stage)
|
|
except (OSError, ValueError) as e:
|
|
print(f'handle 不存在或尚未就绪: {a.stage} ({e.strerror if isinstance(e, OSError) else e})', file=sys.stderr)
|
|
return 1
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|