data-gen-bpu/bin/stageclient.py

112 lines
4.3 KiB
Python

#!/usr/bin/env python3
"""Untrusted, stdlib-only stage ticket client. Authority lives in the outer worker."""
import argparse
import json
import os
import re
import sys
import time
import uuid
from pathlib import Path
NAMES = {n.split('-', 1)[1]: n for n in ('04-calibrate', '05-compile', '06-hbm', '07-board', '08-eval')}
def rollout_dir():
return Path(os.environ['BENCH_ROLLOUT_DIR'])
def submit(stage, config, poll_hint=60):
name = NAMES.get(stage, stage)
if name not in NAMES.values() and name != 'noop':
raise SystemExit('阶段不允许: ' + name)
rd = rollout_dir()
cfg = None
if config:
ws = rd / 'workspace'
cfg = Path(config)
if not cfg.is_absolute():
if cfg.parts[0] == 'workspace':
cfg = Path(*cfg.parts[1:])
cfg = ws / cfg
cfg = cfg.resolve()
if not cfg.is_relative_to(ws.resolve()) or not cfg.is_file():
raise SystemExit('--config 必须是 workspace 内文件')
hid = name + '-' + uuid.uuid4().hex
hd = rd / 'handles' / hid
hd.mkdir()
(hd / 'status.json').write_text(json.dumps({'state': 'QUEUED', 'queued_at': time.time()}))
tmp = hd / 'request.tmp'
tmp.write_text(json.dumps({'stage': name, 'config': str(cfg) if cfg else None, 'submitted_at': time.time()}))
tmp.rename(hd / 'request.json')
print(hid)
print(f'# wait: bench stage wait {hid} --poll {poll_hint}')
return hid
def wait(handle: str, poll: int = 60, quiet: bool = False, max_wait_s: int | None = None) -> int:
"""阻塞到该阶段结束。每轮打一行心跳(排队中 / 运行中 + 已耗时),这样 agent 侧看得见进展,
pi 的 bash 工具也不会因为"长时间无输出"而误判。max_wait_s 到点就返回 2 并提示继续等(不是失败)。"""
hd = rollout_dir() / "handles" / handle
if not hd.exists():
raise SystemExit(f"handle 不存在:{handle}")
t0 = time.time()
while True:
st = json.loads((hd / "status.json").read_text())
if st.get("state") == "DONE":
break
if not quiet:
since = st.get("started_at") or st.get("queued_at") or t0
print(f"[{time.strftime('%H:%M:%S')}] {handle}: {st.get('state')}"
f"{'(等 ' + st.get('pool', '?') + ' 池槽位)' if st.get('state') == 'QUEUED' else ''}"
f" 已 {int(time.time() - since)}s", flush=True)
if max_wait_s and time.time() - t0 > max_wait_s:
print(f"(已等 {int(time.time() - t0)}s 仍未结束;这不是失败,再调一次 bench stage wait {handle} 继续等)")
return 2
time.sleep(poll)
_print_status(st)
return 0 if st["result"] in ("OK", "SKIPPED") else 1
def status(handle: str) -> int:
hd = rollout_dir() / "handles" / handle
st = json.loads((hd / "status.json").read_text())
_print_status(st)
return 0
def _print_status(st: dict) -> None:
if st.get("state") != "DONE":
print(json.dumps(st)); return
print(f"result: {st['result']} exit_code: {st['exit_code']} error_class: {st['error_class']} {st['duration_s']}s")
print(f"stdout: {st['stdout']}\nstderr: {st['stderr']}\nmeta: {st['meta']}")
if st.get("measurements"):
print("measurements: " + json.dumps(st["measurements"], ensure_ascii=False))
if st["result"] != "OK":
try:
tail = Path(st["stderr"]).read_text(errors="replace").splitlines()[-30:]
print("--- stderr tail ---"); print("\n".join(tail))
except OSError:
pass
def main():
p = argparse.ArgumentParser()
p.add_argument('command', choices=['stage'])
p.add_argument('op', choices=['run', 'wait', 'status'])
p.add_argument('stage')
p.add_argument('--config')
p.add_argument('--poll', type=int, default=60)
a = p.parse_args()
if not re.fullmatch(r'[a-zA-Z0-9_-]+', a.stage):
p.error('invalid handle')
try:
if a.op == 'run':
submit(a.stage, a.config); return 0
if a.op == 'wait':
return wait(a.stage, max(1, a.poll))
return status(a.stage)
except (OSError, ValueError) as e:
print(f'handle 不存在或尚未就绪: {a.stage} ({e.strerror if isinstance(e, OSError) else e})', file=sys.stderr)
return 1
if __name__ == '__main__':
sys.exit(main())