- bench dev:受管终端的 development 目标改走 Salty 容器(有完整工具链)。 容器里用 user+mount 命名空间、uid nobody、去掉全部能力,只露工具链/缓存/数据(只读)和本轨迹 workspace; 板卡密钥、其他轨迹、个人空间其余内容不可见(10-14 在 Salty 上实测,含 umount 逃逸被挡)。 本地改动下次 bench dev 时上传,小文件拷回,大文件留在开发环境并列出。 - compile.yaml:model_parameters.onnx_model / calibration_parameters.cal_data_dir 可指向 workspace; 04 在容器里核对输入输出与平台图一致(不一致直接拒、不占编译次数),记录图的 sha256 和校准来源。 - 配置白名单报错列出具体字段;SYSTEM.md 写明开发环境和自带图;agent/VERSION 15→16。 - 生效:stageworker/terminal/stageclient/SYSTEM.md 要重启;新配置要加 salty_remote.stages: terminal、 terminal.enabled: true、pools.dev。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
164 lines
6.9 KiB
Python
164 lines
6.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Untrusted, stdlib-only stage ticket client. Authority lives in the outer worker."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
NAMES = {n.split('-', 1)[1]: n for n in ('01-fetch', '02-baseline', '03-export', '04-calibrate', '05-compile', '06-hbm', '07-board', '08-eval')}
|
|
|
|
def rollout_dir():
|
|
return Path(os.environ['BENCH_ROLLOUT_DIR'])
|
|
|
|
def submit(stage, config, poll_hint=60, *, recipe=None, submission=None):
|
|
name = NAMES.get(stage, stage)
|
|
if name not in NAMES.values() and name != 'noop':
|
|
raise SystemExit('阶段不允许: ' + name)
|
|
rd = rollout_dir()
|
|
if recipe and submission:
|
|
raise ValueError('--recipe and --submission are mutually exclusive')
|
|
if submission and not re.fullmatch(r'[a-zA-Z0-9_-]+', submission):
|
|
raise ValueError('submission must be a handle from this rollout')
|
|
if recipe:
|
|
p = Path(recipe)
|
|
if p.is_absolute():
|
|
p = p.relative_to(rd / 'workspace')
|
|
elif p.parts and p.parts[0] == 'workspace':
|
|
p = Path(*p.parts[1:])
|
|
recipe = str(p)
|
|
cfg = None
|
|
if config:
|
|
ws = rd / 'workspace'
|
|
cfg = Path(config)
|
|
if not cfg.is_absolute():
|
|
if cfg.parts[0] == 'workspace':
|
|
cfg = Path(*cfg.parts[1:])
|
|
cfg = ws / cfg
|
|
cfg = cfg.resolve()
|
|
if not cfg.is_relative_to(ws.resolve()) or not cfg.is_file():
|
|
raise SystemExit('--config 必须是 workspace 内文件')
|
|
hid = name + '-' + uuid.uuid4().hex
|
|
hd = rd / 'handles' / hid
|
|
hd.mkdir()
|
|
(hd / 'status.json').write_text(json.dumps({'state': 'QUEUED', 'queued_at': time.time()}))
|
|
tmp = hd / 'request.tmp'
|
|
tmp.write_text(json.dumps({'stage': name, 'config': str(cfg) if cfg else None,
|
|
'recipe': recipe, 'submission': submission, 'submitted_at': time.time()}))
|
|
tmp.rename(hd / 'request.json')
|
|
print(hid)
|
|
print('# Pi wait: terminal_exec(' + json.dumps({'wait_handle': hid}) + ')')
|
|
print(f'# wait: bench stage wait {hid} --poll {poll_hint}')
|
|
return hid
|
|
|
|
def dev(command, timeout=3600, cwd=None):
|
|
"""bench dev: run one command in the development environment (Salty container with the toolchain)."""
|
|
rd = rollout_dir()
|
|
ws = (rd / 'workspace').resolve()
|
|
here = Path.cwd().resolve()
|
|
cwd = cwd or (str(here.relative_to(ws)) if here.is_relative_to(ws) else '.')
|
|
hid = 'terminal-' + uuid.uuid4().hex
|
|
hd = rd / 'handles' / hid
|
|
hd.mkdir()
|
|
(hd / 'status.json').write_text(json.dumps({'state': 'QUEUED', 'queued_at': time.time()}))
|
|
tmp = hd / 'request.tmp'
|
|
tmp.write_text(json.dumps({'stage': 'terminal', 'target': 'development', 'command': command,
|
|
'timeout': timeout, 'cwd': cwd}))
|
|
tmp.rename(hd / 'request.json')
|
|
print(hid)
|
|
print('# Pi wait: terminal_exec(' + json.dumps({'wait_handle': hid}) + ')')
|
|
return hid
|
|
|
|
def wait(handle: str, poll: int = 60, quiet: bool = False, max_wait_s: float = 60) -> int:
|
|
"""单次最多等 60 秒;返回 2 表示仍在排队/运行,阶段继续执行。"""
|
|
if not 0 < max_wait_s <= 60:
|
|
raise ValueError("stage wait budget must be in (0, 60] seconds")
|
|
hd = rollout_dir() / "handles" / handle
|
|
if not hd.exists():
|
|
raise SystemExit(f"handle 不存在:{handle}")
|
|
t0 = time.time()
|
|
deadline = time.monotonic() + max_wait_s
|
|
while True:
|
|
st = json.loads((hd / "status.json").read_text())
|
|
if st.get("state") == "DONE":
|
|
break
|
|
if not quiet:
|
|
since = st.get("started_at") or st.get("queued_at") or t0
|
|
print(f"[{time.strftime('%H:%M:%S')}] {handle}: {st.get('state')}"
|
|
f"{'(等 ' + st.get('pool', '?') + ' 池槽位)' if st.get('state') == 'QUEUED' else ''}"
|
|
f"{'(阶段计算已结束,等待数据库登记;无需重跑)' if st.get('registration') == 'PENDING' else ''}"
|
|
f" 已 {int(time.time() - since)}s", flush=True)
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
print(f"(已等 {int(max_wait_s - remaining)}s 仍未结束;这不是失败,再调一次 bench stage wait {handle} 继续等)")
|
|
return 2
|
|
time.sleep(min(max(1, poll), remaining))
|
|
_print_status(st)
|
|
return 0 if st["result"] in ("OK", "SKIPPED") else 1
|
|
|
|
|
|
def status(handle: str) -> int:
|
|
hd = rollout_dir() / "handles" / handle
|
|
st = json.loads((hd / "status.json").read_text())
|
|
_print_status(st)
|
|
return 0
|
|
|
|
|
|
def _print_status(st: dict) -> None:
|
|
if st.get("state") != "DONE":
|
|
print(json.dumps(st)); return
|
|
print(f"result: {st['result']} exit_code: {st['exit_code']} error_class: {st['error_class']} {st['duration_s']}s")
|
|
print(f"stdout: {st['stdout']}\nstderr: {st['stderr']}\nmeta: {st['meta']}")
|
|
for name in ('stdout_tail', 'stderr_tail'):
|
|
if st.get(name):
|
|
print(f"--- {name} ---\n{st[name]}")
|
|
if st.get("measurements"):
|
|
print("measurements: " + json.dumps(st["measurements"], ensure_ascii=False))
|
|
if st["result"] != "OK":
|
|
try:
|
|
tail = Path(st["stderr"]).read_text(errors="replace").splitlines()[-30:]
|
|
print("--- stderr tail ---"); print("\n".join(tail))
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def main():
|
|
if sys.argv[1:2] == ['dev']:
|
|
p = argparse.ArgumentParser(prog='bench dev', description='Run a command in the development environment')
|
|
p.add_argument('--timeout', type=int, default=3600, help='seconds, at most 43200')
|
|
p.add_argument('--cwd', help='workspace-relative directory; default: the current one')
|
|
p.add_argument('command', nargs=argparse.REMAINDER)
|
|
a = p.parse_args(sys.argv[2:])
|
|
words = a.command[1:] if a.command[:1] == ['--'] else a.command
|
|
if not words:
|
|
p.error('command required')
|
|
dev(words[0] if len(words) == 1 else shlex.join(words), a.timeout, a.cwd)
|
|
return 0
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('command', choices=['stage'])
|
|
p.add_argument('op', choices=['run', 'wait', 'status'])
|
|
p.add_argument('stage')
|
|
p.add_argument('--config')
|
|
p.add_argument('--recipe')
|
|
p.add_argument('--submission')
|
|
p.add_argument('--poll', type=int, default=60)
|
|
a = p.parse_args()
|
|
if not re.fullmatch(r'[a-zA-Z0-9_-]+', a.stage):
|
|
p.error('invalid handle')
|
|
try:
|
|
if a.op == 'run':
|
|
submit(a.stage, a.config, recipe=a.recipe, submission=a.submission); return 0
|
|
if a.op == 'wait':
|
|
return wait(a.stage, max(1, a.poll))
|
|
return status(a.stage)
|
|
except (OSError, ValueError) as e:
|
|
print(f'handle 不存在或尚未就绪: {a.stage} ({e.strerror if isinstance(e, OSError) else e})', file=sys.stderr)
|
|
return 1
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|