data-gen-bpu/scripts/speed_ref.py
bench 21fce5e2e0 v22:bench stage wait 只看一眼不轮询;板上自测先查大小、拒绝有原因、失败也删副本;速度参考编不出来写 unavailable
- bench stage wait(stageclient 与 stagerun 两个入口)不再睡,没结束就返回 2 并指向 terminal_exec wait_handle。
  10-16 efficientvit.b0 400 轮里 307 轮是轮询。
- bench dev --board:先用 salty storage ls 量开发环境里的目录(>512 MiB 或 >40 个子目录直接拒绝),
  取回后已到时限也明确拒绝;拒绝原因作为命令结果回给 agent;snapshot/dev-files/tar 在任何路径都删。
  10-16 四次自测三次从 1.2 GB 的目录提交,取回 40 分钟、超时、exit 2 无原因、VM 留下 2.5 GB。
- speed_ref:默认配置编不出来时写 latency_ms_est=null、status=unavailable,并补齐以前的 FAILED。
- SYSTEM.md 对应三处;agent/VERSION 22。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 15:24:22 +08:00

157 lines
8.5 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Speed reference per job: compile the platform's default configuration once (in a bench dev container,
outside any rollout) and write the compiler's latency estimate to agent/speed_ref/<job_id>.json: the agent's jail shows bench/agent
read-only and live (not the job directory), so a reference computed after a rollout started still reaches it.
The agent compares its own model against it (SYSTEM.md 速度要求) instead of compiling the default itself
(10-15: 9 of 32 v17 rollouts spent ~4 h each doing that). Jobs that already have the file are skipped.
BENCH_CONFIG=<a run config> python scripts/speed_ref.py r20260929-04 r20260929-05 r20260929-06 [--active --follow] [--workers 3]
--active: only jobs a v18+ rollout is on. Salty is shared with the live rollouts (10-15 20:40: their own dev
jobs queued once 8 of these were added), so references follow demand instead of compiling all 418 up front.
"""
from __future__ import annotations
import argparse
import json
import shlex
import sys
import time
import traceback
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
import yaml
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from harness import paths, salty_dev # noqa: E402
from harness.backends import ptq # noqa: E402
from harness.stages.s01_fetch import weights_dir # noqa: E402
from harness.stages.s03_export import onnx_path # noqa: E402
from harness.stages.s06_hbm import _parse_static_perf # noqa: E402
from harness.stagerun import own_inputs_required # noqa: E402
DEADLINE_S = 10 * 3600
def command(job: dict, ws: Path) -> str:
onnx = onnx_path(job)
weights = weights_dir(job["model"]) # its files are on personal storage, listed in the container
if onnx is None or weights is None:
raise ValueError("weights not fetched yet")
mine = ws / "model.onnx"
calib = Path(job["data"]["calib"]["path"]) / "npy_f32"
cfg = ptq.compile_config(job, Path("model.onnx"), calib, Path("ref"), None)
(ws / "ref.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False))
export = shlex.join(ptq.export_cmd(job, mine, Path("WEIGHTS"))).replace("WEIGHTS", '"$W"')
export = f"W=$(ls -d {shlex.quote(str(weights))}/* | grep -v /meta.json | head -1); {export}"
return (f"set -e; if [ -f {shlex.quote(str(onnx))} ]; then ln -sfn {shlex.quote(str(onnx))} model.onnx; "
f"else {export} > export.log 2>&1; fi; "
"hb_compile -c ref.yaml > ref_compile.log 2>&1 || { tail -40 ref_compile.log; exit 1; }; "
"hb_model_info ref/*.hbm > ref_info.log 2>&1; " # the dev PATH's OE toolchain, as the agent uses it
"sha256sum ref/*.hbm > ref_hbm.sha256; tail -20 ref_info.log; rm -rf ref model.onnx") # keep personal storage small
OUT = paths.AGENT / "speed_ref"
def one(job_dir: Path) -> str:
job = json.loads((job_dir / "job.json").read_text())
jid = job["job_id"]
rd = paths.BENCH_ROOT / "runs" / "zz-speedref" / jid / "rollouts" / "00"
ws = rd / "workspace"
ws.mkdir(parents=True, exist_ok=True)
(rd / "stages").mkdir(exist_ok=True)
(rd.parent.parent / "job.json").write_text(json.dumps(job))
hd = rd / "stages" / "ref"
hd.mkdir(exist_ok=True)
ctx = SimpleNamespace(stage_root=rd / "stages", job=job, job_dir=rd.parent.parent, extra={})
result = salty_dev.execute(ctx, hd, {"command": command(job, ws), "cwd": "."},
time.monotonic() + DEADLINE_S, hd / "out.log", hd / "err.log")
info = (ws / "ref_info.log").read_text(errors="ignore") if (ws / "ref_info.log").exists() else ""
perf = _parse_static_perf(info)
if result.state != "OK" or "latency_ms_est" not in perf:
(rd.parent.parent / "FAILED").write_text((hd / "out.log").read_text(errors="ignore")[-4000:]) # not retried
unavailable(job, f"{DEADLINE_S // 3600} 小时内没编完" if result.exit_code == 124 else f"编译失败,退出码 {result.exit_code}")
return f"{jid}\tFAIL {result.state} {result.exit_code}\t{(hd / 'out.log').read_text(errors='ignore')[-300:]!r}"
sha = (ws / "ref_hbm.sha256").read_text().split()[0] if (ws / "ref_hbm.sha256").exists() else None
ref = {"schema": "bench.speed_ref/1", "job_id": jid, "march": job["march"], "qcfg": job["qcfg"],
"config": "platform default (compile.yaml with no overrides)", **perf, "hbm_sha256": sha,
"measured": "hb_model_info static estimate, not a board run", "computed_at": time.time()}
publish(jid, ref)
return f"{jid}\tOK\t{perf['latency_ms_est']} ms"
def publish(jid: str, ref: dict) -> None:
OUT.mkdir(exist_ok=True)
tmp = OUT / f".{jid}.json.tmp"
tmp.write_text(json.dumps(ref, indent=2, ensure_ascii=False))
tmp.replace(OUT / f"{jid}.json")
def unavailable(job: dict, why: str) -> None:
"""The default configuration did not compile here (10-16: big S600 depth models, >10 h on 6 cores): say so,
so agents stop waiting for the file or compiling the default themselves."""
publish(job["job_id"], {"schema": "bench.speed_ref/1", "job_id": job["job_id"], "march": job["march"], "qcfg": job["qcfg"],
"latency_ms_est": None, "status": "unavailable",
"note": "平台默认配置在这道题上没编出来(" + why + "),本题没有速度参考值:不用再等,也不要自己编默认配置;"
"只要别整模型提到 int16/fp16、别让大段算子落到 CPU,并用 board 阶段实测确认。",
"computed_at": time.time()})
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("runs", nargs="+")
ap.add_argument("--workers", type=int, default=3)
ap.add_argument("--cpu", default="6")
ap.add_argument("--only", action="append", help="job ids")
ap.add_argument("--active", action="store_true", help="only jobs with a rollout under agent v18+")
ap.add_argument("--skip", action="append", default=[], help="job ids (e.g. one another run is doing)")
ap.add_argument("--follow", action="store_true", help="keep scanning every 2 min; at most --workers at a time")
a = ap.parse_args()
cfg = paths.config()
cfg["execution"]["salty_remote"].setdefault("cpu_by_stage", {})["terminal"] = int(a.cpu)
cfg["execution"]["salty_remote"].setdefault("memory_by_stage", {})["terminal"] = "32Gi"
failed = paths.BENCH_ROOT / "runs" / "zz-speedref"
def pending() -> tuple[list[str], dict[str, Path]]:
jobs: dict[str, Path] = {}
started: dict[str, int] = {}
for run in a.runs:
for jd in sorted((paths.BENCH_ROOT / "runs" / run).iterdir()):
if (jd / "job.json").exists() and (not a.only or jd.name in a.only) and (
not a.active or any(own_inputs_required(r) for r in (jd / "rollouts").glob("*"))):
jobs.setdefault(jd.name, jd) # same job id = same model, march and qcfg in every run
started[jd.name] = started.get(jd.name, 0) + len(list((jd / "rollouts").glob("*")))
for j in jobs: # failures recorded before the unavailable file existed (or by an older process)
if (failed / j / "FAILED").exists() and not (OUT / f"{j}.json").exists():
unavailable(json.loads((jobs[j] / "job.json").read_text()), "编译失败或超时")
todo = sorted((j for j in jobs if not (OUT / f"{j}.json").exists() and j not in a.skip),
key=lambda j: started.get(j, 0)) # untouched jobs first: new rollouts go there
return todo, jobs
def safe(jid, jd):
try:
line = one(jd)
except Exception as exc:
line = f"{jid}\tERROR {type(exc).__name__}: {exc}\t{traceback.format_exc(limit=2)[-300:]!r}"
print(time.strftime("%F %T"), line, flush=True)
running: dict[str, object] = {}
with ThreadPoolExecutor(a.workers) as pool:
while True:
running = {j: f for j, f in running.items() if not f.done()}
todo, jobs = pending()
for jid in [j for j in todo if j not in running][:a.workers - len(running)]:
print(time.strftime("%F %T"), "start", jid, flush=True)
running[jid] = pool.submit(safe, jid, jobs[jid])
if not a.follow and not running:
break
time.sleep(120 if a.follow else 5) # --follow: v18 rollouts keep starting on new jobs
return 0
if __name__ == "__main__":
sys.exit(main())