data-gen-bpu/scripts/speed_ref.py
bench 80aa571a0c v18 开发环境给出本题校准图片目录 $BENCH_CALIB_IMAGES;speed_ref 加 --follow 滚动补算
10-15 夜:提示词只说把 /data/calib/ 换成 /data/calib-raw/,agent 去 /data/calib-raw 找,每条花 5-8 次 dev 命令。
speed_ref 按轮跑时一轮要等最慢的那道编完,新题排不上。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 00:57:42 +08:00

140 lines
7.3 KiB
Python

"""Speed reference per job: compile the platform's default configuration once (in a bench dev container,
outside any rollout) and write the compiler's latency estimate to agent/speed_ref/<job_id>.json: the agent's jail shows bench/agent
read-only and live (not the job directory), so a reference computed after a rollout started still reaches it.
The agent compares its own model against it (SYSTEM.md 速度要求) instead of compiling the default itself
(10-15: 9 of 32 v17 rollouts spent ~4 h each doing that). Jobs that already have the file are skipped.
BENCH_CONFIG=<a run config> python scripts/speed_ref.py r20260929-04 r20260929-05 r20260929-06 [--active --follow] [--workers 3]
--active: only jobs a v18+ rollout is on. Salty is shared with the live rollouts (10-15 20:40: their own dev
jobs queued once 8 of these were added), so references follow demand instead of compiling all 418 up front.
"""
from __future__ import annotations
import argparse
import json
import shlex
import sys
import time
import traceback
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
import yaml
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from harness import paths, salty_dev # noqa: E402
from harness.backends import ptq # noqa: E402
from harness.stages.s01_fetch import weights_dir # noqa: E402
from harness.stages.s03_export import onnx_path # noqa: E402
from harness.stages.s06_hbm import _parse_static_perf # noqa: E402
from harness.stagerun import own_inputs_required # noqa: E402
DEADLINE_S = 10 * 3600
def command(job: dict, ws: Path) -> str:
onnx = onnx_path(job)
weights = weights_dir(job["model"]) # its files are on personal storage, listed in the container
if onnx is None or weights is None:
raise ValueError("weights not fetched yet")
mine = ws / "model.onnx"
calib = Path(job["data"]["calib"]["path"]) / "npy_f32"
cfg = ptq.compile_config(job, Path("model.onnx"), calib, Path("ref"), None)
(ws / "ref.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False))
export = shlex.join(ptq.export_cmd(job, mine, Path("WEIGHTS"))).replace("WEIGHTS", '"$W"')
export = f"W=$(ls -d {shlex.quote(str(weights))}/* | grep -v /meta.json | head -1); {export}"
return (f"set -e; if [ -f {shlex.quote(str(onnx))} ]; then ln -sfn {shlex.quote(str(onnx))} model.onnx; "
f"else {export} > export.log 2>&1; fi; "
"hb_compile -c ref.yaml > ref_compile.log 2>&1 || { tail -40 ref_compile.log; exit 1; }; "
"hb_model_info ref/*.hbm > ref_info.log 2>&1; " # the dev PATH's OE toolchain, as the agent uses it
"sha256sum ref/*.hbm > ref_hbm.sha256; tail -20 ref_info.log; rm -rf ref model.onnx") # keep personal storage small
OUT = paths.AGENT / "speed_ref"
def one(job_dir: Path) -> str:
job = json.loads((job_dir / "job.json").read_text())
jid = job["job_id"]
rd = paths.BENCH_ROOT / "runs" / "zz-speedref" / jid / "rollouts" / "00"
ws = rd / "workspace"
ws.mkdir(parents=True, exist_ok=True)
(rd / "stages").mkdir(exist_ok=True)
(rd.parent.parent / "job.json").write_text(json.dumps(job))
hd = rd / "stages" / "ref"
hd.mkdir(exist_ok=True)
ctx = SimpleNamespace(stage_root=rd / "stages", job=job, job_dir=rd.parent.parent, extra={})
result = salty_dev.execute(ctx, hd, {"command": command(job, ws), "cwd": "."},
time.monotonic() + DEADLINE_S, hd / "out.log", hd / "err.log")
info = (ws / "ref_info.log").read_text(errors="ignore") if (ws / "ref_info.log").exists() else ""
perf = _parse_static_perf(info)
if result.state != "OK" or "latency_ms_est" not in perf:
(rd.parent.parent / "FAILED").write_text((hd / "out.log").read_text(errors="ignore")[-4000:]) # not retried
return f"{jid}\tFAIL {result.state} {result.exit_code}\t{(hd / 'out.log').read_text(errors='ignore')[-300:]!r}"
sha = (ws / "ref_hbm.sha256").read_text().split()[0] if (ws / "ref_hbm.sha256").exists() else None
ref = {"schema": "bench.speed_ref/1", "job_id": jid, "march": job["march"], "qcfg": job["qcfg"],
"config": "platform default (compile.yaml with no overrides)", **perf, "hbm_sha256": sha,
"measured": "hb_model_info static estimate, not a board run", "computed_at": time.time()}
OUT.mkdir(exist_ok=True)
tmp = OUT / f".{jid}.json.tmp"
tmp.write_text(json.dumps(ref, indent=2))
tmp.replace(OUT / f"{jid}.json")
return f"{jid}\tOK\t{perf['latency_ms_est']} ms"
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("runs", nargs="+")
ap.add_argument("--workers", type=int, default=3)
ap.add_argument("--cpu", default="6")
ap.add_argument("--only", action="append", help="job ids")
ap.add_argument("--active", action="store_true", help="only jobs with a rollout under agent v18+")
ap.add_argument("--skip", action="append", default=[], help="job ids (e.g. one another run is doing)")
ap.add_argument("--follow", action="store_true", help="keep scanning every 2 min; at most --workers at a time")
a = ap.parse_args()
cfg = paths.config()
cfg["execution"]["salty_remote"].setdefault("cpu_by_stage", {})["terminal"] = int(a.cpu)
cfg["execution"]["salty_remote"].setdefault("memory_by_stage", {})["terminal"] = "32Gi"
failed = paths.BENCH_ROOT / "runs" / "zz-speedref"
def pending() -> tuple[list[str], dict[str, Path]]:
jobs: dict[str, Path] = {}
started: dict[str, int] = {}
for run in a.runs:
for jd in sorted((paths.BENCH_ROOT / "runs" / run).iterdir()):
if (jd / "job.json").exists() and (not a.only or jd.name in a.only) and (
not a.active or any(own_inputs_required(r) for r in (jd / "rollouts").glob("*"))):
jobs.setdefault(jd.name, jd) # same job id = same model, march and qcfg in every run
started[jd.name] = started.get(jd.name, 0) + len(list((jd / "rollouts").glob("*")))
todo = sorted((j for j in jobs if not (OUT / f"{j}.json").exists() and j not in a.skip
and not (failed / j / "FAILED").exists()),
key=lambda j: started.get(j, 0)) # untouched jobs first: new rollouts go there
return todo, jobs
def safe(jid, jd):
try:
line = one(jd)
except Exception as exc:
line = f"{jid}\tERROR {type(exc).__name__}: {exc}\t{traceback.format_exc(limit=2)[-300:]!r}"
print(time.strftime("%F %T"), line, flush=True)
running: dict[str, object] = {}
with ThreadPoolExecutor(a.workers) as pool:
while True:
running = {j: f for j, f in running.items() if not f.done()}
todo, jobs = pending()
for jid in [j for j in todo if j not in running][:a.workers - len(running)]:
print(time.strftime("%F %T"), "start", jid, flush=True)
running[jid] = pool.submit(safe, jid, jobs[jid])
if not a.follow and not running:
break
time.sleep(120 if a.follow else 5) # --follow: v18 rollouts keep starting on new jobs
return 0
if __name__ == "__main__":
sys.exit(main())