data-gen-bpu/scripts/speed_ref.py
bench 2ed2f5fec0 speed_ref:只给有 v18 轨迹的题按需编(--active,默认 3 路),失败的记 FAILED 不重试
Salty 和在跑的轨迹共用:一次加 8 路时 agent 自己的 dev 作业开始排队。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 21:42:43 +08:00

127 lines
6.6 KiB
Python

"""Speed reference per job: compile the platform's default configuration once (in a bench dev container,
outside any rollout) and write the compiler's latency estimate to agent/speed_ref/<job_id>.json: the agent's jail shows bench/agent
read-only and live (not the job directory), so a reference computed after a rollout started still reaches it.
The agent compares its own model against it (SYSTEM.md 速度要求) instead of compiling the default itself
(10-15: 9 of 32 v17 rollouts spent ~4 h each doing that). Jobs that already have the file are skipped.
BENCH_CONFIG=<a run config> python scripts/speed_ref.py r20260929-04 r20260929-05 r20260929-06 [--active] [--workers 3]
--active: only jobs a v18+ rollout is on. Salty is shared with the live rollouts (10-15 20:40: their own dev
jobs queued once 8 of these were added), so references follow demand instead of compiling all 418 up front.
"""
from __future__ import annotations
import argparse
import json
import shlex
import sys
import time
import traceback
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
import yaml
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from harness import paths, salty_dev # noqa: E402
from harness.backends import ptq # noqa: E402
from harness.stages.s01_fetch import weights_dir # noqa: E402
from harness.stages.s03_export import onnx_path # noqa: E402
from harness.stages.s06_hbm import _parse_static_perf # noqa: E402
from harness.stagerun import own_inputs_required # noqa: E402
DEADLINE_S = 10 * 3600
def command(job: dict, ws: Path) -> str:
onnx = onnx_path(job)
weights = weights_dir(job["model"]) # its files are on personal storage, listed in the container
if onnx is None or weights is None:
raise ValueError("weights not fetched yet")
mine = ws / "model.onnx"
calib = Path(job["data"]["calib"]["path"]) / "npy_f32"
cfg = ptq.compile_config(job, Path("model.onnx"), calib, Path("ref"), None)
(ws / "ref.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False))
export = shlex.join(ptq.export_cmd(job, mine, Path("WEIGHTS"))).replace("WEIGHTS", '"$W"')
export = f"W=$(ls -d {shlex.quote(str(weights))}/* | grep -v /meta.json | head -1); {export}"
return (f"set -e; if [ -f {shlex.quote(str(onnx))} ]; then ln -sfn {shlex.quote(str(onnx))} model.onnx; "
f"else {export} > export.log 2>&1; fi; "
"hb_compile -c ref.yaml > ref_compile.log 2>&1 || { tail -40 ref_compile.log; exit 1; }; "
"hb_model_info ref/*.hbm > ref_info.log 2>&1; " # the dev PATH's OE toolchain, as the agent uses it
"sha256sum ref/*.hbm > ref_hbm.sha256; tail -20 ref_info.log; rm -rf ref model.onnx") # keep personal storage small
OUT = paths.AGENT / "speed_ref"
def one(job_dir: Path) -> str:
job = json.loads((job_dir / "job.json").read_text())
jid = job["job_id"]
rd = paths.BENCH_ROOT / "runs" / "zz-speedref" / jid / "rollouts" / "00"
ws = rd / "workspace"
ws.mkdir(parents=True, exist_ok=True)
(rd / "stages").mkdir(exist_ok=True)
(rd.parent.parent / "job.json").write_text(json.dumps(job))
hd = rd / "stages" / "ref"
hd.mkdir(exist_ok=True)
ctx = SimpleNamespace(stage_root=rd / "stages", job=job, job_dir=rd.parent.parent, extra={})
result = salty_dev.execute(ctx, hd, {"command": command(job, ws), "cwd": "."},
time.monotonic() + DEADLINE_S, hd / "out.log", hd / "err.log")
info = (ws / "ref_info.log").read_text(errors="ignore") if (ws / "ref_info.log").exists() else ""
perf = _parse_static_perf(info)
if result.state != "OK" or "latency_ms_est" not in perf:
(rd.parent.parent / "FAILED").write_text((hd / "out.log").read_text(errors="ignore")[-4000:]) # not retried
return f"{jid}\tFAIL {result.state} {result.exit_code}\t{(hd / 'out.log').read_text(errors='ignore')[-300:]!r}"
sha = (ws / "ref_hbm.sha256").read_text().split()[0] if (ws / "ref_hbm.sha256").exists() else None
ref = {"schema": "bench.speed_ref/1", "job_id": jid, "march": job["march"], "qcfg": job["qcfg"],
"config": "platform default (compile.yaml with no overrides)", **perf, "hbm_sha256": sha,
"measured": "hb_model_info static estimate, not a board run", "computed_at": time.time()}
OUT.mkdir(exist_ok=True)
tmp = OUT / f".{jid}.json.tmp"
tmp.write_text(json.dumps(ref, indent=2))
tmp.replace(OUT / f"{jid}.json")
return f"{jid}\tOK\t{perf['latency_ms_est']} ms"
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("runs", nargs="+")
ap.add_argument("--workers", type=int, default=3)
ap.add_argument("--cpu", default="6")
ap.add_argument("--only", action="append", help="job ids")
ap.add_argument("--active", action="store_true", help="only jobs with a rollout under agent v18+")
ap.add_argument("--skip", action="append", default=[], help="job ids (e.g. one another run is doing)")
a = ap.parse_args()
cfg = paths.config()
cfg["execution"]["salty_remote"].setdefault("cpu_by_stage", {})["terminal"] = int(a.cpu)
cfg["execution"]["salty_remote"].setdefault("memory_by_stage", {})["terminal"] = "32Gi"
jobs: dict[str, list[Path]] = {}
started: dict[str, int] = {}
for run in a.runs:
for jd in sorted((paths.BENCH_ROOT / "runs" / run).iterdir()):
if (jd / "job.json").exists() and (not a.only or jd.name in a.only) and (
not a.active or any(own_inputs_required(r) for r in (jd / "rollouts").glob("*"))):
jobs.setdefault(jd.name, []).append(jd)
started[jd.name] = started.get(jd.name, 0) + len(list((jd / "rollouts").glob("*")))
failed = paths.BENCH_ROOT / "runs" / "zz-speedref"
todo = sorted((j for j in jobs if not (OUT / f"{j}.json").exists() and j not in a.skip
and not (failed / j / "FAILED").exists()),
key=lambda j: started.get(j, 0)) # untouched jobs first: new rollouts go there
print(f"{len(todo)} jobs to do of {len(jobs)}", flush=True)
def safe(jid):
try:
return one(jobs[jid][0]) # same job id = same model, march and qcfg in every run
except Exception as exc:
return f"{jid}\tERROR {type(exc).__name__}: {exc}\t{traceback.format_exc(limit=2)[-300:]!r}"
with ThreadPoolExecutor(a.workers) as pool:
for line in pool.map(safe, todo):
print(time.strftime("%F %T"), line, flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())