data-gen-bpu/scripts/speed_ref.py
bench 31817a71fc speed_ref packs the platform code again; agent bench dev still does not
speed_ref exports the default graph with harness/tools inside bench dev; since 8fb9349 the dev bundle carried only
container_site, so a job without a cached export could not get a speed reference. salty_dev.execute takes an explicit
platform_code flag that only speed_ref passes; the agent's terminal path never sets it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-24 12:34:49 +08:00

161 lines
8.9 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Speed reference per job: compile the platform's default configuration once (in a bench dev container,
outside any rollout) and write the compiler's latency estimate to agent/speed_ref/<job_id>.json: the agent's jail shows bench/agent
read-only and live (not the job directory), so a reference computed after a rollout started still reaches it.
The agent compares its own model against it (SYSTEM.md 速度要求) instead of compiling the default itself
(10-15: 9 of 32 v17 rollouts spent ~4 h each doing that). Jobs that already have the file are skipped.
BENCH_CONFIG=<a run config> python scripts/speed_ref.py r20260929-04 r20260929-05 r20260929-06 [--active --follow] [--workers 3]
--active: only jobs a v18+ rollout is on. Salty is shared with the live rollouts (10-15 20:40: their own dev
jobs queued once 8 of these were added), so references follow demand instead of compiling all 418 up front.
"""
from __future__ import annotations
import argparse
import json
import shlex
import sys
import time
import traceback
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
import yaml
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from harness import paths, salty_dev # noqa: E402
from harness.backends import ptq # noqa: E402
from harness.stages.s01_fetch import weights_dir # noqa: E402
from harness.stages.s03_export import onnx_path # noqa: E402
from harness.stages.s06_hbm import _parse_static_perf # noqa: E402
from harness.stagerun import own_inputs_required # noqa: E402
DEADLINE_S = 10 * 3600
def command(job: dict, ws: Path) -> str:
onnx = onnx_path(job)
weights = weights_dir(job["model"]) # its files are on personal storage, listed in the container
if onnx is None or weights is None:
raise ValueError("weights not fetched yet")
mine = ws / "model.onnx"
calib = Path(job["data"]["calib"]["path"]) / "npy_f32"
cfg = ptq.compile_config(job, Path("model.onnx"), calib, Path("ref"), None)
(ws / "ref.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False))
export = shlex.join(ptq.export_cmd(job, mine, Path("WEIGHTS"))).replace("WEIGHTS", '"$W"')
export = f"W=$(ls -d {shlex.quote(str(weights))}/* | grep -v /meta.json | head -1); {export}"
return (f"set -e; if [ -f {shlex.quote(str(onnx))} ]; then ln -sfn {shlex.quote(str(onnx))} model.onnx; "
f"else {export} > export.log 2>&1; fi; "
"hb_compile -c ref.yaml > ref_compile.log 2>&1 || { tail -40 ref_compile.log; exit 1; }; "
"hb_model_info ref/*.hbm > ref_info.log 2>&1; " # the dev PATH's OE toolchain, as the agent uses it
"sha256sum ref/*.hbm > ref_hbm.sha256; tail -20 ref_info.log; rm -rf ref model.onnx") # keep personal storage small
OUT = paths.AGENT / "speed_ref"
def one(job_dir: Path) -> str:
job = json.loads((job_dir / "job.json").read_text())
jid = job["job_id"]
rd = paths.BENCH_ROOT / "runs" / "zz-speedref" / jid / "rollouts" / "00"
ws = rd / "workspace"
ws.mkdir(parents=True, exist_ok=True)
(rd / "stages").mkdir(exist_ok=True)
(rd.parent.parent / "job.json").write_text(json.dumps(job))
hd = rd / "stages" / "ref"
if hd.exists():
# Left by a process that died mid-compile (salty_dev creates bundle.tar exclusively): every retry failed with
# FileExistsError and the 3 workers spun on those jobs (10-21 reboot). Kept aside: salty/remote.json names its job.
hd.rename(hd.with_name(f"ref.interrupted-{int(time.time())}"))
hd.mkdir()
ctx = SimpleNamespace(stage_root=rd / "stages", job=job, job_dir=rd.parent.parent, extra={})
result = salty_dev.execute(ctx, hd, {"command": command(job, ws), "cwd": "."},
time.monotonic() + DEADLINE_S, hd / "out.log", hd / "err.log", platform_code=True)
info = (ws / "ref_info.log").read_text(errors="ignore") if (ws / "ref_info.log").exists() else ""
perf = _parse_static_perf(info)
if result.state != "OK" or "latency_ms_est" not in perf:
(rd.parent.parent / "FAILED").write_text((hd / "out.log").read_text(errors="ignore")[-4000:]) # not retried
unavailable(job, f"{DEADLINE_S // 3600} 小时内没编完" if result.exit_code == 124 else f"编译失败,退出码 {result.exit_code}")
return f"{jid}\tFAIL {result.state} {result.exit_code}\t{(hd / 'out.log').read_text(errors='ignore')[-300:]!r}"
sha = (ws / "ref_hbm.sha256").read_text().split()[0] if (ws / "ref_hbm.sha256").exists() else None
ref = {"schema": "bench.speed_ref/1", "job_id": jid, "march": job["march"], "qcfg": job["qcfg"],
"config": "platform default (compile.yaml with no overrides)", **perf, "hbm_sha256": sha,
"measured": "hb_model_info static estimate, not a board run", "computed_at": time.time()}
publish(jid, ref)
return f"{jid}\tOK\t{perf['latency_ms_est']} ms"
def publish(jid: str, ref: dict) -> None:
OUT.mkdir(exist_ok=True)
tmp = OUT / f".{jid}.json.tmp"
tmp.write_text(json.dumps(ref, indent=2, ensure_ascii=False))
tmp.replace(OUT / f"{jid}.json")
def unavailable(job: dict, why: str) -> None:
"""The default configuration did not compile here (10-16: big S600 depth models, >10 h on 6 cores): say so,
so agents stop waiting for the file or compiling the default themselves."""
publish(job["job_id"], {"schema": "bench.speed_ref/1", "job_id": job["job_id"], "march": job["march"], "qcfg": job["qcfg"],
"latency_ms_est": None, "status": "unavailable",
"note": "平台默认配置在这道题上没编出来(" + why + "),本题没有速度参考值:不用再等,也不要自己编默认配置;"
"只要别整模型提到 int16/fp16、别让大段算子落到 CPU,并用 board 阶段实测确认。",
"computed_at": time.time()})
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("runs", nargs="+")
ap.add_argument("--workers", type=int, default=3)
ap.add_argument("--cpu", default="6")
ap.add_argument("--only", action="append", help="job ids")
ap.add_argument("--active", action="store_true", help="only jobs with a rollout under agent v18+")
ap.add_argument("--skip", action="append", default=[], help="job ids (e.g. one another run is doing)")
ap.add_argument("--follow", action="store_true", help="keep scanning every 2 min; at most --workers at a time")
a = ap.parse_args()
cfg = paths.config()
cfg["execution"]["salty_remote"].setdefault("cpu_by_stage", {})["terminal"] = int(a.cpu)
cfg["execution"]["salty_remote"].setdefault("memory_by_stage", {})["terminal"] = "32Gi"
failed = paths.BENCH_ROOT / "runs" / "zz-speedref"
def pending() -> tuple[list[str], dict[str, Path]]:
jobs: dict[str, Path] = {}
started: dict[str, int] = {}
for run in a.runs:
for jd in sorted((paths.BENCH_ROOT / "runs" / run).iterdir()):
if (jd / "job.json").exists() and (not a.only or jd.name in a.only) and (
not a.active or any(own_inputs_required(r) for r in (jd / "rollouts").glob("*"))):
jobs.setdefault(jd.name, jd) # same job id = same model, march and qcfg in every run
started[jd.name] = started.get(jd.name, 0) + len(list((jd / "rollouts").glob("*")))
for j in jobs: # failures recorded before the unavailable file existed (or by an older process)
if (failed / j / "FAILED").exists() and not (OUT / f"{j}.json").exists():
unavailable(json.loads((jobs[j] / "job.json").read_text()), "编译失败或超时")
todo = sorted((j for j in jobs if not (OUT / f"{j}.json").exists() and j not in a.skip),
key=lambda j: started.get(j, 0)) # untouched jobs first: new rollouts go there
return todo, jobs
def safe(jid, jd):
try:
line = one(jd)
except Exception as exc:
line = f"{jid}\tERROR {type(exc).__name__}: {exc}\t{traceback.format_exc(limit=2)[-300:]!r}"
print(time.strftime("%F %T"), line, flush=True)
running: dict[str, object] = {}
with ThreadPoolExecutor(a.workers) as pool:
while True:
running = {j: f for j, f in running.items() if not f.done()}
todo, jobs = pending()
for jid in [j for j in todo if j not in running][:a.workers - len(running)]:
print(time.strftime("%F %T"), "start", jid, flush=True)
running[jid] = pool.submit(safe, jid, jobs[jid])
if not a.follow and not running:
break
time.sleep(120 if a.follow else 5) # --follow: v18 rollouts keep starting on new jobs
return 0
if __name__ == "__main__":
sys.exit(main())