"""Speed reference per job: compile the platform's default configuration once (in a bench dev container, outside any rollout) and write the compiler's latency estimate to agent/speed_ref/.json: the agent's jail shows bench/agent read-only and live (not the job directory), so a reference computed after a rollout started still reaches it. The agent compares its own model against it (SYSTEM.md 速度要求) instead of compiling the default itself (10-15: 9 of 32 v17 rollouts spent ~4 h each doing that). Jobs that already have the file are skipped. BENCH_CONFIG= python scripts/speed_ref.py r20260929-04 r20260929-05 r20260929-06 [--active --follow] [--workers 3] --active: only jobs a v18+ rollout is on. Salty is shared with the live rollouts (10-15 20:40: their own dev jobs queued once 8 of these were added), so references follow demand instead of compiling all 418 up front. """ from __future__ import annotations import argparse import json import shlex import sys import time import traceback from concurrent.futures import ThreadPoolExecutor from pathlib import Path from types import SimpleNamespace import yaml sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from harness import paths, salty_dev # noqa: E402 from harness.backends import ptq # noqa: E402 from harness.stages.s01_fetch import weights_dir # noqa: E402 from harness.stages.s03_export import onnx_path # noqa: E402 from harness.stages.s06_hbm import _parse_static_perf # noqa: E402 from harness.stagerun import own_inputs_required # noqa: E402 DEADLINE_S = 10 * 3600 def command(job: dict, ws: Path) -> str: onnx = onnx_path(job) weights = weights_dir(job["model"]) # its files are on personal storage, listed in the container if onnx is None or weights is None: raise ValueError("weights not fetched yet") mine = ws / "model.onnx" calib = Path(job["data"]["calib"]["path"]) / "npy_f32" cfg = ptq.compile_config(job, Path("model.onnx"), calib, Path("ref"), None) (ws / "ref.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False)) export = shlex.join(ptq.export_cmd(job, mine, Path("WEIGHTS"))).replace("WEIGHTS", '"$W"') export = f"W=$(ls -d {shlex.quote(str(weights))}/* | grep -v /meta.json | head -1); {export}" return (f"set -e; if [ -f {shlex.quote(str(onnx))} ]; then ln -sfn {shlex.quote(str(onnx))} model.onnx; " f"else {export} > export.log 2>&1; fi; " "hb_compile -c ref.yaml > ref_compile.log 2>&1 || { tail -40 ref_compile.log; exit 1; }; " "hb_model_info ref/*.hbm > ref_info.log 2>&1; " # the dev PATH's OE toolchain, as the agent uses it "sha256sum ref/*.hbm > ref_hbm.sha256; tail -20 ref_info.log; rm -rf ref model.onnx") # keep personal storage small OUT = paths.AGENT / "speed_ref" def one(job_dir: Path) -> str: job = json.loads((job_dir / "job.json").read_text()) jid = job["job_id"] rd = paths.BENCH_ROOT / "runs" / "zz-speedref" / jid / "rollouts" / "00" ws = rd / "workspace" ws.mkdir(parents=True, exist_ok=True) (rd / "stages").mkdir(exist_ok=True) (rd.parent.parent / "job.json").write_text(json.dumps(job)) hd = rd / "stages" / "ref" if hd.exists(): # Left by a process that died mid-compile (salty_dev creates bundle.tar exclusively): every retry failed with # FileExistsError and the 3 workers spun on those jobs (10-21 reboot). Kept aside: salty/remote.json names its job. hd.rename(hd.with_name(f"ref.interrupted-{int(time.time())}")) hd.mkdir() ctx = SimpleNamespace(stage_root=rd / "stages", job=job, job_dir=rd.parent.parent, extra={}) result = salty_dev.execute(ctx, hd, {"command": command(job, ws), "cwd": "."}, time.monotonic() + DEADLINE_S, hd / "out.log", hd / "err.log") info = (ws / "ref_info.log").read_text(errors="ignore") if (ws / "ref_info.log").exists() else "" perf = _parse_static_perf(info) if result.state != "OK" or "latency_ms_est" not in perf: (rd.parent.parent / "FAILED").write_text((hd / "out.log").read_text(errors="ignore")[-4000:]) # not retried unavailable(job, f"{DEADLINE_S // 3600} 小时内没编完" if result.exit_code == 124 else f"编译失败,退出码 {result.exit_code}") return f"{jid}\tFAIL {result.state} {result.exit_code}\t{(hd / 'out.log').read_text(errors='ignore')[-300:]!r}" sha = (ws / "ref_hbm.sha256").read_text().split()[0] if (ws / "ref_hbm.sha256").exists() else None ref = {"schema": "bench.speed_ref/1", "job_id": jid, "march": job["march"], "qcfg": job["qcfg"], "config": "platform default (compile.yaml with no overrides)", **perf, "hbm_sha256": sha, "measured": "hb_model_info static estimate, not a board run", "computed_at": time.time()} publish(jid, ref) return f"{jid}\tOK\t{perf['latency_ms_est']} ms" def publish(jid: str, ref: dict) -> None: OUT.mkdir(exist_ok=True) tmp = OUT / f".{jid}.json.tmp" tmp.write_text(json.dumps(ref, indent=2, ensure_ascii=False)) tmp.replace(OUT / f"{jid}.json") def unavailable(job: dict, why: str) -> None: """The default configuration did not compile here (10-16: big S600 depth models, >10 h on 6 cores): say so, so agents stop waiting for the file or compiling the default themselves.""" publish(job["job_id"], {"schema": "bench.speed_ref/1", "job_id": job["job_id"], "march": job["march"], "qcfg": job["qcfg"], "latency_ms_est": None, "status": "unavailable", "note": "平台默认配置在这道题上没编出来(" + why + "),本题没有速度参考值:不用再等,也不要自己编默认配置;" "只要别整模型提到 int16/fp16、别让大段算子落到 CPU,并用 board 阶段实测确认。", "computed_at": time.time()}) def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("runs", nargs="+") ap.add_argument("--workers", type=int, default=3) ap.add_argument("--cpu", default="6") ap.add_argument("--only", action="append", help="job ids") ap.add_argument("--active", action="store_true", help="only jobs with a rollout under agent v18+") ap.add_argument("--skip", action="append", default=[], help="job ids (e.g. one another run is doing)") ap.add_argument("--follow", action="store_true", help="keep scanning every 2 min; at most --workers at a time") a = ap.parse_args() cfg = paths.config() cfg["execution"]["salty_remote"].setdefault("cpu_by_stage", {})["terminal"] = int(a.cpu) cfg["execution"]["salty_remote"].setdefault("memory_by_stage", {})["terminal"] = "32Gi" failed = paths.BENCH_ROOT / "runs" / "zz-speedref" def pending() -> tuple[list[str], dict[str, Path]]: jobs: dict[str, Path] = {} started: dict[str, int] = {} for run in a.runs: for jd in sorted((paths.BENCH_ROOT / "runs" / run).iterdir()): if (jd / "job.json").exists() and (not a.only or jd.name in a.only) and ( not a.active or any(own_inputs_required(r) for r in (jd / "rollouts").glob("*"))): jobs.setdefault(jd.name, jd) # same job id = same model, march and qcfg in every run started[jd.name] = started.get(jd.name, 0) + len(list((jd / "rollouts").glob("*"))) for j in jobs: # failures recorded before the unavailable file existed (or by an older process) if (failed / j / "FAILED").exists() and not (OUT / f"{j}.json").exists(): unavailable(json.loads((jobs[j] / "job.json").read_text()), "编译失败或超时") todo = sorted((j for j in jobs if not (OUT / f"{j}.json").exists() and j not in a.skip), key=lambda j: started.get(j, 0)) # untouched jobs first: new rollouts go there return todo, jobs def safe(jid, jd): try: line = one(jd) except Exception as exc: line = f"{jid}\tERROR {type(exc).__name__}: {exc}\t{traceback.format_exc(limit=2)[-300:]!r}" print(time.strftime("%F %T"), line, flush=True) running: dict[str, object] = {} with ThreadPoolExecutor(a.workers) as pool: while True: running = {j: f for j, f in running.items() if not f.done()} todo, jobs = pending() for jid in [j for j in todo if j not in running][:a.workers - len(running)]: print(time.strftime("%F %T"), "start", jid, flush=True) running[jid] = pool.submit(safe, jid, jobs[jid]) if not a.follow and not running: break time.sleep(120 if a.follow else 5) # --follow: v18 rollouts keep starting on new jobs return 0 if __name__ == "__main__": sys.exit(main())