data-gen-bpu/scripts/board_refs.py
bench 1dcf331c90 Gates: compute must run on the BPU; board latency budget and the original model on the board CPU as speed floors
- 06-hbm records the convolutions/matmuls hb_compile left on the CPU (its node table's ON column); G1 fails on any
  when the batch's gates say require_bpu_compute (new in gates.yaml; frozen batches without it are not judged)
- batch creation freezes agent/speed_ref/<job>.json into G4: latency_budget_ms = 1.5 x board_latency_ms (median
  board latency of plain int8 builds) and cpu_fp32_latency_ms; G4 also fails when the model is not faster than
  the original FP32 model on the board CPU, and records the speedup
- scripts/board_refs.py computes both; the CPU number is onnxruntime on the board itself under the board lease
- e2e prompt states the three rules; agent/VERSION 25

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-24 00:41:02 +08:00

133 lines
6.5 KiB
Python

"""Board speed references per job, merged into agent/speed_ref/<job_id>.json; registry.gate_snapshot freezes them into
each new batch's G4 (budget = 1.5 x board_latency_ms; the model must also beat cpu_fp32_latency_ms).
- board_latency_ms: median board latency (07-board, latency_ms_mean) of this job's plain int8 builds in any run:
compile.yaml without int16/float qtypes or run_on_cpu. Rollouts whose compile.yaml was offloaded are not seen.
- cpu_fp32_latency_ms: the platform's FP32 export (03-export cache, fetched from Salty storage) run with onnxruntime on
the board's own CPU, all cores, under the board lease so no stage shares the board. onnxruntime lives in
/userdata/bench-ort on the board (installed 10-24 with pip --target; the system python is untouched).
BENCH_CONFIG=<a run config> python3 scripts/board_refs.py RUN [--only JOB_ID ...] [--no-cpu]
"""
from __future__ import annotations
import argparse
import json
import os
import re
import shlex
import sqlite3
import statistics
import subprocess
import sys
import tempfile
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from harness import paths, scheduler # noqa: E402
from harness.stages.s03_export import onnx_path # noqa: E402
from harness.tools.board_perf import ssh_base # noqa: E402
ORT_DIR = "/userdata/bench-ort"
CPU_BENCH = r'''
import json, os, sys, time
import numpy as np
import onnxruntime as ort
o = ort.SessionOptions(); o.intra_op_num_threads = os.cpu_count()
s = ort.InferenceSession(sys.argv[1], o, providers=["CPUExecutionProvider"])
feed = {i.name: np.random.rand(*[d if isinstance(d, int) else 1 for d in i.shape]).astype(np.float32) for i in s.get_inputs()}
s.run(None, feed) # warm-up
t, start = [], time.time()
while len(t) < 10 and (len(t) < 3 or time.time() - start < 120):
a = time.perf_counter(); s.run(None, feed); t.append((time.perf_counter() - a) * 1000)
print(json.dumps({"latency_ms_median": sorted(t)[len(t) // 2], "runs": len(t), "threads": os.cpu_count()}))
'''
def plain_int8(yaml_text: str | None) -> bool:
if not yaml_text:
return False
body = re.sub(r"model_output_type:.*", "", yaml_text)
return not re.search(r"int16|float", body) and "run_on_cpu" not in body
def board_latency(job_id: str) -> tuple[float | None, int]:
db = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True, timeout=60)
roots = [paths.RUNS, paths.BENCH_ROOT.parent / "bench" / "runs", paths.BENCH_ROOT.parent / "bench-staging" / "runs"]
lat = []
for run, kk, verdict in db.execute("SELECT run_id, kk, verdict FROM rollouts WHERE job_id=? AND verdict IS NOT NULL", (job_id,)):
value = ((json.loads(verdict).get("gates") or {}).get("G4") or {}).get("latency_ms_mean")
yaml_text = next((p.read_text() for r in dict.fromkeys(roots)
if (p := r / run / job_id / "rollouts" / f"{kk:02d}" / "stages/04-calibrate/compile.yaml").exists()), None)
if value and plain_int8(yaml_text):
lat.append(value)
return (round(statistics.median(lat), 3), len(lat)) if lat else (None, 0)
def cpu_latency(job: dict) -> dict:
onnx = onnx_path(job)
if onnx is None:
raise ValueError("no platform export for this model")
remote = "personal:/bench/cache/" + onnx.relative_to(paths.CACHE).as_posix()
salty = paths.config()["execution"]["salty_remote"]["salty"]
with tempfile.TemporaryDirectory() as tmp:
local = Path(tmp) / "model.onnx"
subprocess.run([salty, "storage", "cp", remote, str(local)], check=True, capture_output=True, timeout=1800)
(Path(tmp) / "cpu_bench.py").write_text(CPU_BENCH)
with scheduler._acquire_board(job["march"], 5.0, "normal") as (slot, board):
if board is None:
raise ValueError(f"no enabled board for {job['march']}")
env = {k: os.environ.get(board[f"{k.split('_', 1)[1].lower()}_env"], "")
for k in ("BOARD_HOST", "BOARD_PORT", "BOARD_USER")}
env["BOARD_SSH_KEY"] = os.environ.get(board["ssh_key_env"], "")
os.environ.update(env)
opts, target = ssh_base()
work = f"{ORT_DIR}/run-{job['job_id']}"
subprocess.run(["ssh", *opts, target, f"mkdir -p {shlex.quote(work)}"], check=True, timeout=120)
subprocess.run(["scp", *[o.replace("-p", "-P") if o == "-p" else o for o in opts],
str(local), str(Path(tmp) / "cpu_bench.py"), f"{target}:{work}/"], check=True, timeout=3600)
try:
out = subprocess.run(["ssh", *opts, target, f"cd {shlex.quote(work)} && PYTHONPATH={ORT_DIR} python3 cpu_bench.py model.onnx"],
check=True, capture_output=True, text=True, timeout=3600).stdout
finally:
subprocess.run(["ssh", *opts, target, f"rm -rf {shlex.quote(work)}"], timeout=120)
return {**json.loads(out.strip().splitlines()[-1]), "board": board.get("name")}
def merge(job_id: str, fields: dict) -> None:
out = paths.AGENT / "speed_ref" / f"{job_id}.json"
ref = json.loads(out.read_text()) if out.exists() else {"schema": "bench.speed_ref/1", "job_id": job_id}
ref.update(fields)
tmp = out.with_name(f".{out.name}.tmp")
tmp.write_text(json.dumps(ref, indent=2, ensure_ascii=False))
tmp.replace(out)
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("run")
ap.add_argument("--only", action="append")
ap.add_argument("--no-cpu", action="store_true")
a = ap.parse_args()
for jd in sorted((paths.RUNS / a.run).iterdir()):
job = json.loads((jd / "job.json").read_text())
if a.only and job["job_id"] not in a.only:
continue
lat, n = board_latency(job["job_id"])
fields = {"board_latency_ms": lat, "board_latency_n": n,
"board_latency_rule": "median 07-board latency of plain int8 builds", "board_refs_at": time.time()}
if not a.no_cpu:
try:
cpu = cpu_latency(job)
fields.update(cpu_fp32_latency_ms=round(cpu["latency_ms_median"], 3), cpu_fp32_measured=cpu)
except Exception as e: # one job's missing export or board hiccup must not stop the others
fields["cpu_fp32_error"] = f"{type(e).__name__}: {e}"[:300]
merge(job["job_id"], fields)
print(job["job_id"], json.dumps({k: v for k, v in fields.items() if k != "board_refs_at"}, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())