data-gen-bpu/scripts/board_refs.py
bench ec047480ba gates: G1 fails only when no conv/matmul runs on the BPU; G4 judged against board-CPU FP32 only
The agent decides what goes on the BPU and what stays on the CPU. Drop the
1.5x budget built from a platform int8 reference: that reference was an answer.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 14:40:22 +08:00

161 lines
8.5 KiB
Python

"""Board speed references per job, merged into agent/speed_ref/<job_id>.json; registry.gate_snapshot freezes them into
each new batch's G4 (the model must beat cpu_fp32_latency_ms; board_latency_ms is recorded only, not a budget).
- board_latency_ms: median board latency (07-board, latency_ms_mean) of this job's plain int8 builds in any run:
compile.yaml without int16/float qtypes or run_on_cpu. Rollouts whose compile.yaml was offloaded are not seen.
- cpu_fp32_latency_ms: the platform's FP32 export (03-export cache, fetched from Salty storage) run with onnxruntime on
the board's own CPU, all cores, under the board lease so no stage shares the board. onnxruntime lives in
/userdata/bench-ort on the board (installed 10-24 with pip --target; the system python is untouched).
BENCH_CONFIG=<a run config> python3 scripts/board_refs.py RUN [--only JOB_ID ...] [--no-cpu]
"""
from __future__ import annotations
import argparse
import json
import os
import re
import shlex
import sqlite3
import statistics
import subprocess
import sys
import tempfile
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from harness import paths, scheduler # noqa: E402
from harness.stages.s03_export import onnx_path # noqa: E402
from harness.tools.board_perf import ssh_base # noqa: E402
ORT_DIR = os.environ.get("BENCH_ORT_DIR", "/userdata/bench-ort")
# Boards have no internet: onnxruntime wheels for each board python (cp310 S100P, cp312 S600) are kept here and
# installed on first use, so a board that comes back or is added gets it without anyone remembering.
ORT_WHEELS = Path(os.environ.get("BENCH_ORT_WHEELS", "/mnt/data/ming01.wang/configs/board-ort"))
CPU_BENCH = r'''
import json, os, sys, time
import numpy as np
import onnxruntime as ort
o = ort.SessionOptions(); o.intra_op_num_threads = os.cpu_count()
s = ort.InferenceSession(sys.argv[1], o, providers=["CPUExecutionProvider"])
feed = {i.name: np.random.rand(*[d if isinstance(d, int) else 1 for d in i.shape]).astype(np.float32) for i in s.get_inputs()}
s.run(None, feed) # warm-up
t, start = [], time.time()
while len(t) < 10 and (len(t) < 3 or time.time() - start < 120):
a = time.perf_counter(); s.run(None, feed); t.append((time.perf_counter() - a) * 1000)
print(json.dumps({"latency_ms_median": sorted(t)[len(t) // 2], "runs": len(t), "threads": os.cpu_count()}))
'''
def ensure_ort(opts: list[str], target: str, board: str) -> None:
"""onnxruntime importable from ORT_DIR on this board, installing the matching kept wheel if needed."""
probe = (f"PYTHONPATH={ORT_DIR} python3 -c 'import onnxruntime' 2>/dev/null && echo ok || "
"python3 -c 'import sys; print(\"cp%d%d\" % sys.version_info[:2])'")
tag = subprocess.run(["ssh", *opts, target, probe], check=True, capture_output=True, text=True, timeout=120).stdout.strip()
if tag == "ok":
return
wheels = sorted(ORT_WHEELS.glob(f"onnxruntime-*-{tag}-{tag}-*aarch64*.whl"))
if not wheels:
raise ValueError(f"{board}: no onnxruntime wheel for {tag} in {ORT_WHEELS} (pip download --no-deps --only-binary=:all: "
f"--platform manylinux_2_28_aarch64 --python-version {tag[2]}.{tag[3:]} onnxruntime)")
scp = [o if o != "-p" else "-P" for o in opts]
subprocess.run(["scp", *scp, str(wheels[-1]), f"{target}:/tmp/"], check=True, timeout=1800)
subprocess.run(["ssh", *opts, target, f"python3 -m pip install -q --no-deps --no-index --target {ORT_DIR} /tmp/{wheels[-1].name}; "
f"rm -f /tmp/{wheels[-1].name}; PYTHONPATH={ORT_DIR} python3 -c 'import onnxruntime'"],
check=True, capture_output=True, timeout=1800)
print(f"installed {wheels[-1].name} on {board} into {ORT_DIR}", file=sys.stderr)
def plain_int8(yaml_text: str | None) -> bool:
if not yaml_text:
return False
body = re.sub(r"model_output_type:.*", "", yaml_text)
return not re.search(r"int16|float", body) and "run_on_cpu" not in body
def board_latency(job_id: str) -> tuple[float | None, int]:
db = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True, timeout=60)
roots = [paths.RUNS, paths.BENCH_ROOT.parent / "bench" / "runs", paths.BENCH_ROOT.parent / "bench-staging" / "runs"]
lat = []
for run, kk, verdict in db.execute("SELECT run_id, kk, verdict FROM rollouts WHERE job_id=? AND verdict IS NOT NULL", (job_id,)):
value = ((json.loads(verdict).get("gates") or {}).get("G4") or {}).get("latency_ms_mean")
yaml_text = next((p.read_text() for r in dict.fromkeys(roots)
if (p := r / run / job_id / "rollouts" / f"{kk:02d}" / "stages/04-calibrate/compile.yaml").exists()), None)
if value and plain_int8(yaml_text):
lat.append(value)
return (round(statistics.median(lat), 3), len(lat)) if lat else (None, 0)
def cpu_latency(job: dict) -> dict:
onnx = onnx_path(job)
if onnx is None:
raise ValueError("no platform export for this model")
remote = "personal:/bench/cache/" + onnx.relative_to(paths.CACHE).as_posix()
salty = paths.config()["execution"]["salty_remote"]["salty"]
with tempfile.TemporaryDirectory() as tmp:
local = Path(tmp) / "model.onnx"
subprocess.run([salty, "storage", "cp", remote, str(local)], check=True, capture_output=True, timeout=1800)
(Path(tmp) / "cpu_bench.py").write_text(CPU_BENCH)
with scheduler._acquire_board(job["march"], 5.0, "normal") as (slot, board):
if board is None:
raise ValueError(f"no enabled board for {job['march']}")
env = {k: os.environ.get(board[f"{k.split('_', 1)[1].lower()}_env"], "")
for k in ("BOARD_HOST", "BOARD_PORT", "BOARD_USER")}
env["BOARD_SSH_KEY"] = os.environ.get(board["ssh_key_env"], "")
saved = {k: os.environ.get(k) for k in env}
os.environ.update(env) # ssh_base reads BOARD_*; registry validation reads them too, so restore
try:
opts, target = ssh_base()
finally:
for k, v in saved.items():
os.environ.pop(k, None) if v is None else os.environ.__setitem__(k, v)
ensure_ort(opts, target, board.get("name"))
work = f"{ORT_DIR}/run-{job['job_id']}"
subprocess.run(["ssh", *opts, target, f"mkdir -p {shlex.quote(work)}"], check=True, timeout=120)
subprocess.run(["scp", *[o if o != "-p" else "-P" for o in opts],
str(local), str(Path(tmp) / "cpu_bench.py"), f"{target}:{work}/"], check=True, timeout=3600)
try:
out = subprocess.run(["ssh", *opts, target, f"cd {shlex.quote(work)} && PYTHONPATH={ORT_DIR} python3 cpu_bench.py model.onnx"],
check=True, capture_output=True, text=True, timeout=3600).stdout
finally:
subprocess.run(["ssh", *opts, target, f"rm -rf {shlex.quote(work)}"], timeout=120)
return {**json.loads(out.strip().splitlines()[-1]), "board": board.get("name")}
def merge(job_id: str, fields: dict) -> None:
out = paths.AGENT / "speed_ref" / f"{job_id}.json"
ref = json.loads(out.read_text()) if out.exists() else {"schema": "bench.speed_ref/1", "job_id": job_id}
ref.update(fields)
tmp = out.with_name(f".{out.name}.tmp")
tmp.write_text(json.dumps(ref, indent=2, ensure_ascii=False))
tmp.replace(out)
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("run")
ap.add_argument("--only", action="append")
ap.add_argument("--no-cpu", action="store_true")
a = ap.parse_args()
for jd in sorted((paths.RUNS / a.run).iterdir()):
job = json.loads((jd / "job.json").read_text())
if a.only and job["job_id"] not in a.only:
continue
lat, n = board_latency(job["job_id"])
fields = {"board_latency_ms": lat, "board_latency_n": n,
"board_latency_rule": "median 07-board latency of plain int8 builds", "board_refs_at": time.time()}
if not a.no_cpu:
try:
cpu = cpu_latency(job)
fields.update(cpu_fp32_latency_ms=round(cpu["latency_ms_median"], 3), cpu_fp32_measured=cpu)
except Exception as e: # one job's missing export or board hiccup must not stop the others
fields["cpu_fp32_error"] = f"{type(e).__name__}: {e}"[:300]
merge(job["job_id"], fields)
print(job["job_id"], json.dumps({k: v for k, v in fields.items() if k != "board_refs_at"}, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())