"""Board speed references per job, merged into agent/speed_ref/.json; registry.gate_snapshot freezes them into each new batch's G4 (budget = 1.5 x board_latency_ms; the model must also beat cpu_fp32_latency_ms). - board_latency_ms: median board latency (07-board, latency_ms_mean) of this job's plain int8 builds in any run: compile.yaml without int16/float qtypes or run_on_cpu. Rollouts whose compile.yaml was offloaded are not seen. - cpu_fp32_latency_ms: the platform's FP32 export (03-export cache, fetched from Salty storage) run with onnxruntime on the board's own CPU, all cores, under the board lease so no stage shares the board. onnxruntime lives in /userdata/bench-ort on the board (installed 10-24 with pip --target; the system python is untouched). BENCH_CONFIG= python3 scripts/board_refs.py RUN [--only JOB_ID ...] [--no-cpu] """ from __future__ import annotations import argparse import json import os import re import shlex import sqlite3 import statistics import subprocess import sys import tempfile import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from harness import paths, scheduler # noqa: E402 from harness.stages.s03_export import onnx_path # noqa: E402 from harness.tools.board_perf import ssh_base # noqa: E402 ORT_DIR = "/userdata/bench-ort" CPU_BENCH = r''' import json, os, sys, time import numpy as np import onnxruntime as ort o = ort.SessionOptions(); o.intra_op_num_threads = os.cpu_count() s = ort.InferenceSession(sys.argv[1], o, providers=["CPUExecutionProvider"]) feed = {i.name: np.random.rand(*[d if isinstance(d, int) else 1 for d in i.shape]).astype(np.float32) for i in s.get_inputs()} s.run(None, feed) # warm-up t, start = [], time.time() while len(t) < 10 and (len(t) < 3 or time.time() - start < 120): a = time.perf_counter(); s.run(None, feed); t.append((time.perf_counter() - a) * 1000) print(json.dumps({"latency_ms_median": sorted(t)[len(t) // 2], "runs": len(t), "threads": os.cpu_count()})) ''' def plain_int8(yaml_text: str | None) -> bool: if not yaml_text: return False body = re.sub(r"model_output_type:.*", "", yaml_text) return not re.search(r"int16|float", body) and "run_on_cpu" not in body def board_latency(job_id: str) -> tuple[float | None, int]: db = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True, timeout=60) roots = [paths.RUNS, paths.BENCH_ROOT.parent / "bench" / "runs", paths.BENCH_ROOT.parent / "bench-staging" / "runs"] lat = [] for run, kk, verdict in db.execute("SELECT run_id, kk, verdict FROM rollouts WHERE job_id=? AND verdict IS NOT NULL", (job_id,)): value = ((json.loads(verdict).get("gates") or {}).get("G4") or {}).get("latency_ms_mean") yaml_text = next((p.read_text() for r in dict.fromkeys(roots) if (p := r / run / job_id / "rollouts" / f"{kk:02d}" / "stages/04-calibrate/compile.yaml").exists()), None) if value and plain_int8(yaml_text): lat.append(value) return (round(statistics.median(lat), 3), len(lat)) if lat else (None, 0) def cpu_latency(job: dict) -> dict: onnx = onnx_path(job) if onnx is None: raise ValueError("no platform export for this model") remote = "personal:/bench/cache/" + onnx.relative_to(paths.CACHE).as_posix() salty = paths.config()["execution"]["salty_remote"]["salty"] with tempfile.TemporaryDirectory() as tmp: local = Path(tmp) / "model.onnx" subprocess.run([salty, "storage", "cp", remote, str(local)], check=True, capture_output=True, timeout=1800) (Path(tmp) / "cpu_bench.py").write_text(CPU_BENCH) with scheduler._acquire_board(job["march"], 5.0, "normal") as (slot, board): if board is None: raise ValueError(f"no enabled board for {job['march']}") env = {k: os.environ.get(board[f"{k.split('_', 1)[1].lower()}_env"], "") for k in ("BOARD_HOST", "BOARD_PORT", "BOARD_USER")} env["BOARD_SSH_KEY"] = os.environ.get(board["ssh_key_env"], "") os.environ.update(env) opts, target = ssh_base() work = f"{ORT_DIR}/run-{job['job_id']}" subprocess.run(["ssh", *opts, target, f"mkdir -p {shlex.quote(work)}"], check=True, timeout=120) subprocess.run(["scp", *[o.replace("-p", "-P") if o == "-p" else o for o in opts], str(local), str(Path(tmp) / "cpu_bench.py"), f"{target}:{work}/"], check=True, timeout=3600) try: out = subprocess.run(["ssh", *opts, target, f"cd {shlex.quote(work)} && PYTHONPATH={ORT_DIR} python3 cpu_bench.py model.onnx"], check=True, capture_output=True, text=True, timeout=3600).stdout finally: subprocess.run(["ssh", *opts, target, f"rm -rf {shlex.quote(work)}"], timeout=120) return {**json.loads(out.strip().splitlines()[-1]), "board": board.get("name")} def merge(job_id: str, fields: dict) -> None: out = paths.AGENT / "speed_ref" / f"{job_id}.json" ref = json.loads(out.read_text()) if out.exists() else {"schema": "bench.speed_ref/1", "job_id": job_id} ref.update(fields) tmp = out.with_name(f".{out.name}.tmp") tmp.write_text(json.dumps(ref, indent=2, ensure_ascii=False)) tmp.replace(out) def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("run") ap.add_argument("--only", action="append") ap.add_argument("--no-cpu", action="store_true") a = ap.parse_args() for jd in sorted((paths.RUNS / a.run).iterdir()): job = json.loads((jd / "job.json").read_text()) if a.only and job["job_id"] not in a.only: continue lat, n = board_latency(job["job_id"]) fields = {"board_latency_ms": lat, "board_latency_n": n, "board_latency_rule": "median 07-board latency of plain int8 builds", "board_refs_at": time.time()} if not a.no_cpu: try: cpu = cpu_latency(job) fields.update(cpu_fp32_latency_ms=round(cpu["latency_ms_median"], 3), cpu_fp32_measured=cpu) except Exception as e: # one job's missing export or board hiccup must not stop the others fields["cpu_fp32_error"] = f"{type(e).__name__}: {e}"[:300] merge(job["job_id"], fields) print(job["job_id"], json.dumps({k: v for k, v in fields.items() if k != "board_refs_at"}, ensure_ascii=False)) return 0 if __name__ == "__main__": sys.exit(main())