The agent decides what goes on the BPU and what stays on the CPU. Drop the 1.5x budget built from a platform int8 reference: that reference was an answer. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
161 lines
8.5 KiB
Python
161 lines
8.5 KiB
Python
"""Board speed references per job, merged into agent/speed_ref/<job_id>.json; registry.gate_snapshot freezes them into
|
|
each new batch's G4 (the model must beat cpu_fp32_latency_ms; board_latency_ms is recorded only, not a budget).
|
|
|
|
- board_latency_ms: median board latency (07-board, latency_ms_mean) of this job's plain int8 builds in any run:
|
|
compile.yaml without int16/float qtypes or run_on_cpu. Rollouts whose compile.yaml was offloaded are not seen.
|
|
- cpu_fp32_latency_ms: the platform's FP32 export (03-export cache, fetched from Salty storage) run with onnxruntime on
|
|
the board's own CPU, all cores, under the board lease so no stage shares the board. onnxruntime lives in
|
|
/userdata/bench-ort on the board (installed 10-24 with pip --target; the system python is untouched).
|
|
|
|
BENCH_CONFIG=<a run config> python3 scripts/board_refs.py RUN [--only JOB_ID ...] [--no-cpu]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import sqlite3
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
from harness import paths, scheduler # noqa: E402
|
|
from harness.stages.s03_export import onnx_path # noqa: E402
|
|
from harness.tools.board_perf import ssh_base # noqa: E402
|
|
|
|
ORT_DIR = os.environ.get("BENCH_ORT_DIR", "/userdata/bench-ort")
|
|
# Boards have no internet: onnxruntime wheels for each board python (cp310 S100P, cp312 S600) are kept here and
|
|
# installed on first use, so a board that comes back or is added gets it without anyone remembering.
|
|
ORT_WHEELS = Path(os.environ.get("BENCH_ORT_WHEELS", "/mnt/data/ming01.wang/configs/board-ort"))
|
|
CPU_BENCH = r'''
|
|
import json, os, sys, time
|
|
import numpy as np
|
|
import onnxruntime as ort
|
|
o = ort.SessionOptions(); o.intra_op_num_threads = os.cpu_count()
|
|
s = ort.InferenceSession(sys.argv[1], o, providers=["CPUExecutionProvider"])
|
|
feed = {i.name: np.random.rand(*[d if isinstance(d, int) else 1 for d in i.shape]).astype(np.float32) for i in s.get_inputs()}
|
|
s.run(None, feed) # warm-up
|
|
t, start = [], time.time()
|
|
while len(t) < 10 and (len(t) < 3 or time.time() - start < 120):
|
|
a = time.perf_counter(); s.run(None, feed); t.append((time.perf_counter() - a) * 1000)
|
|
print(json.dumps({"latency_ms_median": sorted(t)[len(t) // 2], "runs": len(t), "threads": os.cpu_count()}))
|
|
'''
|
|
|
|
|
|
def ensure_ort(opts: list[str], target: str, board: str) -> None:
|
|
"""onnxruntime importable from ORT_DIR on this board, installing the matching kept wheel if needed."""
|
|
probe = (f"PYTHONPATH={ORT_DIR} python3 -c 'import onnxruntime' 2>/dev/null && echo ok || "
|
|
"python3 -c 'import sys; print(\"cp%d%d\" % sys.version_info[:2])'")
|
|
tag = subprocess.run(["ssh", *opts, target, probe], check=True, capture_output=True, text=True, timeout=120).stdout.strip()
|
|
if tag == "ok":
|
|
return
|
|
wheels = sorted(ORT_WHEELS.glob(f"onnxruntime-*-{tag}-{tag}-*aarch64*.whl"))
|
|
if not wheels:
|
|
raise ValueError(f"{board}: no onnxruntime wheel for {tag} in {ORT_WHEELS} (pip download --no-deps --only-binary=:all: "
|
|
f"--platform manylinux_2_28_aarch64 --python-version {tag[2]}.{tag[3:]} onnxruntime)")
|
|
scp = [o if o != "-p" else "-P" for o in opts]
|
|
subprocess.run(["scp", *scp, str(wheels[-1]), f"{target}:/tmp/"], check=True, timeout=1800)
|
|
subprocess.run(["ssh", *opts, target, f"python3 -m pip install -q --no-deps --no-index --target {ORT_DIR} /tmp/{wheels[-1].name}; "
|
|
f"rm -f /tmp/{wheels[-1].name}; PYTHONPATH={ORT_DIR} python3 -c 'import onnxruntime'"],
|
|
check=True, capture_output=True, timeout=1800)
|
|
print(f"installed {wheels[-1].name} on {board} into {ORT_DIR}", file=sys.stderr)
|
|
|
|
|
|
def plain_int8(yaml_text: str | None) -> bool:
|
|
if not yaml_text:
|
|
return False
|
|
body = re.sub(r"model_output_type:.*", "", yaml_text)
|
|
return not re.search(r"int16|float", body) and "run_on_cpu" not in body
|
|
|
|
|
|
def board_latency(job_id: str) -> tuple[float | None, int]:
|
|
db = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True, timeout=60)
|
|
roots = [paths.RUNS, paths.BENCH_ROOT.parent / "bench" / "runs", paths.BENCH_ROOT.parent / "bench-staging" / "runs"]
|
|
lat = []
|
|
for run, kk, verdict in db.execute("SELECT run_id, kk, verdict FROM rollouts WHERE job_id=? AND verdict IS NOT NULL", (job_id,)):
|
|
value = ((json.loads(verdict).get("gates") or {}).get("G4") or {}).get("latency_ms_mean")
|
|
yaml_text = next((p.read_text() for r in dict.fromkeys(roots)
|
|
if (p := r / run / job_id / "rollouts" / f"{kk:02d}" / "stages/04-calibrate/compile.yaml").exists()), None)
|
|
if value and plain_int8(yaml_text):
|
|
lat.append(value)
|
|
return (round(statistics.median(lat), 3), len(lat)) if lat else (None, 0)
|
|
|
|
|
|
def cpu_latency(job: dict) -> dict:
|
|
onnx = onnx_path(job)
|
|
if onnx is None:
|
|
raise ValueError("no platform export for this model")
|
|
remote = "personal:/bench/cache/" + onnx.relative_to(paths.CACHE).as_posix()
|
|
salty = paths.config()["execution"]["salty_remote"]["salty"]
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
local = Path(tmp) / "model.onnx"
|
|
subprocess.run([salty, "storage", "cp", remote, str(local)], check=True, capture_output=True, timeout=1800)
|
|
(Path(tmp) / "cpu_bench.py").write_text(CPU_BENCH)
|
|
with scheduler._acquire_board(job["march"], 5.0, "normal") as (slot, board):
|
|
if board is None:
|
|
raise ValueError(f"no enabled board for {job['march']}")
|
|
env = {k: os.environ.get(board[f"{k.split('_', 1)[1].lower()}_env"], "")
|
|
for k in ("BOARD_HOST", "BOARD_PORT", "BOARD_USER")}
|
|
env["BOARD_SSH_KEY"] = os.environ.get(board["ssh_key_env"], "")
|
|
saved = {k: os.environ.get(k) for k in env}
|
|
os.environ.update(env) # ssh_base reads BOARD_*; registry validation reads them too, so restore
|
|
try:
|
|
opts, target = ssh_base()
|
|
finally:
|
|
for k, v in saved.items():
|
|
os.environ.pop(k, None) if v is None else os.environ.__setitem__(k, v)
|
|
ensure_ort(opts, target, board.get("name"))
|
|
work = f"{ORT_DIR}/run-{job['job_id']}"
|
|
subprocess.run(["ssh", *opts, target, f"mkdir -p {shlex.quote(work)}"], check=True, timeout=120)
|
|
subprocess.run(["scp", *[o if o != "-p" else "-P" for o in opts],
|
|
str(local), str(Path(tmp) / "cpu_bench.py"), f"{target}:{work}/"], check=True, timeout=3600)
|
|
try:
|
|
out = subprocess.run(["ssh", *opts, target, f"cd {shlex.quote(work)} && PYTHONPATH={ORT_DIR} python3 cpu_bench.py model.onnx"],
|
|
check=True, capture_output=True, text=True, timeout=3600).stdout
|
|
finally:
|
|
subprocess.run(["ssh", *opts, target, f"rm -rf {shlex.quote(work)}"], timeout=120)
|
|
return {**json.loads(out.strip().splitlines()[-1]), "board": board.get("name")}
|
|
|
|
|
|
def merge(job_id: str, fields: dict) -> None:
|
|
out = paths.AGENT / "speed_ref" / f"{job_id}.json"
|
|
ref = json.loads(out.read_text()) if out.exists() else {"schema": "bench.speed_ref/1", "job_id": job_id}
|
|
ref.update(fields)
|
|
tmp = out.with_name(f".{out.name}.tmp")
|
|
tmp.write_text(json.dumps(ref, indent=2, ensure_ascii=False))
|
|
tmp.replace(out)
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("run")
|
|
ap.add_argument("--only", action="append")
|
|
ap.add_argument("--no-cpu", action="store_true")
|
|
a = ap.parse_args()
|
|
for jd in sorted((paths.RUNS / a.run).iterdir()):
|
|
job = json.loads((jd / "job.json").read_text())
|
|
if a.only and job["job_id"] not in a.only:
|
|
continue
|
|
lat, n = board_latency(job["job_id"])
|
|
fields = {"board_latency_ms": lat, "board_latency_n": n,
|
|
"board_latency_rule": "median 07-board latency of plain int8 builds", "board_refs_at": time.time()}
|
|
if not a.no_cpu:
|
|
try:
|
|
cpu = cpu_latency(job)
|
|
fields.update(cpu_fp32_latency_ms=round(cpu["latency_ms_median"], 3), cpu_fp32_measured=cpu)
|
|
except Exception as e: # one job's missing export or board hiccup must not stop the others
|
|
fields["cpu_fp32_error"] = f"{type(e).__name__}: {e}"[:300]
|
|
merge(job["job_id"], fields)
|
|
print(job["job_id"], json.dumps({k: v for k, v in fields.items() if k != "board_refs_at"}, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|