Gates: compute must run on the BPU; board latency budget and the original model on the board CPU as speed floors

- 06-hbm records the convolutions/matmuls hb_compile left on the CPU (its node table's ON column); G1 fails on any
  when the batch's gates say require_bpu_compute (new in gates.yaml; frozen batches without it are not judged)
- batch creation freezes agent/speed_ref/<job>.json into G4: latency_budget_ms = 1.5 x board_latency_ms (median
  board latency of plain int8 builds) and cpu_fp32_latency_ms; G4 also fails when the model is not faster than
  the original FP32 model on the board CPU, and records the speedup
- scripts/board_refs.py computes both; the CPU number is onnxruntime on the board itself under the board lease
- e2e prompt states the three rules; agent/VERSION 25

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bench 2026-10-24 00:41:02 +08:00
parent f6e7dfe56a
commit 1dcf331c90
8 changed files with 240 additions and 9 deletions

View File

@ -1 +1 @@
24
25

View File

@ -7,11 +7,12 @@ job.json:`{job_json}`(只读) 数据:`{data_dir}`(只读) 你的
1. G1 编译:产出 `.hbm`,march = {march},输入 {input_spec},`hb_verifier` 通过
2. G2 数值:校准外样本的逐输出余弦,参考阈值 {g2_threshold},只作诊断
3. G3 任务精度:板上 T1 子集({t1_n} 张)的 {g3_metric} 与同一子集的 FP32 基线比较,容差 {g3_tol}({g3_kind})
4. G4 速度:不比平台默认量化配置(compile.yaml 只写两项必填)编出的模型慢 1.5 倍以上。参考在 `$BENCH_ROOT/agent/speed_ref/{job_id}.json`:`latency_ms_est` 是默认配置的编译器静态估计,不含落到 CPU 的算子;文件还没有表示还在算,值为 null 表示默认配置没编出来、本题没有参考。
4. G4 速度:以 job.json 的 `gates.G4` 为准。`latency_budget_ms` 是本题板上实测延迟的上限(同题普通 int8 配置板上实测 `latency_ref_ms` 的 1.5 倍),`cpu_fp32_latency_ms` 是原始 FP32 模型在本题板卡 CPU 上的延迟,你的模型必须比它快。没有这两个字段的题不判速度。`$BENCH_ROOT/agent/speed_ref/{job_id}.json` 里还有默认配置的编译器静态估计 `latency_ms_est`,可以当量级参考。
两条硬规定,违反任何一条都不算通过:
三条硬规定,违反任何一条都不算通过:
- 本题是 int8 量化。`all_node_type`,或者整类计算算子(Conv、MatMul、Gemm 等;用 op_config,或者逐节点列出超过同类节点的一半)设成 int16、float16 或 float32,都判不通过。只把少数敏感节点设成 int16 可以。
- 板上实测延迟超过同题默认 int8 配置板上实测的 1.5 倍,判不通过(G4 参考文件里是默认配置的编译器静态估计,可以当量级参考)。
- 卷积、矩阵乘(Conv、ConvTranspose、MatMul、Gemm)必须跑在 BPU 上:hb_compile 节点表 `ON` 一列里有这类节点是 `CPU`,G1 判不通过。
- 板上实测延迟超过 `latency_budget_ms`,或者不比 `cpu_fp32_latency_ms` 快,G4 判不通过。
缺少板上精度结果不能报完整 PASS;缺权重、工具或材料时如实保留实际错误。

View File

@ -53,9 +53,13 @@ def g1(m: dict | None, cfg: dict) -> dict:
"verifier_ok": m.get("verifier_ok") if cfg.get("run_hb_verifier", True) else True,
"log_clean": (not m.get("log_has_error")) if m.get("log_has_error") is not None else None,
}
if cfg.get("require_bpu_compute") and "cpu_compute_nodes" in m: # absent: 06-hbm predates the check
cpu = m["cpu_compute_nodes"]
checks["bpu_compute"] = None if cpu is None else not cpu
if any(v is False for v in checks.values()):
failed = [k for k, v in checks.items() if v is False]
return {"verdict": "fail", "failed": failed, **checks, "march_read": m.get("march_read")}
return {"verdict": "fail", "failed": failed, **checks, "march_read": m.get("march_read"),
**({"cpu_compute_nodes": m["cpu_compute_nodes"][:20]} if m.get("cpu_compute_nodes") else {})}
if any(v is None for v in checks.values()):
return {"verdict": "unverified", "missing": [k for k, v in checks.items() if v is None], **checks}
return {"verdict": "pass", **checks, "march_read": m.get("march_read")}
@ -267,7 +271,12 @@ def g4(m: dict | None, cfg: dict) -> dict:
return {"verdict": "na", **rec}
if not m or m.get("latency_ms_mean") is None:
return {"verdict": "unverified", **rec}
return {"verdict": "pass" if m["latency_ms_mean"] <= budget else "fail", "budget_ms": budget, **rec}
lat, cpu = m["latency_ms_mean"], cfg.get("cpu_fp32_latency_ms")
if cpu:
rec.update(cpu_fp32_latency_ms=cpu, speedup_vs_cpu=round(cpu / lat, 2) if lat else None)
ok = lat <= budget and (not cpu or lat < cpu)
return {"verdict": "pass" if ok else "fail", "budget_ms": budget, **rec,
**({"latency_ref_ms": cfg["latency_ref_ms"]} if cfg.get("latency_ref_ms") else {})}
def evaluate(measurements: dict, gates_cfg: dict, *, provenance: dict | None = None) -> dict:

View File

@ -377,11 +377,28 @@ def gate_snapshot(spec: JobSpec) -> dict:
"G1": g["G1"],
"G2": g2,
"G3": g3,
"G4": {"mandatory": False, "latency_budget_ms": spec.overrides.get("latency_budget_ms"),
"record": g["G4"]["record"]},
"G4": {"mandatory": False, "record": g["G4"]["record"], **speed_budget(spec, g["G4"])},
}
def speed_budget(spec: JobSpec, cfg: dict) -> dict:
"""agent/speed_ref/<job>.json (scripts/board_refs.py): the board latency of a plain int8 build and the original
FP32 model's latency on the board CPU, frozen at batch creation. An explicit override still wins."""
if spec.overrides.get("latency_budget_ms"):
return {"latency_budget_ms": spec.overrides["latency_budget_ms"]}
try:
ref = json.loads((paths.AGENT / "speed_ref" / f"{spec.job_id}.json").read_text())
except (OSError, ValueError):
return {"latency_budget_ms": None}
out = {"latency_budget_ms": None}
if ref.get("board_latency_ms") and cfg.get("latency_ratio_max"):
out.update(latency_ref_ms=ref["board_latency_ms"],
latency_budget_ms=round(ref["board_latency_ms"] * cfg["latency_ratio_max"], 3))
if ref.get("cpu_fp32_latency_ms"):
out["cpu_fp32_latency_ms"] = ref["cpu_fp32_latency_ms"]
return out
def data_snapshot(spec: JobSpec, row: dict | None = None) -> dict:
"""job.json.data:calib / t1 / t2 的 pool 路径与 manifest sha。缺的记 null,preflight 会报。"""
row = model_row(spec.model) if row is None else row

View File

@ -135,7 +135,8 @@ class Hbm(Stage):
last = clog.rfind("Start hb_compile")
tail = clog[last:] if last >= 0 else clog
m["log_has_error"] = bool(re.search(r"\bERROR\b", tail))
m["hmct_output_cosine"] = _hmct_output_cosine(tail) # hmct 校准阶段自己报的模型输出余弦(校准/量化),PTQ 崩了这里先看到
m["hmct_output_cosine"] = _hmct_output_cosine(tail)
m["cpu_compute_nodes"] = _cpu_compute_nodes(tail) # hmct 校准阶段自己报的模型输出余弦(校准/量化),PTQ 崩了这里先看到
(stage_dir / "stdout.log").write_text(json.dumps(m, indent=2, ensure_ascii=False) + "\n")
(stage_dir / "stderr.log").touch()
@ -215,6 +216,17 @@ def _check_io(io: dict, input_spec: str) -> bool | None:
return any(i["shape"] == dims for i in ins)
COMPUTE_OPS = {"Conv", "ConvTranspose", "MatMul", "Gemm"}
_NODE_ROW = re.compile(r"\|\s*(\S+)\s*\|\s*(\S+)\s*\|\s*(BPU|CPU|CPU\+BPU)\s*\|")
def _cpu_compute_nodes(text: str) -> list[str] | None:
"""hb_compile's node table (| Node | NodeType | ON | ...): the convolutions/matmuls it left on the CPU, where a
float16/float32 or unsupported layer ends up (10-14: FPN convs in float16 ran 50-87x slower). None: no table."""
rows = _NODE_ROW.findall(text)
return [node for node, kind, on in rows if on == "CPU" and kind in COMPUTE_OPS] if rows else None
def _parse_static_perf(info: str) -> dict:
"""hb_model_info stdout:FPS=2318.51, latency = 431.3 us, DDR = 12709504 bytes(编译器静态估算,不是板端实测)"""
d = {}

View File

@ -10,6 +10,7 @@ G1:
run_hb_verifier: true
verifier_cosine_threshold: 0.99 # 编译前后量化产物一致性,与 G2 的 FP32 比较分开
log_must_not_contain: ["ERROR"]
require_bpu_compute: true # 卷积/矩阵乘不许留在 CPU(hb_compile 节点表的 ON 列)
G2:
default_threshold: 0.99
@ -87,3 +88,4 @@ G4:
mandatory: false
judge_only_if: latency_budget_ms # job.json 声明才判
record: [latency_ms_mean, latency_ms_p95, bpu_util, ddr_bw_mbps]
latency_ratio_max: 1.5 # 预算 = 同题普通 int8 板上实测 × 1.5;另须快于原始 FP32 模型在板上 CPU 的速度

132
scripts/board_refs.py Normal file
View File

@ -0,0 +1,132 @@
"""Board speed references per job, merged into agent/speed_ref/<job_id>.json; registry.gate_snapshot freezes them into
each new batch's G4 (budget = 1.5 x board_latency_ms; the model must also beat cpu_fp32_latency_ms).
- board_latency_ms: median board latency (07-board, latency_ms_mean) of this job's plain int8 builds in any run:
compile.yaml without int16/float qtypes or run_on_cpu. Rollouts whose compile.yaml was offloaded are not seen.
- cpu_fp32_latency_ms: the platform's FP32 export (03-export cache, fetched from Salty storage) run with onnxruntime on
the board's own CPU, all cores, under the board lease so no stage shares the board. onnxruntime lives in
/userdata/bench-ort on the board (installed 10-24 with pip --target; the system python is untouched).
BENCH_CONFIG=<a run config> python3 scripts/board_refs.py RUN [--only JOB_ID ...] [--no-cpu]
"""
from __future__ import annotations
import argparse
import json
import os
import re
import shlex
import sqlite3
import statistics
import subprocess
import sys
import tempfile
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from harness import paths, scheduler # noqa: E402
from harness.stages.s03_export import onnx_path # noqa: E402
from harness.tools.board_perf import ssh_base # noqa: E402
ORT_DIR = "/userdata/bench-ort"
CPU_BENCH = r'''
import json, os, sys, time
import numpy as np
import onnxruntime as ort
o = ort.SessionOptions(); o.intra_op_num_threads = os.cpu_count()
s = ort.InferenceSession(sys.argv[1], o, providers=["CPUExecutionProvider"])
feed = {i.name: np.random.rand(*[d if isinstance(d, int) else 1 for d in i.shape]).astype(np.float32) for i in s.get_inputs()}
s.run(None, feed) # warm-up
t, start = [], time.time()
while len(t) < 10 and (len(t) < 3 or time.time() - start < 120):
a = time.perf_counter(); s.run(None, feed); t.append((time.perf_counter() - a) * 1000)
print(json.dumps({"latency_ms_median": sorted(t)[len(t) // 2], "runs": len(t), "threads": os.cpu_count()}))
'''
def plain_int8(yaml_text: str | None) -> bool:
if not yaml_text:
return False
body = re.sub(r"model_output_type:.*", "", yaml_text)
return not re.search(r"int16|float", body) and "run_on_cpu" not in body
def board_latency(job_id: str) -> tuple[float | None, int]:
db = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True, timeout=60)
roots = [paths.RUNS, paths.BENCH_ROOT.parent / "bench" / "runs", paths.BENCH_ROOT.parent / "bench-staging" / "runs"]
lat = []
for run, kk, verdict in db.execute("SELECT run_id, kk, verdict FROM rollouts WHERE job_id=? AND verdict IS NOT NULL", (job_id,)):
value = ((json.loads(verdict).get("gates") or {}).get("G4") or {}).get("latency_ms_mean")
yaml_text = next((p.read_text() for r in dict.fromkeys(roots)
if (p := r / run / job_id / "rollouts" / f"{kk:02d}" / "stages/04-calibrate/compile.yaml").exists()), None)
if value and plain_int8(yaml_text):
lat.append(value)
return (round(statistics.median(lat), 3), len(lat)) if lat else (None, 0)
def cpu_latency(job: dict) -> dict:
onnx = onnx_path(job)
if onnx is None:
raise ValueError("no platform export for this model")
remote = "personal:/bench/cache/" + onnx.relative_to(paths.CACHE).as_posix()
salty = paths.config()["execution"]["salty_remote"]["salty"]
with tempfile.TemporaryDirectory() as tmp:
local = Path(tmp) / "model.onnx"
subprocess.run([salty, "storage", "cp", remote, str(local)], check=True, capture_output=True, timeout=1800)
(Path(tmp) / "cpu_bench.py").write_text(CPU_BENCH)
with scheduler._acquire_board(job["march"], 5.0, "normal") as (slot, board):
if board is None:
raise ValueError(f"no enabled board for {job['march']}")
env = {k: os.environ.get(board[f"{k.split('_', 1)[1].lower()}_env"], "")
for k in ("BOARD_HOST", "BOARD_PORT", "BOARD_USER")}
env["BOARD_SSH_KEY"] = os.environ.get(board["ssh_key_env"], "")
os.environ.update(env)
opts, target = ssh_base()
work = f"{ORT_DIR}/run-{job['job_id']}"
subprocess.run(["ssh", *opts, target, f"mkdir -p {shlex.quote(work)}"], check=True, timeout=120)
subprocess.run(["scp", *[o.replace("-p", "-P") if o == "-p" else o for o in opts],
str(local), str(Path(tmp) / "cpu_bench.py"), f"{target}:{work}/"], check=True, timeout=3600)
try:
out = subprocess.run(["ssh", *opts, target, f"cd {shlex.quote(work)} && PYTHONPATH={ORT_DIR} python3 cpu_bench.py model.onnx"],
check=True, capture_output=True, text=True, timeout=3600).stdout
finally:
subprocess.run(["ssh", *opts, target, f"rm -rf {shlex.quote(work)}"], timeout=120)
return {**json.loads(out.strip().splitlines()[-1]), "board": board.get("name")}
def merge(job_id: str, fields: dict) -> None:
out = paths.AGENT / "speed_ref" / f"{job_id}.json"
ref = json.loads(out.read_text()) if out.exists() else {"schema": "bench.speed_ref/1", "job_id": job_id}
ref.update(fields)
tmp = out.with_name(f".{out.name}.tmp")
tmp.write_text(json.dumps(ref, indent=2, ensure_ascii=False))
tmp.replace(out)
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("run")
ap.add_argument("--only", action="append")
ap.add_argument("--no-cpu", action="store_true")
a = ap.parse_args()
for jd in sorted((paths.RUNS / a.run).iterdir()):
job = json.loads((jd / "job.json").read_text())
if a.only and job["job_id"] not in a.only:
continue
lat, n = board_latency(job["job_id"])
fields = {"board_latency_ms": lat, "board_latency_n": n,
"board_latency_rule": "median 07-board latency of plain int8 builds", "board_refs_at": time.time()}
if not a.no_cpu:
try:
cpu = cpu_latency(job)
fields.update(cpu_fp32_latency_ms=round(cpu["latency_ms_median"], 3), cpu_fp32_measured=cpu)
except Exception as e: # one job's missing export or board hiccup must not stop the others
fields["cpu_fp32_error"] = f"{type(e).__name__}: {e}"[:300]
merge(job["job_id"], fields)
print(job["job_id"], json.dumps({k: v for k, v in fields.items() if k != "board_refs_at"}, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@ -0,0 +1,58 @@
"""BPU-compute (G1) and board speed (G4) gates. Run: python -m unittest scripts.test_bpu_speed_gates"""
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from harness import gates, registry
from harness.stages.s06_hbm import _cpu_compute_nodes
TABLE = """INFO | Node | NodeType | ON | Threshold |
INFO | /backbone/Conv | Conv | BPU | 1.0 |
INFO | /head/Conv_1 | Conv | CPU | -- |
INFO | /Softmax | Softmax | CPU+BPU | -- |
INFO | /post/Mul | Mul | CPU | -- |
"""
G1 = {"check_march": True, "check_input_dtype_layout": True, "run_hb_verifier": True}
M1 = {"hbm_exists": True, "march_ok": True, "io_ok": True, "verifier_ok": True, "log_has_error": False}
class BpuCompute(unittest.TestCase):
def test_table_lists_only_compute_nodes_on_cpu(self):
self.assertEqual(_cpu_compute_nodes(TABLE), ["/head/Conv_1"])
self.assertIsNone(_cpu_compute_nodes("no table here"))
def test_g1_fails_compute_on_cpu_only_when_required(self):
m = {**M1, "cpu_compute_nodes": ["/head/Conv_1"]}
self.assertEqual(gates.g1(m, G1)["verdict"], "pass") # old batches: not judged
r = gates.g1(m, {**G1, "require_bpu_compute": True})
self.assertEqual((r["verdict"], r["failed"]), ("fail", ["bpu_compute"]))
self.assertEqual(gates.g1({**M1, "cpu_compute_nodes": []}, {**G1, "require_bpu_compute": True})["verdict"], "pass")
self.assertEqual(gates.g1({**M1, "cpu_compute_nodes": None}, {**G1, "require_bpu_compute": True})["verdict"], "unverified")
self.assertEqual(gates.g1(M1, {**G1, "require_bpu_compute": True})["verdict"], "pass") # measured before the check
class BoardSpeed(unittest.TestCase):
CFG = {"record": ["latency_ms_mean"], "latency_budget_ms": 3.0, "latency_ref_ms": 2.0, "cpu_fp32_latency_ms": 40.0}
def test_budget_and_cpu(self):
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, self.CFG)["verdict"], "pass")
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, self.CFG)["speedup_vs_cpu"], 16.0)
self.assertEqual(gates.g4({"latency_ms_mean": 3.5}, self.CFG)["verdict"], "fail") # over 1.5x reference
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, {**self.CFG, "cpu_fp32_latency_ms": 2.0})["verdict"], "fail")
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, {"record": []})["verdict"], "na")
def test_batch_creation_freezes_the_reference(self):
spec = registry.JobSpec("det--m--nash-m--ptq--int8", "det", "m", "nash-m", "ptq", "int8", "A", "e2e")
with tempfile.TemporaryDirectory() as tmp, patch.object(registry.paths, "AGENT", Path(tmp)):
(Path(tmp) / "speed_ref").mkdir()
self.assertEqual(registry.speed_budget(spec, {"latency_ratio_max": 1.5}), {"latency_budget_ms": None})
(Path(tmp) / "speed_ref" / f"{spec.job_id}.json").write_text(json.dumps(
{"board_latency_ms": 2.0, "cpu_fp32_latency_ms": 40.0}))
self.assertEqual(registry.speed_budget(spec, {"latency_ratio_max": 1.5}),
{"latency_budget_ms": 3.0, "latency_ref_ms": 2.0, "cpu_fp32_latency_ms": 40.0})
if __name__ == "__main__":
unittest.main()