From 1dcf331c9095e24324ea68dc4aefa0e5cc2c1935 Mon Sep 17 00:00:00 2001 From: bench Date: Sat, 24 Oct 2026 00:41:02 +0800 Subject: [PATCH] Gates: compute must run on the BPU; board latency budget and the original model on the board CPU as speed floors - 06-hbm records the convolutions/matmuls hb_compile left on the CPU (its node table's ON column); G1 fails on any when the batch's gates say require_bpu_compute (new in gates.yaml; frozen batches without it are not judged) - batch creation freezes agent/speed_ref/.json into G4: latency_budget_ms = 1.5 x board_latency_ms (median board latency of plain int8 builds) and cpu_fp32_latency_ms; G4 also fails when the model is not faster than the original FP32 model on the board CPU, and records the speedup - scripts/board_refs.py computes both; the CPU number is onnxruntime on the board itself under the board lease - e2e prompt states the three rules; agent/VERSION 25 Co-Authored-By: Claude Opus 5.5 --- agent/VERSION | 2 +- agent/prompts/e2e.md | 7 +- harness/gates.py | 13 +++- harness/registry.py | 21 ++++- harness/stages/s06_hbm.py | 14 +++- registry/gates.yaml | 2 + scripts/board_refs.py | 132 ++++++++++++++++++++++++++++++++ scripts/test_bpu_speed_gates.py | 58 ++++++++++++++ 8 files changed, 240 insertions(+), 9 deletions(-) create mode 100644 scripts/board_refs.py create mode 100644 scripts/test_bpu_speed_gates.py diff --git a/agent/VERSION b/agent/VERSION index a45fd52..7273c0f 100644 --- a/agent/VERSION +++ b/agent/VERSION @@ -1 +1 @@ -24 +25 diff --git a/agent/prompts/e2e.md b/agent/prompts/e2e.md index b758cfb..138009a 100644 --- a/agent/prompts/e2e.md +++ b/agent/prompts/e2e.md @@ -7,11 +7,12 @@ job.json:`{job_json}`(只读) 数据:`{data_dir}`(只读) 你的 1. G1 编译:产出 `.hbm`,march = {march},输入 {input_spec},`hb_verifier` 通过 2. G2 数值:校准外样本的逐输出余弦,参考阈值 {g2_threshold},只作诊断 3. G3 任务精度:板上 T1 子集({t1_n} 张)的 {g3_metric} 与同一子集的 FP32 基线比较,容差 {g3_tol}({g3_kind}) -4. G4 速度:不比平台默认量化配置(compile.yaml 只写两项必填)编出的模型慢 1.5 倍以上。参考在 `$BENCH_ROOT/agent/speed_ref/{job_id}.json`:`latency_ms_est` 是默认配置的编译器静态估计,不含落到 CPU 的算子;文件还没有表示还在算,值为 null 表示默认配置没编出来、本题没有参考。 +4. G4 速度:以 job.json 的 `gates.G4` 为准。`latency_budget_ms` 是本题板上实测延迟的上限(同题普通 int8 配置板上实测 `latency_ref_ms` 的 1.5 倍),`cpu_fp32_latency_ms` 是原始 FP32 模型在本题板卡 CPU 上的延迟,你的模型必须比它快。没有这两个字段的题不判速度。`$BENCH_ROOT/agent/speed_ref/{job_id}.json` 里还有默认配置的编译器静态估计 `latency_ms_est`,可以当量级参考。 -两条硬规定,违反任何一条都不算通过: +三条硬规定,违反任何一条都不算通过: - 本题是 int8 量化。`all_node_type`,或者整类计算算子(Conv、MatMul、Gemm 等;用 op_config,或者逐节点列出超过同类节点的一半)设成 int16、float16 或 float32,都判不通过。只把少数敏感节点设成 int16 可以。 -- 板上实测延迟超过同题默认 int8 配置板上实测的 1.5 倍,判不通过(G4 参考文件里是默认配置的编译器静态估计,可以当量级参考)。 +- 卷积、矩阵乘(Conv、ConvTranspose、MatMul、Gemm)必须跑在 BPU 上:hb_compile 节点表 `ON` 一列里有这类节点是 `CPU`,G1 判不通过。 +- 板上实测延迟超过 `latency_budget_ms`,或者不比 `cpu_fp32_latency_ms` 快,G4 判不通过。 缺少板上精度结果不能报完整 PASS;缺权重、工具或材料时如实保留实际错误。 diff --git a/harness/gates.py b/harness/gates.py index 2645571..111f596 100644 --- a/harness/gates.py +++ b/harness/gates.py @@ -53,9 +53,13 @@ def g1(m: dict | None, cfg: dict) -> dict: "verifier_ok": m.get("verifier_ok") if cfg.get("run_hb_verifier", True) else True, "log_clean": (not m.get("log_has_error")) if m.get("log_has_error") is not None else None, } + if cfg.get("require_bpu_compute") and "cpu_compute_nodes" in m: # absent: 06-hbm predates the check + cpu = m["cpu_compute_nodes"] + checks["bpu_compute"] = None if cpu is None else not cpu if any(v is False for v in checks.values()): failed = [k for k, v in checks.items() if v is False] - return {"verdict": "fail", "failed": failed, **checks, "march_read": m.get("march_read")} + return {"verdict": "fail", "failed": failed, **checks, "march_read": m.get("march_read"), + **({"cpu_compute_nodes": m["cpu_compute_nodes"][:20]} if m.get("cpu_compute_nodes") else {})} if any(v is None for v in checks.values()): return {"verdict": "unverified", "missing": [k for k, v in checks.items() if v is None], **checks} return {"verdict": "pass", **checks, "march_read": m.get("march_read")} @@ -267,7 +271,12 @@ def g4(m: dict | None, cfg: dict) -> dict: return {"verdict": "na", **rec} if not m or m.get("latency_ms_mean") is None: return {"verdict": "unverified", **rec} - return {"verdict": "pass" if m["latency_ms_mean"] <= budget else "fail", "budget_ms": budget, **rec} + lat, cpu = m["latency_ms_mean"], cfg.get("cpu_fp32_latency_ms") + if cpu: + rec.update(cpu_fp32_latency_ms=cpu, speedup_vs_cpu=round(cpu / lat, 2) if lat else None) + ok = lat <= budget and (not cpu or lat < cpu) + return {"verdict": "pass" if ok else "fail", "budget_ms": budget, **rec, + **({"latency_ref_ms": cfg["latency_ref_ms"]} if cfg.get("latency_ref_ms") else {})} def evaluate(measurements: dict, gates_cfg: dict, *, provenance: dict | None = None) -> dict: diff --git a/harness/registry.py b/harness/registry.py index 7af05eb..7486d12 100644 --- a/harness/registry.py +++ b/harness/registry.py @@ -377,11 +377,28 @@ def gate_snapshot(spec: JobSpec) -> dict: "G1": g["G1"], "G2": g2, "G3": g3, - "G4": {"mandatory": False, "latency_budget_ms": spec.overrides.get("latency_budget_ms"), - "record": g["G4"]["record"]}, + "G4": {"mandatory": False, "record": g["G4"]["record"], **speed_budget(spec, g["G4"])}, } +def speed_budget(spec: JobSpec, cfg: dict) -> dict: + """agent/speed_ref/.json (scripts/board_refs.py): the board latency of a plain int8 build and the original + FP32 model's latency on the board CPU, frozen at batch creation. An explicit override still wins.""" + if spec.overrides.get("latency_budget_ms"): + return {"latency_budget_ms": spec.overrides["latency_budget_ms"]} + try: + ref = json.loads((paths.AGENT / "speed_ref" / f"{spec.job_id}.json").read_text()) + except (OSError, ValueError): + return {"latency_budget_ms": None} + out = {"latency_budget_ms": None} + if ref.get("board_latency_ms") and cfg.get("latency_ratio_max"): + out.update(latency_ref_ms=ref["board_latency_ms"], + latency_budget_ms=round(ref["board_latency_ms"] * cfg["latency_ratio_max"], 3)) + if ref.get("cpu_fp32_latency_ms"): + out["cpu_fp32_latency_ms"] = ref["cpu_fp32_latency_ms"] + return out + + def data_snapshot(spec: JobSpec, row: dict | None = None) -> dict: """job.json.data:calib / t1 / t2 的 pool 路径与 manifest sha。缺的记 null,preflight 会报。""" row = model_row(spec.model) if row is None else row diff --git a/harness/stages/s06_hbm.py b/harness/stages/s06_hbm.py index ee56116..f7b0752 100644 --- a/harness/stages/s06_hbm.py +++ b/harness/stages/s06_hbm.py @@ -135,7 +135,8 @@ class Hbm(Stage): last = clog.rfind("Start hb_compile") tail = clog[last:] if last >= 0 else clog m["log_has_error"] = bool(re.search(r"\bERROR\b", tail)) - m["hmct_output_cosine"] = _hmct_output_cosine(tail) # hmct 校准阶段自己报的模型输出余弦(校准/量化),PTQ 崩了这里先看到 + m["hmct_output_cosine"] = _hmct_output_cosine(tail) + m["cpu_compute_nodes"] = _cpu_compute_nodes(tail) # hmct 校准阶段自己报的模型输出余弦(校准/量化),PTQ 崩了这里先看到 (stage_dir / "stdout.log").write_text(json.dumps(m, indent=2, ensure_ascii=False) + "\n") (stage_dir / "stderr.log").touch() @@ -215,6 +216,17 @@ def _check_io(io: dict, input_spec: str) -> bool | None: return any(i["shape"] == dims for i in ins) +COMPUTE_OPS = {"Conv", "ConvTranspose", "MatMul", "Gemm"} +_NODE_ROW = re.compile(r"\|\s*(\S+)\s*\|\s*(\S+)\s*\|\s*(BPU|CPU|CPU\+BPU)\s*\|") + + +def _cpu_compute_nodes(text: str) -> list[str] | None: + """hb_compile's node table (| Node | NodeType | ON | ...): the convolutions/matmuls it left on the CPU, where a + float16/float32 or unsupported layer ends up (10-14: FPN convs in float16 ran 50-87x slower). None: no table.""" + rows = _NODE_ROW.findall(text) + return [node for node, kind, on in rows if on == "CPU" and kind in COMPUTE_OPS] if rows else None + + def _parse_static_perf(info: str) -> dict: """hb_model_info stdout:FPS=2318.51, latency = 431.3 us, DDR = 12709504 bytes(编译器静态估算,不是板端实测)""" d = {} diff --git a/registry/gates.yaml b/registry/gates.yaml index 797b8c1..e91c201 100644 --- a/registry/gates.yaml +++ b/registry/gates.yaml @@ -10,6 +10,7 @@ G1: run_hb_verifier: true verifier_cosine_threshold: 0.99 # 编译前后量化产物一致性,与 G2 的 FP32 比较分开 log_must_not_contain: ["ERROR"] + require_bpu_compute: true # 卷积/矩阵乘不许留在 CPU(hb_compile 节点表的 ON 列) G2: default_threshold: 0.99 @@ -87,3 +88,4 @@ G4: mandatory: false judge_only_if: latency_budget_ms # job.json 声明才判 record: [latency_ms_mean, latency_ms_p95, bpu_util, ddr_bw_mbps] + latency_ratio_max: 1.5 # 预算 = 同题普通 int8 板上实测 × 1.5;另须快于原始 FP32 模型在板上 CPU 的速度 diff --git a/scripts/board_refs.py b/scripts/board_refs.py new file mode 100644 index 0000000..713b52e --- /dev/null +++ b/scripts/board_refs.py @@ -0,0 +1,132 @@ +"""Board speed references per job, merged into agent/speed_ref/.json; registry.gate_snapshot freezes them into +each new batch's G4 (budget = 1.5 x board_latency_ms; the model must also beat cpu_fp32_latency_ms). + +- board_latency_ms: median board latency (07-board, latency_ms_mean) of this job's plain int8 builds in any run: + compile.yaml without int16/float qtypes or run_on_cpu. Rollouts whose compile.yaml was offloaded are not seen. +- cpu_fp32_latency_ms: the platform's FP32 export (03-export cache, fetched from Salty storage) run with onnxruntime on + the board's own CPU, all cores, under the board lease so no stage shares the board. onnxruntime lives in + /userdata/bench-ort on the board (installed 10-24 with pip --target; the system python is untouched). + + BENCH_CONFIG= python3 scripts/board_refs.py RUN [--only JOB_ID ...] [--no-cpu] +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import shlex +import sqlite3 +import statistics +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from harness import paths, scheduler # noqa: E402 +from harness.stages.s03_export import onnx_path # noqa: E402 +from harness.tools.board_perf import ssh_base # noqa: E402 + +ORT_DIR = "/userdata/bench-ort" +CPU_BENCH = r''' +import json, os, sys, time +import numpy as np +import onnxruntime as ort +o = ort.SessionOptions(); o.intra_op_num_threads = os.cpu_count() +s = ort.InferenceSession(sys.argv[1], o, providers=["CPUExecutionProvider"]) +feed = {i.name: np.random.rand(*[d if isinstance(d, int) else 1 for d in i.shape]).astype(np.float32) for i in s.get_inputs()} +s.run(None, feed) # warm-up +t, start = [], time.time() +while len(t) < 10 and (len(t) < 3 or time.time() - start < 120): + a = time.perf_counter(); s.run(None, feed); t.append((time.perf_counter() - a) * 1000) +print(json.dumps({"latency_ms_median": sorted(t)[len(t) // 2], "runs": len(t), "threads": os.cpu_count()})) +''' + + +def plain_int8(yaml_text: str | None) -> bool: + if not yaml_text: + return False + body = re.sub(r"model_output_type:.*", "", yaml_text) + return not re.search(r"int16|float", body) and "run_on_cpu" not in body + + +def board_latency(job_id: str) -> tuple[float | None, int]: + db = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True, timeout=60) + roots = [paths.RUNS, paths.BENCH_ROOT.parent / "bench" / "runs", paths.BENCH_ROOT.parent / "bench-staging" / "runs"] + lat = [] + for run, kk, verdict in db.execute("SELECT run_id, kk, verdict FROM rollouts WHERE job_id=? AND verdict IS NOT NULL", (job_id,)): + value = ((json.loads(verdict).get("gates") or {}).get("G4") or {}).get("latency_ms_mean") + yaml_text = next((p.read_text() for r in dict.fromkeys(roots) + if (p := r / run / job_id / "rollouts" / f"{kk:02d}" / "stages/04-calibrate/compile.yaml").exists()), None) + if value and plain_int8(yaml_text): + lat.append(value) + return (round(statistics.median(lat), 3), len(lat)) if lat else (None, 0) + + +def cpu_latency(job: dict) -> dict: + onnx = onnx_path(job) + if onnx is None: + raise ValueError("no platform export for this model") + remote = "personal:/bench/cache/" + onnx.relative_to(paths.CACHE).as_posix() + salty = paths.config()["execution"]["salty_remote"]["salty"] + with tempfile.TemporaryDirectory() as tmp: + local = Path(tmp) / "model.onnx" + subprocess.run([salty, "storage", "cp", remote, str(local)], check=True, capture_output=True, timeout=1800) + (Path(tmp) / "cpu_bench.py").write_text(CPU_BENCH) + with scheduler._acquire_board(job["march"], 5.0, "normal") as (slot, board): + if board is None: + raise ValueError(f"no enabled board for {job['march']}") + env = {k: os.environ.get(board[f"{k.split('_', 1)[1].lower()}_env"], "") + for k in ("BOARD_HOST", "BOARD_PORT", "BOARD_USER")} + env["BOARD_SSH_KEY"] = os.environ.get(board["ssh_key_env"], "") + os.environ.update(env) + opts, target = ssh_base() + work = f"{ORT_DIR}/run-{job['job_id']}" + subprocess.run(["ssh", *opts, target, f"mkdir -p {shlex.quote(work)}"], check=True, timeout=120) + subprocess.run(["scp", *[o.replace("-p", "-P") if o == "-p" else o for o in opts], + str(local), str(Path(tmp) / "cpu_bench.py"), f"{target}:{work}/"], check=True, timeout=3600) + try: + out = subprocess.run(["ssh", *opts, target, f"cd {shlex.quote(work)} && PYTHONPATH={ORT_DIR} python3 cpu_bench.py model.onnx"], + check=True, capture_output=True, text=True, timeout=3600).stdout + finally: + subprocess.run(["ssh", *opts, target, f"rm -rf {shlex.quote(work)}"], timeout=120) + return {**json.loads(out.strip().splitlines()[-1]), "board": board.get("name")} + + +def merge(job_id: str, fields: dict) -> None: + out = paths.AGENT / "speed_ref" / f"{job_id}.json" + ref = json.loads(out.read_text()) if out.exists() else {"schema": "bench.speed_ref/1", "job_id": job_id} + ref.update(fields) + tmp = out.with_name(f".{out.name}.tmp") + tmp.write_text(json.dumps(ref, indent=2, ensure_ascii=False)) + tmp.replace(out) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("run") + ap.add_argument("--only", action="append") + ap.add_argument("--no-cpu", action="store_true") + a = ap.parse_args() + for jd in sorted((paths.RUNS / a.run).iterdir()): + job = json.loads((jd / "job.json").read_text()) + if a.only and job["job_id"] not in a.only: + continue + lat, n = board_latency(job["job_id"]) + fields = {"board_latency_ms": lat, "board_latency_n": n, + "board_latency_rule": "median 07-board latency of plain int8 builds", "board_refs_at": time.time()} + if not a.no_cpu: + try: + cpu = cpu_latency(job) + fields.update(cpu_fp32_latency_ms=round(cpu["latency_ms_median"], 3), cpu_fp32_measured=cpu) + except Exception as e: # one job's missing export or board hiccup must not stop the others + fields["cpu_fp32_error"] = f"{type(e).__name__}: {e}"[:300] + merge(job["job_id"], fields) + print(job["job_id"], json.dumps({k: v for k, v in fields.items() if k != "board_refs_at"}, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_bpu_speed_gates.py b/scripts/test_bpu_speed_gates.py new file mode 100644 index 0000000..5b37f7d --- /dev/null +++ b/scripts/test_bpu_speed_gates.py @@ -0,0 +1,58 @@ +"""BPU-compute (G1) and board speed (G4) gates. Run: python -m unittest scripts.test_bpu_speed_gates""" +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from harness import gates, registry +from harness.stages.s06_hbm import _cpu_compute_nodes + +TABLE = """INFO | Node | NodeType | ON | Threshold | +INFO | /backbone/Conv | Conv | BPU | 1.0 | +INFO | /head/Conv_1 | Conv | CPU | -- | +INFO | /Softmax | Softmax | CPU+BPU | -- | +INFO | /post/Mul | Mul | CPU | -- | +""" +G1 = {"check_march": True, "check_input_dtype_layout": True, "run_hb_verifier": True} +M1 = {"hbm_exists": True, "march_ok": True, "io_ok": True, "verifier_ok": True, "log_has_error": False} + + +class BpuCompute(unittest.TestCase): + def test_table_lists_only_compute_nodes_on_cpu(self): + self.assertEqual(_cpu_compute_nodes(TABLE), ["/head/Conv_1"]) + self.assertIsNone(_cpu_compute_nodes("no table here")) + + def test_g1_fails_compute_on_cpu_only_when_required(self): + m = {**M1, "cpu_compute_nodes": ["/head/Conv_1"]} + self.assertEqual(gates.g1(m, G1)["verdict"], "pass") # old batches: not judged + r = gates.g1(m, {**G1, "require_bpu_compute": True}) + self.assertEqual((r["verdict"], r["failed"]), ("fail", ["bpu_compute"])) + self.assertEqual(gates.g1({**M1, "cpu_compute_nodes": []}, {**G1, "require_bpu_compute": True})["verdict"], "pass") + self.assertEqual(gates.g1({**M1, "cpu_compute_nodes": None}, {**G1, "require_bpu_compute": True})["verdict"], "unverified") + self.assertEqual(gates.g1(M1, {**G1, "require_bpu_compute": True})["verdict"], "pass") # measured before the check + + +class BoardSpeed(unittest.TestCase): + CFG = {"record": ["latency_ms_mean"], "latency_budget_ms": 3.0, "latency_ref_ms": 2.0, "cpu_fp32_latency_ms": 40.0} + + def test_budget_and_cpu(self): + self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, self.CFG)["verdict"], "pass") + self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, self.CFG)["speedup_vs_cpu"], 16.0) + self.assertEqual(gates.g4({"latency_ms_mean": 3.5}, self.CFG)["verdict"], "fail") # over 1.5x reference + self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, {**self.CFG, "cpu_fp32_latency_ms": 2.0})["verdict"], "fail") + self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, {"record": []})["verdict"], "na") + + def test_batch_creation_freezes_the_reference(self): + spec = registry.JobSpec("det--m--nash-m--ptq--int8", "det", "m", "nash-m", "ptq", "int8", "A", "e2e") + with tempfile.TemporaryDirectory() as tmp, patch.object(registry.paths, "AGENT", Path(tmp)): + (Path(tmp) / "speed_ref").mkdir() + self.assertEqual(registry.speed_budget(spec, {"latency_ratio_max": 1.5}), {"latency_budget_ms": None}) + (Path(tmp) / "speed_ref" / f"{spec.job_id}.json").write_text(json.dumps( + {"board_latency_ms": 2.0, "cpu_fp32_latency_ms": 40.0})) + self.assertEqual(registry.speed_budget(spec, {"latency_ratio_max": 1.5}), + {"latency_budget_ms": 3.0, "latency_ref_ms": 2.0, "cpu_fp32_latency_ms": 40.0}) + + +if __name__ == "__main__": + unittest.main()