data-gen-bpu/scripts/test_bpu_speed_gates.py
bench 1dcf331c90 Gates: compute must run on the BPU; board latency budget and the original model on the board CPU as speed floors
- 06-hbm records the convolutions/matmuls hb_compile left on the CPU (its node table's ON column); G1 fails on any
  when the batch's gates say require_bpu_compute (new in gates.yaml; frozen batches without it are not judged)
- batch creation freezes agent/speed_ref/<job>.json into G4: latency_budget_ms = 1.5 x board_latency_ms (median
  board latency of plain int8 builds) and cpu_fp32_latency_ms; G4 also fails when the model is not faster than
  the original FP32 model on the board CPU, and records the speedup
- scripts/board_refs.py computes both; the CPU number is onnxruntime on the board itself under the board lease
- e2e prompt states the three rules; agent/VERSION 25

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-24 00:41:02 +08:00

59 lines
3.2 KiB
Python

"""BPU-compute (G1) and board speed (G4) gates. Run: python -m unittest scripts.test_bpu_speed_gates"""
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from harness import gates, registry
from harness.stages.s06_hbm import _cpu_compute_nodes
TABLE = """INFO | Node | NodeType | ON | Threshold |
INFO | /backbone/Conv | Conv | BPU | 1.0 |
INFO | /head/Conv_1 | Conv | CPU | -- |
INFO | /Softmax | Softmax | CPU+BPU | -- |
INFO | /post/Mul | Mul | CPU | -- |
"""
G1 = {"check_march": True, "check_input_dtype_layout": True, "run_hb_verifier": True}
M1 = {"hbm_exists": True, "march_ok": True, "io_ok": True, "verifier_ok": True, "log_has_error": False}
class BpuCompute(unittest.TestCase):
def test_table_lists_only_compute_nodes_on_cpu(self):
self.assertEqual(_cpu_compute_nodes(TABLE), ["/head/Conv_1"])
self.assertIsNone(_cpu_compute_nodes("no table here"))
def test_g1_fails_compute_on_cpu_only_when_required(self):
m = {**M1, "cpu_compute_nodes": ["/head/Conv_1"]}
self.assertEqual(gates.g1(m, G1)["verdict"], "pass") # old batches: not judged
r = gates.g1(m, {**G1, "require_bpu_compute": True})
self.assertEqual((r["verdict"], r["failed"]), ("fail", ["bpu_compute"]))
self.assertEqual(gates.g1({**M1, "cpu_compute_nodes": []}, {**G1, "require_bpu_compute": True})["verdict"], "pass")
self.assertEqual(gates.g1({**M1, "cpu_compute_nodes": None}, {**G1, "require_bpu_compute": True})["verdict"], "unverified")
self.assertEqual(gates.g1(M1, {**G1, "require_bpu_compute": True})["verdict"], "pass") # measured before the check
class BoardSpeed(unittest.TestCase):
CFG = {"record": ["latency_ms_mean"], "latency_budget_ms": 3.0, "latency_ref_ms": 2.0, "cpu_fp32_latency_ms": 40.0}
def test_budget_and_cpu(self):
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, self.CFG)["verdict"], "pass")
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, self.CFG)["speedup_vs_cpu"], 16.0)
self.assertEqual(gates.g4({"latency_ms_mean": 3.5}, self.CFG)["verdict"], "fail") # over 1.5x reference
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, {**self.CFG, "cpu_fp32_latency_ms": 2.0})["verdict"], "fail")
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, {"record": []})["verdict"], "na")
def test_batch_creation_freezes_the_reference(self):
spec = registry.JobSpec("det--m--nash-m--ptq--int8", "det", "m", "nash-m", "ptq", "int8", "A", "e2e")
with tempfile.TemporaryDirectory() as tmp, patch.object(registry.paths, "AGENT", Path(tmp)):
(Path(tmp) / "speed_ref").mkdir()
self.assertEqual(registry.speed_budget(spec, {"latency_ratio_max": 1.5}), {"latency_budget_ms": None})
(Path(tmp) / "speed_ref" / f"{spec.job_id}.json").write_text(json.dumps(
{"board_latency_ms": 2.0, "cpu_fp32_latency_ms": 40.0}))
self.assertEqual(registry.speed_budget(spec, {"latency_ratio_max": 1.5}),
{"latency_budget_ms": 3.0, "latency_ref_ms": 2.0, "cpu_fp32_latency_ms": 40.0})
if __name__ == "__main__":
unittest.main()