- 06-hbm records the convolutions/matmuls hb_compile left on the CPU (its node table's ON column); G1 fails on any when the batch's gates say require_bpu_compute (new in gates.yaml; frozen batches without it are not judged) - batch creation freezes agent/speed_ref/<job>.json into G4: latency_budget_ms = 1.5 x board_latency_ms (median board latency of plain int8 builds) and cpu_fp32_latency_ms; G4 also fails when the model is not faster than the original FP32 model on the board CPU, and records the speedup - scripts/board_refs.py computes both; the CPU number is onnxruntime on the board itself under the board lease - e2e prompt states the three rules; agent/VERSION 25 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
59 lines
3.2 KiB
Python
59 lines
3.2 KiB
Python
"""BPU-compute (G1) and board speed (G4) gates. Run: python -m unittest scripts.test_bpu_speed_gates"""
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from harness import gates, registry
|
|
from harness.stages.s06_hbm import _cpu_compute_nodes
|
|
|
|
TABLE = """INFO | Node | NodeType | ON | Threshold |
|
|
INFO | /backbone/Conv | Conv | BPU | 1.0 |
|
|
INFO | /head/Conv_1 | Conv | CPU | -- |
|
|
INFO | /Softmax | Softmax | CPU+BPU | -- |
|
|
INFO | /post/Mul | Mul | CPU | -- |
|
|
"""
|
|
G1 = {"check_march": True, "check_input_dtype_layout": True, "run_hb_verifier": True}
|
|
M1 = {"hbm_exists": True, "march_ok": True, "io_ok": True, "verifier_ok": True, "log_has_error": False}
|
|
|
|
|
|
class BpuCompute(unittest.TestCase):
|
|
def test_table_lists_only_compute_nodes_on_cpu(self):
|
|
self.assertEqual(_cpu_compute_nodes(TABLE), ["/head/Conv_1"])
|
|
self.assertIsNone(_cpu_compute_nodes("no table here"))
|
|
|
|
def test_g1_fails_compute_on_cpu_only_when_required(self):
|
|
m = {**M1, "cpu_compute_nodes": ["/head/Conv_1"]}
|
|
self.assertEqual(gates.g1(m, G1)["verdict"], "pass") # old batches: not judged
|
|
r = gates.g1(m, {**G1, "require_bpu_compute": True})
|
|
self.assertEqual((r["verdict"], r["failed"]), ("fail", ["bpu_compute"]))
|
|
self.assertEqual(gates.g1({**M1, "cpu_compute_nodes": []}, {**G1, "require_bpu_compute": True})["verdict"], "pass")
|
|
self.assertEqual(gates.g1({**M1, "cpu_compute_nodes": None}, {**G1, "require_bpu_compute": True})["verdict"], "unverified")
|
|
self.assertEqual(gates.g1(M1, {**G1, "require_bpu_compute": True})["verdict"], "pass") # measured before the check
|
|
|
|
|
|
class BoardSpeed(unittest.TestCase):
|
|
CFG = {"record": ["latency_ms_mean"], "latency_budget_ms": 3.0, "latency_ref_ms": 2.0, "cpu_fp32_latency_ms": 40.0}
|
|
|
|
def test_budget_and_cpu(self):
|
|
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, self.CFG)["verdict"], "pass")
|
|
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, self.CFG)["speedup_vs_cpu"], 16.0)
|
|
self.assertEqual(gates.g4({"latency_ms_mean": 3.5}, self.CFG)["verdict"], "fail") # over 1.5x reference
|
|
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, {**self.CFG, "cpu_fp32_latency_ms": 2.0})["verdict"], "fail")
|
|
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, {"record": []})["verdict"], "na")
|
|
|
|
def test_batch_creation_freezes_the_reference(self):
|
|
spec = registry.JobSpec("det--m--nash-m--ptq--int8", "det", "m", "nash-m", "ptq", "int8", "A", "e2e")
|
|
with tempfile.TemporaryDirectory() as tmp, patch.object(registry.paths, "AGENT", Path(tmp)):
|
|
(Path(tmp) / "speed_ref").mkdir()
|
|
self.assertEqual(registry.speed_budget(spec, {"latency_ratio_max": 1.5}), {"latency_budget_ms": None})
|
|
(Path(tmp) / "speed_ref" / f"{spec.job_id}.json").write_text(json.dumps(
|
|
{"board_latency_ms": 2.0, "cpu_fp32_latency_ms": 40.0}))
|
|
self.assertEqual(registry.speed_budget(spec, {"latency_ratio_max": 1.5}),
|
|
{"latency_budget_ms": 3.0, "latency_ref_ms": 2.0, "cpu_fp32_latency_ms": 40.0})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|