data-gen-bpu/scripts/test_explicit_batch.py

42 lines
2.1 KiB
Python

import json
from unittest.mock import patch
import unittest
from harness import registry
from scripts.test_collection import Collection
class ExplicitBatch(unittest.TestCase):
def test_explicit_admission_keeps_sampling_and_gates_without_changing_csv(self):
before = registry.model_row("yolov8n.seg").copy()
normal = registry.expand_matrix(["oracle", "e2e"])
self.assertFalse(any(s.model == "yolov8n.seg" for s in normal))
specs = registry.expand_matrix(["oracle", "e2e"], ["yolov8n.seg"])
self.assertEqual([(s.mode, s.k, s.policy, s.march) for s in specs],
[("oracle", 1, None, "nash-m"), ("e2e", 4, "teacher", "nash-m")])
self.assertEqual(registry.gate_snapshot(specs[0])["G3"]["tol"], 5)
self.assertEqual(registry.model_row("yolov8n.seg"), before)
with self.assertRaises(ValueError):
registry.expand_matrix(["oracle"], ["nonexistent"])
class ExplicitRepair(Collection):
def test_admitted_model_does_not_generate_oracle_repairs(self):
from harness import pipeline
self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2,
"policies": ["teacher"], "error_classes": ["accuracy_drop"], "marches": ["nash-m"], "backends": ["ptq"]}
self.job("segment", task="iseg", status="needs_eval", state="FAIL", k=4)
jd = self.root / "runs/run/segment"
jd.mkdir(parents=True)
(jd / "job.json").write_text(json.dumps({"admission": {"method": "explicit_model_batch"}}))
with self.connect() as con:
con.execute("UPDATE jobs SET error_class='accuracy_drop',finished_at=1 WHERE job_id='segment'")
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
with self.connect() as con:
rows = db_rows = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE job_id='segment' ORDER BY kk")]
self.assertEqual([(r["kk"], r["mode"]) for r in rows], [(kk, "e2e") for kk in range(4)])
if __name__ == "__main__":
unittest.main()