data-gen-bpu/scripts/test_generation.py

121 lines
5.4 KiB
Python

"""Conditional repair generation preserves the existing matrix and rollout numbering."""
from contextlib import contextmanager
from copy import deepcopy
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from harness import db, pipeline
class RepairGeneration(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.db_path = Path(self.tmp.name) / "jobs.sqlite"
self.mx = {
"marches": ["nash-m"], "backends": ["ptq"],
"modes": {"repair": {
"enabled": True, "when": "oracle_fail", "marches": "all", "backends": "all",
"error_classes": ["accuracy_drop"], "policies": ["teacher"], "k": 2,
}},
}
self.model_rows = [{"model_id": "resnet18", "task": "cls", "priority": "A", "status": "ready"}]
self.addCleanup(patch.stopall)
patch.object(pipeline.db, "connect", self.connect).start()
patch.object(pipeline, "matrix", lambda: self.mx).start()
patch.object(pipeline, "models", lambda: self.model_rows).start()
@contextmanager
def connect(self):
con = db._open(self.db_path)
con.executescript(db.SCHEMA)
try:
yield con
con.commit()
finally:
con.close()
def job(self, jid="job", *, state="FAIL", error_class="accuracy_drop", run="run",
march="nash-m", backend="ptq", finished=1):
with self.connect() as con:
con.execute(
"INSERT INTO jobs (run_id,job_id,model,march,backend,mode,state,error_class,finished_at)"
" VALUES (?,?,?,?,?,'oracle',?,?,?)",
(run, jid, "resnet18", march, backend, state, error_class, finished))
def rollouts(self, jid="job"):
with self.connect() as con:
return db.list_rollouts(con, "run", jid)
def test_idempotent_and_e2e_numbers_preserved(self):
self.job()
original_matrix = deepcopy(self.mx)
with self.connect() as con:
for kk in range(4):
db.rollout_upsert(con, "run", "job", kk, "e2e", "teacher")
self.assertEqual(pipeline.enqueue_repairs("run"), 2)
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
self.assertEqual([(r["kk"], r["mode"]) for r in self.rollouts()],
[(0, "e2e"), (1, "e2e"), (2, "e2e"), (3, "e2e"), (4, "repair"), (5, "repair")])
self.assertEqual(self.mx, original_matrix)
def test_only_finished_eligible_oracles_and_selected_jobs(self):
self.job("eligible")
self.job("not_selected")
self.job("passed", state="PASS")
self.job("unfinished", finished=None)
self.job("unknown_error", error_class="timeout")
self.job("wrong_march", march="nash-e")
self.job("wrong_backend", backend="qat")
self.job("other_run", run="other")
chosen = ["eligible", "passed", "unfinished", "unknown_error", "wrong_march", "wrong_backend", "other_run"]
self.assertEqual(pipeline.enqueue_repairs("run", chosen), 2)
with self.connect() as con:
self.assertEqual({r[0] for r in con.execute("SELECT job_id FROM rollouts")}, {"eligible"})
def test_mode_enabled_when_and_select_filters(self):
self.job()
self.assertEqual(pipeline.enqueue_repairs("run", modes=["e2e"]), 0)
cfg = self.mx["modes"]["repair"]
for key, value in [("enabled", False), ("when", "always"), ("select", {"tasks": ["det"]}),
("select", {"priority": ["B"]}), ("select", {"models": ["resnet50"]}),
("marches", ["nash-e"]), ("backends", ["qat"])]:
with self.subTest(key=key, value=value):
old = deepcopy(cfg)
cfg[key] = value
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
cfg.clear()
cfg.update(old)
self.model_rows[0]["status"] = "needs_adapter"
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
cfg["select"] = {"status": ["needs_adapter"]}
self.assertEqual(pipeline.enqueue_repairs("run", modes=["repair"]), 2)
def test_fill_missing_per_policy_without_replacing_completed_rollouts(self):
self.job()
self.mx["modes"]["repair"]["policies"] = ["teacher", "student"]
with self.connect() as con:
db.rollout_upsert(con, "run", "job", 7, "repair", "teacher", state="FAIL")
db.rollout_upsert(con, "run", "job", 10, "e2e", "teacher", state="PASS")
self.assertEqual(pipeline.enqueue_repairs("run"), 3)
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
rows = self.rollouts()
self.assertEqual([(r["kk"], r["mode"], r["policy"], r["state"]) for r in rows], [
(7, "repair", "teacher", "FAIL"), (10, "e2e", "teacher", "PASS"),
(11, "repair", "teacher", "QUEUED"), (12, "repair", "student", "QUEUED"),
(13, "repair", "student", "QUEUED"),
])
def test_collect_dry_run_does_not_enqueue(self):
self.job()
with patch.object(pipeline, "enqueue_repairs") as enqueue:
self.assertEqual(pipeline.collect_rollouts("run", dry_run=True, parallel=1, log=lambda _: None), {})
enqueue.assert_not_called()
self.assertEqual(self.rollouts(), [])
if __name__ == "__main__":
unittest.main()