121 lines
5.4 KiB
Python
121 lines
5.4 KiB
Python
"""Conditional repair generation preserves the existing matrix and rollout numbering."""
|
|
from contextlib import contextmanager
|
|
from copy import deepcopy
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import db, pipeline
|
|
|
|
|
|
class RepairGeneration(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.db_path = Path(self.tmp.name) / "jobs.sqlite"
|
|
self.mx = {
|
|
"marches": ["nash-m"], "backends": ["ptq"],
|
|
"modes": {"repair": {
|
|
"enabled": True, "when": "oracle_fail", "marches": "all", "backends": "all",
|
|
"error_classes": ["accuracy_drop"], "policies": ["teacher"], "k": 2,
|
|
}},
|
|
}
|
|
self.model_rows = [{"model_id": "resnet18", "task": "cls", "priority": "A", "status": "ready"}]
|
|
self.addCleanup(patch.stopall)
|
|
patch.object(pipeline.db, "connect", self.connect).start()
|
|
patch.object(pipeline, "matrix", lambda: self.mx).start()
|
|
patch.object(pipeline, "models", lambda: self.model_rows).start()
|
|
|
|
@contextmanager
|
|
def connect(self):
|
|
con = db._open(self.db_path)
|
|
con.executescript(db.SCHEMA)
|
|
try:
|
|
yield con
|
|
con.commit()
|
|
finally:
|
|
con.close()
|
|
|
|
def job(self, jid="job", *, state="FAIL", error_class="accuracy_drop", run="run",
|
|
march="nash-m", backend="ptq", finished=1):
|
|
with self.connect() as con:
|
|
con.execute(
|
|
"INSERT INTO jobs (run_id,job_id,model,march,backend,mode,state,error_class,finished_at)"
|
|
" VALUES (?,?,?,?,?,'oracle',?,?,?)",
|
|
(run, jid, "resnet18", march, backend, state, error_class, finished))
|
|
|
|
def rollouts(self, jid="job"):
|
|
with self.connect() as con:
|
|
return db.list_rollouts(con, "run", jid)
|
|
|
|
def test_idempotent_and_e2e_numbers_preserved(self):
|
|
self.job()
|
|
original_matrix = deepcopy(self.mx)
|
|
with self.connect() as con:
|
|
for kk in range(4):
|
|
db.rollout_upsert(con, "run", "job", kk, "e2e", "teacher")
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 2)
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
|
|
self.assertEqual([(r["kk"], r["mode"]) for r in self.rollouts()],
|
|
[(0, "e2e"), (1, "e2e"), (2, "e2e"), (3, "e2e"), (4, "repair"), (5, "repair")])
|
|
self.assertEqual(self.mx, original_matrix)
|
|
|
|
def test_only_finished_eligible_oracles_and_selected_jobs(self):
|
|
self.job("eligible")
|
|
self.job("not_selected")
|
|
self.job("passed", state="PASS")
|
|
self.job("unfinished", finished=None)
|
|
self.job("unknown_error", error_class="timeout")
|
|
self.job("wrong_march", march="nash-e")
|
|
self.job("wrong_backend", backend="qat")
|
|
self.job("other_run", run="other")
|
|
chosen = ["eligible", "passed", "unfinished", "unknown_error", "wrong_march", "wrong_backend", "other_run"]
|
|
self.assertEqual(pipeline.enqueue_repairs("run", chosen), 2)
|
|
with self.connect() as con:
|
|
self.assertEqual({r[0] for r in con.execute("SELECT job_id FROM rollouts")}, {"eligible"})
|
|
|
|
def test_mode_enabled_when_and_select_filters(self):
|
|
self.job()
|
|
self.assertEqual(pipeline.enqueue_repairs("run", modes=["e2e"]), 0)
|
|
cfg = self.mx["modes"]["repair"]
|
|
for key, value in [("enabled", False), ("when", "always"), ("select", {"tasks": ["det"]}),
|
|
("select", {"priority": ["B"]}), ("select", {"models": ["resnet50"]}),
|
|
("marches", ["nash-e"]), ("backends", ["qat"])]:
|
|
with self.subTest(key=key, value=value):
|
|
old = deepcopy(cfg)
|
|
cfg[key] = value
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
|
|
cfg.clear()
|
|
cfg.update(old)
|
|
self.model_rows[0]["status"] = "needs_adapter"
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
|
|
cfg["select"] = {"status": ["needs_adapter"]}
|
|
self.assertEqual(pipeline.enqueue_repairs("run", modes=["repair"]), 2)
|
|
|
|
def test_fill_missing_per_policy_without_replacing_completed_rollouts(self):
|
|
self.job()
|
|
self.mx["modes"]["repair"]["policies"] = ["teacher", "student"]
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "job", 7, "repair", "teacher", state="FAIL")
|
|
db.rollout_upsert(con, "run", "job", 10, "e2e", "teacher", state="PASS")
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 3)
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
|
|
rows = self.rollouts()
|
|
self.assertEqual([(r["kk"], r["mode"], r["policy"], r["state"]) for r in rows], [
|
|
(7, "repair", "teacher", "FAIL"), (10, "e2e", "teacher", "PASS"),
|
|
(11, "repair", "teacher", "QUEUED"), (12, "repair", "student", "QUEUED"),
|
|
(13, "repair", "student", "QUEUED"),
|
|
])
|
|
|
|
def test_collect_dry_run_does_not_enqueue(self):
|
|
self.job()
|
|
with patch.object(pipeline, "enqueue_repairs") as enqueue:
|
|
self.assertEqual(pipeline.collect_rollouts("run", dry_run=True, parallel=1, log=lambda _: None), {})
|
|
enqueue.assert_not_called()
|
|
self.assertEqual(self.rollouts(), [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|