data-gen-bpu/scripts/test_generation.py

178 lines
8.0 KiB
Python

"""Conditional repair generation preserves the existing matrix and rollout numbering."""
from contextlib import contextmanager
from copy import deepcopy
from pathlib import Path
import json
import tempfile
import unittest
from unittest.mock import patch
from harness import db, pipeline, registry
class RepairGeneration(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.db_path = Path(self.tmp.name) / "jobs.sqlite"
self.mx = {
"marches": ["nash-m"], "backends": ["ptq"],
"modes": {"repair": {
"enabled": True, "when": "oracle_fail", "marches": "all", "backends": "all",
"error_classes": ["accuracy_drop"], "policies": ["teacher"], "k": 2,
}},
}
self.model_rows = [{"model_id": "resnet18", "task": "cls", "priority": "A", "status": "ready"}]
self.addCleanup(patch.stopall)
patch.object(pipeline.db, "connect", self.connect).start()
patch.object(pipeline, "matrix", lambda: self.mx).start()
patch.object(pipeline, "models", lambda: self.model_rows).start()
@contextmanager
def connect(self):
con = db._open(self.db_path)
con.executescript(db.SCHEMA)
try:
yield con
con.commit()
finally:
con.close()
def job(self, jid="job", *, state="FAIL", error_class="accuracy_drop", run="run",
march="nash-m", backend="ptq", finished=1):
with self.connect() as con:
con.execute(
"INSERT INTO jobs (run_id,job_id,model,march,backend,mode,state,error_class,finished_at)"
" VALUES (?,?,?,?,?,'oracle',?,?,?)",
(run, jid, "resnet18", march, backend, state, error_class, finished))
def rollouts(self, jid="job"):
with self.connect() as con:
return db.list_rollouts(con, "run", jid)
def test_idempotent_and_e2e_numbers_preserved(self):
self.job()
original_matrix = deepcopy(self.mx)
with self.connect() as con:
for kk in range(4):
db.rollout_upsert(con, "run", "job", kk, "e2e", "teacher")
self.assertEqual(pipeline.enqueue_repairs("run"), 2)
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
self.assertEqual([(r["kk"], r["mode"]) for r in self.rollouts()],
[(0, "e2e"), (1, "e2e"), (2, "e2e"), (3, "e2e"), (4, "repair"), (5, "repair")])
self.assertEqual(self.mx, original_matrix)
def test_only_finished_eligible_oracles_and_selected_jobs(self):
self.job("eligible")
self.job("not_selected")
self.job("passed", state="PASS")
self.job("unfinished", finished=None)
self.job("unknown_error", error_class="timeout")
self.job("wrong_march", march="nash-e")
self.job("wrong_backend", backend="qat")
self.job("other_run", run="other")
chosen = ["eligible", "passed", "unfinished", "unknown_error", "wrong_march", "wrong_backend", "other_run"]
self.assertEqual(pipeline.enqueue_repairs("run", chosen), 2)
with self.connect() as con:
self.assertEqual({r[0] for r in con.execute("SELECT job_id FROM rollouts")}, {"eligible"})
def test_mode_enabled_when_and_select_filters(self):
self.job()
self.assertEqual(pipeline.enqueue_repairs("run", modes=["e2e"]), 0)
cfg = self.mx["modes"]["repair"]
for key, value in [("enabled", False), ("when", "always"), ("select", {"tasks": ["det"]}),
("select", {"priority": ["B"]}), ("select", {"models": ["resnet50"]}),
("marches", ["nash-e"]), ("backends", ["qat"])]:
with self.subTest(key=key, value=value):
old = deepcopy(cfg)
cfg[key] = value
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
cfg.clear()
cfg.update(old)
self.model_rows[0]["status"] = "needs_adapter"
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
cfg["select"] = {"status": ["needs_adapter"]}
self.assertEqual(pipeline.enqueue_repairs("run", modes=["repair"]), 2)
def test_fill_missing_per_policy_without_replacing_completed_rollouts(self):
self.job()
self.mx["modes"]["repair"]["policies"] = ["teacher", "student"]
with self.connect() as con:
db.rollout_upsert(con, "run", "job", 7, "repair", "teacher", state="FAIL")
db.rollout_upsert(con, "run", "job", 10, "e2e", "teacher", state="PASS")
self.assertEqual(pipeline.enqueue_repairs("run"), 3)
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
rows = self.rollouts()
self.assertEqual([(r["kk"], r["mode"], r["policy"], r["state"]) for r in rows], [
(7, "repair", "teacher", "FAIL"), (10, "e2e", "teacher", "PASS"),
(11, "repair", "teacher", "QUEUED"), (12, "repair", "student", "QUEUED"),
(13, "repair", "student", "QUEUED"),
])
def test_collect_dry_run_does_not_enqueue(self):
self.job()
with patch.object(pipeline, "enqueue_repairs") as enqueue:
self.assertEqual(pipeline.collect_rollouts("run", dry_run=True, parallel=1, log=lambda _: None), {})
enqueue.assert_not_called()
self.assertEqual(self.rollouts(), [])
class BoardResourceManifest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.reg = self.root / "registry"
self.reg.mkdir()
self.run = self.root / "runs/r"
self.run.mkdir(parents=True)
self.initial = json.dumps({"march_by_board": {"s100p": "nash-m"}, "boards": {"first": {"slot": 0}}}).encode()
(self.reg / "boards.yaml").write_bytes(self.initial)
(self.reg / "gates.yaml").write_text("tolerance: 5\n")
self.addCleanup(patch.stopall)
patch.object(registry, "REGISTRY", self.reg).start()
patch.object(pipeline.paths, "RUNS", self.root / "runs").start()
patch.object(pipeline, "build_manifest", self.current).start()
self.manifest = self.current()
def current(self):
return {"registry_fingerprint": registry.registry_fingerprint(),
"registry_task_fingerprint": registry.registry_fingerprint(include_board_resources=False),
"venv": {}, "evaluator_version": "0.5.0"}
def save(self):
(self.run / "manifest.json").write_text(json.dumps(self.manifest))
def add_board(self):
config = json.loads(self.initial)
config["boards"]["second"] = {"slot": 1}
(self.reg / "boards.yaml").write_text(json.dumps(config))
def test_new_manifest_allows_resources_but_rejects_changed_task_inputs(self):
self.save(); self.add_board()
self.assertEqual(pipeline.check_manifest("r"), [])
(self.reg / "gates.yaml").write_text("tolerance: 10\n")
self.assertIn("registry 改了:开新 run", pipeline.check_manifest("r"))
def test_legacy_manifest_requires_valid_original_snapshot(self):
self.manifest.pop("registry_task_fingerprint")
self.save(); self.add_board()
self.assertTrue(pipeline.check_manifest("r"))
(self.run / "board_registry.initial.yaml").write_bytes(self.initial)
self.assertEqual(pipeline.check_manifest("r"), [])
(self.reg / "gates.yaml").write_text("tolerance: 10\n")
self.assertTrue(pipeline.check_manifest("r"))
def test_board_snapshot_cannot_allow_changed_march_mapping(self):
self.manifest.pop("registry_task_fingerprint")
self.save()
(self.run / "board_registry.initial.yaml").write_bytes(self.initial)
config = json.loads(self.initial)
config["march_by_board"]["s100p"] = "nash-p"
(self.reg / "boards.yaml").write_text(json.dumps(config))
self.assertTrue(pipeline.check_manifest("r"))
if __name__ == "__main__":
unittest.main()