data-gen-bpu/scripts/test_generation.py

184 lines
8.3 KiB
Python

"""Retired Oracle repair generation preserves historical rows and identities."""
from contextlib import contextmanager
from copy import deepcopy
from pathlib import Path
import json
import tempfile
import unittest
from unittest.mock import patch
from harness import db, pipeline, registry
class RepairGeneration(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.db_path = Path(self.tmp.name) / "jobs.sqlite"
self.mx = {
"marches": ["nash-m"], "backends": ["ptq"],
"modes": {"repair": {
"enabled": True, "when": "oracle_fail", "marches": "all", "backends": "all",
"error_classes": ["accuracy_drop"], "policies": ["teacher"], "k": 2,
}},
}
self.model_rows = [{"model_id": "resnet18", "task": "cls", "priority": "A", "status": "ready"}]
self.addCleanup(patch.stopall)
readonly = db.readonly
patch.object(db, "readonly", lambda: readonly(self.db_path)).start()
patch.object(pipeline.db, "connect", self.connect).start()
patch.object(pipeline, "matrix", lambda: self.mx).start()
patch.object(pipeline, "models", lambda: self.model_rows).start()
@contextmanager
def connect(self):
con = db._open(self.db_path)
con.executescript(db.SCHEMA)
try:
yield con
con.commit()
finally:
con.close()
def job(self, jid="job", *, state="FAIL", error_class="accuracy_drop", run="run",
march="nash-m", backend="ptq", finished=1):
with self.connect() as con:
con.execute(
"INSERT INTO jobs (run_id,job_id,model,march,backend,mode,state,error_class,finished_at)"
" VALUES (?,?,?,?,?,'oracle',?,?,?)",
(run, jid, "resnet18", march, backend, state, error_class, finished))
def rollouts(self, jid="job"):
with self.connect() as con:
return db.list_rollouts(con, "run", jid)
def test_idempotent_and_e2e_numbers_preserved(self):
self.job()
original_matrix = deepcopy(self.mx)
with self.connect() as con:
for kk in range(4):
db.rollout_upsert(con, "run", "job", kk, "e2e", "teacher")
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
self.assertEqual([(r["kk"], r["mode"]) for r in self.rollouts()],
[(0, "e2e"), (1, "e2e"), (2, "e2e"), (3, "e2e")])
self.assertEqual(self.mx, original_matrix)
def test_finished_oracle_failures_do_not_generate_any_repairs(self):
self.job("eligible")
self.job("not_selected")
self.job("passed", state="PASS")
self.job("unfinished", finished=None)
self.job("unknown_error", error_class="timeout")
self.job("wrong_march", march="nash-e")
self.job("wrong_backend", backend="qat")
self.job("other_run", run="other")
chosen = ["eligible", "passed", "unfinished", "unknown_error", "wrong_march", "wrong_backend", "other_run"]
self.assertEqual(pipeline.enqueue_repairs("run", chosen), 0)
with self.connect() as con:
self.assertEqual({r[0] for r in con.execute("SELECT job_id FROM rollouts")}, set())
def test_mode_enabled_when_and_select_filters(self):
self.job()
self.assertEqual(pipeline.enqueue_repairs("run", modes=["e2e"]), 0)
cfg = self.mx["modes"]["repair"]
for key, value in [("enabled", False), ("when", "always"), ("select", {"tasks": ["det"]}),
("select", {"priority": ["B"]}), ("select", {"models": ["resnet50"]}),
("marches", ["nash-e"]), ("backends", ["qat"])]:
with self.subTest(key=key, value=value):
old = deepcopy(cfg)
cfg[key] = value
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
cfg.clear()
cfg.update(old)
self.model_rows[0]["status"] = "needs_adapter"
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
cfg["select"] = {"status": ["needs_adapter"]}
self.assertEqual(pipeline.enqueue_repairs("run", modes=["repair"]), 0)
def test_historical_rows_and_remaining_quotas_are_not_changed(self):
self.job()
self.mx["modes"]["repair"]["policies"] = ["teacher", "student"]
with self.connect() as con:
db.rollout_upsert(con, "run", "job", 7, "repair", "teacher", state="FAIL")
db.rollout_upsert(con, "run", "job", 10, "e2e", "teacher", state="PASS")
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
rows = self.rollouts()
self.assertEqual([(r["kk"], r["mode"], r["policy"], r["state"]) for r in rows], [
(7, "repair", "teacher", "FAIL"), (10, "e2e", "teacher", "PASS"),
])
def test_expand_cannot_reenable_repair_through_legacy_matrix(self):
self.mx["modes"]["repair"].update(when="always", enabled=True)
with patch.object(registry, "matrix", return_value=self.mx), \
patch.object(registry, "models", return_value=self.model_rows):
self.assertEqual(registry.expand_matrix(["repair"]), [])
def test_collect_dry_run_does_not_enqueue(self):
self.job()
with patch.object(pipeline, "enqueue_repairs") as enqueue:
self.assertEqual(pipeline.collect_rollouts("run", dry_run=True, parallel=1, log=lambda _: None), {})
enqueue.assert_not_called()
self.assertEqual(self.rollouts(), [])
class BoardResourceManifest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.reg = self.root / "registry"
self.reg.mkdir()
self.run = self.root / "runs/r"
self.run.mkdir(parents=True)
self.initial = json.dumps({"march_by_board": {"s100p": "nash-m"}, "boards": {"first": {"slot": 0}}}).encode()
(self.reg / "boards.yaml").write_bytes(self.initial)
(self.reg / "gates.yaml").write_text("tolerance: 5\n")
self.addCleanup(patch.stopall)
patch.object(registry, "REGISTRY", self.reg).start()
patch.object(pipeline.paths, "RUNS", self.root / "runs").start()
patch.object(pipeline, "build_manifest", self.current).start()
self.manifest = self.current()
def current(self):
return {"registry_fingerprint": registry.registry_fingerprint(),
"registry_task_fingerprint": registry.registry_fingerprint(include_board_resources=False),
"venv": {}, "evaluator_version": "0.5.0"}
def save(self):
(self.run / "manifest.json").write_text(json.dumps(self.manifest))
def add_board(self):
config = json.loads(self.initial)
config["boards"]["second"] = {"slot": 1}
(self.reg / "boards.yaml").write_text(json.dumps(config))
def test_new_manifest_allows_resources_but_rejects_changed_task_inputs(self):
self.save(); self.add_board()
self.assertEqual(pipeline.check_manifest("r"), [])
(self.reg / "gates.yaml").write_text("tolerance: 10\n")
self.assertIn("registry 改了:开新 run", pipeline.check_manifest("r"))
def test_legacy_manifest_requires_valid_original_snapshot(self):
self.manifest.pop("registry_task_fingerprint")
self.save(); self.add_board()
self.assertTrue(pipeline.check_manifest("r"))
(self.run / "board_registry.initial.yaml").write_bytes(self.initial)
self.assertEqual(pipeline.check_manifest("r"), [])
(self.reg / "gates.yaml").write_text("tolerance: 10\n")
self.assertTrue(pipeline.check_manifest("r"))
def test_board_snapshot_cannot_allow_changed_march_mapping(self):
self.manifest.pop("registry_task_fingerprint")
self.save()
(self.run / "board_registry.initial.yaml").write_bytes(self.initial)
config = json.loads(self.initial)
config["march_by_board"]["s100p"] = "nash-p"
(self.reg / "boards.yaml").write_text(json.dumps(config))
self.assertTrue(pipeline.check_manifest("r"))
if __name__ == "__main__":
unittest.main()