"""Retired Oracle repair generation preserves historical rows and identities.""" from contextlib import contextmanager from copy import deepcopy from pathlib import Path import json import tempfile import unittest from unittest.mock import patch from harness import db, pipeline, registry class RepairGeneration(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.db_path = Path(self.tmp.name) / "jobs.sqlite" self.mx = { "marches": ["nash-m"], "backends": ["ptq"], "modes": {"repair": { "enabled": True, "when": "oracle_fail", "marches": "all", "backends": "all", "error_classes": ["accuracy_drop"], "policies": ["teacher"], "k": 2, }}, } self.model_rows = [{"model_id": "resnet18", "task": "cls", "priority": "A", "status": "ready"}] self.addCleanup(patch.stopall) patch.object(pipeline.db, "connect", self.connect).start() patch.object(pipeline, "matrix", lambda: self.mx).start() patch.object(pipeline, "models", lambda: self.model_rows).start() @contextmanager def connect(self): con = db._open(self.db_path) con.executescript(db.SCHEMA) try: yield con con.commit() finally: con.close() def job(self, jid="job", *, state="FAIL", error_class="accuracy_drop", run="run", march="nash-m", backend="ptq", finished=1): with self.connect() as con: con.execute( "INSERT INTO jobs (run_id,job_id,model,march,backend,mode,state,error_class,finished_at)" " VALUES (?,?,?,?,?,'oracle',?,?,?)", (run, jid, "resnet18", march, backend, state, error_class, finished)) def rollouts(self, jid="job"): with self.connect() as con: return db.list_rollouts(con, "run", jid) def test_idempotent_and_e2e_numbers_preserved(self): self.job() original_matrix = deepcopy(self.mx) with self.connect() as con: for kk in range(4): db.rollout_upsert(con, "run", "job", kk, "e2e", "teacher") self.assertEqual(pipeline.enqueue_repairs("run"), 0) self.assertEqual(pipeline.enqueue_repairs("run"), 0) self.assertEqual([(r["kk"], r["mode"]) for r in self.rollouts()], [(0, "e2e"), (1, "e2e"), (2, "e2e"), (3, "e2e")]) self.assertEqual(self.mx, original_matrix) def test_finished_oracle_failures_do_not_generate_any_repairs(self): self.job("eligible") self.job("not_selected") self.job("passed", state="PASS") self.job("unfinished", finished=None) self.job("unknown_error", error_class="timeout") self.job("wrong_march", march="nash-e") self.job("wrong_backend", backend="qat") self.job("other_run", run="other") chosen = ["eligible", "passed", "unfinished", "unknown_error", "wrong_march", "wrong_backend", "other_run"] self.assertEqual(pipeline.enqueue_repairs("run", chosen), 0) with self.connect() as con: self.assertEqual({r[0] for r in con.execute("SELECT job_id FROM rollouts")}, set()) def test_mode_enabled_when_and_select_filters(self): self.job() self.assertEqual(pipeline.enqueue_repairs("run", modes=["e2e"]), 0) cfg = self.mx["modes"]["repair"] for key, value in [("enabled", False), ("when", "always"), ("select", {"tasks": ["det"]}), ("select", {"priority": ["B"]}), ("select", {"models": ["resnet50"]}), ("marches", ["nash-e"]), ("backends", ["qat"])]: with self.subTest(key=key, value=value): old = deepcopy(cfg) cfg[key] = value self.assertEqual(pipeline.enqueue_repairs("run"), 0) cfg.clear() cfg.update(old) self.model_rows[0]["status"] = "needs_adapter" self.assertEqual(pipeline.enqueue_repairs("run"), 0) cfg["select"] = {"status": ["needs_adapter"]} self.assertEqual(pipeline.enqueue_repairs("run", modes=["repair"]), 0) def test_historical_rows_and_remaining_quotas_are_not_changed(self): self.job() self.mx["modes"]["repair"]["policies"] = ["teacher", "student"] with self.connect() as con: db.rollout_upsert(con, "run", "job", 7, "repair", "teacher", state="FAIL") db.rollout_upsert(con, "run", "job", 10, "e2e", "teacher", state="PASS") self.assertEqual(pipeline.enqueue_repairs("run"), 0) self.assertEqual(pipeline.enqueue_repairs("run"), 0) rows = self.rollouts() self.assertEqual([(r["kk"], r["mode"], r["policy"], r["state"]) for r in rows], [ (7, "repair", "teacher", "FAIL"), (10, "e2e", "teacher", "PASS"), ]) def test_expand_cannot_reenable_repair_through_legacy_matrix(self): self.mx["modes"]["repair"].update(when="always", enabled=True) with patch.object(registry, "matrix", return_value=self.mx), \ patch.object(registry, "models", return_value=self.model_rows): self.assertEqual(registry.expand_matrix(["repair"]), []) def test_collect_dry_run_does_not_enqueue(self): self.job() with patch.object(pipeline, "enqueue_repairs") as enqueue: self.assertEqual(pipeline.collect_rollouts("run", dry_run=True, parallel=1, log=lambda _: None), {}) enqueue.assert_not_called() self.assertEqual(self.rollouts(), []) class BoardResourceManifest(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.root = Path(self.tmp.name) self.reg = self.root / "registry" self.reg.mkdir() self.run = self.root / "runs/r" self.run.mkdir(parents=True) self.initial = json.dumps({"march_by_board": {"s100p": "nash-m"}, "boards": {"first": {"slot": 0}}}).encode() (self.reg / "boards.yaml").write_bytes(self.initial) (self.reg / "gates.yaml").write_text("tolerance: 5\n") self.addCleanup(patch.stopall) patch.object(registry, "REGISTRY", self.reg).start() patch.object(pipeline.paths, "RUNS", self.root / "runs").start() patch.object(pipeline, "build_manifest", self.current).start() self.manifest = self.current() def current(self): return {"registry_fingerprint": registry.registry_fingerprint(), "registry_task_fingerprint": registry.registry_fingerprint(include_board_resources=False), "venv": {}, "evaluator_version": "0.5.0"} def save(self): (self.run / "manifest.json").write_text(json.dumps(self.manifest)) def add_board(self): config = json.loads(self.initial) config["boards"]["second"] = {"slot": 1} (self.reg / "boards.yaml").write_text(json.dumps(config)) def test_new_manifest_allows_resources_but_rejects_changed_task_inputs(self): self.save(); self.add_board() self.assertEqual(pipeline.check_manifest("r"), []) (self.reg / "gates.yaml").write_text("tolerance: 10\n") self.assertIn("registry 改了:开新 run", pipeline.check_manifest("r")) def test_legacy_manifest_requires_valid_original_snapshot(self): self.manifest.pop("registry_task_fingerprint") self.save(); self.add_board() self.assertTrue(pipeline.check_manifest("r")) (self.run / "board_registry.initial.yaml").write_bytes(self.initial) self.assertEqual(pipeline.check_manifest("r"), []) (self.reg / "gates.yaml").write_text("tolerance: 10\n") self.assertTrue(pipeline.check_manifest("r")) def test_board_snapshot_cannot_allow_changed_march_mapping(self): self.manifest.pop("registry_task_fingerprint") self.save() (self.run / "board_registry.initial.yaml").write_bytes(self.initial) config = json.loads(self.initial) config["march_by_board"]["s100p"] = "nash-p" (self.reg / "boards.yaml").write_text(json.dumps(config)) self.assertTrue(pipeline.check_manifest("r")) if __name__ == "__main__": unittest.main()