182 lines
8.2 KiB
Python
182 lines
8.2 KiB
Python
"""Retired Oracle repair generation preserves historical rows and identities."""
|
|
from contextlib import contextmanager
|
|
from copy import deepcopy
|
|
from pathlib import Path
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import db, pipeline, registry
|
|
|
|
|
|
class RepairGeneration(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.db_path = Path(self.tmp.name) / "jobs.sqlite"
|
|
self.mx = {
|
|
"marches": ["nash-m"], "backends": ["ptq"],
|
|
"modes": {"repair": {
|
|
"enabled": True, "when": "oracle_fail", "marches": "all", "backends": "all",
|
|
"error_classes": ["accuracy_drop"], "policies": ["teacher"], "k": 2,
|
|
}},
|
|
}
|
|
self.model_rows = [{"model_id": "resnet18", "task": "cls", "priority": "A", "status": "ready"}]
|
|
self.addCleanup(patch.stopall)
|
|
patch.object(pipeline.db, "connect", self.connect).start()
|
|
patch.object(pipeline, "matrix", lambda: self.mx).start()
|
|
patch.object(pipeline, "models", lambda: self.model_rows).start()
|
|
|
|
@contextmanager
|
|
def connect(self):
|
|
con = db._open(self.db_path)
|
|
con.executescript(db.SCHEMA)
|
|
try:
|
|
yield con
|
|
con.commit()
|
|
finally:
|
|
con.close()
|
|
|
|
def job(self, jid="job", *, state="FAIL", error_class="accuracy_drop", run="run",
|
|
march="nash-m", backend="ptq", finished=1):
|
|
with self.connect() as con:
|
|
con.execute(
|
|
"INSERT INTO jobs (run_id,job_id,model,march,backend,mode,state,error_class,finished_at)"
|
|
" VALUES (?,?,?,?,?,'oracle',?,?,?)",
|
|
(run, jid, "resnet18", march, backend, state, error_class, finished))
|
|
|
|
def rollouts(self, jid="job"):
|
|
with self.connect() as con:
|
|
return db.list_rollouts(con, "run", jid)
|
|
|
|
def test_idempotent_and_e2e_numbers_preserved(self):
|
|
self.job()
|
|
original_matrix = deepcopy(self.mx)
|
|
with self.connect() as con:
|
|
for kk in range(4):
|
|
db.rollout_upsert(con, "run", "job", kk, "e2e", "teacher")
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
|
|
self.assertEqual([(r["kk"], r["mode"]) for r in self.rollouts()],
|
|
[(0, "e2e"), (1, "e2e"), (2, "e2e"), (3, "e2e")])
|
|
self.assertEqual(self.mx, original_matrix)
|
|
|
|
def test_finished_oracle_failures_do_not_generate_any_repairs(self):
|
|
self.job("eligible")
|
|
self.job("not_selected")
|
|
self.job("passed", state="PASS")
|
|
self.job("unfinished", finished=None)
|
|
self.job("unknown_error", error_class="timeout")
|
|
self.job("wrong_march", march="nash-e")
|
|
self.job("wrong_backend", backend="qat")
|
|
self.job("other_run", run="other")
|
|
chosen = ["eligible", "passed", "unfinished", "unknown_error", "wrong_march", "wrong_backend", "other_run"]
|
|
self.assertEqual(pipeline.enqueue_repairs("run", chosen), 0)
|
|
with self.connect() as con:
|
|
self.assertEqual({r[0] for r in con.execute("SELECT job_id FROM rollouts")}, set())
|
|
|
|
def test_mode_enabled_when_and_select_filters(self):
|
|
self.job()
|
|
self.assertEqual(pipeline.enqueue_repairs("run", modes=["e2e"]), 0)
|
|
cfg = self.mx["modes"]["repair"]
|
|
for key, value in [("enabled", False), ("when", "always"), ("select", {"tasks": ["det"]}),
|
|
("select", {"priority": ["B"]}), ("select", {"models": ["resnet50"]}),
|
|
("marches", ["nash-e"]), ("backends", ["qat"])]:
|
|
with self.subTest(key=key, value=value):
|
|
old = deepcopy(cfg)
|
|
cfg[key] = value
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
|
|
cfg.clear()
|
|
cfg.update(old)
|
|
self.model_rows[0]["status"] = "needs_adapter"
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
|
|
cfg["select"] = {"status": ["needs_adapter"]}
|
|
self.assertEqual(pipeline.enqueue_repairs("run", modes=["repair"]), 0)
|
|
|
|
def test_historical_rows_and_remaining_quotas_are_not_changed(self):
|
|
self.job()
|
|
self.mx["modes"]["repair"]["policies"] = ["teacher", "student"]
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "job", 7, "repair", "teacher", state="FAIL")
|
|
db.rollout_upsert(con, "run", "job", 10, "e2e", "teacher", state="PASS")
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
|
|
self.assertEqual(pipeline.enqueue_repairs("run"), 0)
|
|
rows = self.rollouts()
|
|
self.assertEqual([(r["kk"], r["mode"], r["policy"], r["state"]) for r in rows], [
|
|
(7, "repair", "teacher", "FAIL"), (10, "e2e", "teacher", "PASS"),
|
|
])
|
|
|
|
def test_expand_cannot_reenable_repair_through_legacy_matrix(self):
|
|
self.mx["modes"]["repair"].update(when="always", enabled=True)
|
|
with patch.object(registry, "matrix", return_value=self.mx), \
|
|
patch.object(registry, "models", return_value=self.model_rows):
|
|
self.assertEqual(registry.expand_matrix(["repair"]), [])
|
|
|
|
def test_collect_dry_run_does_not_enqueue(self):
|
|
self.job()
|
|
with patch.object(pipeline, "enqueue_repairs") as enqueue:
|
|
self.assertEqual(pipeline.collect_rollouts("run", dry_run=True, parallel=1, log=lambda _: None), {})
|
|
enqueue.assert_not_called()
|
|
self.assertEqual(self.rollouts(), [])
|
|
|
|
|
|
class BoardResourceManifest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.root = Path(self.tmp.name)
|
|
self.reg = self.root / "registry"
|
|
self.reg.mkdir()
|
|
self.run = self.root / "runs/r"
|
|
self.run.mkdir(parents=True)
|
|
self.initial = json.dumps({"march_by_board": {"s100p": "nash-m"}, "boards": {"first": {"slot": 0}}}).encode()
|
|
(self.reg / "boards.yaml").write_bytes(self.initial)
|
|
(self.reg / "gates.yaml").write_text("tolerance: 5\n")
|
|
self.addCleanup(patch.stopall)
|
|
patch.object(registry, "REGISTRY", self.reg).start()
|
|
patch.object(pipeline.paths, "RUNS", self.root / "runs").start()
|
|
patch.object(pipeline, "build_manifest", self.current).start()
|
|
self.manifest = self.current()
|
|
|
|
def current(self):
|
|
return {"registry_fingerprint": registry.registry_fingerprint(),
|
|
"registry_task_fingerprint": registry.registry_fingerprint(include_board_resources=False),
|
|
"venv": {}, "evaluator_version": "0.5.0"}
|
|
|
|
def save(self):
|
|
(self.run / "manifest.json").write_text(json.dumps(self.manifest))
|
|
|
|
def add_board(self):
|
|
config = json.loads(self.initial)
|
|
config["boards"]["second"] = {"slot": 1}
|
|
(self.reg / "boards.yaml").write_text(json.dumps(config))
|
|
|
|
def test_new_manifest_allows_resources_but_rejects_changed_task_inputs(self):
|
|
self.save(); self.add_board()
|
|
self.assertEqual(pipeline.check_manifest("r"), [])
|
|
(self.reg / "gates.yaml").write_text("tolerance: 10\n")
|
|
self.assertIn("registry 改了:开新 run", pipeline.check_manifest("r"))
|
|
|
|
def test_legacy_manifest_requires_valid_original_snapshot(self):
|
|
self.manifest.pop("registry_task_fingerprint")
|
|
self.save(); self.add_board()
|
|
self.assertTrue(pipeline.check_manifest("r"))
|
|
(self.run / "board_registry.initial.yaml").write_bytes(self.initial)
|
|
self.assertEqual(pipeline.check_manifest("r"), [])
|
|
(self.reg / "gates.yaml").write_text("tolerance: 10\n")
|
|
self.assertTrue(pipeline.check_manifest("r"))
|
|
|
|
def test_board_snapshot_cannot_allow_changed_march_mapping(self):
|
|
self.manifest.pop("registry_task_fingerprint")
|
|
self.save()
|
|
(self.run / "board_registry.initial.yaml").write_bytes(self.initial)
|
|
config = json.loads(self.initial)
|
|
config["march_by_board"]["s100p"] = "nash-p"
|
|
(self.reg / "boards.yaml").write_text(json.dumps(config))
|
|
self.assertTrue(pipeline.check_manifest("r"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|