data-gen-bpu/scripts/test_collection.py

365 lines
18 KiB
Python

"""Collection must cover ready models promptly without duplicating or losing work."""
from contextlib import contextmanager, nullcontext
import json
import signal
from pathlib import Path
import tempfile
import threading
import time
import unittest
from unittest.mock import patch
from harness import db, pipeline, registry, rollout
class Collection(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.db_path = self.root / "jobs.sqlite"
self.mx = {"marches": ["nash-m"], "backends": ["ptq"], "modes": {
"e2e": {"enabled": True, "select": {"priority": ["A"], "tasks": ["cls", "det"]},
"marches": ["nash-m"], "backends": ["ptq"], "policies": ["teacher"], "k": 4},
"repair": {"enabled": False},
}}
self.rows = []
self.addCleanup(patch.stopall)
patch.object(db, "connect", self.connect).start()
patch.object(pipeline, "matrix", lambda: self.mx).start()
patch.object(pipeline, "models", lambda: self.rows).start()
patch.object(pipeline.paths, "STATE", self.root / "state").start()
patch.object(pipeline.paths, "RUNS", self.root / "runs").start()
patch.object(pipeline.paths, "config", lambda: {"pools": {"agent": {"workers": 3}}}).start()
patch.object(rollout, "setup_models_json").start()
self.ready = patch.object(rollout, "inputs_ready", return_value=True).start()
patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext()).start()
self.export = patch.object(pipeline, "refresh_run_dataset").start()
with self.connect():
pass
@contextmanager
def connect(self):
con = db._open(self.db_path)
con.executescript(db.SCHEMA)
try:
yield con
con.commit()
finally:
con.close()
def job(self, jid, *, state="PASS", priority="A", task="cls", march="nash-m", backend="ptq",
estimate=10, k=0, status="ready"):
self.rows.append({"model_id": jid, "task": task, "priority": priority, "status": status})
with self.connect() as con:
con.execute(
"INSERT INTO jobs (run_id,job_id,task,model,march,backend,qcfg,priority,mode,state,estimate_s)"
" VALUES ('run',?,?,?,?,?,?,?,'oracle',?,?)",
(jid, task, jid, march, backend, "int8", priority, state, estimate))
for kk in range(k):
db.rollout_upsert(con, "run", jid, kk, "e2e", "teacher")
def complete(self, run_id, job_id, kk, mode, policy, **_):
with self.connect() as con:
db.rollout_set(con, run_id, job_id, kk, state="PASS", finished_at=time.time())
return {"terminal": "PASS", "claim_consistent": True}
def collect(self, **kw):
return pipeline.collect_rollouts("run", log=lambda _: None, **kw)
def test_all_coverage_is_idempotent_preserves_repairs_and_non_priority_filters(self):
self.job("eligible", priority="C")
for jid, kw in [("wrong-task", {"task": "occ"}), ("wrong-march", {"march": "nash-p"}),
("wrong-backend", {"backend": "qat"}), ("not-ready", {"status": "needs_adapter"})]:
self.job(jid, **kw)
with self.connect() as con:
db.rollout_upsert(con, "run", "eligible", 0, "repair", "teacher", "FAIL")
db.rollout_upsert(con, "run", "eligible", 4, "e2e", "teacher", "PASS")
self.assertEqual(pipeline.enqueue_e2e_coverage("run"), 3)
self.assertEqual(pipeline.enqueue_e2e_coverage("run"), 0)
with self.connect() as con:
rows = db.list_rollouts(con, "run", "eligible")
self.assertEqual([(r["kk"], r["mode"]) for r in rows],
[(0, "repair"), (4, "e2e"), (5, "e2e"), (6, "e2e"), (7, "e2e")])
self.assertEqual(con.execute("SELECT COUNT(*) FROM annotations").fetchone()[0], 1)
self.assertEqual(con.execute("SELECT COUNT(*) FROM jobs WHERE mode='e2e'").fetchone()[0], 1)
self.assertEqual(self.mx["modes"]["e2e"]["select"]["priority"], ["A"])
self.assertFalse((self.root / "runs").exists())
def test_coverage_fills_each_policy_and_respects_job_selection(self):
self.job("chosen", priority="B")
self.job("other", priority="B")
self.mx["modes"]["e2e"]["policies"] = ["teacher", "student"]
with self.connect() as con:
db.rollout_upsert(con, "run", "chosen", 9, "e2e", "teacher", "TIMEOUT")
self.assertEqual(pipeline.enqueue_e2e_coverage("run", ["chosen"]), 7)
with self.connect() as con:
self.assertEqual(len(db.list_rollouts(con, "run", "chosen")), 8)
self.assertEqual(db.list_rollouts(con, "run", "other"), [])
def test_atomic_claim_prevents_duplicate_job_and_exceeding_pool(self):
self.job("a", k=2)
self.job("b", k=1)
outcomes = []
barrier = threading.Barrier(2)
def claim(kk):
barrier.wait(timeout=2)
with self.connect() as con:
outcomes.append(db.claim_rollout(con, "run", "a", kk, max_active=3))
threads = [threading.Thread(target=claim, args=(kk,)) for kk in range(2)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=3)
self.assertEqual(sorted(outcomes), [False, True])
with self.connect() as con:
self.assertFalse(db.claim_rollout(con, "run", "b", 0, max_active=1))
con.execute("UPDATE rollouts SET state='PASS' WHERE job_id='a' AND state='RUNNING'")
con.commit()
self.assertTrue(db.claim_rollout(con, "run", "b", 0, max_active=1))
self.assertFalse(db.claim_rollout(con, "run", "b", 0, max_active=3))
def test_missing_inputs_do_not_delay_first_agent_attempt(self):
self.job("first", k=4)
self.job("late", state="ORACLE_RUNNING", estimate=1, k=4)
self.job("covered", estimate=0, k=4)
with self.connect() as con:
db.rollout_set(con, "run", "covered", 0, state="PASS", finished_at=1)
started = []
self.ready.side_effect = lambda rid, jid: jid != "late" or bool(started)
def run(*args, **kw):
started.append(args[1])
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
self.assertEqual(self.collect(parallel=1, limit=2), {"PASS": 2})
self.assertEqual(started, ["late", "first"])
def test_three_slots_cover_three_distinct_models(self):
for jid in ("a", "b", "c"):
self.job(jid, k=4)
barrier = threading.Barrier(3)
started = []
def run(*args, **kw):
started.append(args[1])
barrier.wait(timeout=3)
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
self.assertEqual(self.collect(parallel=3, limit=3), {"PASS": 3})
self.assertEqual(set(started), {"a", "b", "c"})
def test_prefers_shorter_ready_inputs_independent_of_oracle_verdict(self):
self.job("failed", state="FAIL", estimate=0, k=1)
self.job("slow", estimate=100, k=1)
self.job("fast", estimate=1, k=1)
rows, pending = pipeline._collection_snapshot("run", None, None)
self.assertFalse(pending)
self.assertEqual([r["job_id"] for r in rows], ["failed", "fast", "slow"])
def test_e2e_prepares_its_own_inputs_and_retired_repair_is_never_ready(self):
self.job("ready", state="ORACLE_RUNNING", k=1)
self.job("missing", state="PASS", k=1)
self.ready.side_effect = lambda rid, jid: jid == "ready"
with self.connect() as con:
db.rollout_upsert(con, "run", "ready", 1, "repair", "teacher")
rows, pending = pipeline._collection_snapshot("run", None, None)
self.assertEqual([(r["job_id"], r["mode"]) for r in rows], [("missing", "e2e"), ("ready", "e2e")])
self.assertFalse(pending)
with self.connect() as con:
self.assertEqual(con.execute("SELECT COUNT(*) FROM rollouts WHERE state!='QUEUED'").fetchone()[0], 0)
def test_collect_preserves_and_skips_historical_queued_repairs(self):
self.job("failed", state="FAIL", k=1)
with self.connect() as con:
db.rollout_upsert(con, "run", "failed", 7, "repair", "teacher")
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
with patch.object(rollout, "run_rollout", self.complete), \
patch.object(pipeline, "enqueue_repairs") as enqueue:
self.assertEqual(self.collect(parallel=1, modes=["e2e", "repair"]), {"PASS": 1})
enqueue.assert_not_called()
with self.connect() as con:
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
rows, pending = pipeline._collection_snapshot("run", None, ["repair"])
self.assertEqual(rows, [])
self.assertFalse(pending)
def test_exception_releases_claim_and_does_not_retry_same_row(self):
self.job("broken", k=1)
with patch.object(rollout, "run_rollout", side_effect=SystemExit("missing cache")):
self.assertEqual(self.collect(parallel=1), {"UNVERIFIED": 1})
with self.connect() as con:
row = db.list_rollouts(con, "run", "broken")[0]
self.assertEqual(row["state"], "UNVERIFIED")
self.assertIsNotNone(row["finished_at"])
def test_exception_after_measurements_still_verifies_and_exports_real_result(self):
self.job("measured", k=1)
jd = self.root / "runs" / "run" / "measured"
rd = jd / "rollouts" / "00"
rd.mkdir(parents=True)
spec = registry.JobSpec(job_id="measured", task="cls", model="measured", march="nash-m",
backend="ptq", qcfg="int8", priority="A", mode="oracle")
(jd / "job.json").write_text(json.dumps({"job_id": "measured", "run_id": "run",
"gates": registry.gate_snapshot(spec)}))
(rd / "rollout.json").write_text(json.dumps({"state": "RUNNING", "mode": "e2e", "policy": "teacher"}))
(rd / "claim.json").write_text('{"terminal":"PASS"}')
for stage, measurements in [
("05-compile", {}),
("06-hbm", {"G1": dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False)}),
("07-board", {"G4": {"latency_ms_mean": 1}}),
("08-eval", {"G3": {"fp32": 80, "quant": 79, "source": "board", "tier": "t1", "n": 2000}}),
]:
directory = rd / "stages" / stage
directory.mkdir(parents=True)
(directory / "meta.json").write_text(json.dumps({"state": "OK", "measurements": measurements, "attempt": 0}))
with patch.object(rollout, "run_rollout", side_effect=RuntimeError("agent stream closed")):
self.assertEqual(self.collect(parallel=1), {"PASS": 1})
verdict = json.loads((rd / "verdict.json").read_text())
self.assertEqual(verdict["terminal"], "PASS")
self.assertNotIn("oracle_ref", verdict)
self.assertEqual(verdict["execution_terminal"], "ERROR")
self.assertTrue(verdict["completion"]["workflow_completed"])
self.export.assert_called_with("run")
def test_dry_run_watch_neither_claims_nor_adds_coverage(self):
self.job("example", priority="B", k=1)
with patch.object(rollout, "run_rollout", return_value={"dry_run": True}), \
patch.object(pipeline, "enqueue_e2e_coverage") as enqueue:
self.assertEqual(self.collect(dry_run=True, watch=True, coverage="all"), {"DRY_RUN": 1})
enqueue.assert_not_called()
self.export.assert_not_called()
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, "run", "example")[0]["state"], "QUEUED")
def test_singleton_collector_cannot_duplicate_inflight_work(self):
self.job("model", k=1)
entered, release = threading.Event(), threading.Event()
def run(*args, **kw):
entered.set()
release.wait(timeout=3)
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
thread = threading.Thread(target=self.collect, kwargs={"parallel": 1})
thread.start()
try:
self.assertTrue(entered.wait(timeout=2))
self.assertEqual(self.collect(parallel=1), {})
finally:
release.set()
thread.join(timeout=3)
self.assertFalse(thread.is_alive())
def test_drain_finishes_inflight_without_claiming_repeats(self):
self.job("model", k=4)
def run(*args, **kw):
pipeline.collect_drain_file("run").touch()
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
with self.connect() as con:
states = [r["state"] for r in db.list_rollouts(con, "run", "model")]
self.assertEqual(states, ["PASS", "QUEUED", "QUEUED", "QUEUED"])
def test_first_pass_preserves_repeats_and_failed_attempts_consume_one_slot(self):
self.job("fresh", k=4)
self.job("covered", k=4)
self.job("failed", k=4)
with self.connect() as con:
db.rollout_set(con, "run", "covered", 0, state="PASS", started_at=1, finished_at=2)
db.rollout_set(con, "run", "failed", 0, state="UNVERIFIED", started_at=1, finished_at=2)
with patch.object(rollout, "run_rollout", self.complete):
self.assertEqual(self.collect(parallel=1, first_pass=True), {"PASS": 1})
with self.connect() as con:
for model in ("fresh", "covered", "failed"):
rows = db.list_rollouts(con, "run", model)
self.assertEqual([r["state"] for r in rows[1:]], ["QUEUED"] * 3)
def test_first_pass_ignores_completed_repair_but_keeps_e2e_history(self):
self.job("repair-only", k=2)
self.job("e2e-covered", k=2)
with self.connect() as con:
db.rollout_upsert(con, "run", "repair-only", 7, "repair", "teacher", "PASS")
db.rollout_set(con, "run", "repair-only", 7, started_at=1, finished_at=2)
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
db.rollout_set(con, "run", "e2e-covered", 0, state="PASS", started_at=1, finished_at=2)
with patch.object(rollout, "run_rollout", side_effect=self.complete) as run:
self.assertEqual(self.collect(parallel=1, modes=["e2e"], first_pass=True), {"PASS": 1})
self.assertEqual(run.call_args.args[1:4], ("repair-only", 0, "e2e"))
with self.connect() as con:
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
for jid in ("repair-only", "e2e-covered"):
self.assertEqual(db.list_rollouts(con, "run", jid)[1]["state"], "QUEUED")
def test_sigterm_drains_inflight_and_restores_previous_handler(self):
self.job("model", k=2)
old_handler = signal.getsignal(signal.SIGTERM)
def run(*args, **kw):
signal.getsignal(signal.SIGTERM)(signal.SIGTERM, None)
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
self.assertEqual(signal.getsignal(signal.SIGTERM), old_handler)
def test_last_export_exception_is_reported_without_losing_completed_rollout(self):
self.job("model", k=1)
messages = []
with patch.object(rollout, "run_rollout", self.complete), \
patch.object(pipeline, "refresh_run_dataset", side_effect=RuntimeError("export unavailable")):
self.assertEqual(pipeline.collect_rollouts("run", parallel=1, log=messages.append), {"PASS": 1})
self.assertTrue(any("export unavailable" in message for message in messages))
def test_slow_export_does_not_delay_next_dispatch(self):
self.job("model", k=2)
second_started = threading.Event()
export_saw_second = []
def run(*args, **kw):
if args[2] == 1:
second_started.set()
return self.complete(*args, **kw)
def export(_):
export_saw_second.append(second_started.wait(timeout=2))
with patch.object(rollout, "run_rollout", run), patch.object(pipeline, "refresh_run_dataset", export):
self.assertEqual(self.collect(parallel=1), {"PASS": 2})
self.assertTrue(export_saw_second)
self.assertTrue(all(export_saw_second))
def test_existing_active_rollout_counts_against_slots_and_blocks_same_job(self):
self.job("old", k=2)
self.job("fresh", k=1)
with self.connect() as con:
db.rollout_set(con, "run", "old", 0, state="RUNNING", started_at=1)
started = []
def run(*args, **kw):
started.append(args[1])
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
self.assertEqual(self.collect(parallel=3), {"PASS": 1})
self.assertEqual(started, ["fresh"])
def test_watch_finishes_independent_of_oracle_running_state(self):
self.job("late", state="ORACLE_RUNNING", k=1)
with patch.object(rollout, "run_rollout", self.complete):
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
with self.connect() as con:
self.assertEqual(db.get_job(con, "run", "late")["state"], "ORACLE_RUNNING")
if __name__ == "__main__":
unittest.main()