402 lines
20 KiB
Python
402 lines
20 KiB
Python
"""Collection must cover queued models promptly without duplicating or losing work."""
|
|
from contextlib import contextmanager, nullcontext
|
|
import hashlib
|
|
import json
|
|
import signal
|
|
from pathlib import Path
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import db, pipeline, registry, rollout
|
|
|
|
|
|
class Collection(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.root = Path(self.tmp.name)
|
|
self.db_path = self.root / "jobs.sqlite"
|
|
self.mx = {"marches": ["nash-m"], "backends": ["ptq"], "modes": {
|
|
"e2e": {"enabled": True, "select": {"priority": ["A"], "tasks": ["cls", "det"]},
|
|
"marches": ["nash-m"], "backends": ["ptq"], "policies": ["teacher"], "k": 4},
|
|
"repair": {"enabled": False},
|
|
}}
|
|
self.rows = []
|
|
self.addCleanup(patch.stopall)
|
|
readonly = db.readonly
|
|
patch.object(db, "readonly", lambda: readonly(self.db_path)).start()
|
|
patch.object(db, "connect", self.connect).start()
|
|
patch.object(pipeline, "matrix", lambda: self.mx).start()
|
|
patch.object(pipeline, "models", lambda: self.rows).start()
|
|
patch.object(pipeline.paths, "STATE", self.root / "state").start()
|
|
patch.object(pipeline.paths, "RUNS", self.root / "runs").start()
|
|
patch.object(pipeline.paths, "config", lambda: {"pools": {"agent": {"workers": 3}}, "endpoints": {"teacher": {}, "student": {}}}).start()
|
|
patch.object(rollout, "setup_models_json").start()
|
|
patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext()).start()
|
|
self.runtime_config = patch('harness.collection_runtime.config', return_value=({'credential_groups': {'teacher': 'fixture'}}, {})).start()
|
|
self.runtime_preflight = patch('harness.collection_runtime.preflight').start()
|
|
self.reconciliation = patch('harness.recovery.reconcile_run', return_value={'ready': True}).start()
|
|
self.export = patch.object(pipeline, "refresh_run_dataset").start()
|
|
with self.connect():
|
|
pass
|
|
|
|
@contextmanager
|
|
def connect(self):
|
|
con = db._open(self.db_path)
|
|
con.executescript(db.SCHEMA)
|
|
try:
|
|
yield con
|
|
con.commit()
|
|
finally:
|
|
con.close()
|
|
|
|
def job(self, jid, *, state="PASS", priority="A", task="cls", march="nash-m", backend="ptq",
|
|
estimate=10, k=0, status="ready"):
|
|
self.rows.append({"model_id": jid, "task": task, "priority": priority, "status": status})
|
|
with self.connect() as con:
|
|
con.execute(
|
|
"INSERT INTO jobs (run_id,job_id,task,model,march,backend,qcfg,priority,mode,state,estimate_s)"
|
|
" VALUES ('run',?,?,?,?,?,?,?,'oracle',?,?)",
|
|
(jid, task, jid, march, backend, "int8", priority, state, estimate))
|
|
for kk in range(k):
|
|
db.rollout_upsert(con, "run", jid, kk, "e2e", "teacher")
|
|
|
|
def test_missing_provider_prerequisite_does_not_claim_any_attempt(self):
|
|
self.job('j', k=3)
|
|
self.runtime_preflight.side_effect = ValueError('CREDENTIAL_UNAVAILABLE')
|
|
with patch.object(rollout, 'run_rollout') as launch:
|
|
with self.assertRaisesRegex(ValueError, 'CREDENTIAL_UNAVAILABLE'):
|
|
pipeline.collect_rollouts('run', log=lambda _: None)
|
|
self.assertFalse(launch.called)
|
|
with self.connect() as con:
|
|
self.assertEqual([r[0] for r in con.execute('SELECT state FROM rollouts')], ['QUEUED'] * 3)
|
|
|
|
def complete(self, run_id, job_id, kk, mode, policy, **_):
|
|
with self.connect() as con:
|
|
db.rollout_set(con, run_id, job_id, kk, state="PASS", finished_at=time.time())
|
|
return {"terminal": "PASS", "claim_consistent": True}
|
|
|
|
def collect(self, **kw):
|
|
return pipeline.collect_rollouts("run", log=lambda _: None, **kw)
|
|
|
|
def test_missing_runtime_budget_refuses_before_claim(self):
|
|
self.job('eligible', k=1)
|
|
self.runtime_config.side_effect = ValueError('QUOTA_CONFIG_INCOMPLETE')
|
|
with self.assertRaisesRegex(ValueError, 'QUOTA_CONFIG_INCOMPLETE'):
|
|
self.collect()
|
|
with self.connect() as con:
|
|
self.assertEqual(db.list_rollouts(con, 'run', 'eligible')[0]['state'], 'QUEUED')
|
|
|
|
def test_unreconciled_execution_refuses_before_claim(self):
|
|
self.job('eligible', k=1)
|
|
self.reconciliation.return_value = {'ready': False}
|
|
with self.assertRaisesRegex(ValueError, 'RECONCILIATION_REQUIRED'):
|
|
self.collect()
|
|
with self.connect() as con:
|
|
self.assertEqual(db.list_rollouts(con, 'run', 'eligible')[0]['state'], 'QUEUED')
|
|
|
|
def test_all_coverage_is_idempotent_preserves_repairs_and_non_priority_filters(self):
|
|
self.job("eligible", priority="C")
|
|
for jid, kw in [("wrong-task", {"task": "occ"}), ("wrong-march", {"march": "nash-p"}),
|
|
("wrong-backend", {"backend": "qat"}), ("not-ready", {"status": "needs_adapter"})]:
|
|
self.job(jid, **kw)
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "eligible", 0, "repair", "teacher", "FAIL")
|
|
db.rollout_upsert(con, "run", "eligible", 4, "e2e", "teacher", "PASS")
|
|
self.assertEqual(pipeline.enqueue_e2e_coverage("run"), 3)
|
|
self.assertEqual(pipeline.enqueue_e2e_coverage("run"), 0)
|
|
with self.connect() as con:
|
|
rows = db.list_rollouts(con, "run", "eligible")
|
|
self.assertEqual([(r["kk"], r["mode"]) for r in rows],
|
|
[(0, "repair"), (4, "e2e"), (5, "e2e"), (6, "e2e"), (7, "e2e")])
|
|
self.assertEqual(con.execute("SELECT COUNT(*) FROM annotations").fetchone()[0], 1)
|
|
self.assertEqual(con.execute("SELECT COUNT(*) FROM jobs WHERE mode='e2e'").fetchone()[0], 1)
|
|
self.assertEqual(self.mx["modes"]["e2e"]["select"]["priority"], ["A"])
|
|
self.assertFalse((self.root / "runs").exists())
|
|
|
|
def test_coverage_fills_each_policy_and_respects_job_selection(self):
|
|
self.job("chosen", priority="B")
|
|
self.job("other", priority="B")
|
|
self.mx["modes"]["e2e"]["policies"] = ["teacher", "student"]
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "chosen", 9, "e2e", "teacher", "TIMEOUT")
|
|
self.assertEqual(pipeline.enqueue_e2e_coverage("run", ["chosen"]), 7)
|
|
with self.connect() as con:
|
|
self.assertEqual(len(db.list_rollouts(con, "run", "chosen")), 8)
|
|
self.assertEqual(db.list_rollouts(con, "run", "other"), [])
|
|
|
|
def test_atomic_claim_prevents_duplicate_job_and_exceeding_pool(self):
|
|
self.job("a", k=2)
|
|
self.job("b", k=1)
|
|
outcomes = []
|
|
barrier = threading.Barrier(2)
|
|
|
|
def claim(kk):
|
|
barrier.wait(timeout=2)
|
|
with self.connect() as con:
|
|
outcomes.append(db.claim_rollout(con, "run", "a", kk, max_active=3))
|
|
|
|
threads = [threading.Thread(target=claim, args=(kk,)) for kk in range(2)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join(timeout=3)
|
|
self.assertEqual(sorted(outcomes), [False, True])
|
|
with self.connect() as con:
|
|
self.assertFalse(db.claim_rollout(con, "run", "b", 0, max_active=1))
|
|
con.execute("UPDATE rollouts SET state='PASS' WHERE job_id='a' AND state='RUNNING'")
|
|
con.commit()
|
|
self.assertTrue(db.claim_rollout(con, "run", "b", 0, max_active=1))
|
|
self.assertFalse(db.claim_rollout(con, "run", "b", 0, max_active=3))
|
|
|
|
def test_missing_inputs_do_not_delay_first_agent_attempt(self):
|
|
self.job("first", k=4)
|
|
self.job("late", state="ORACLE_RUNNING", estimate=1, k=4)
|
|
self.job("covered", estimate=0, k=4)
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "run", "covered", 0, state="PASS", finished_at=1)
|
|
started = []
|
|
self.assertFalse((self.root / "runs").exists())
|
|
|
|
def run(*args, **kw):
|
|
started.append(args[1])
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
self.assertEqual(self.collect(parallel=1, limit=2), {"PASS": 2})
|
|
self.assertEqual(started, ["late", "first"])
|
|
|
|
def test_three_slots_cover_three_distinct_models(self):
|
|
for jid in ("a", "b", "c"):
|
|
self.job(jid, k=4)
|
|
barrier = threading.Barrier(3)
|
|
started = []
|
|
|
|
def run(*args, **kw):
|
|
started.append(args[1])
|
|
barrier.wait(timeout=3)
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
self.assertEqual(self.collect(parallel=3, limit=3), {"PASS": 3})
|
|
self.assertEqual(set(started), {"a", "b", "c"})
|
|
|
|
def test_prefers_shorter_jobs_independent_of_oracle_verdict(self):
|
|
self.job("failed", state="FAIL", estimate=0, k=1)
|
|
self.job("slow", estimate=100, k=1)
|
|
self.job("fast", estimate=1, k=1)
|
|
rows, pending = pipeline._collection_snapshot("run", None, None)
|
|
self.assertFalse(pending)
|
|
self.assertEqual([r["job_id"] for r in rows], ["failed", "fast", "slow"])
|
|
|
|
def test_e2e_prepares_its_own_inputs_and_retired_repair_is_never_ready(self):
|
|
self.job("ready", state="ORACLE_RUNNING", k=1)
|
|
self.job("missing", state="PASS", k=1)
|
|
self.assertFalse((self.root / "runs").exists())
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "ready", 1, "repair", "teacher")
|
|
rows, pending = pipeline._collection_snapshot("run", None, None)
|
|
self.assertEqual([(r["job_id"], r["mode"]) for r in rows], [("missing", "e2e"), ("ready", "e2e")])
|
|
self.assertFalse(pending)
|
|
with self.connect() as con:
|
|
self.assertEqual(con.execute("SELECT COUNT(*) FROM rollouts WHERE state!='QUEUED'").fetchone()[0], 0)
|
|
|
|
def test_collect_preserves_and_skips_historical_queued_repairs(self):
|
|
self.job("failed", state="FAIL", k=1)
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "failed", 7, "repair", "teacher")
|
|
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
|
|
with patch.object(rollout, "run_rollout", self.complete), \
|
|
patch.object(pipeline, "enqueue_repairs") as enqueue:
|
|
self.assertEqual(self.collect(parallel=1, modes=["e2e", "repair"]), {"PASS": 1})
|
|
enqueue.assert_not_called()
|
|
with self.connect() as con:
|
|
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
|
|
rows, pending = pipeline._collection_snapshot("run", None, ["repair"])
|
|
self.assertEqual(rows, [])
|
|
self.assertFalse(pending)
|
|
|
|
def test_exception_releases_claim_and_does_not_retry_same_row(self):
|
|
self.job("broken", k=1)
|
|
with patch.object(rollout, "run_rollout", side_effect=SystemExit("missing cache")):
|
|
self.assertEqual(self.collect(parallel=1), {"UNVERIFIED": 1})
|
|
with self.connect() as con:
|
|
row = db.list_rollouts(con, "run", "broken")[0]
|
|
self.assertEqual(row["state"], "UNVERIFIED")
|
|
self.assertIsNotNone(row["finished_at"])
|
|
|
|
def test_exception_after_measurements_still_verifies_and_exports_real_result(self):
|
|
self.job("measured", k=1)
|
|
jd = self.root / "runs" / "run" / "measured"
|
|
rd = jd / "rollouts" / "00"
|
|
rd.mkdir(parents=True)
|
|
spec = registry.JobSpec(job_id="measured", task="cls", model="measured", march="nash-m",
|
|
backend="ptq", qcfg="int8", priority="A", mode="oracle")
|
|
(jd / "job.json").write_text(json.dumps({"job_id": "measured", "run_id": "run",
|
|
"gates": registry.gate_snapshot(spec)}))
|
|
(rd / "rollout.json").write_text(json.dumps({"state": "RUNNING", "mode": "e2e", "policy": "teacher"}))
|
|
(rd / "claim.json").write_text('{"terminal":"PASS"}')
|
|
hbm = rd / "artifacts/measured.hbm"
|
|
hbm.parent.mkdir()
|
|
hbm.write_bytes(b"collection fixture")
|
|
sha = hashlib.sha256(hbm.read_bytes()).hexdigest()
|
|
for stage, measurements in [
|
|
("05-compile", {}),
|
|
("06-hbm", {"G1": dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False)}),
|
|
("07-board", {"G4": {"latency_ms_mean": 1}}),
|
|
("08-eval", {"G3": {"fp32": 80, "quant": 79, "source": "board", "tier": "t1", "n": 2000}}),
|
|
]:
|
|
for value in measurements.values():
|
|
value["hbm_sha256"] = sha
|
|
directory = rd / "stages" / stage
|
|
directory.mkdir(parents=True)
|
|
(directory / "meta.json").write_text(json.dumps({"state": "OK", "measurements": measurements, "attempt": 0}))
|
|
with patch.object(rollout, "run_rollout", side_effect=RuntimeError("agent stream closed")):
|
|
self.assertEqual(self.collect(parallel=1), {"PASS": 1})
|
|
verdict = json.loads((rd / "verdict.json").read_text())
|
|
self.assertEqual(verdict["terminal"], "PASS")
|
|
self.assertNotIn("oracle_ref", verdict)
|
|
self.assertEqual(verdict["execution_terminal"], "ERROR")
|
|
self.assertTrue(verdict["completion"]["workflow_completed"])
|
|
self.export.assert_called_with("run")
|
|
|
|
def test_dry_run_watch_neither_claims_nor_adds_coverage(self):
|
|
self.job("example", priority="B", k=1)
|
|
with patch.object(rollout, "run_rollout", return_value={"dry_run": True}), \
|
|
patch.object(pipeline, "enqueue_e2e_coverage") as enqueue:
|
|
self.assertEqual(self.collect(dry_run=True, watch=True, coverage="all"), {"DRY_RUN": 1})
|
|
enqueue.assert_not_called()
|
|
self.export.assert_not_called()
|
|
with self.connect() as con:
|
|
self.assertEqual(db.list_rollouts(con, "run", "example")[0]["state"], "QUEUED")
|
|
|
|
def test_singleton_collector_cannot_duplicate_inflight_work(self):
|
|
self.job("model", k=1)
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
def run(*args, **kw):
|
|
entered.set()
|
|
release.wait(timeout=3)
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
thread = threading.Thread(target=self.collect, kwargs={"parallel": 1})
|
|
thread.start()
|
|
try:
|
|
self.assertTrue(entered.wait(timeout=2))
|
|
self.assertEqual(self.collect(parallel=1), {})
|
|
finally:
|
|
release.set()
|
|
thread.join(timeout=3)
|
|
self.assertFalse(thread.is_alive())
|
|
|
|
def test_drain_finishes_inflight_without_claiming_repeats(self):
|
|
self.job("model", k=4)
|
|
|
|
def run(*args, **kw):
|
|
pipeline.collect_drain_file("run").touch()
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
|
|
with self.connect() as con:
|
|
states = [r["state"] for r in db.list_rollouts(con, "run", "model")]
|
|
self.assertEqual(states, ["PASS", "QUEUED", "QUEUED", "QUEUED"])
|
|
|
|
def test_first_pass_preserves_repeats_and_failed_attempts_consume_one_slot(self):
|
|
self.job("fresh", k=4)
|
|
self.job("covered", k=4)
|
|
self.job("failed", k=4)
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "run", "covered", 0, state="PASS", started_at=1, finished_at=2)
|
|
db.rollout_set(con, "run", "failed", 0, state="UNVERIFIED", started_at=1, finished_at=2)
|
|
with patch.object(rollout, "run_rollout", self.complete):
|
|
self.assertEqual(self.collect(parallel=1, first_pass=True), {"PASS": 1})
|
|
with self.connect() as con:
|
|
for model in ("fresh", "covered", "failed"):
|
|
rows = db.list_rollouts(con, "run", model)
|
|
self.assertEqual([r["state"] for r in rows[1:]], ["QUEUED"] * 3)
|
|
|
|
def test_first_pass_ignores_completed_repair_but_keeps_e2e_history(self):
|
|
self.job("repair-only", k=2)
|
|
self.job("e2e-covered", k=2)
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "repair-only", 7, "repair", "teacher", "PASS")
|
|
db.rollout_set(con, "run", "repair-only", 7, started_at=1, finished_at=2)
|
|
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
|
|
db.rollout_set(con, "run", "e2e-covered", 0, state="PASS", started_at=1, finished_at=2)
|
|
with patch.object(rollout, "run_rollout", side_effect=self.complete) as run:
|
|
self.assertEqual(self.collect(parallel=1, modes=["e2e"], first_pass=True), {"PASS": 1})
|
|
self.assertEqual(run.call_args.args[1:4], ("repair-only", 0, "e2e"))
|
|
with self.connect() as con:
|
|
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
|
|
for jid in ("repair-only", "e2e-covered"):
|
|
self.assertEqual(db.list_rollouts(con, "run", jid)[1]["state"], "QUEUED")
|
|
|
|
def test_sigterm_drains_inflight_and_restores_previous_handler(self):
|
|
self.job("model", k=2)
|
|
old_handler = signal.getsignal(signal.SIGTERM)
|
|
|
|
def run(*args, **kw):
|
|
signal.getsignal(signal.SIGTERM)(signal.SIGTERM, None)
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
|
|
self.assertEqual(signal.getsignal(signal.SIGTERM), old_handler)
|
|
|
|
def test_last_export_exception_is_reported_without_losing_completed_rollout(self):
|
|
self.job("model", k=1)
|
|
messages = []
|
|
with patch.object(rollout, "run_rollout", self.complete), \
|
|
patch.object(pipeline, "refresh_run_dataset", side_effect=RuntimeError("export unavailable")):
|
|
self.assertEqual(pipeline.collect_rollouts("run", parallel=1, log=messages.append), {"PASS": 1})
|
|
self.assertTrue(any("export unavailable" in message for message in messages))
|
|
|
|
def test_slow_export_does_not_delay_next_dispatch(self):
|
|
self.job("model", k=2)
|
|
second_started = threading.Event()
|
|
export_saw_second = []
|
|
|
|
def run(*args, **kw):
|
|
if args[2] == 1:
|
|
second_started.set()
|
|
return self.complete(*args, **kw)
|
|
|
|
def export(_):
|
|
export_saw_second.append(second_started.wait(timeout=2))
|
|
|
|
with patch.object(rollout, "run_rollout", run), patch.object(pipeline, "refresh_run_dataset", export):
|
|
self.assertEqual(self.collect(parallel=1), {"PASS": 2})
|
|
self.assertTrue(export_saw_second)
|
|
self.assertTrue(all(export_saw_second))
|
|
|
|
def test_existing_active_rollout_counts_against_slots_and_blocks_same_job(self):
|
|
self.job("old", k=2)
|
|
self.job("fresh", k=1)
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "run", "old", 0, state="RUNNING", started_at=1)
|
|
started = []
|
|
|
|
def run(*args, **kw):
|
|
started.append(args[1])
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
self.assertEqual(self.collect(parallel=3), {"PASS": 1})
|
|
self.assertEqual(started, ["fresh"])
|
|
|
|
def test_watch_finishes_independent_of_oracle_running_state(self):
|
|
self.job("late", state="ORACLE_RUNNING", k=1)
|
|
with patch.object(rollout, "run_rollout", self.complete):
|
|
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
|
|
with self.connect() as con:
|
|
self.assertEqual(db.get_job(con, "run", "late")["state"], "ORACLE_RUNNING")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|