data-gen-bpu/scripts/test_collection.py
ming01.wang 498187569c fix: close rollout finalization and review publication gaps
Reuse execution, stage and review evidence to recover database registration and
SFT publication without replaying completed work. Keep finished futures during
SQLite contention, and hand durable results to startup recovery when draining.
Release proven local leases independently of unresolved board isolation.

Keep UNKNOWN review retries explicit and archived, publish completed receipts
after transient export failures, and prevent a malformed sample from aborting
the entire dataset. Scope routine reconciliation to newly finished attempts.
Add failure-injection coverage and the production acceptance plan.
2026-09-29 16:09:01 +08:00

698 lines
34 KiB
Python

"""Collection must cover queued models promptly without duplicating or losing work."""
from contextlib import contextmanager, nullcontext
import hashlib
import json
import os
import signal
import subprocess
import sys
import sqlite3
from pathlib import Path
import tempfile
import threading
import time
import unittest
from unittest.mock import patch
from harness import db, pipeline, registry, rollout, recovery
REAL_RECONCILE = recovery.reconcile_run
class Collection(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.db_path = self.root / "jobs.sqlite"
self.mx = {"marches": ["nash-m"], "backends": ["ptq"], "modes": {
"e2e": {"enabled": True, "select": {"priority": ["A"], "tasks": ["cls", "det"]},
"marches": ["nash-m"], "backends": ["ptq"], "policies": ["teacher"], "k": 4},
"repair": {"enabled": False},
}}
self.rows = []
self.addCleanup(patch.stopall)
readonly = db.readonly
patch.object(db, "readonly", lambda: readonly(self.db_path)).start()
patch.object(db, "connect", self.connect).start()
patch.object(pipeline, "matrix", lambda: self.mx).start()
patch.object(pipeline, "models", lambda: self.rows).start()
patch.object(pipeline.paths, "STATE", self.root / "state").start()
patch.object(pipeline.paths, "RUNS", self.root / "runs").start()
patch.object(pipeline.paths, "config", lambda: {"pools": {"agent": {"workers": 3}}, "endpoints": {"teacher": {}, "student": {}}}).start()
patch.object(rollout, "setup_models_json").start()
patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext()).start()
self.runtime_config = patch('harness.collection_runtime.config', return_value=({'credential_groups': {'teacher': 'fixture'}}, {})).start()
self.runtime_preflight = patch('harness.collection_runtime.preflight').start()
self.reconciliation = patch('harness.recovery.reconcile_run', return_value={'ready': True}).start()
self.export = patch.object(pipeline, "refresh_run_dataset").start()
with self.connect():
pass
@contextmanager
def connect(self, *, timeout_s=60):
con = db._open(self.db_path, timeout_s=timeout_s)
try:
con.executescript(db.SCHEMA)
yield con
con.commit()
finally:
con.close()
def job(self, jid, *, state="PASS", priority="A", task="cls", march="nash-m", backend="ptq",
estimate=10, k=0, status="ready"):
self.rows.append({"model_id": jid, "task": task, "priority": priority, "status": status})
with self.connect() as con:
con.execute(
"INSERT INTO jobs (run_id,job_id,task,model,march,backend,qcfg,priority,mode,state,estimate_s)"
" VALUES ('run',?,?,?,?,?,?,?,'oracle',?,?)",
(jid, task, jid, march, backend, "int8", priority, state, estimate))
for kk in range(k):
db.rollout_upsert(con, "run", jid, kk, "e2e", "teacher")
def test_missing_provider_prerequisite_does_not_claim_any_attempt(self):
self.job('j', k=3)
self.runtime_preflight.side_effect = ValueError('CREDENTIAL_UNAVAILABLE')
with patch.object(rollout, 'run_rollout') as launch:
with self.assertRaisesRegex(ValueError, 'CREDENTIAL_UNAVAILABLE'):
pipeline.collect_rollouts('run', log=lambda _: None)
self.assertFalse(launch.called)
with self.connect() as con:
self.assertEqual([r[0] for r in con.execute('SELECT state FROM rollouts')], ['QUEUED'] * 3)
def complete(self, run_id, job_id, kk, mode, policy, **_):
with self.connect() as con:
db.rollout_set(con, run_id, job_id, kk, state="PASS", finished_at=time.time())
return {"terminal": "PASS", "claim_consistent": True}
def collect(self, **kw):
return pipeline.collect_rollouts("run", log=lambda _: None, **kw)
def test_missing_runtime_budget_refuses_before_claim(self):
self.job('eligible', k=1)
self.runtime_config.side_effect = ValueError('QUOTA_CONFIG_INCOMPLETE')
with self.assertRaisesRegex(ValueError, 'QUOTA_CONFIG_INCOMPLETE'):
self.collect()
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, 'run', 'eligible')[0]['state'], 'QUEUED')
def test_unreconciled_execution_refuses_before_claim(self):
self.job('eligible', k=1)
self.reconciliation.return_value = {'ready': False}
with self.assertRaisesRegex(ValueError, 'RECONCILIATION_REQUIRED'):
self.collect()
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, 'run', 'eligible')[0]['state'], 'QUEUED')
def test_real_database_lock_keeps_failed_future_without_replaying_or_claiming(self):
for jid in ('a', 'b', 'c'):
self.job(jid, k=1)
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
self.addCleanup(locker.close)
entered = threading.Barrier(2)
good_committed, locked, waiting = (threading.Event() for _ in range(3))
launched, logs, outcome, errors = [], [], [], []
def run(*args, **kw):
jid = args[1]
launched.append(jid)
if jid in ('a', 'b'):
entered.wait(timeout=3)
if jid == 'a':
result = self.complete(*args, **kw)
good_committed.set()
self.assertTrue(locked.wait(3))
return result
if jid == 'b':
self.assertTrue(good_committed.wait(3))
locker.execute('BEGIN EXCLUSIVE')
locked.set()
raise RuntimeError('fixture agent ended after accepted work')
return self.complete(*args, **kw)
def log(message):
logs.append(message)
if 'database busy:' in message:
waiting.set()
def collect():
try:
outcome.append(pipeline.collect_rollouts('run', parallel=2, log=log))
except BaseException as exc:
errors.append(exc)
with patch.object(rollout, 'run_rollout', run):
thread = threading.Thread(target=collect)
thread.start()
try:
self.assertTrue(waiting.wait(4), logs)
self.assertTrue(thread.is_alive())
self.assertCountEqual(launched, ['a', 'b'])
self.assertTrue(any('→ PASS' in message for message in logs), logs)
finally:
locker.rollback()
thread.join(timeout=8)
self.assertFalse(thread.is_alive())
self.assertEqual(errors, [])
self.assertEqual(outcome, [{'PASS': 2, 'UNVERIFIED': 1}])
self.assertCountEqual(launched, ['a', 'b', 'c'])
self.assertTrue(any('database recovered:' in message for message in logs))
with self.connect() as con:
self.assertEqual([(r['job_id'], r['state']) for r in con.execute('SELECT * FROM rollouts ORDER BY job_id')],
[('a', 'PASS'), ('b', 'UNVERIFIED'), ('c', 'PASS')])
def test_real_database_lock_at_start_snapshot_and_claim_waits_without_launch(self):
self.job('a', k=1)
for boundary in ('start', 'snapshot', 'claim'):
with self.subTest(boundary=boundary):
with self.connect() as con:
con.execute("UPDATE rollouts SET state='QUEUED', started_at=NULL, finished_at=NULL")
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
waiting, logs, outcomes, errors, calls = threading.Event(), [], [], [], []
target = pipeline if boundary == 'snapshot' else db
method = '_collection_snapshot' if boundary == 'snapshot' else 'claim_rollout'
original = getattr(target, method)
locked = boundary == 'start'
if locked:
locker.execute('BEGIN EXCLUSIVE')
def inject(*args, **kwargs):
nonlocal locked
if not locked:
locker.execute('BEGIN EXCLUSIVE')
locked = True
return original(*args, **kwargs)
def log(message):
logs.append(message)
if 'database busy:' in message:
waiting.set()
def launch(*args, **kw):
calls.append(args[1])
return self.complete(*args, **kw)
def collect():
try:
outcomes.append(pipeline.collect_rollouts('run', parallel=1, log=log))
except BaseException as exc:
errors.append(exc)
with patch.object(rollout, 'run_rollout', launch), \
patch.object(target, method, inject) if boundary != 'start' else nullcontext():
thread = threading.Thread(target=collect)
thread.start()
try:
self.assertTrue(waiting.wait(4), (logs, errors))
self.assertEqual(calls, [])
self.assertTrue(thread.is_alive())
finally:
locker.rollback()
thread.join(timeout=8)
locker.close()
self.assertFalse(thread.is_alive())
self.assertEqual(errors, [])
self.assertEqual(outcomes, [{'PASS': 1}])
self.assertEqual(calls, ['a'])
def test_drain_during_database_wait_preserves_unclaimed_work(self):
self.job('a', k=1)
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
locker.execute('BEGIN EXCLUSIVE')
waiting, outcomes, errors = threading.Event(), [], []
def log(message):
if 'database busy:' in message:
waiting.set()
def collect():
try:
outcomes.append(pipeline.collect_rollouts('run', log=log))
except BaseException as exc:
errors.append(exc)
with patch.object(rollout, 'run_rollout') as launch:
thread = threading.Thread(target=collect)
thread.start()
try:
self.assertTrue(waiting.wait(3))
pipeline.collect_drain_file('run').touch()
thread.join(timeout=7)
self.assertFalse(thread.is_alive(), 'empty collector must drain while database remains locked')
finally:
locker.rollback()
locker.close()
thread.join(timeout=8)
self.assertFalse(thread.is_alive())
self.assertEqual(errors, [])
self.assertEqual(outcomes, [{}])
launch.assert_not_called()
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'QUEUED')
def test_runtime_recovery_scans_only_just_finished_attempts(self):
for jid in ('a', 'b'):
self.job(jid, k=1)
with patch.object(rollout, 'run_rollout', self.complete):
self.assertEqual(self.collect(parallel=1), {'PASS': 2})
calls = self.reconciliation.call_args_list
self.assertEqual(calls[0].args, ('run',))
self.assertEqual(calls[0].kwargs, {})
self.assertEqual([call.kwargs['attempts'] for call in calls[1:]], [(('a', 0),), (('b', 0),)])
def test_drain_persists_pending_prelaunch_failure_and_exits_before_unlock(self):
self.job('a', k=1)
self.job('b', k=1)
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
waiting, outcomes, errors, launched = threading.Event(), [], [], []
def launch(*args, **kw):
launched.append(args[1])
locker.execute('BEGIN EXCLUSIVE')
raise RuntimeError('completed worker needs failure registration')
def log(message):
if 'database busy:' in message:
waiting.set()
def collect():
try:
outcomes.append(pipeline.collect_rollouts('run', parallel=1, log=log))
except BaseException as exc:
errors.append(exc)
with patch.object(rollout, 'run_rollout', launch):
thread = threading.Thread(target=collect)
thread.start()
try:
self.assertTrue(waiting.wait(3))
pipeline.collect_drain_file('run').touch()
thread.join(timeout=7)
self.assertFalse(thread.is_alive(), 'drain must persist completed work and exit while DB stays locked')
finally:
locker.rollback()
locker.close()
thread.join(timeout=8)
self.assertFalse(thread.is_alive())
self.assertEqual(errors, [])
self.assertEqual(outcomes, [{}])
self.assertEqual(launched, ['a'])
from harness import recovery
with patch.object(recovery, 'reconcile_run', REAL_RECONCILE), \
patch.object(rollout, 'run_rollout', side_effect=AssertionError('restart dispatched work')):
self.assertEqual(self.collect(parallel=1, limit=0), {})
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'INFRA_ERROR')
self.assertEqual(db.list_rollouts(con, 'run', 'b')[0]['state'], 'QUEUED')
def test_sigterm_hands_off_completed_execution_and_restart_reverifies_once(self):
program = """
import sys
from scripts.test_collection import Collection
case = Collection(); case.setUp()
try: case.sigterm_restart_probe(sys.argv[1])
finally: case.doCleanups()
"""
for boundary in ('first_db', 'during_verify'):
with self.subTest(boundary=boundary):
result = subprocess.run([sys.executable, '-c', program, boundary], cwd=Path(__file__).resolve().parents[1],
capture_output=True, text=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('SIGTERM_RECOVERY_ONCE', result.stdout)
def sigterm_restart_probe(self, boundary):
from harness import execution, verify
from scripts.test_execution import policy
self.job('a', k=1); self.job('b', k=1)
locker = sqlite3.connect(self.db_path, timeout=0, check_same_thread=False)
self.addCleanup(locker.close)
rd = self.root/'runs/run/a/rollouts/00'
observed = []
def interrupt():
locker.execute('BEGIN EXCLUSIVE')
os.kill(os.getpid(), signal.SIGTERM)
real_verify = verify.verify_rollout
def interrupted_verify(*args, **kwargs):
with self.connect() as con:
observed.append(db.list_rollouts(con, 'run', 'a')[0]['state'])
interrupt()
return real_verify(*args, **kwargs)
def launch(*args, **kw):
owner = execution.Execution(rd); owner.start(policy())
(rd/'effects').write_text('one\n')
owner.finish(0, protocol_complete=True)
execution.atomic_json(rd/'rollout.json', {'state': 'DONE', 'mode': 'e2e', 'policy': 'teacher',
'execution_managed': True, 'execution_id': owner.snapshot()['execution_id']})
spec = registry.JobSpec('a', 'cls', 'fixture', 'nash-m', 'ptq', 'int8', 'A', 'e2e')
execution.atomic_json(rd.parent.parent/'job.json', {'run_id': 'run', 'job_id': 'a',
'gates': registry.gate_snapshot(spec)})
if boundary == 'first_db':
interrupt()
raise RuntimeError('fixture final registration interrupted')
with patch.object(rollout, 'run_rollout', launch), \
patch.object(verify, 'verify_rollout', interrupted_verify) if boundary == 'during_verify' else nullcontext():
self.assertEqual(self.collect(parallel=1), {})
if boundary == 'during_verify':
self.assertEqual(observed, ['VERIFYING'])
self.assertTrue(locker.in_transaction) # Collector exited before DB recovery.
locker.rollback()
with patch.object(recovery, 'reconcile_run', REAL_RECONCILE), \
patch.object(rollout, 'run_rollout', side_effect=AssertionError('restart dispatched work')):
self.assertEqual(self.collect(parallel=1, limit=0), {})
self.assertEqual((rd/'effects').read_text(), 'one\n')
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'UNVERIFIED')
self.assertIsNotNone(db.list_rollouts(con, 'run', 'a')[0]['verdict'])
self.assertEqual(db.list_rollouts(con, 'run', 'b')[0]['state'], 'QUEUED')
print('SIGTERM_RECOVERY_ONCE')
def test_error_after_committed_verdict_preserves_database_terminal(self):
self.job('a', k=1)
def launch(*args, **kw):
with self.connect() as con:
db.rollout_set(con, 'run', 'a', 0, state='FAIL', verdict='{"terminal":"FAIL"}', finished_at=1)
rd = self.root / 'runs/run/a/rollouts/00'
rd.mkdir(parents=True)
# A pre-existing verdict wins; control-plane outcome is not task quality.
(rd / 'execution.json').write_text('{"execution_outcome":"AGENT_ERROR"}')
raise RuntimeError('fixture failure after verdict commit')
with patch.object(rollout, 'run_rollout', launch), patch.object(rollout, 'settle_execution') as settle:
self.assertEqual(self.collect(), {'FAIL': 1})
settle.assert_not_called()
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'FAIL')
def test_nonbusy_database_error_is_not_retried(self):
self.job('a', k=1)
with patch.object(pipeline, '_collection_snapshot', side_effect=sqlite3.OperationalError('disk I/O error')) as snapshot, \
patch.object(rollout, 'run_rollout') as launch:
with self.assertRaisesRegex(sqlite3.OperationalError, 'disk I/O error'):
self.collect()
self.assertEqual(snapshot.call_count, 1)
launch.assert_not_called()
def test_recovery_reason_and_released_resources_are_visible(self):
self.job('a', k=1)
logs = []
self.reconciliation.return_value = {'ready': False, 'executions': [
{'ready': False, 'reason': 'LEGACY_EXECUTION_OWNERSHIP_UNKNOWN'}],
'resources': [{'lease': '/fixture/agent-2.lease.json', 'state': 'RELEASED'}]}
with self.assertRaisesRegex(ValueError, 'LEGACY_EXECUTION_OWNERSHIP_UNKNOWN'):
pipeline.collect_rollouts('run', log=logs.append)
self.assertTrue(any('released=1' in message and 'agent-2.lease.json' in message for message in logs))
def test_all_coverage_is_idempotent_preserves_repairs_and_non_priority_filters(self):
self.job("eligible", priority="C")
for jid, kw in [("wrong-task", {"task": "occ"}), ("wrong-march", {"march": "nash-p"}),
("wrong-backend", {"backend": "qat"}), ("not-ready", {"status": "needs_adapter"})]:
self.job(jid, **kw)
with self.connect() as con:
db.rollout_upsert(con, "run", "eligible", 0, "repair", "teacher", "FAIL")
db.rollout_upsert(con, "run", "eligible", 4, "e2e", "teacher", "PASS")
self.assertEqual(pipeline.enqueue_e2e_coverage("run"), 3)
self.assertEqual(pipeline.enqueue_e2e_coverage("run"), 0)
with self.connect() as con:
rows = db.list_rollouts(con, "run", "eligible")
self.assertEqual([(r["kk"], r["mode"]) for r in rows],
[(0, "repair"), (4, "e2e"), (5, "e2e"), (6, "e2e"), (7, "e2e")])
self.assertEqual(con.execute("SELECT COUNT(*) FROM annotations").fetchone()[0], 1)
self.assertEqual(con.execute("SELECT COUNT(*) FROM jobs WHERE mode='e2e'").fetchone()[0], 1)
self.assertEqual(self.mx["modes"]["e2e"]["select"]["priority"], ["A"])
self.assertFalse((self.root / "runs").exists())
def test_coverage_fills_each_policy_and_respects_job_selection(self):
self.job("chosen", priority="B")
self.job("other", priority="B")
self.mx["modes"]["e2e"]["policies"] = ["teacher", "student"]
with self.connect() as con:
db.rollout_upsert(con, "run", "chosen", 9, "e2e", "teacher", "TIMEOUT")
self.assertEqual(pipeline.enqueue_e2e_coverage("run", ["chosen"]), 7)
with self.connect() as con:
self.assertEqual(len(db.list_rollouts(con, "run", "chosen")), 8)
self.assertEqual(db.list_rollouts(con, "run", "other"), [])
def test_atomic_claim_prevents_duplicate_job_and_exceeding_pool(self):
self.job("a", k=2)
self.job("b", k=1)
outcomes = []
barrier = threading.Barrier(2)
def claim(kk):
barrier.wait(timeout=2)
with self.connect() as con:
outcomes.append(db.claim_rollout(con, "run", "a", kk, max_active=3))
threads = [threading.Thread(target=claim, args=(kk,)) for kk in range(2)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=3)
self.assertEqual(sorted(outcomes), [False, True])
with self.connect() as con:
self.assertFalse(db.claim_rollout(con, "run", "b", 0, max_active=1))
con.execute("UPDATE rollouts SET state='PASS' WHERE job_id='a' AND state='RUNNING'")
con.commit()
self.assertTrue(db.claim_rollout(con, "run", "b", 0, max_active=1))
self.assertFalse(db.claim_rollout(con, "run", "b", 0, max_active=3))
def test_missing_inputs_do_not_delay_first_agent_attempt(self):
self.job("first", k=4)
self.job("late", state="ORACLE_RUNNING", estimate=1, k=4)
self.job("covered", estimate=0, k=4)
with self.connect() as con:
db.rollout_set(con, "run", "covered", 0, state="PASS", finished_at=1)
started = []
self.assertFalse((self.root / "runs").exists())
def run(*args, **kw):
started.append(args[1])
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
self.assertEqual(self.collect(parallel=1, limit=2), {"PASS": 2})
self.assertEqual(started, ["late", "first"])
def test_three_slots_cover_three_distinct_models(self):
for jid in ("a", "b", "c"):
self.job(jid, k=4)
barrier = threading.Barrier(3)
started = []
def run(*args, **kw):
started.append(args[1])
barrier.wait(timeout=3)
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
self.assertEqual(self.collect(parallel=3, limit=3), {"PASS": 3})
self.assertEqual(set(started), {"a", "b", "c"})
def test_prefers_shorter_jobs_independent_of_oracle_verdict(self):
self.job("failed", state="FAIL", estimate=0, k=1)
self.job("slow", estimate=100, k=1)
self.job("fast", estimate=1, k=1)
rows, pending = pipeline._collection_snapshot("run", None, None)
self.assertFalse(pending)
self.assertEqual([r["job_id"] for r in rows], ["failed", "fast", "slow"])
def test_e2e_prepares_its_own_inputs_and_retired_repair_is_never_ready(self):
self.job("ready", state="ORACLE_RUNNING", k=1)
self.job("missing", state="PASS", k=1)
self.assertFalse((self.root / "runs").exists())
with self.connect() as con:
db.rollout_upsert(con, "run", "ready", 1, "repair", "teacher")
rows, pending = pipeline._collection_snapshot("run", None, None)
self.assertEqual([(r["job_id"], r["mode"]) for r in rows], [("missing", "e2e"), ("ready", "e2e")])
self.assertFalse(pending)
with self.connect() as con:
self.assertEqual(con.execute("SELECT COUNT(*) FROM rollouts WHERE state!='QUEUED'").fetchone()[0], 0)
def test_collect_preserves_and_skips_historical_queued_repairs(self):
self.job("failed", state="FAIL", k=1)
with self.connect() as con:
db.rollout_upsert(con, "run", "failed", 7, "repair", "teacher")
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
with patch.object(rollout, "run_rollout", self.complete), \
patch.object(pipeline, "enqueue_repairs") as enqueue:
self.assertEqual(self.collect(parallel=1, modes=["e2e", "repair"]), {"PASS": 1})
enqueue.assert_not_called()
with self.connect() as con:
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
rows, pending = pipeline._collection_snapshot("run", None, ["repair"])
self.assertEqual(rows, [])
self.assertFalse(pending)
def test_exception_releases_claim_and_does_not_retry_same_row(self):
self.job("broken", k=1)
with patch.object(rollout, "run_rollout", side_effect=SystemExit("missing cache")):
self.assertEqual(self.collect(parallel=1), {"UNVERIFIED": 1})
with self.connect() as con:
row = db.list_rollouts(con, "run", "broken")[0]
self.assertEqual(row["state"], "UNVERIFIED")
self.assertIsNotNone(row["finished_at"])
def test_exception_after_measurements_still_verifies_and_exports_real_result(self):
self.job("measured", k=1)
jd = self.root / "runs" / "run" / "measured"
rd = jd / "rollouts" / "00"
rd.mkdir(parents=True)
spec = registry.JobSpec(job_id="measured", task="cls", model="measured", march="nash-m",
backend="ptq", qcfg="int8", priority="A", mode="oracle")
(jd / "job.json").write_text(json.dumps({"job_id": "measured", "run_id": "run",
"gates": registry.gate_snapshot(spec)}))
(rd / "rollout.json").write_text(json.dumps({"state": "RUNNING", "mode": "e2e", "policy": "teacher"}))
(rd / "claim.json").write_text('{"terminal":"PASS"}')
hbm = rd / "artifacts/measured.hbm"
hbm.parent.mkdir()
hbm.write_bytes(b"collection fixture")
sha = hashlib.sha256(hbm.read_bytes()).hexdigest()
for stage, measurements in [
("05-compile", {}),
("06-hbm", {"G1": dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False)}),
("07-board", {"G4": {"latency_ms_mean": 1}}),
("08-eval", {"G3": {"fp32": 80, "quant": 79, "source": "board", "tier": "t1", "n": 2000}}),
]:
for value in measurements.values():
value["hbm_sha256"] = sha
directory = rd / "stages" / stage
directory.mkdir(parents=True)
(directory / "meta.json").write_text(json.dumps({"state": "OK", "measurements": measurements, "attempt": 0}))
with patch.object(rollout, "run_rollout", side_effect=RuntimeError("agent stream closed")):
self.assertEqual(self.collect(parallel=1), {"PASS": 1})
verdict = json.loads((rd / "verdict.json").read_text())
self.assertEqual(verdict["terminal"], "PASS")
self.assertNotIn("oracle_ref", verdict)
self.assertEqual(verdict["execution_terminal"], "ERROR")
self.assertTrue(verdict["completion"]["workflow_completed"])
self.export.assert_called_with("run")
def test_dry_run_watch_neither_claims_nor_adds_coverage(self):
self.job("example", priority="B", k=1)
with patch.object(rollout, "run_rollout", return_value={"dry_run": True}), \
patch.object(pipeline, "enqueue_e2e_coverage") as enqueue:
self.assertEqual(self.collect(dry_run=True, watch=True, coverage="all"), {"DRY_RUN": 1})
enqueue.assert_not_called()
self.export.assert_not_called()
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, "run", "example")[0]["state"], "QUEUED")
def test_singleton_collector_cannot_duplicate_inflight_work(self):
self.job("model", k=1)
entered, release = threading.Event(), threading.Event()
def run(*args, **kw):
entered.set()
release.wait(timeout=3)
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
thread = threading.Thread(target=self.collect, kwargs={"parallel": 1})
thread.start()
try:
self.assertTrue(entered.wait(timeout=2))
self.assertEqual(self.collect(parallel=1), {})
finally:
release.set()
thread.join(timeout=3)
self.assertFalse(thread.is_alive())
def test_drain_finishes_inflight_without_claiming_repeats(self):
self.job("model", k=4)
def run(*args, **kw):
pipeline.collect_drain_file("run").touch()
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
with self.connect() as con:
states = [r["state"] for r in db.list_rollouts(con, "run", "model")]
self.assertEqual(states, ["PASS", "QUEUED", "QUEUED", "QUEUED"])
def test_first_pass_preserves_repeats_and_failed_attempts_consume_one_slot(self):
self.job("fresh", k=4)
self.job("covered", k=4)
self.job("failed", k=4)
with self.connect() as con:
db.rollout_set(con, "run", "covered", 0, state="PASS", started_at=1, finished_at=2)
db.rollout_set(con, "run", "failed", 0, state="UNVERIFIED", started_at=1, finished_at=2)
with patch.object(rollout, "run_rollout", self.complete):
self.assertEqual(self.collect(parallel=1, first_pass=True), {"PASS": 1})
with self.connect() as con:
for model in ("fresh", "covered", "failed"):
rows = db.list_rollouts(con, "run", model)
self.assertEqual([r["state"] for r in rows[1:]], ["QUEUED"] * 3)
def test_first_pass_ignores_completed_repair_but_keeps_e2e_history(self):
self.job("repair-only", k=2)
self.job("e2e-covered", k=2)
with self.connect() as con:
db.rollout_upsert(con, "run", "repair-only", 7, "repair", "teacher", "PASS")
db.rollout_set(con, "run", "repair-only", 7, started_at=1, finished_at=2)
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
db.rollout_set(con, "run", "e2e-covered", 0, state="PASS", started_at=1, finished_at=2)
with patch.object(rollout, "run_rollout", side_effect=self.complete) as run:
self.assertEqual(self.collect(parallel=1, modes=["e2e"], first_pass=True), {"PASS": 1})
self.assertEqual(run.call_args.args[1:4], ("repair-only", 0, "e2e"))
with self.connect() as con:
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
for jid in ("repair-only", "e2e-covered"):
self.assertEqual(db.list_rollouts(con, "run", jid)[1]["state"], "QUEUED")
def test_sigterm_drains_inflight_and_restores_previous_handler(self):
self.job("model", k=2)
old_handler = signal.getsignal(signal.SIGTERM)
def run(*args, **kw):
signal.getsignal(signal.SIGTERM)(signal.SIGTERM, None)
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
self.assertEqual(signal.getsignal(signal.SIGTERM), old_handler)
def test_last_export_exception_is_reported_without_losing_completed_rollout(self):
self.job("model", k=1)
messages = []
with patch.object(rollout, "run_rollout", self.complete), \
patch.object(pipeline, "refresh_run_dataset", side_effect=RuntimeError("export unavailable")):
self.assertEqual(pipeline.collect_rollouts("run", parallel=1, log=messages.append), {"PASS": 1})
self.assertTrue(any("export unavailable" in message for message in messages))
def test_slow_export_does_not_delay_next_dispatch(self):
self.job("model", k=2)
second_started = threading.Event()
export_saw_second = []
def run(*args, **kw):
if args[2] == 1:
second_started.set()
return self.complete(*args, **kw)
def export(_):
export_saw_second.append(second_started.wait(timeout=2))
with patch.object(rollout, "run_rollout", run), patch.object(pipeline, "refresh_run_dataset", export):
self.assertEqual(self.collect(parallel=1), {"PASS": 2})
self.assertTrue(export_saw_second)
self.assertTrue(all(export_saw_second))
def test_existing_active_rollout_counts_against_slots_and_blocks_same_job(self):
self.job("old", k=2)
self.job("fresh", k=1)
with self.connect() as con:
db.rollout_set(con, "run", "old", 0, state="RUNNING", started_at=1)
started = []
def run(*args, **kw):
started.append(args[1])
return self.complete(*args, **kw)
with patch.object(rollout, "run_rollout", run):
self.assertEqual(self.collect(parallel=3), {"PASS": 1})
self.assertEqual(started, ["fresh"])
def test_watch_finishes_independent_of_oracle_running_state(self):
self.job("late", state="ORACLE_RUNNING", k=1)
with patch.object(rollout, "run_rollout", self.complete):
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
with self.connect() as con:
self.assertEqual(db.get_job(con, "run", "late")["state"], "ORACLE_RUNNING")
if __name__ == "__main__":
unittest.main()