Reuse execution, stage and review evidence to recover database registration and SFT publication without replaying completed work. Keep finished futures during SQLite contention, and hand durable results to startup recovery when draining. Release proven local leases independently of unresolved board isolation. Keep UNKNOWN review retries explicit and archived, publish completed receipts after transient export failures, and prevent a malformed sample from aborting the entire dataset. Scope routine reconciliation to newly finished attempts. Add failure-injection coverage and the production acceptance plan.
698 lines
34 KiB
Python
698 lines
34 KiB
Python
"""Collection must cover queued models promptly without duplicating or losing work."""
|
|
from contextlib import contextmanager, nullcontext
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import sqlite3
|
|
from pathlib import Path
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import db, pipeline, registry, rollout, recovery
|
|
|
|
REAL_RECONCILE = recovery.reconcile_run
|
|
|
|
|
|
class Collection(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.root = Path(self.tmp.name)
|
|
self.db_path = self.root / "jobs.sqlite"
|
|
self.mx = {"marches": ["nash-m"], "backends": ["ptq"], "modes": {
|
|
"e2e": {"enabled": True, "select": {"priority": ["A"], "tasks": ["cls", "det"]},
|
|
"marches": ["nash-m"], "backends": ["ptq"], "policies": ["teacher"], "k": 4},
|
|
"repair": {"enabled": False},
|
|
}}
|
|
self.rows = []
|
|
self.addCleanup(patch.stopall)
|
|
readonly = db.readonly
|
|
patch.object(db, "readonly", lambda: readonly(self.db_path)).start()
|
|
patch.object(db, "connect", self.connect).start()
|
|
patch.object(pipeline, "matrix", lambda: self.mx).start()
|
|
patch.object(pipeline, "models", lambda: self.rows).start()
|
|
patch.object(pipeline.paths, "STATE", self.root / "state").start()
|
|
patch.object(pipeline.paths, "RUNS", self.root / "runs").start()
|
|
patch.object(pipeline.paths, "config", lambda: {"pools": {"agent": {"workers": 3}}, "endpoints": {"teacher": {}, "student": {}}}).start()
|
|
patch.object(rollout, "setup_models_json").start()
|
|
patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext()).start()
|
|
self.runtime_config = patch('harness.collection_runtime.config', return_value=({'credential_groups': {'teacher': 'fixture'}}, {})).start()
|
|
self.runtime_preflight = patch('harness.collection_runtime.preflight').start()
|
|
self.reconciliation = patch('harness.recovery.reconcile_run', return_value={'ready': True}).start()
|
|
self.export = patch.object(pipeline, "refresh_run_dataset").start()
|
|
with self.connect():
|
|
pass
|
|
|
|
@contextmanager
|
|
def connect(self, *, timeout_s=60):
|
|
con = db._open(self.db_path, timeout_s=timeout_s)
|
|
try:
|
|
con.executescript(db.SCHEMA)
|
|
yield con
|
|
con.commit()
|
|
finally:
|
|
con.close()
|
|
|
|
def job(self, jid, *, state="PASS", priority="A", task="cls", march="nash-m", backend="ptq",
|
|
estimate=10, k=0, status="ready"):
|
|
self.rows.append({"model_id": jid, "task": task, "priority": priority, "status": status})
|
|
with self.connect() as con:
|
|
con.execute(
|
|
"INSERT INTO jobs (run_id,job_id,task,model,march,backend,qcfg,priority,mode,state,estimate_s)"
|
|
" VALUES ('run',?,?,?,?,?,?,?,'oracle',?,?)",
|
|
(jid, task, jid, march, backend, "int8", priority, state, estimate))
|
|
for kk in range(k):
|
|
db.rollout_upsert(con, "run", jid, kk, "e2e", "teacher")
|
|
|
|
def test_missing_provider_prerequisite_does_not_claim_any_attempt(self):
|
|
self.job('j', k=3)
|
|
self.runtime_preflight.side_effect = ValueError('CREDENTIAL_UNAVAILABLE')
|
|
with patch.object(rollout, 'run_rollout') as launch:
|
|
with self.assertRaisesRegex(ValueError, 'CREDENTIAL_UNAVAILABLE'):
|
|
pipeline.collect_rollouts('run', log=lambda _: None)
|
|
self.assertFalse(launch.called)
|
|
with self.connect() as con:
|
|
self.assertEqual([r[0] for r in con.execute('SELECT state FROM rollouts')], ['QUEUED'] * 3)
|
|
|
|
def complete(self, run_id, job_id, kk, mode, policy, **_):
|
|
with self.connect() as con:
|
|
db.rollout_set(con, run_id, job_id, kk, state="PASS", finished_at=time.time())
|
|
return {"terminal": "PASS", "claim_consistent": True}
|
|
|
|
def collect(self, **kw):
|
|
return pipeline.collect_rollouts("run", log=lambda _: None, **kw)
|
|
|
|
def test_missing_runtime_budget_refuses_before_claim(self):
|
|
self.job('eligible', k=1)
|
|
self.runtime_config.side_effect = ValueError('QUOTA_CONFIG_INCOMPLETE')
|
|
with self.assertRaisesRegex(ValueError, 'QUOTA_CONFIG_INCOMPLETE'):
|
|
self.collect()
|
|
with self.connect() as con:
|
|
self.assertEqual(db.list_rollouts(con, 'run', 'eligible')[0]['state'], 'QUEUED')
|
|
|
|
def test_unreconciled_execution_refuses_before_claim(self):
|
|
self.job('eligible', k=1)
|
|
self.reconciliation.return_value = {'ready': False}
|
|
with self.assertRaisesRegex(ValueError, 'RECONCILIATION_REQUIRED'):
|
|
self.collect()
|
|
with self.connect() as con:
|
|
self.assertEqual(db.list_rollouts(con, 'run', 'eligible')[0]['state'], 'QUEUED')
|
|
|
|
def test_real_database_lock_keeps_failed_future_without_replaying_or_claiming(self):
|
|
for jid in ('a', 'b', 'c'):
|
|
self.job(jid, k=1)
|
|
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
|
|
self.addCleanup(locker.close)
|
|
entered = threading.Barrier(2)
|
|
good_committed, locked, waiting = (threading.Event() for _ in range(3))
|
|
launched, logs, outcome, errors = [], [], [], []
|
|
|
|
def run(*args, **kw):
|
|
jid = args[1]
|
|
launched.append(jid)
|
|
if jid in ('a', 'b'):
|
|
entered.wait(timeout=3)
|
|
if jid == 'a':
|
|
result = self.complete(*args, **kw)
|
|
good_committed.set()
|
|
self.assertTrue(locked.wait(3))
|
|
return result
|
|
if jid == 'b':
|
|
self.assertTrue(good_committed.wait(3))
|
|
locker.execute('BEGIN EXCLUSIVE')
|
|
locked.set()
|
|
raise RuntimeError('fixture agent ended after accepted work')
|
|
return self.complete(*args, **kw)
|
|
|
|
def log(message):
|
|
logs.append(message)
|
|
if 'database busy:' in message:
|
|
waiting.set()
|
|
|
|
def collect():
|
|
try:
|
|
outcome.append(pipeline.collect_rollouts('run', parallel=2, log=log))
|
|
except BaseException as exc:
|
|
errors.append(exc)
|
|
|
|
with patch.object(rollout, 'run_rollout', run):
|
|
thread = threading.Thread(target=collect)
|
|
thread.start()
|
|
try:
|
|
self.assertTrue(waiting.wait(4), logs)
|
|
self.assertTrue(thread.is_alive())
|
|
self.assertCountEqual(launched, ['a', 'b'])
|
|
self.assertTrue(any('→ PASS' in message for message in logs), logs)
|
|
finally:
|
|
locker.rollback()
|
|
thread.join(timeout=8)
|
|
self.assertFalse(thread.is_alive())
|
|
self.assertEqual(errors, [])
|
|
self.assertEqual(outcome, [{'PASS': 2, 'UNVERIFIED': 1}])
|
|
self.assertCountEqual(launched, ['a', 'b', 'c'])
|
|
self.assertTrue(any('database recovered:' in message for message in logs))
|
|
with self.connect() as con:
|
|
self.assertEqual([(r['job_id'], r['state']) for r in con.execute('SELECT * FROM rollouts ORDER BY job_id')],
|
|
[('a', 'PASS'), ('b', 'UNVERIFIED'), ('c', 'PASS')])
|
|
|
|
def test_real_database_lock_at_start_snapshot_and_claim_waits_without_launch(self):
|
|
self.job('a', k=1)
|
|
for boundary in ('start', 'snapshot', 'claim'):
|
|
with self.subTest(boundary=boundary):
|
|
with self.connect() as con:
|
|
con.execute("UPDATE rollouts SET state='QUEUED', started_at=NULL, finished_at=NULL")
|
|
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
|
|
waiting, logs, outcomes, errors, calls = threading.Event(), [], [], [], []
|
|
target = pipeline if boundary == 'snapshot' else db
|
|
method = '_collection_snapshot' if boundary == 'snapshot' else 'claim_rollout'
|
|
original = getattr(target, method)
|
|
locked = boundary == 'start'
|
|
if locked:
|
|
locker.execute('BEGIN EXCLUSIVE')
|
|
|
|
def inject(*args, **kwargs):
|
|
nonlocal locked
|
|
if not locked:
|
|
locker.execute('BEGIN EXCLUSIVE')
|
|
locked = True
|
|
return original(*args, **kwargs)
|
|
|
|
def log(message):
|
|
logs.append(message)
|
|
if 'database busy:' in message:
|
|
waiting.set()
|
|
|
|
def launch(*args, **kw):
|
|
calls.append(args[1])
|
|
return self.complete(*args, **kw)
|
|
|
|
def collect():
|
|
try:
|
|
outcomes.append(pipeline.collect_rollouts('run', parallel=1, log=log))
|
|
except BaseException as exc:
|
|
errors.append(exc)
|
|
|
|
with patch.object(rollout, 'run_rollout', launch), \
|
|
patch.object(target, method, inject) if boundary != 'start' else nullcontext():
|
|
thread = threading.Thread(target=collect)
|
|
thread.start()
|
|
try:
|
|
self.assertTrue(waiting.wait(4), (logs, errors))
|
|
self.assertEqual(calls, [])
|
|
self.assertTrue(thread.is_alive())
|
|
finally:
|
|
locker.rollback()
|
|
thread.join(timeout=8)
|
|
locker.close()
|
|
self.assertFalse(thread.is_alive())
|
|
self.assertEqual(errors, [])
|
|
self.assertEqual(outcomes, [{'PASS': 1}])
|
|
self.assertEqual(calls, ['a'])
|
|
|
|
def test_drain_during_database_wait_preserves_unclaimed_work(self):
|
|
self.job('a', k=1)
|
|
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
|
|
locker.execute('BEGIN EXCLUSIVE')
|
|
waiting, outcomes, errors = threading.Event(), [], []
|
|
def log(message):
|
|
if 'database busy:' in message:
|
|
waiting.set()
|
|
def collect():
|
|
try:
|
|
outcomes.append(pipeline.collect_rollouts('run', log=log))
|
|
except BaseException as exc:
|
|
errors.append(exc)
|
|
with patch.object(rollout, 'run_rollout') as launch:
|
|
thread = threading.Thread(target=collect)
|
|
thread.start()
|
|
try:
|
|
self.assertTrue(waiting.wait(3))
|
|
pipeline.collect_drain_file('run').touch()
|
|
thread.join(timeout=7)
|
|
self.assertFalse(thread.is_alive(), 'empty collector must drain while database remains locked')
|
|
finally:
|
|
locker.rollback()
|
|
locker.close()
|
|
thread.join(timeout=8)
|
|
self.assertFalse(thread.is_alive())
|
|
self.assertEqual(errors, [])
|
|
self.assertEqual(outcomes, [{}])
|
|
launch.assert_not_called()
|
|
with self.connect() as con:
|
|
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'QUEUED')
|
|
|
|
def test_runtime_recovery_scans_only_just_finished_attempts(self):
|
|
for jid in ('a', 'b'):
|
|
self.job(jid, k=1)
|
|
with patch.object(rollout, 'run_rollout', self.complete):
|
|
self.assertEqual(self.collect(parallel=1), {'PASS': 2})
|
|
calls = self.reconciliation.call_args_list
|
|
self.assertEqual(calls[0].args, ('run',))
|
|
self.assertEqual(calls[0].kwargs, {})
|
|
self.assertEqual([call.kwargs['attempts'] for call in calls[1:]], [(('a', 0),), (('b', 0),)])
|
|
|
|
def test_drain_persists_pending_prelaunch_failure_and_exits_before_unlock(self):
|
|
self.job('a', k=1)
|
|
self.job('b', k=1)
|
|
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
|
|
waiting, outcomes, errors, launched = threading.Event(), [], [], []
|
|
def launch(*args, **kw):
|
|
launched.append(args[1])
|
|
locker.execute('BEGIN EXCLUSIVE')
|
|
raise RuntimeError('completed worker needs failure registration')
|
|
def log(message):
|
|
if 'database busy:' in message:
|
|
waiting.set()
|
|
def collect():
|
|
try:
|
|
outcomes.append(pipeline.collect_rollouts('run', parallel=1, log=log))
|
|
except BaseException as exc:
|
|
errors.append(exc)
|
|
with patch.object(rollout, 'run_rollout', launch):
|
|
thread = threading.Thread(target=collect)
|
|
thread.start()
|
|
try:
|
|
self.assertTrue(waiting.wait(3))
|
|
pipeline.collect_drain_file('run').touch()
|
|
thread.join(timeout=7)
|
|
self.assertFalse(thread.is_alive(), 'drain must persist completed work and exit while DB stays locked')
|
|
finally:
|
|
locker.rollback()
|
|
locker.close()
|
|
thread.join(timeout=8)
|
|
self.assertFalse(thread.is_alive())
|
|
self.assertEqual(errors, [])
|
|
self.assertEqual(outcomes, [{}])
|
|
self.assertEqual(launched, ['a'])
|
|
from harness import recovery
|
|
with patch.object(recovery, 'reconcile_run', REAL_RECONCILE), \
|
|
patch.object(rollout, 'run_rollout', side_effect=AssertionError('restart dispatched work')):
|
|
self.assertEqual(self.collect(parallel=1, limit=0), {})
|
|
with self.connect() as con:
|
|
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'INFRA_ERROR')
|
|
self.assertEqual(db.list_rollouts(con, 'run', 'b')[0]['state'], 'QUEUED')
|
|
|
|
def test_sigterm_hands_off_completed_execution_and_restart_reverifies_once(self):
|
|
program = """
|
|
import sys
|
|
from scripts.test_collection import Collection
|
|
case = Collection(); case.setUp()
|
|
try: case.sigterm_restart_probe(sys.argv[1])
|
|
finally: case.doCleanups()
|
|
"""
|
|
for boundary in ('first_db', 'during_verify'):
|
|
with self.subTest(boundary=boundary):
|
|
result = subprocess.run([sys.executable, '-c', program, boundary], cwd=Path(__file__).resolve().parents[1],
|
|
capture_output=True, text=True, timeout=10)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn('SIGTERM_RECOVERY_ONCE', result.stdout)
|
|
|
|
def sigterm_restart_probe(self, boundary):
|
|
from harness import execution, verify
|
|
from scripts.test_execution import policy
|
|
self.job('a', k=1); self.job('b', k=1)
|
|
locker = sqlite3.connect(self.db_path, timeout=0, check_same_thread=False)
|
|
self.addCleanup(locker.close)
|
|
rd = self.root/'runs/run/a/rollouts/00'
|
|
observed = []
|
|
def interrupt():
|
|
locker.execute('BEGIN EXCLUSIVE')
|
|
os.kill(os.getpid(), signal.SIGTERM)
|
|
real_verify = verify.verify_rollout
|
|
def interrupted_verify(*args, **kwargs):
|
|
with self.connect() as con:
|
|
observed.append(db.list_rollouts(con, 'run', 'a')[0]['state'])
|
|
interrupt()
|
|
return real_verify(*args, **kwargs)
|
|
def launch(*args, **kw):
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd/'effects').write_text('one\n')
|
|
owner.finish(0, protocol_complete=True)
|
|
execution.atomic_json(rd/'rollout.json', {'state': 'DONE', 'mode': 'e2e', 'policy': 'teacher',
|
|
'execution_managed': True, 'execution_id': owner.snapshot()['execution_id']})
|
|
spec = registry.JobSpec('a', 'cls', 'fixture', 'nash-m', 'ptq', 'int8', 'A', 'e2e')
|
|
execution.atomic_json(rd.parent.parent/'job.json', {'run_id': 'run', 'job_id': 'a',
|
|
'gates': registry.gate_snapshot(spec)})
|
|
if boundary == 'first_db':
|
|
interrupt()
|
|
raise RuntimeError('fixture final registration interrupted')
|
|
with patch.object(rollout, 'run_rollout', launch), \
|
|
patch.object(verify, 'verify_rollout', interrupted_verify) if boundary == 'during_verify' else nullcontext():
|
|
self.assertEqual(self.collect(parallel=1), {})
|
|
if boundary == 'during_verify':
|
|
self.assertEqual(observed, ['VERIFYING'])
|
|
self.assertTrue(locker.in_transaction) # Collector exited before DB recovery.
|
|
locker.rollback()
|
|
with patch.object(recovery, 'reconcile_run', REAL_RECONCILE), \
|
|
patch.object(rollout, 'run_rollout', side_effect=AssertionError('restart dispatched work')):
|
|
self.assertEqual(self.collect(parallel=1, limit=0), {})
|
|
self.assertEqual((rd/'effects').read_text(), 'one\n')
|
|
with self.connect() as con:
|
|
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'UNVERIFIED')
|
|
self.assertIsNotNone(db.list_rollouts(con, 'run', 'a')[0]['verdict'])
|
|
self.assertEqual(db.list_rollouts(con, 'run', 'b')[0]['state'], 'QUEUED')
|
|
print('SIGTERM_RECOVERY_ONCE')
|
|
|
|
def test_error_after_committed_verdict_preserves_database_terminal(self):
|
|
self.job('a', k=1)
|
|
def launch(*args, **kw):
|
|
with self.connect() as con:
|
|
db.rollout_set(con, 'run', 'a', 0, state='FAIL', verdict='{"terminal":"FAIL"}', finished_at=1)
|
|
rd = self.root / 'runs/run/a/rollouts/00'
|
|
rd.mkdir(parents=True)
|
|
# A pre-existing verdict wins; control-plane outcome is not task quality.
|
|
(rd / 'execution.json').write_text('{"execution_outcome":"AGENT_ERROR"}')
|
|
raise RuntimeError('fixture failure after verdict commit')
|
|
with patch.object(rollout, 'run_rollout', launch), patch.object(rollout, 'settle_execution') as settle:
|
|
self.assertEqual(self.collect(), {'FAIL': 1})
|
|
settle.assert_not_called()
|
|
with self.connect() as con:
|
|
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'FAIL')
|
|
|
|
def test_nonbusy_database_error_is_not_retried(self):
|
|
self.job('a', k=1)
|
|
with patch.object(pipeline, '_collection_snapshot', side_effect=sqlite3.OperationalError('disk I/O error')) as snapshot, \
|
|
patch.object(rollout, 'run_rollout') as launch:
|
|
with self.assertRaisesRegex(sqlite3.OperationalError, 'disk I/O error'):
|
|
self.collect()
|
|
self.assertEqual(snapshot.call_count, 1)
|
|
launch.assert_not_called()
|
|
|
|
def test_recovery_reason_and_released_resources_are_visible(self):
|
|
self.job('a', k=1)
|
|
logs = []
|
|
self.reconciliation.return_value = {'ready': False, 'executions': [
|
|
{'ready': False, 'reason': 'LEGACY_EXECUTION_OWNERSHIP_UNKNOWN'}],
|
|
'resources': [{'lease': '/fixture/agent-2.lease.json', 'state': 'RELEASED'}]}
|
|
with self.assertRaisesRegex(ValueError, 'LEGACY_EXECUTION_OWNERSHIP_UNKNOWN'):
|
|
pipeline.collect_rollouts('run', log=logs.append)
|
|
self.assertTrue(any('released=1' in message and 'agent-2.lease.json' in message for message in logs))
|
|
|
|
def test_all_coverage_is_idempotent_preserves_repairs_and_non_priority_filters(self):
|
|
self.job("eligible", priority="C")
|
|
for jid, kw in [("wrong-task", {"task": "occ"}), ("wrong-march", {"march": "nash-p"}),
|
|
("wrong-backend", {"backend": "qat"}), ("not-ready", {"status": "needs_adapter"})]:
|
|
self.job(jid, **kw)
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "eligible", 0, "repair", "teacher", "FAIL")
|
|
db.rollout_upsert(con, "run", "eligible", 4, "e2e", "teacher", "PASS")
|
|
self.assertEqual(pipeline.enqueue_e2e_coverage("run"), 3)
|
|
self.assertEqual(pipeline.enqueue_e2e_coverage("run"), 0)
|
|
with self.connect() as con:
|
|
rows = db.list_rollouts(con, "run", "eligible")
|
|
self.assertEqual([(r["kk"], r["mode"]) for r in rows],
|
|
[(0, "repair"), (4, "e2e"), (5, "e2e"), (6, "e2e"), (7, "e2e")])
|
|
self.assertEqual(con.execute("SELECT COUNT(*) FROM annotations").fetchone()[0], 1)
|
|
self.assertEqual(con.execute("SELECT COUNT(*) FROM jobs WHERE mode='e2e'").fetchone()[0], 1)
|
|
self.assertEqual(self.mx["modes"]["e2e"]["select"]["priority"], ["A"])
|
|
self.assertFalse((self.root / "runs").exists())
|
|
|
|
def test_coverage_fills_each_policy_and_respects_job_selection(self):
|
|
self.job("chosen", priority="B")
|
|
self.job("other", priority="B")
|
|
self.mx["modes"]["e2e"]["policies"] = ["teacher", "student"]
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "chosen", 9, "e2e", "teacher", "TIMEOUT")
|
|
self.assertEqual(pipeline.enqueue_e2e_coverage("run", ["chosen"]), 7)
|
|
with self.connect() as con:
|
|
self.assertEqual(len(db.list_rollouts(con, "run", "chosen")), 8)
|
|
self.assertEqual(db.list_rollouts(con, "run", "other"), [])
|
|
|
|
def test_atomic_claim_prevents_duplicate_job_and_exceeding_pool(self):
|
|
self.job("a", k=2)
|
|
self.job("b", k=1)
|
|
outcomes = []
|
|
barrier = threading.Barrier(2)
|
|
|
|
def claim(kk):
|
|
barrier.wait(timeout=2)
|
|
with self.connect() as con:
|
|
outcomes.append(db.claim_rollout(con, "run", "a", kk, max_active=3))
|
|
|
|
threads = [threading.Thread(target=claim, args=(kk,)) for kk in range(2)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join(timeout=3)
|
|
self.assertEqual(sorted(outcomes), [False, True])
|
|
with self.connect() as con:
|
|
self.assertFalse(db.claim_rollout(con, "run", "b", 0, max_active=1))
|
|
con.execute("UPDATE rollouts SET state='PASS' WHERE job_id='a' AND state='RUNNING'")
|
|
con.commit()
|
|
self.assertTrue(db.claim_rollout(con, "run", "b", 0, max_active=1))
|
|
self.assertFalse(db.claim_rollout(con, "run", "b", 0, max_active=3))
|
|
|
|
def test_missing_inputs_do_not_delay_first_agent_attempt(self):
|
|
self.job("first", k=4)
|
|
self.job("late", state="ORACLE_RUNNING", estimate=1, k=4)
|
|
self.job("covered", estimate=0, k=4)
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "run", "covered", 0, state="PASS", finished_at=1)
|
|
started = []
|
|
self.assertFalse((self.root / "runs").exists())
|
|
|
|
def run(*args, **kw):
|
|
started.append(args[1])
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
self.assertEqual(self.collect(parallel=1, limit=2), {"PASS": 2})
|
|
self.assertEqual(started, ["late", "first"])
|
|
|
|
def test_three_slots_cover_three_distinct_models(self):
|
|
for jid in ("a", "b", "c"):
|
|
self.job(jid, k=4)
|
|
barrier = threading.Barrier(3)
|
|
started = []
|
|
|
|
def run(*args, **kw):
|
|
started.append(args[1])
|
|
barrier.wait(timeout=3)
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
self.assertEqual(self.collect(parallel=3, limit=3), {"PASS": 3})
|
|
self.assertEqual(set(started), {"a", "b", "c"})
|
|
|
|
def test_prefers_shorter_jobs_independent_of_oracle_verdict(self):
|
|
self.job("failed", state="FAIL", estimate=0, k=1)
|
|
self.job("slow", estimate=100, k=1)
|
|
self.job("fast", estimate=1, k=1)
|
|
rows, pending = pipeline._collection_snapshot("run", None, None)
|
|
self.assertFalse(pending)
|
|
self.assertEqual([r["job_id"] for r in rows], ["failed", "fast", "slow"])
|
|
|
|
def test_e2e_prepares_its_own_inputs_and_retired_repair_is_never_ready(self):
|
|
self.job("ready", state="ORACLE_RUNNING", k=1)
|
|
self.job("missing", state="PASS", k=1)
|
|
self.assertFalse((self.root / "runs").exists())
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "ready", 1, "repair", "teacher")
|
|
rows, pending = pipeline._collection_snapshot("run", None, None)
|
|
self.assertEqual([(r["job_id"], r["mode"]) for r in rows], [("missing", "e2e"), ("ready", "e2e")])
|
|
self.assertFalse(pending)
|
|
with self.connect() as con:
|
|
self.assertEqual(con.execute("SELECT COUNT(*) FROM rollouts WHERE state!='QUEUED'").fetchone()[0], 0)
|
|
|
|
def test_collect_preserves_and_skips_historical_queued_repairs(self):
|
|
self.job("failed", state="FAIL", k=1)
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "failed", 7, "repair", "teacher")
|
|
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
|
|
with patch.object(rollout, "run_rollout", self.complete), \
|
|
patch.object(pipeline, "enqueue_repairs") as enqueue:
|
|
self.assertEqual(self.collect(parallel=1, modes=["e2e", "repair"]), {"PASS": 1})
|
|
enqueue.assert_not_called()
|
|
with self.connect() as con:
|
|
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
|
|
rows, pending = pipeline._collection_snapshot("run", None, ["repair"])
|
|
self.assertEqual(rows, [])
|
|
self.assertFalse(pending)
|
|
|
|
def test_exception_releases_claim_and_does_not_retry_same_row(self):
|
|
self.job("broken", k=1)
|
|
with patch.object(rollout, "run_rollout", side_effect=SystemExit("missing cache")):
|
|
self.assertEqual(self.collect(parallel=1), {"UNVERIFIED": 1})
|
|
with self.connect() as con:
|
|
row = db.list_rollouts(con, "run", "broken")[0]
|
|
self.assertEqual(row["state"], "UNVERIFIED")
|
|
self.assertIsNotNone(row["finished_at"])
|
|
|
|
def test_exception_after_measurements_still_verifies_and_exports_real_result(self):
|
|
self.job("measured", k=1)
|
|
jd = self.root / "runs" / "run" / "measured"
|
|
rd = jd / "rollouts" / "00"
|
|
rd.mkdir(parents=True)
|
|
spec = registry.JobSpec(job_id="measured", task="cls", model="measured", march="nash-m",
|
|
backend="ptq", qcfg="int8", priority="A", mode="oracle")
|
|
(jd / "job.json").write_text(json.dumps({"job_id": "measured", "run_id": "run",
|
|
"gates": registry.gate_snapshot(spec)}))
|
|
(rd / "rollout.json").write_text(json.dumps({"state": "RUNNING", "mode": "e2e", "policy": "teacher"}))
|
|
(rd / "claim.json").write_text('{"terminal":"PASS"}')
|
|
hbm = rd / "artifacts/measured.hbm"
|
|
hbm.parent.mkdir()
|
|
hbm.write_bytes(b"collection fixture")
|
|
sha = hashlib.sha256(hbm.read_bytes()).hexdigest()
|
|
for stage, measurements in [
|
|
("05-compile", {}),
|
|
("06-hbm", {"G1": dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False)}),
|
|
("07-board", {"G4": {"latency_ms_mean": 1}}),
|
|
("08-eval", {"G3": {"fp32": 80, "quant": 79, "source": "board", "tier": "t1", "n": 2000}}),
|
|
]:
|
|
for value in measurements.values():
|
|
value["hbm_sha256"] = sha
|
|
directory = rd / "stages" / stage
|
|
directory.mkdir(parents=True)
|
|
(directory / "meta.json").write_text(json.dumps({"state": "OK", "measurements": measurements, "attempt": 0}))
|
|
with patch.object(rollout, "run_rollout", side_effect=RuntimeError("agent stream closed")):
|
|
self.assertEqual(self.collect(parallel=1), {"PASS": 1})
|
|
verdict = json.loads((rd / "verdict.json").read_text())
|
|
self.assertEqual(verdict["terminal"], "PASS")
|
|
self.assertNotIn("oracle_ref", verdict)
|
|
self.assertEqual(verdict["execution_terminal"], "ERROR")
|
|
self.assertTrue(verdict["completion"]["workflow_completed"])
|
|
self.export.assert_called_with("run")
|
|
|
|
def test_dry_run_watch_neither_claims_nor_adds_coverage(self):
|
|
self.job("example", priority="B", k=1)
|
|
with patch.object(rollout, "run_rollout", return_value={"dry_run": True}), \
|
|
patch.object(pipeline, "enqueue_e2e_coverage") as enqueue:
|
|
self.assertEqual(self.collect(dry_run=True, watch=True, coverage="all"), {"DRY_RUN": 1})
|
|
enqueue.assert_not_called()
|
|
self.export.assert_not_called()
|
|
with self.connect() as con:
|
|
self.assertEqual(db.list_rollouts(con, "run", "example")[0]["state"], "QUEUED")
|
|
|
|
def test_singleton_collector_cannot_duplicate_inflight_work(self):
|
|
self.job("model", k=1)
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
def run(*args, **kw):
|
|
entered.set()
|
|
release.wait(timeout=3)
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
thread = threading.Thread(target=self.collect, kwargs={"parallel": 1})
|
|
thread.start()
|
|
try:
|
|
self.assertTrue(entered.wait(timeout=2))
|
|
self.assertEqual(self.collect(parallel=1), {})
|
|
finally:
|
|
release.set()
|
|
thread.join(timeout=3)
|
|
self.assertFalse(thread.is_alive())
|
|
|
|
def test_drain_finishes_inflight_without_claiming_repeats(self):
|
|
self.job("model", k=4)
|
|
|
|
def run(*args, **kw):
|
|
pipeline.collect_drain_file("run").touch()
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
|
|
with self.connect() as con:
|
|
states = [r["state"] for r in db.list_rollouts(con, "run", "model")]
|
|
self.assertEqual(states, ["PASS", "QUEUED", "QUEUED", "QUEUED"])
|
|
|
|
def test_first_pass_preserves_repeats_and_failed_attempts_consume_one_slot(self):
|
|
self.job("fresh", k=4)
|
|
self.job("covered", k=4)
|
|
self.job("failed", k=4)
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "run", "covered", 0, state="PASS", started_at=1, finished_at=2)
|
|
db.rollout_set(con, "run", "failed", 0, state="UNVERIFIED", started_at=1, finished_at=2)
|
|
with patch.object(rollout, "run_rollout", self.complete):
|
|
self.assertEqual(self.collect(parallel=1, first_pass=True), {"PASS": 1})
|
|
with self.connect() as con:
|
|
for model in ("fresh", "covered", "failed"):
|
|
rows = db.list_rollouts(con, "run", model)
|
|
self.assertEqual([r["state"] for r in rows[1:]], ["QUEUED"] * 3)
|
|
|
|
def test_first_pass_ignores_completed_repair_but_keeps_e2e_history(self):
|
|
self.job("repair-only", k=2)
|
|
self.job("e2e-covered", k=2)
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "run", "repair-only", 7, "repair", "teacher", "PASS")
|
|
db.rollout_set(con, "run", "repair-only", 7, started_at=1, finished_at=2)
|
|
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
|
|
db.rollout_set(con, "run", "e2e-covered", 0, state="PASS", started_at=1, finished_at=2)
|
|
with patch.object(rollout, "run_rollout", side_effect=self.complete) as run:
|
|
self.assertEqual(self.collect(parallel=1, modes=["e2e"], first_pass=True), {"PASS": 1})
|
|
self.assertEqual(run.call_args.args[1:4], ("repair-only", 0, "e2e"))
|
|
with self.connect() as con:
|
|
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
|
|
for jid in ("repair-only", "e2e-covered"):
|
|
self.assertEqual(db.list_rollouts(con, "run", jid)[1]["state"], "QUEUED")
|
|
|
|
def test_sigterm_drains_inflight_and_restores_previous_handler(self):
|
|
self.job("model", k=2)
|
|
old_handler = signal.getsignal(signal.SIGTERM)
|
|
|
|
def run(*args, **kw):
|
|
signal.getsignal(signal.SIGTERM)(signal.SIGTERM, None)
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
|
|
self.assertEqual(signal.getsignal(signal.SIGTERM), old_handler)
|
|
|
|
def test_last_export_exception_is_reported_without_losing_completed_rollout(self):
|
|
self.job("model", k=1)
|
|
messages = []
|
|
with patch.object(rollout, "run_rollout", self.complete), \
|
|
patch.object(pipeline, "refresh_run_dataset", side_effect=RuntimeError("export unavailable")):
|
|
self.assertEqual(pipeline.collect_rollouts("run", parallel=1, log=messages.append), {"PASS": 1})
|
|
self.assertTrue(any("export unavailable" in message for message in messages))
|
|
|
|
def test_slow_export_does_not_delay_next_dispatch(self):
|
|
self.job("model", k=2)
|
|
second_started = threading.Event()
|
|
export_saw_second = []
|
|
|
|
def run(*args, **kw):
|
|
if args[2] == 1:
|
|
second_started.set()
|
|
return self.complete(*args, **kw)
|
|
|
|
def export(_):
|
|
export_saw_second.append(second_started.wait(timeout=2))
|
|
|
|
with patch.object(rollout, "run_rollout", run), patch.object(pipeline, "refresh_run_dataset", export):
|
|
self.assertEqual(self.collect(parallel=1), {"PASS": 2})
|
|
self.assertTrue(export_saw_second)
|
|
self.assertTrue(all(export_saw_second))
|
|
|
|
def test_existing_active_rollout_counts_against_slots_and_blocks_same_job(self):
|
|
self.job("old", k=2)
|
|
self.job("fresh", k=1)
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "run", "old", 0, state="RUNNING", started_at=1)
|
|
started = []
|
|
|
|
def run(*args, **kw):
|
|
started.append(args[1])
|
|
return self.complete(*args, **kw)
|
|
|
|
with patch.object(rollout, "run_rollout", run):
|
|
self.assertEqual(self.collect(parallel=3), {"PASS": 1})
|
|
self.assertEqual(started, ["fresh"])
|
|
|
|
def test_watch_finishes_independent_of_oracle_running_state(self):
|
|
self.job("late", state="ORACLE_RUNNING", k=1)
|
|
with patch.object(rollout, "run_rollout", self.complete):
|
|
self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1})
|
|
with self.connect() as con:
|
|
self.assertEqual(db.get_job(con, "run", "late")["state"], "ORACLE_RUNNING")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|