Reuse execution, stage and review evidence to recover database registration and SFT publication without replaying completed work. Keep finished futures during SQLite contention, and hand durable results to startup recovery when draining. Release proven local leases independently of unresolved board isolation. Keep UNKNOWN review retries explicit and archived, publish completed receipts after transient export failures, and prevent a malformed sample from aborting the entire dataset. Scope routine reconciliation to newly finished attempts. Add failure-injection coverage and the production acceptance plan.
248 lines
15 KiB
Python
248 lines
15 KiB
Python
"""Actual lifecycle boundaries with owned processes, queues and read-only UI API."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import ExitStack
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import urllib.request
|
|
|
|
from harness import collection_runtime, db, execution, monitor, paths, rollout, scheduler, watchdog
|
|
from scripts import test_execution, test_quota
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux process identities and kernel containment')
|
|
class LifecycleJoint(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory(prefix='bench-lifecycle-joint-')
|
|
self.addCleanup(temp.cleanup)
|
|
self.root = Path(temp.name)
|
|
self.rd = self.root/'runs/r/j/rollouts/00'; self.rd.mkdir(parents=True)
|
|
self.database = self.root/'state/jobs.sqlite'
|
|
self.connect, self.readonly = db.connect, db.readonly
|
|
with self.connect(self.database) as con:
|
|
db.create_run(con, 'r', {})
|
|
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING')
|
|
(self.rd/'rollout.json').write_text('{"state":"RUNNING","mode":"e2e","policy":"teacher"}')
|
|
self.config = {'pools': {'agent': {'workers': 1}, 'cpu': {'workers': 1}, 'board': {'workers': 1}}}
|
|
self.stack = ExitStack(); self.addCleanup(self.stack.close)
|
|
self.stack.enter_context(patch.object(paths, 'STATE', self.root/'state'))
|
|
self.stack.enter_context(patch.object(paths, 'RUNS', self.root/'runs'))
|
|
self.stack.enter_context(patch.object(paths, 'config', return_value=self.config))
|
|
self.stack.enter_context(patch.object(db, 'connect', lambda: self.connect(self.database)))
|
|
self.stack.enter_context(patch.object(db, 'readonly', lambda: self.readonly(self.database)))
|
|
self.stack.enter_context(patch.object(scheduler, 'board_slots', return_value={0: {'name': 'local-lock-only'}}))
|
|
|
|
def wait(self, predicate, timeout=5):
|
|
deadline = time.monotonic()+timeout
|
|
while time.monotonic() < deadline:
|
|
result = predicate()
|
|
if result:
|
|
return result
|
|
time.sleep(.005)
|
|
self.fail('bounded fixture condition did not arrive')
|
|
|
|
@staticmethod
|
|
def receipt(case, **facts):
|
|
print('JOINT_RECEIPT '+json.dumps({'case': case, **facts}, sort_keys=True))
|
|
|
|
def test_cancel_same_execution_fences_quota_compile_and_board_queues(self):
|
|
owner = execution.Execution(self.rd)
|
|
policy = test_execution.policy(); policy['queue_deadline_s'] = 5
|
|
for phase in policy['phases'].values(): phase['queue_deadline_s'] = 5
|
|
owner.start(policy)
|
|
quota = test_quota.Quota(); quota.setUp(); self.addCleanup(quota.doCleanups)
|
|
entered, release = threading.Event(), threading.Event()
|
|
def held_provider(n, body):
|
|
entered.set()
|
|
if not release.wait(5): raise TimeoutError('fixture provider release missing')
|
|
return test_quota.ok(n, body)
|
|
provider = quota.provider(held_provider)
|
|
broker = quota.broker(provider, quota.policy(max_concurrency=1, wait_budget_s=5,
|
|
first_byte_timeout_s=5, request_total_s=6))
|
|
self.stack.enter_context(patch.object(collection_runtime, 'broker', return_value=broker))
|
|
hold_scope = broker.issue_scope('fixture-slot-holder', 'teacher', 'glm-test')
|
|
target = broker.issue_scope(owner.snapshot()['execution_id'], 'teacher', 'glm-test')
|
|
attempted = {name: threading.Event() for name in ('cpu', 'board')}
|
|
def queued_stage(pool, phase):
|
|
operation = 'queued-'+pool
|
|
owner.begin_phase(operation, phase)
|
|
def observed_wait():
|
|
attempted[pool].set()
|
|
return None
|
|
try:
|
|
with owner.scope(phase, operation), scheduler.acquire(pool, poll_s=.01, stop_check=observed_wait):
|
|
return watchdog.run([sys.executable, '-c', 'from pathlib import Path;Path("unexpected-spawn").touch()'],
|
|
cwd=self.rd, stdout=self.rd/(pool+'.out'), stderr=self.rd/(pool+'.err'), execution=owner,
|
|
role='stage', poll_s=.01).state
|
|
except execution.ExecutionStopped as error:
|
|
return str(error)
|
|
with ExitStack() as holds:
|
|
holds.enter_context(scheduler.acquire('cpu'))
|
|
holds.enter_context(scheduler.acquire('board'))
|
|
with ThreadPoolExecutor(max_workers=4) as workers:
|
|
active = workers.submit(quota.request, hold_scope, 'held')
|
|
pending = []
|
|
try:
|
|
self.assertTrue(entered.wait(3))
|
|
request = workers.submit(quota.request, target, 'queued')
|
|
pending = [workers.submit(queued_stage, pool, phase)
|
|
for pool, phase in [('cpu', '05-compile'), ('board', '07-board')]]
|
|
self.wait(lambda: all(event.is_set() for event in attempted.values()))
|
|
self.wait(lambda: any(g['state']=='QUEUED' for g in broker.execution_status(target.execution_id)['generations']))
|
|
before = len(provider.requests)
|
|
settled = rollout.settle_execution('r', 'j', 0, owner, 'joint-cancel', cancel=True)
|
|
self.assertEqual(settled['execution_outcome'], 'CANCELLED')
|
|
self.assertEqual([future.result(timeout=3) for future in pending], ['CANCEL_REQUESTED']*2)
|
|
self.assertEqual(request.result(timeout=3)[0], 409)
|
|
self.assertEqual(len(provider.requests), before)
|
|
self.assertEqual(owner.snapshot()['processes'], [])
|
|
self.assertFalse((self.rd/'unexpected-spawn').exists())
|
|
generation = broker.execution_status(target.execution_id)['generations'][0]
|
|
self.assertEqual((generation['state'], generation['attempts']), ('CANCELLED', 0))
|
|
finally:
|
|
release.set()
|
|
self.assertEqual(active.result(timeout=3)[0], 200)
|
|
self.assertEqual(scheduler.pool_usage(), {'agent': 0, 'cpu': 0, 'board': 0})
|
|
self.assertEqual(list((self.root/'state/locks').glob('*.lease.json')), [])
|
|
self.assertTrue(all(a['ended_at'] is not None for a in broker.snapshot()['attempts']))
|
|
self.assertEqual(owner.snapshot()['cleanup_state'], 'VERIFIED')
|
|
self.receipt('JOINT-005', queued_pools=['quota','cpu','board'], target_http_attempts=0,
|
|
target_spawns=0, upstream_requests=1, owned_slots_after=0, cleanup='VERIFIED')
|
|
|
|
def test_silent_work_spin_and_hang_report_real_progress_and_absolute_deadlines(self):
|
|
observer = monitor.Observer(self.root)
|
|
server = monitor.make_server(observer, ('127.0.0.1', 0))
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start()
|
|
self.addCleanup(server.server_close); self.addCleanup(server.shutdown)
|
|
facts = []
|
|
programs = {
|
|
'working': 'import os,time;f=open("work.bin","wb");end=time.monotonic()+.35\n'
|
|
'while time.monotonic()<end:\n f.write(b"x"*4096);f.flush();os.fsync(f.fileno());sum(i*i for i in range(20000))\n',
|
|
'spinning': 'while True: pass',
|
|
'hanging': 'import signal;signal.pause()',
|
|
}
|
|
for kind, program in programs.items():
|
|
with self.subTest(kind=kind):
|
|
rd = self.root/'runs/r'/kind/'rollouts/00'; rd.mkdir(parents=True)
|
|
with self.connect(self.database) as con:
|
|
db.rollout_upsert(con, 'r', kind, 0, 'e2e', 'teacher', state='RUNNING')
|
|
owner = execution.Execution(rd)
|
|
policy = test_execution.policy()
|
|
policy['phases']['05-compile'].update(execution_deadline_s=1.5 if kind=='working' else .35,
|
|
queue_deadline_s=1, suspect_after_s=.08)
|
|
owner.start(policy); owner.begin_phase('compile', '05-compile'); owner.phase_running('compile')
|
|
owner.activity('COMPILING', operation_id='compile')
|
|
observed = []
|
|
def sample():
|
|
observer.sample()
|
|
observed.append(observer.detail('r', kind, 0))
|
|
started = time.monotonic()
|
|
with owner.scope('05-compile', 'compile'):
|
|
result = watchdog.run([sys.executable, '-c', program], cwd=rd, stdout=rd/'out', stderr=rd/'err',
|
|
execution=owner, role='stage', poll_s=.02, stop_check=sample)
|
|
elapsed = time.monotonic()-started
|
|
state = owner.finish(result.exit_code, protocol_complete=result.state=='OK',
|
|
stop_reason=result.stalled_reason, cleanup_state=result.cleanup_state)
|
|
self.assertLess(elapsed, 3)
|
|
self.assertEqual(result.cleanup_state, 'VERIFIED')
|
|
self.assertEqual(execution.owned_processes(state), [])
|
|
self.assertEqual((rd/'out').read_bytes(), b'')
|
|
if kind=='working':
|
|
self.assertEqual(result.state, 'OK')
|
|
self.assertGreater((rd/'work.bin').stat().st_size, 0)
|
|
else:
|
|
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
|
|
self.assertIn('DEADLINE', result.stalled_reason)
|
|
progress = [row['progress'] for row in observed if (row.get('progress') or {}).get('signals')]
|
|
self.assertTrue(progress)
|
|
self.assertTrue(all(row['suspect_after_s']==.08 for row in progress))
|
|
self.assertTrue(all(row['signals']['business_log_bytes']==0 for row in progress))
|
|
if kind in ('working','spinning'):
|
|
self.assertGreater(max(row['signals']['cpu_ticks'] or 0 for row in progress),
|
|
min(row['signals']['cpu_ticks'] or 0 for row in progress))
|
|
else:
|
|
self.assertTrue(any(row['reason']=='ALL_SIGNALS_IDLE' for row in progress))
|
|
observer.sample()
|
|
url=f'http://127.0.0.1:{server.server_port}/api/runs/r/rollouts/{kind}/0'
|
|
with urllib.request.urlopen(url, timeout=3) as response: displayed=json.load(response)
|
|
phase=displayed['phases']['compile']
|
|
self.assertEqual(phase['phase'], '05-compile')
|
|
self.assertAlmostEqual(phase['execution_deadline_monotonic']-phase['execution_started_monotonic'],
|
|
policy['phases']['05-compile']['execution_deadline_s'])
|
|
self.assertEqual(displayed['progress']['suspect_after_s'], .08)
|
|
self.assertEqual(displayed['stop_reason'], state['stop_reason'])
|
|
self.assertTrue(displayed['progress']['reason'])
|
|
facts.append({'kind':kind,'outcome':state['execution_outcome'],'cleanup':result.cleanup_state,
|
|
'elapsed_s':round(elapsed,3),'phase':'05-compile','suspect_after_s':.08,
|
|
'display_reason':displayed['progress']['reason'],'log_bytes':0})
|
|
self.receipt('JOINT-006', processes=facts, scope='actual process telemetry and monitor HTTP detail; no compiler/board model workload')
|
|
|
|
def test_old_boot_identity_is_quarantined_before_real_collector_claims(self):
|
|
victim = subprocess.Popen([sys.executable, '-c', 'import time;time.sleep(15)'])
|
|
def stop():
|
|
if victim.poll() is None: victim.kill()
|
|
victim.wait(timeout=3)
|
|
self.addCleanup(stop)
|
|
owner = execution.Execution(self.rd); owner.start(test_execution.policy())
|
|
before = owner.snapshot()
|
|
wrong = execution.process_identity(victim.pid)
|
|
wrong.update(boot_id='old-boot-fixture', proc_start_ticks=wrong['proc_start_ticks']+1)
|
|
before.update(boot_id='old-boot-fixture', owner=wrong, processes=[dict(wrong, role='stage')])
|
|
execution.atomic_json(self.rd/'execution.json', before)
|
|
lockdir=self.root/'state/locks';lockdir.mkdir()
|
|
lease=lockdir/'board-0.lease.json'
|
|
execution.atomic_json(lease, {'execution_id':before['execution_id'],'execution_dir':str(self.rd),
|
|
'epoch':before['epoch'],'holder':wrong,'pool':'board','slot':0,'operation_id':'unknown-board'})
|
|
lease_bytes=lease.read_bytes()
|
|
with self.connect(self.database) as con:
|
|
db.rollout_upsert(con, 'r', 'next', 0, 'e2e', 'teacher')
|
|
program = '''
|
|
import json
|
|
from harness import collection_runtime,db,paths,pipeline
|
|
paths.config=lambda:{'pools':{'agent':{'workers':1}}}
|
|
collection_runtime.config=lambda:({}, {})
|
|
claims=[]
|
|
def forbidden(*args,**kwargs):
|
|
claims.append(True);raise AssertionError('new task admitted before reconciliation')
|
|
db.claim_rollout=forbidden
|
|
try:pipeline.collect_rollouts('r',log=lambda _:None)
|
|
except ValueError as error:
|
|
assert str(error)=='EXECUTION_RECONCILIATION_REQUIRED: OWNERSHIP_UNKNOWN',str(error)
|
|
else:raise AssertionError('unknown old boot admitted')
|
|
print(json.dumps({'new_claims':len(claims),'result':'QUARANTINED'}))
|
|
'''
|
|
env={k:v for k,v in os.environ.items() if not k.startswith('BENCH_')}
|
|
env.update(BENCH_ROOT=str(self.root), BENCH_STATE_DIR=str(self.root/'state'),
|
|
PYTHONPATH=str(ROOT), PYTHONDONTWRITEBYTECODE='1')
|
|
child=subprocess.run([sys.executable,'-c',program],cwd=ROOT,env=env,capture_output=True,text=True,timeout=8)
|
|
self.assertEqual(child.returncode,0,child.stderr)
|
|
self.assertEqual(json.loads(child.stdout)['new_claims'],0)
|
|
after=owner.snapshot()
|
|
self.assertEqual(after['cleanup_state'],'QUARANTINED')
|
|
self.assertEqual(after['stop_reason'],'OWNERSHIP_UNKNOWN')
|
|
self.assertGreater(after['epoch'],before['epoch'])
|
|
self.assertEqual((after['execution_id'],after['deadline_monotonic']),
|
|
(before['execution_id'],before['deadline_monotonic']))
|
|
with self.assertRaisesRegex(execution.ExecutionStopped,'OWNER_FENCED'):
|
|
with owner.admission(): self.fail('old epoch reopened admission')
|
|
self.assertIsNone(victim.poll())
|
|
self.assertEqual(lease.read_bytes(),lease_bytes)
|
|
with self.readonly(self.database) as con:
|
|
self.assertEqual([(r['job_id'],r['state']) for r in con.execute('SELECT job_id,state FROM rollouts ORDER BY job_id')],
|
|
[('j','RUNNING'),('next','QUEUED')])
|
|
self.receipt('JOINT-010-boot', collector_processes=1,new_claims=0,cleanup='QUARANTINED',
|
|
unrelated_birth_identity_alive=True,board_lease_preserved=True,old_epoch_fenced=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|