"""Actual lifecycle boundaries with owned processes, queues and read-only UI API.""" from concurrent.futures import ThreadPoolExecutor from contextlib import ExitStack import json import os from pathlib import Path import subprocess import sys import tempfile import threading import time import unittest from unittest.mock import patch import urllib.request from harness import collection_runtime, db, execution, monitor, paths, rollout, scheduler, watchdog from scripts import test_execution, test_quota ROOT = Path(__file__).resolve().parents[1] @unittest.skipUnless(sys.platform == 'linux', 'Linux process identities and kernel containment') class LifecycleJoint(unittest.TestCase): def setUp(self): temp = tempfile.TemporaryDirectory(prefix='bench-lifecycle-joint-') self.addCleanup(temp.cleanup) self.root = Path(temp.name) self.rd = self.root/'runs/r/j/rollouts/00'; self.rd.mkdir(parents=True) self.database = self.root/'state/jobs.sqlite' self.connect, self.readonly = db.connect, db.readonly with self.connect(self.database) as con: db.create_run(con, 'r', {}) db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING') (self.rd/'rollout.json').write_text('{"state":"RUNNING","mode":"e2e","policy":"teacher"}') self.config = {'pools': {'agent': {'workers': 1}, 'cpu': {'workers': 1}, 'board': {'workers': 1}}} self.stack = ExitStack(); self.addCleanup(self.stack.close) self.stack.enter_context(patch.object(paths, 'STATE', self.root/'state')) self.stack.enter_context(patch.object(paths, 'RUNS', self.root/'runs')) self.stack.enter_context(patch.object(paths, 'config', return_value=self.config)) self.stack.enter_context(patch.object(db, 'connect', lambda: self.connect(self.database))) self.stack.enter_context(patch.object(db, 'readonly', lambda: self.readonly(self.database))) self.stack.enter_context(patch.object(scheduler, 'board_slots', return_value={0: {'name': 'local-lock-only'}})) def wait(self, predicate, timeout=5): deadline = time.monotonic()+timeout while time.monotonic() < deadline: result = predicate() if result: return result time.sleep(.005) self.fail('bounded fixture condition did not arrive') @staticmethod def receipt(case, **facts): print('JOINT_RECEIPT '+json.dumps({'case': case, **facts}, sort_keys=True)) def test_cancel_same_execution_fences_quota_compile_and_board_queues(self): owner = execution.Execution(self.rd) policy = test_execution.policy(); policy['queue_deadline_s'] = 5 for phase in policy['phases'].values(): phase['queue_deadline_s'] = 5 owner.start(policy) quota = test_quota.Quota(); quota.setUp(); self.addCleanup(quota.doCleanups) entered, release = threading.Event(), threading.Event() def held_provider(n, body): entered.set() if not release.wait(5): raise TimeoutError('fixture provider release missing') return test_quota.ok(n, body) provider = quota.provider(held_provider) broker = quota.broker(provider, quota.policy(max_concurrency=1, wait_budget_s=5, first_byte_timeout_s=5, request_total_s=6)) self.stack.enter_context(patch.object(collection_runtime, 'broker', return_value=broker)) hold_scope = broker.issue_scope('fixture-slot-holder', 'teacher', 'glm-test') target = broker.issue_scope(owner.snapshot()['execution_id'], 'teacher', 'glm-test') attempted = {name: threading.Event() for name in ('cpu', 'board')} def queued_stage(pool, phase): operation = 'queued-'+pool owner.begin_phase(operation, phase) def observed_wait(): attempted[pool].set() return None try: with owner.scope(phase, operation), scheduler.acquire(pool, poll_s=.01, stop_check=observed_wait): return watchdog.run([sys.executable, '-c', 'from pathlib import Path;Path("unexpected-spawn").touch()'], cwd=self.rd, stdout=self.rd/(pool+'.out'), stderr=self.rd/(pool+'.err'), execution=owner, role='stage', poll_s=.01).state except execution.ExecutionStopped as error: return str(error) with ExitStack() as holds: holds.enter_context(scheduler.acquire('cpu')) holds.enter_context(scheduler.acquire('board')) with ThreadPoolExecutor(max_workers=4) as workers: active = workers.submit(quota.request, hold_scope, 'held') pending = [] try: self.assertTrue(entered.wait(3)) request = workers.submit(quota.request, target, 'queued') pending = [workers.submit(queued_stage, pool, phase) for pool, phase in [('cpu', '05-compile'), ('board', '07-board')]] self.wait(lambda: all(event.is_set() for event in attempted.values())) self.wait(lambda: any(g['state']=='QUEUED' for g in broker.execution_status(target.execution_id)['generations'])) before = len(provider.requests) settled = rollout.settle_execution('r', 'j', 0, owner, 'joint-cancel', cancel=True) self.assertEqual(settled['execution_outcome'], 'CANCELLED') self.assertEqual([future.result(timeout=3) for future in pending], ['CANCEL_REQUESTED']*2) self.assertEqual(request.result(timeout=3)[0], 409) self.assertEqual(len(provider.requests), before) self.assertEqual(owner.snapshot()['processes'], []) self.assertFalse((self.rd/'unexpected-spawn').exists()) generation = broker.execution_status(target.execution_id)['generations'][0] self.assertEqual((generation['state'], generation['attempts']), ('CANCELLED', 0)) finally: release.set() self.assertEqual(active.result(timeout=3)[0], 200) self.assertEqual(scheduler.pool_usage(), {'agent': 0, 'cpu': 0, 'board': 0}) self.assertEqual(list((self.root/'state/locks').glob('*.lease.json')), []) self.assertTrue(all(a['ended_at'] is not None for a in broker.snapshot()['attempts'])) self.assertEqual(owner.snapshot()['cleanup_state'], 'VERIFIED') self.receipt('JOINT-005', queued_pools=['quota','cpu','board'], target_http_attempts=0, target_spawns=0, upstream_requests=1, owned_slots_after=0, cleanup='VERIFIED') def test_silent_work_spin_and_hang_report_real_progress_and_absolute_deadlines(self): observer = monitor.Observer(self.root) server = monitor.make_server(observer, ('127.0.0.1', 0)) thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() self.addCleanup(server.server_close); self.addCleanup(server.shutdown) facts = [] programs = { 'working': 'import os,time;f=open("work.bin","wb");end=time.monotonic()+.35\n' 'while time.monotonic()