248 lines
15 KiB
Python
248 lines
15 KiB
Python
"""Actual lifecycle boundaries with owned processes, queues and read-only UI API."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import ExitStack
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import urllib.request
|
|
|
|
from harness import collection_runtime, db, execution, monitor, paths, rollout, scheduler, watchdog
|
|
from scripts import test_execution, test_quota
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux process identities and kernel containment')
|
|
class LifecycleJoint(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory(prefix='bench-lifecycle-joint-')
|
|
self.addCleanup(temp.cleanup)
|
|
self.root = Path(temp.name)
|
|
self.rd = self.root/'runs/r/j/rollouts/00'; self.rd.mkdir(parents=True)
|
|
self.database = self.root/'state/jobs.sqlite'
|
|
self.connect, self.readonly = db.connect, db.readonly
|
|
with self.connect(self.database) as con:
|
|
db.create_run(con, 'r', {})
|
|
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING')
|
|
(self.rd/'rollout.json').write_text('{"state":"RUNNING","mode":"e2e","policy":"teacher"}')
|
|
self.config = {'pools': {'agent': {'workers': 1}, 'cpu': {'workers': 1}, 'board': {'workers': 1}}}
|
|
self.stack = ExitStack(); self.addCleanup(self.stack.close)
|
|
self.stack.enter_context(patch.object(paths, 'STATE', self.root/'state'))
|
|
self.stack.enter_context(patch.object(paths, 'RUNS', self.root/'runs'))
|
|
self.stack.enter_context(patch.object(paths, 'config', return_value=self.config))
|
|
self.stack.enter_context(patch.object(db, 'connect', lambda: self.connect(self.database)))
|
|
self.stack.enter_context(patch.object(db, 'readonly', lambda: self.readonly(self.database)))
|
|
self.stack.enter_context(patch.object(scheduler, 'board_slots', return_value={0: {'name': 'local-lock-only'}}))
|
|
|
|
def wait(self, predicate, timeout=5):
|
|
deadline = time.monotonic()+timeout
|
|
while time.monotonic() < deadline:
|
|
result = predicate()
|
|
if result:
|
|
return result
|
|
time.sleep(.005)
|
|
self.fail('bounded fixture condition did not arrive')
|
|
|
|
@staticmethod
|
|
def receipt(case, **facts):
|
|
print('JOINT_RECEIPT '+json.dumps({'case': case, **facts}, sort_keys=True))
|
|
|
|
def test_cancel_same_execution_fences_quota_compile_and_board_queues(self):
|
|
owner = execution.Execution(self.rd)
|
|
policy = test_execution.policy(); policy['queue_deadline_s'] = 5
|
|
for phase in policy['phases'].values(): phase['queue_deadline_s'] = 5
|
|
owner.start(policy)
|
|
quota = test_quota.Quota(); quota.setUp(); self.addCleanup(quota.doCleanups)
|
|
entered, release = threading.Event(), threading.Event()
|
|
def held_provider(n, body):
|
|
entered.set()
|
|
if not release.wait(5): raise TimeoutError('fixture provider release missing')
|
|
return test_quota.ok(n, body)
|
|
provider = quota.provider(held_provider)
|
|
broker = quota.broker(provider, quota.policy(max_concurrency=1, wait_budget_s=5,
|
|
first_byte_timeout_s=5, request_total_s=6))
|
|
self.stack.enter_context(patch.object(collection_runtime, 'broker', return_value=broker))
|
|
hold_scope = broker.issue_scope('fixture-slot-holder', 'teacher', 'glm-test')
|
|
target = broker.issue_scope(owner.snapshot()['execution_id'], 'teacher', 'glm-test')
|
|
attempted = {name: threading.Event() for name in ('cpu', 'board')}
|
|
def queued_stage(pool, phase):
|
|
operation = 'queued-'+pool
|
|
owner.begin_phase(operation, phase)
|
|
def observed_wait():
|
|
attempted[pool].set()
|
|
return None
|
|
try:
|
|
with owner.scope(phase, operation), scheduler.acquire(pool, poll_s=.01, stop_check=observed_wait):
|
|
return watchdog.run([sys.executable, '-c', 'from pathlib import Path;Path("unexpected-spawn").touch()'],
|
|
cwd=self.rd, stdout=self.rd/(pool+'.out'), stderr=self.rd/(pool+'.err'), execution=owner,
|
|
role='stage', poll_s=.01).state
|
|
except execution.ExecutionStopped as error:
|
|
return str(error)
|
|
with ExitStack() as holds:
|
|
holds.enter_context(scheduler.acquire('cpu'))
|
|
holds.enter_context(scheduler.acquire('board'))
|
|
with ThreadPoolExecutor(max_workers=4) as workers:
|
|
active = workers.submit(quota.request, hold_scope, 'held')
|
|
pending = []
|
|
try:
|
|
self.assertTrue(entered.wait(3))
|
|
request = workers.submit(quota.request, target, 'queued')
|
|
pending = [workers.submit(queued_stage, pool, phase)
|
|
for pool, phase in [('cpu', '05-compile'), ('board', '07-board')]]
|
|
self.wait(lambda: all(event.is_set() for event in attempted.values()))
|
|
self.wait(lambda: any(g['state']=='QUEUED' for g in broker.execution_status(target.execution_id)['generations']))
|
|
before = len(provider.requests)
|
|
settled = rollout.settle_execution('r', 'j', 0, owner, 'joint-cancel', cancel=True)
|
|
self.assertEqual(settled['execution_outcome'], 'CANCELLED')
|
|
self.assertEqual([future.result(timeout=3) for future in pending], ['CANCEL_REQUESTED']*2)
|
|
self.assertEqual(request.result(timeout=3)[0], 409)
|
|
self.assertEqual(len(provider.requests), before)
|
|
self.assertEqual(owner.snapshot()['processes'], [])
|
|
self.assertFalse((self.rd/'unexpected-spawn').exists())
|
|
generation = broker.execution_status(target.execution_id)['generations'][0]
|
|
self.assertEqual((generation['state'], generation['attempts']), ('CANCELLED', 0))
|
|
finally:
|
|
release.set()
|
|
self.assertEqual(active.result(timeout=3)[0], 200)
|
|
self.assertEqual(scheduler.pool_usage(), {'agent': 0, 'cpu': 0, 'board': 0})
|
|
self.assertEqual(list((self.root/'state/locks').glob('*.lease.json')), [])
|
|
self.assertTrue(all(a['ended_at'] is not None for a in broker.snapshot()['attempts']))
|
|
self.assertEqual(owner.snapshot()['cleanup_state'], 'VERIFIED')
|
|
self.receipt('JOINT-005', queued_pools=['quota','cpu','board'], target_http_attempts=0,
|
|
target_spawns=0, upstream_requests=1, owned_slots_after=0, cleanup='VERIFIED')
|
|
|
|
def test_silent_work_spin_and_hang_report_real_progress_and_absolute_deadlines(self):
|
|
observer = monitor.Observer(self.root)
|
|
server = monitor.make_server(observer, ('127.0.0.1', 0))
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start()
|
|
self.addCleanup(server.server_close); self.addCleanup(server.shutdown)
|
|
facts = []
|
|
programs = {
|
|
'working': 'import os,time;f=open("work.bin","wb");end=time.monotonic()+.35\n'
|
|
'while time.monotonic()<end:\n f.write(b"x"*4096);f.flush();os.fsync(f.fileno());sum(i*i for i in range(20000))\n',
|
|
'spinning': 'while True: pass',
|
|
'hanging': 'import signal;signal.pause()',
|
|
}
|
|
for kind, program in programs.items():
|
|
with self.subTest(kind=kind):
|
|
rd = self.root/'runs/r'/kind/'rollouts/00'; rd.mkdir(parents=True)
|
|
with self.connect(self.database) as con:
|
|
db.rollout_upsert(con, 'r', kind, 0, 'e2e', 'teacher', state='RUNNING')
|
|
owner = execution.Execution(rd)
|
|
policy = test_execution.policy()
|
|
policy['phases']['05-compile'].update(execution_deadline_s=1.5 if kind=='working' else .35,
|
|
queue_deadline_s=1, suspect_after_s=.08)
|
|
owner.start(policy); owner.begin_phase('compile', '05-compile'); owner.phase_running('compile')
|
|
owner.activity('COMPILING', operation_id='compile')
|
|
observed = []
|
|
def sample():
|
|
observer.sample()
|
|
observed.append(observer.detail('r', kind, 0))
|
|
started = time.monotonic()
|
|
with owner.scope('05-compile', 'compile'):
|
|
result = watchdog.run([sys.executable, '-c', program], cwd=rd, stdout=rd/'out', stderr=rd/'err',
|
|
execution=owner, role='stage', poll_s=.02, stop_check=sample)
|
|
elapsed = time.monotonic()-started
|
|
state = owner.finish(result.exit_code, protocol_complete=result.state=='OK',
|
|
stop_reason=result.stalled_reason, cleanup_state=result.cleanup_state)
|
|
self.assertLess(elapsed, 3)
|
|
self.assertEqual(result.cleanup_state, 'VERIFIED')
|
|
self.assertEqual(execution.owned_processes(state), [])
|
|
self.assertEqual((rd/'out').read_bytes(), b'')
|
|
if kind=='working':
|
|
self.assertEqual(result.state, 'OK')
|
|
self.assertGreater((rd/'work.bin').stat().st_size, 0)
|
|
else:
|
|
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
|
|
self.assertIn('DEADLINE', result.stalled_reason)
|
|
progress = [row['progress'] for row in observed if (row.get('progress') or {}).get('signals')]
|
|
self.assertTrue(progress)
|
|
self.assertTrue(all(row['suspect_after_s']==.08 for row in progress))
|
|
self.assertTrue(all(row['signals']['business_log_bytes']==0 for row in progress))
|
|
if kind in ('working','spinning'):
|
|
self.assertGreater(max(row['signals']['cpu_ticks'] or 0 for row in progress),
|
|
min(row['signals']['cpu_ticks'] or 0 for row in progress))
|
|
else:
|
|
self.assertTrue(any(row['reason']=='ALL_SIGNALS_IDLE' for row in progress))
|
|
observer.sample()
|
|
url=f'http://127.0.0.1:{server.server_port}/api/runs/r/rollouts/{kind}/0'
|
|
with urllib.request.urlopen(url, timeout=3) as response: displayed=json.load(response)
|
|
phase=displayed['phases']['compile']
|
|
self.assertEqual(phase['phase'], '05-compile')
|
|
self.assertAlmostEqual(phase['execution_deadline_monotonic']-phase['execution_started_monotonic'],
|
|
policy['phases']['05-compile']['execution_deadline_s'])
|
|
self.assertEqual(displayed['progress']['suspect_after_s'], .08)
|
|
self.assertEqual(displayed['stop_reason'], state['stop_reason'])
|
|
self.assertTrue(displayed['progress']['reason'])
|
|
facts.append({'kind':kind,'outcome':state['execution_outcome'],'cleanup':result.cleanup_state,
|
|
'elapsed_s':round(elapsed,3),'phase':'05-compile','suspect_after_s':.08,
|
|
'display_reason':displayed['progress']['reason'],'log_bytes':0})
|
|
self.receipt('JOINT-006', processes=facts, scope='actual process telemetry and monitor HTTP detail; no compiler/board model workload')
|
|
|
|
def test_old_boot_identity_is_quarantined_before_real_collector_claims(self):
|
|
victim = subprocess.Popen([sys.executable, '-c', 'import time;time.sleep(15)'])
|
|
def stop():
|
|
if victim.poll() is None: victim.kill()
|
|
victim.wait(timeout=3)
|
|
self.addCleanup(stop)
|
|
owner = execution.Execution(self.rd); owner.start(test_execution.policy())
|
|
before = owner.snapshot()
|
|
wrong = execution.process_identity(victim.pid)
|
|
wrong.update(boot_id='old-boot-fixture', proc_start_ticks=wrong['proc_start_ticks']+1)
|
|
before.update(boot_id='old-boot-fixture', owner=wrong, processes=[dict(wrong, role='stage')])
|
|
execution.atomic_json(self.rd/'execution.json', before)
|
|
lockdir=self.root/'state/locks';lockdir.mkdir()
|
|
lease=lockdir/'board-0.lease.json'
|
|
execution.atomic_json(lease, {'execution_id':before['execution_id'],'execution_dir':str(self.rd),
|
|
'epoch':before['epoch'],'holder':wrong,'pool':'board','slot':0,'operation_id':'unknown-board'})
|
|
lease_bytes=lease.read_bytes()
|
|
with self.connect(self.database) as con:
|
|
db.rollout_upsert(con, 'r', 'next', 0, 'e2e', 'teacher')
|
|
program = '''
|
|
import json
|
|
from harness import collection_runtime,db,paths,pipeline
|
|
paths.config=lambda:{'pools':{'agent':{'workers':1}}}
|
|
collection_runtime.config=lambda:({}, {})
|
|
claims=[]
|
|
def forbidden(*args,**kwargs):
|
|
claims.append(True);raise AssertionError('new task admitted before reconciliation')
|
|
db.claim_rollout=forbidden
|
|
try:pipeline.collect_rollouts('r',log=lambda _:None)
|
|
except ValueError as error:
|
|
assert str(error)=='EXECUTION_RECONCILIATION_REQUIRED',str(error)
|
|
else:raise AssertionError('unknown old boot admitted')
|
|
print(json.dumps({'new_claims':len(claims),'result':'QUARANTINED'}))
|
|
'''
|
|
env={k:v for k,v in os.environ.items() if not k.startswith('BENCH_')}
|
|
env.update(BENCH_ROOT=str(self.root), BENCH_STATE_DIR=str(self.root/'state'),
|
|
PYTHONPATH=str(ROOT), PYTHONDONTWRITEBYTECODE='1')
|
|
child=subprocess.run([sys.executable,'-c',program],cwd=ROOT,env=env,capture_output=True,text=True,timeout=8)
|
|
self.assertEqual(child.returncode,0,child.stderr)
|
|
self.assertEqual(json.loads(child.stdout)['new_claims'],0)
|
|
after=owner.snapshot()
|
|
self.assertEqual(after['cleanup_state'],'QUARANTINED')
|
|
self.assertEqual(after['stop_reason'],'OWNERSHIP_UNKNOWN')
|
|
self.assertGreater(after['epoch'],before['epoch'])
|
|
self.assertEqual((after['execution_id'],after['deadline_monotonic']),
|
|
(before['execution_id'],before['deadline_monotonic']))
|
|
with self.assertRaisesRegex(execution.ExecutionStopped,'OWNER_FENCED'):
|
|
with owner.admission(): self.fail('old epoch reopened admission')
|
|
self.assertIsNone(victim.poll())
|
|
self.assertEqual(lease.read_bytes(),lease_bytes)
|
|
with self.readonly(self.database) as con:
|
|
self.assertEqual([(r['job_id'],r['state']) for r in con.execute('SELECT job_id,state FROM rollouts ORDER BY job_id')],
|
|
[('j','RUNNING'),('next','QUEUED')])
|
|
self.receipt('JOINT-010-boot', collector_processes=1,new_claims=0,cleanup='QUARANTINED',
|
|
unrelated_birth_identity_alive=True,board_lease_preserved=True,old_epoch_fenced=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|