data-gen-bpu/scripts/test_lifecycle_joint.py

248 lines
15 KiB
Python

"""Actual lifecycle boundaries with owned processes, queues and read-only UI API."""
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from unittest.mock import patch
import urllib.request
from harness import collection_runtime, db, execution, monitor, paths, rollout, scheduler, watchdog
from scripts import test_execution, test_quota
ROOT = Path(__file__).resolve().parents[1]
@unittest.skipUnless(sys.platform == 'linux', 'Linux process identities and kernel containment')
class LifecycleJoint(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(prefix='bench-lifecycle-joint-')
self.addCleanup(temp.cleanup)
self.root = Path(temp.name)
self.rd = self.root/'runs/r/j/rollouts/00'; self.rd.mkdir(parents=True)
self.database = self.root/'state/jobs.sqlite'
self.connect, self.readonly = db.connect, db.readonly
with self.connect(self.database) as con:
db.create_run(con, 'r', {})
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING')
(self.rd/'rollout.json').write_text('{"state":"RUNNING","mode":"e2e","policy":"teacher"}')
self.config = {'pools': {'agent': {'workers': 1}, 'cpu': {'workers': 1}, 'board': {'workers': 1}}}
self.stack = ExitStack(); self.addCleanup(self.stack.close)
self.stack.enter_context(patch.object(paths, 'STATE', self.root/'state'))
self.stack.enter_context(patch.object(paths, 'RUNS', self.root/'runs'))
self.stack.enter_context(patch.object(paths, 'config', return_value=self.config))
self.stack.enter_context(patch.object(db, 'connect', lambda: self.connect(self.database)))
self.stack.enter_context(patch.object(db, 'readonly', lambda: self.readonly(self.database)))
self.stack.enter_context(patch.object(scheduler, 'board_slots', return_value={0: {'name': 'local-lock-only'}}))
def wait(self, predicate, timeout=5):
deadline = time.monotonic()+timeout
while time.monotonic() < deadline:
result = predicate()
if result:
return result
time.sleep(.005)
self.fail('bounded fixture condition did not arrive')
@staticmethod
def receipt(case, **facts):
print('JOINT_RECEIPT '+json.dumps({'case': case, **facts}, sort_keys=True))
def test_cancel_same_execution_fences_quota_compile_and_board_queues(self):
owner = execution.Execution(self.rd)
policy = test_execution.policy(); policy['queue_deadline_s'] = 5
for phase in policy['phases'].values(): phase['queue_deadline_s'] = 5
owner.start(policy)
quota = test_quota.Quota(); quota.setUp(); self.addCleanup(quota.doCleanups)
entered, release = threading.Event(), threading.Event()
def held_provider(n, body):
entered.set()
if not release.wait(5): raise TimeoutError('fixture provider release missing')
return test_quota.ok(n, body)
provider = quota.provider(held_provider)
broker = quota.broker(provider, quota.policy(max_concurrency=1, wait_budget_s=5,
first_byte_timeout_s=5, request_total_s=6))
self.stack.enter_context(patch.object(collection_runtime, 'broker', return_value=broker))
hold_scope = broker.issue_scope('fixture-slot-holder', 'teacher', 'glm-test')
target = broker.issue_scope(owner.snapshot()['execution_id'], 'teacher', 'glm-test')
attempted = {name: threading.Event() for name in ('cpu', 'board')}
def queued_stage(pool, phase):
operation = 'queued-'+pool
owner.begin_phase(operation, phase)
def observed_wait():
attempted[pool].set()
return None
try:
with owner.scope(phase, operation), scheduler.acquire(pool, poll_s=.01, stop_check=observed_wait):
return watchdog.run([sys.executable, '-c', 'from pathlib import Path;Path("unexpected-spawn").touch()'],
cwd=self.rd, stdout=self.rd/(pool+'.out'), stderr=self.rd/(pool+'.err'), execution=owner,
role='stage', poll_s=.01).state
except execution.ExecutionStopped as error:
return str(error)
with ExitStack() as holds:
holds.enter_context(scheduler.acquire('cpu'))
holds.enter_context(scheduler.acquire('board'))
with ThreadPoolExecutor(max_workers=4) as workers:
active = workers.submit(quota.request, hold_scope, 'held')
pending = []
try:
self.assertTrue(entered.wait(3))
request = workers.submit(quota.request, target, 'queued')
pending = [workers.submit(queued_stage, pool, phase)
for pool, phase in [('cpu', '05-compile'), ('board', '07-board')]]
self.wait(lambda: all(event.is_set() for event in attempted.values()))
self.wait(lambda: any(g['state']=='QUEUED' for g in broker.execution_status(target.execution_id)['generations']))
before = len(provider.requests)
settled = rollout.settle_execution('r', 'j', 0, owner, 'joint-cancel', cancel=True)
self.assertEqual(settled['execution_outcome'], 'CANCELLED')
self.assertEqual([future.result(timeout=3) for future in pending], ['CANCEL_REQUESTED']*2)
self.assertEqual(request.result(timeout=3)[0], 409)
self.assertEqual(len(provider.requests), before)
self.assertEqual(owner.snapshot()['processes'], [])
self.assertFalse((self.rd/'unexpected-spawn').exists())
generation = broker.execution_status(target.execution_id)['generations'][0]
self.assertEqual((generation['state'], generation['attempts']), ('CANCELLED', 0))
finally:
release.set()
self.assertEqual(active.result(timeout=3)[0], 200)
self.assertEqual(scheduler.pool_usage(), {'agent': 0, 'cpu': 0, 'board': 0})
self.assertEqual(list((self.root/'state/locks').glob('*.lease.json')), [])
self.assertTrue(all(a['ended_at'] is not None for a in broker.snapshot()['attempts']))
self.assertEqual(owner.snapshot()['cleanup_state'], 'VERIFIED')
self.receipt('JOINT-005', queued_pools=['quota','cpu','board'], target_http_attempts=0,
target_spawns=0, upstream_requests=1, owned_slots_after=0, cleanup='VERIFIED')
def test_silent_work_spin_and_hang_report_real_progress_and_absolute_deadlines(self):
observer = monitor.Observer(self.root)
server = monitor.make_server(observer, ('127.0.0.1', 0))
thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start()
self.addCleanup(server.server_close); self.addCleanup(server.shutdown)
facts = []
programs = {
'working': 'import os,time;f=open("work.bin","wb");end=time.monotonic()+.35\n'
'while time.monotonic()<end:\n f.write(b"x"*4096);f.flush();os.fsync(f.fileno());sum(i*i for i in range(20000))\n',
'spinning': 'while True: pass',
'hanging': 'import signal;signal.pause()',
}
for kind, program in programs.items():
with self.subTest(kind=kind):
rd = self.root/'runs/r'/kind/'rollouts/00'; rd.mkdir(parents=True)
with self.connect(self.database) as con:
db.rollout_upsert(con, 'r', kind, 0, 'e2e', 'teacher', state='RUNNING')
owner = execution.Execution(rd)
policy = test_execution.policy()
policy['phases']['05-compile'].update(execution_deadline_s=1.5 if kind=='working' else .35,
queue_deadline_s=1, suspect_after_s=.08)
owner.start(policy); owner.begin_phase('compile', '05-compile'); owner.phase_running('compile')
owner.activity('COMPILING', operation_id='compile')
observed = []
def sample():
observer.sample()
observed.append(observer.detail('r', kind, 0))
started = time.monotonic()
with owner.scope('05-compile', 'compile'):
result = watchdog.run([sys.executable, '-c', program], cwd=rd, stdout=rd/'out', stderr=rd/'err',
execution=owner, role='stage', poll_s=.02, stop_check=sample)
elapsed = time.monotonic()-started
state = owner.finish(result.exit_code, protocol_complete=result.state=='OK',
stop_reason=result.stalled_reason, cleanup_state=result.cleanup_state)
self.assertLess(elapsed, 3)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertEqual(execution.owned_processes(state), [])
self.assertEqual((rd/'out').read_bytes(), b'')
if kind=='working':
self.assertEqual(result.state, 'OK')
self.assertGreater((rd/'work.bin').stat().st_size, 0)
else:
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
self.assertIn('DEADLINE', result.stalled_reason)
progress = [row['progress'] for row in observed if (row.get('progress') or {}).get('signals')]
self.assertTrue(progress)
self.assertTrue(all(row['suspect_after_s']==.08 for row in progress))
self.assertTrue(all(row['signals']['business_log_bytes']==0 for row in progress))
if kind in ('working','spinning'):
self.assertGreater(max(row['signals']['cpu_ticks'] or 0 for row in progress),
min(row['signals']['cpu_ticks'] or 0 for row in progress))
else:
self.assertTrue(any(row['reason']=='ALL_SIGNALS_IDLE' for row in progress))
observer.sample()
url=f'http://127.0.0.1:{server.server_port}/api/runs/r/rollouts/{kind}/0'
with urllib.request.urlopen(url, timeout=3) as response: displayed=json.load(response)
phase=displayed['phases']['compile']
self.assertEqual(phase['phase'], '05-compile')
self.assertAlmostEqual(phase['execution_deadline_monotonic']-phase['execution_started_monotonic'],
policy['phases']['05-compile']['execution_deadline_s'])
self.assertEqual(displayed['progress']['suspect_after_s'], .08)
self.assertEqual(displayed['stop_reason'], state['stop_reason'])
self.assertTrue(displayed['progress']['reason'])
facts.append({'kind':kind,'outcome':state['execution_outcome'],'cleanup':result.cleanup_state,
'elapsed_s':round(elapsed,3),'phase':'05-compile','suspect_after_s':.08,
'display_reason':displayed['progress']['reason'],'log_bytes':0})
self.receipt('JOINT-006', processes=facts, scope='actual process telemetry and monitor HTTP detail; no compiler/board model workload')
def test_old_boot_identity_is_quarantined_before_real_collector_claims(self):
victim = subprocess.Popen([sys.executable, '-c', 'import time;time.sleep(15)'])
def stop():
if victim.poll() is None: victim.kill()
victim.wait(timeout=3)
self.addCleanup(stop)
owner = execution.Execution(self.rd); owner.start(test_execution.policy())
before = owner.snapshot()
wrong = execution.process_identity(victim.pid)
wrong.update(boot_id='old-boot-fixture', proc_start_ticks=wrong['proc_start_ticks']+1)
before.update(boot_id='old-boot-fixture', owner=wrong, processes=[dict(wrong, role='stage')])
execution.atomic_json(self.rd/'execution.json', before)
lockdir=self.root/'state/locks';lockdir.mkdir()
lease=lockdir/'board-0.lease.json'
execution.atomic_json(lease, {'execution_id':before['execution_id'],'execution_dir':str(self.rd),
'epoch':before['epoch'],'holder':wrong,'pool':'board','slot':0,'operation_id':'unknown-board'})
lease_bytes=lease.read_bytes()
with self.connect(self.database) as con:
db.rollout_upsert(con, 'r', 'next', 0, 'e2e', 'teacher')
program = '''
import json
from harness import collection_runtime,db,paths,pipeline
paths.config=lambda:{'pools':{'agent':{'workers':1}}}
collection_runtime.config=lambda:({}, {})
claims=[]
def forbidden(*args,**kwargs):
claims.append(True);raise AssertionError('new task admitted before reconciliation')
db.claim_rollout=forbidden
try:pipeline.collect_rollouts('r',log=lambda _:None)
except ValueError as error:
assert str(error)=='EXECUTION_RECONCILIATION_REQUIRED',str(error)
else:raise AssertionError('unknown old boot admitted')
print(json.dumps({'new_claims':len(claims),'result':'QUARANTINED'}))
'''
env={k:v for k,v in os.environ.items() if not k.startswith('BENCH_')}
env.update(BENCH_ROOT=str(self.root), BENCH_STATE_DIR=str(self.root/'state'),
PYTHONPATH=str(ROOT), PYTHONDONTWRITEBYTECODE='1')
child=subprocess.run([sys.executable,'-c',program],cwd=ROOT,env=env,capture_output=True,text=True,timeout=8)
self.assertEqual(child.returncode,0,child.stderr)
self.assertEqual(json.loads(child.stdout)['new_claims'],0)
after=owner.snapshot()
self.assertEqual(after['cleanup_state'],'QUARANTINED')
self.assertEqual(after['stop_reason'],'OWNERSHIP_UNKNOWN')
self.assertGreater(after['epoch'],before['epoch'])
self.assertEqual((after['execution_id'],after['deadline_monotonic']),
(before['execution_id'],before['deadline_monotonic']))
with self.assertRaisesRegex(execution.ExecutionStopped,'OWNER_FENCED'):
with owner.admission(): self.fail('old epoch reopened admission')
self.assertIsNone(victim.poll())
self.assertEqual(lease.read_bytes(),lease_bytes)
with self.readonly(self.database) as con:
self.assertEqual([(r['job_id'],r['state']) for r in con.execute('SELECT job_id,state FROM rollouts ORDER BY job_id')],
[('j','RUNNING'),('next','QUEUED')])
self.receipt('JOINT-010-boot', collector_processes=1,new_claims=0,cleanup='QUARANTINED',
unrelated_birth_identity_alive=True,board_lease_preserved=True,old_epoch_fenced=True)
if __name__ == '__main__':
unittest.main()