data-gen-bpu/scripts/test_lifecycle_joint.py
bench 0da66edde0 Resume rollouts after the collector dies (crash, kill, reboot, planned handover)
Processes are disposable; rollout state is resumable. When the collector dies,
the next one reconciles each running rollout from durable state instead of
quarantining or losing it:

- execution: a process of an earlier boot on this host is DEAD; a top-level
  reaper whose owner (the collector) died still proves its subtree empty
  (scopes record the owner that started them); reconcile() pauses a rollout
  with a proven-clean crash checkpoint as CONTROLLER_RESTARTED and marks its
  running tool calls INTERRUPTED; resume() across a reboot rebases monotonic
  clocks by wall time and credits the downtime as a pause
- stageworker.settle_interrupted: an admitted step whose executor is gone ends
  as STAGE_EXECUTOR_EXIT (free compile) and its handle is told directly; a
  board step without a clean board receipt still quarantines; nothing replays
- collection_runtime.crash_checkpoint: capture the real system prompt, freeze
  the session at the last saved message (mid-work phases only); configure_pi
  skips a generation ID the ledger already completed
- pi_session.js: checkpoint after every saved message (and compaction); on a
  recovered resume drop an unfinished trailing reply and answer open tool
  calls with an honest "interrupted by a platform restart" result
- scheduler: a paused rollout's local slot held by a dead holder is released
  once every process is proven gone
- pipeline: SIGUSR1 = planned handover (stop claiming, let board steps
  finish, shorten in-flight request holds, exit); SIGTERM still drains
- export: bench_recovery closes the cut segment; the dropped unfinished reply
  is excluded (UNFINISHED_BEFORE_RESTART)
- tests: test_controller_restart (real Pi on Linux: kill -9 mid-command,
  kill -9 mid-reply, SIGUSR1 handover, power loss + reboot), test_settle_
  interrupted; three tests of the old quarantine-everything rule now assert
  the new outcome while keeping their guarantees (no replay, unrelated PID
  untouched, board lease kept)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 09:29:26 +08:00

235 lines
14 KiB
Python

"""Actual lifecycle boundaries with owned processes, queues and read-only UI API."""
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from unittest.mock import patch
import urllib.request
from harness import collection_runtime, db, execution, monitor, paths, rollout, scheduler, watchdog
from scripts import test_execution, test_quota
ROOT = Path(__file__).resolve().parents[1]
@unittest.skipUnless(sys.platform == 'linux', 'Linux process identities and kernel containment')
class LifecycleJoint(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(prefix='bench-lifecycle-joint-')
self.addCleanup(temp.cleanup)
self.root = Path(temp.name)
self.rd = self.root/'runs/r/j/rollouts/00'; self.rd.mkdir(parents=True)
self.database = self.root/'state/jobs.sqlite'
self.connect, self.readonly = db.connect, db.readonly
with self.connect(self.database) as con:
db.create_run(con, 'r', {})
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING')
(self.rd/'rollout.json').write_text('{"state":"RUNNING","mode":"e2e","policy":"teacher"}')
self.config = {'pools': {'agent': {'workers': 1}, 'cpu': {'workers': 1}, 'board': {'workers': 1}}}
self.stack = ExitStack(); self.addCleanup(self.stack.close)
self.stack.enter_context(patch.object(paths, 'STATE', self.root/'state'))
self.stack.enter_context(patch.object(paths, 'RUNS', self.root/'runs'))
self.stack.enter_context(patch.object(paths, 'config', return_value=self.config))
self.stack.enter_context(patch.object(db, 'connect', lambda: self.connect(self.database)))
self.stack.enter_context(patch.object(db, 'readonly', lambda: self.readonly(self.database)))
self.stack.enter_context(patch.object(scheduler, 'board_slots', return_value={0: {'name': 'local-lock-only'}}))
def wait(self, predicate, timeout=5):
deadline = time.monotonic()+timeout
while time.monotonic() < deadline:
result = predicate()
if result:
return result
time.sleep(.005)
self.fail('bounded fixture condition did not arrive')
@staticmethod
def receipt(case, **facts):
print('JOINT_RECEIPT '+json.dumps({'case': case, **facts}, sort_keys=True))
def test_cancel_same_execution_fences_quota_compile_and_board_queues(self):
owner = execution.Execution(self.rd)
policy = test_execution.policy(); policy['queue_deadline_s'] = 5
for phase in policy['phases'].values(): phase['queue_deadline_s'] = 5
owner.start(policy)
quota = test_quota.Quota(); quota.setUp(); self.addCleanup(quota.doCleanups)
entered, release = threading.Event(), threading.Event()
def held_provider(n, body):
entered.set()
if not release.wait(5): raise TimeoutError('fixture provider release missing')
return test_quota.ok(n, body)
provider = quota.provider(held_provider)
broker = quota.broker(provider, quota.policy(max_concurrency=1, wait_budget_s=5,
first_byte_timeout_s=5, request_total_s=6))
self.stack.enter_context(patch.object(collection_runtime, 'broker', return_value=broker))
hold_scope = broker.issue_scope('fixture-slot-holder', 'teacher', 'glm-test')
target = broker.issue_scope(owner.snapshot()['execution_id'], 'teacher', 'glm-test')
attempted = {name: threading.Event() for name in ('cpu', 'board')}
def queued_stage(pool, phase):
operation = 'queued-'+pool
owner.begin_phase(operation, phase)
def observed_wait():
attempted[pool].set()
return None
try:
with owner.scope(phase, operation), scheduler.acquire(pool, poll_s=.01, stop_check=observed_wait):
return watchdog.run([sys.executable, '-c', 'from pathlib import Path;Path("unexpected-spawn").touch()'],
cwd=self.rd, stdout=self.rd/(pool+'.out'), stderr=self.rd/(pool+'.err'), execution=owner,
role='stage', poll_s=.01).state
except execution.ExecutionStopped as error:
return str(error)
with ExitStack() as holds:
holds.enter_context(scheduler.acquire('cpu'))
holds.enter_context(scheduler.acquire('board'))
with ThreadPoolExecutor(max_workers=4) as workers:
active = workers.submit(quota.request, hold_scope, 'held')
pending = []
try:
self.assertTrue(entered.wait(3))
request = workers.submit(quota.request, target, 'queued')
pending = [workers.submit(queued_stage, pool, phase)
for pool, phase in [('cpu', '05-compile'), ('board', '07-board')]]
self.wait(lambda: all(event.is_set() for event in attempted.values()))
self.wait(lambda: any(g['state']=='QUEUED' for g in broker.execution_status(target.execution_id)['generations']))
before = len(provider.requests)
settled = rollout.settle_execution('r', 'j', 0, owner, 'joint-cancel', cancel=True)
self.assertEqual(settled['execution_outcome'], 'CANCELLED')
self.assertEqual([future.result(timeout=3) for future in pending], ['CANCEL_REQUESTED']*2)
self.assertEqual(request.result(timeout=3)[0], 409)
self.assertEqual(len(provider.requests), before)
self.assertEqual(owner.snapshot()['processes'], [])
self.assertFalse((self.rd/'unexpected-spawn').exists())
generation = broker.execution_status(target.execution_id)['generations'][0]
self.assertEqual((generation['state'], generation['attempts']), ('CANCELLED', 0))
finally:
release.set()
self.assertEqual(active.result(timeout=3)[0], 200)
self.assertEqual(scheduler.pool_usage(), {'agent': 0, 'cpu': 0, 'board': 0})
self.assertEqual(list((self.root/'state/locks').glob('*.lease.json')), [])
self.assertTrue(all(a['ended_at'] is not None for a in broker.snapshot()['attempts']))
self.assertEqual(owner.snapshot()['cleanup_state'], 'VERIFIED')
self.receipt('JOINT-005', queued_pools=['quota','cpu','board'], target_http_attempts=0,
target_spawns=0, upstream_requests=1, owned_slots_after=0, cleanup='VERIFIED')
def test_silent_work_spin_and_hang_report_real_progress_and_absolute_deadlines(self):
observer = monitor.Observer(self.root)
server = monitor.make_server(observer, ('127.0.0.1', 0))
thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start()
self.addCleanup(server.server_close); self.addCleanup(server.shutdown)
facts = []
programs = {
'working': 'import os,time;f=open("work.bin","wb");end=time.monotonic()+.35\n'
'while time.monotonic()<end:\n f.write(b"x"*4096);f.flush();os.fsync(f.fileno());sum(i*i for i in range(20000))\n',
'spinning': 'while True: pass',
'hanging': 'import signal;signal.pause()',
}
for kind, program in programs.items():
with self.subTest(kind=kind):
rd = self.root/'runs/r'/kind/'rollouts/00'; rd.mkdir(parents=True)
with self.connect(self.database) as con:
db.rollout_upsert(con, 'r', kind, 0, 'e2e', 'teacher', state='RUNNING')
owner = execution.Execution(rd)
policy = test_execution.policy()
policy['phases']['05-compile'].update(execution_deadline_s=1.5 if kind=='working' else .35,
queue_deadline_s=1, suspect_after_s=.08)
owner.start(policy); owner.begin_phase('compile', '05-compile'); owner.phase_running('compile')
owner.activity('COMPILING', operation_id='compile')
observed = []
def sample():
observer.sample()
observed.append(observer.detail('r', kind, 0))
started = time.monotonic()
with owner.scope('05-compile', 'compile'):
result = watchdog.run([sys.executable, '-c', program], cwd=rd, stdout=rd/'out', stderr=rd/'err',
execution=owner, role='stage', poll_s=.02, stop_check=sample)
elapsed = time.monotonic()-started
state = owner.finish(result.exit_code, protocol_complete=result.state=='OK',
stop_reason=result.stalled_reason, cleanup_state=result.cleanup_state)
self.assertLess(elapsed, 3)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertEqual(execution.owned_processes(state), [])
self.assertEqual((rd/'out').read_bytes(), b'')
if kind=='working':
self.assertEqual(result.state, 'OK')
self.assertGreater((rd/'work.bin').stat().st_size, 0)
else:
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
self.assertIn('DEADLINE', result.stalled_reason)
progress = [row['progress'] for row in observed if (row.get('progress') or {}).get('signals')]
self.assertTrue(progress)
self.assertTrue(all(row['suspect_after_s']==.08 for row in progress))
self.assertTrue(all(row['signals']['business_log_bytes']==0 for row in progress))
if kind in ('working','spinning'):
self.assertGreater(max(row['signals']['cpu_ticks'] or 0 for row in progress),
min(row['signals']['cpu_ticks'] or 0 for row in progress))
else:
self.assertTrue(any(row['reason']=='ALL_SIGNALS_IDLE' for row in progress))
observer.sample()
url=f'http://127.0.0.1:{server.server_port}/api/runs/r/rollouts/{kind}/0'
with urllib.request.urlopen(url, timeout=3) as response: displayed=json.load(response)
phase=displayed['phases']['compile']
self.assertEqual(phase['phase'], '05-compile')
self.assertAlmostEqual(phase['execution_deadline_monotonic']-phase['execution_started_monotonic'],
policy['phases']['05-compile']['execution_deadline_s'])
self.assertEqual(displayed['progress']['suspect_after_s'], .08)
self.assertEqual(displayed['stop_reason'], state['stop_reason'])
self.assertTrue(displayed['progress']['reason'])
facts.append({'kind':kind,'outcome':state['execution_outcome'],'cleanup':result.cleanup_state,
'elapsed_s':round(elapsed,3),'phase':'05-compile','suspect_after_s':.08,
'display_reason':displayed['progress']['reason'],'log_bytes':0})
self.receipt('JOINT-006', processes=facts, scope='actual process telemetry and monitor HTTP detail; no compiler/board model workload')
def test_old_boot_identity_is_dead_and_the_process_now_on_its_pid_is_untouched(self):
victim = subprocess.Popen([sys.executable, '-c', 'import time;time.sleep(15)'])
def stop():
if victim.poll() is None: victim.kill()
victim.wait(timeout=3)
self.addCleanup(stop)
owner = execution.Execution(self.rd); owner.start(test_execution.policy())
before = owner.snapshot()
wrong = execution.process_identity(victim.pid)
wrong.update(boot_id='old-boot-fixture', proc_start_ticks=wrong['proc_start_ticks']+1)
before.update(boot_id='old-boot-fixture', owner=wrong, processes=[dict(wrong, role='stage')])
execution.atomic_json(self.rd/'execution.json', before)
lockdir=self.root/'state/locks';lockdir.mkdir()
lease=lockdir/'board-0.lease.json'
execution.atomic_json(lease, {'execution_id':before['execution_id'],'execution_dir':str(self.rd),
'epoch':before['epoch'],'holder':wrong,'pool':'board','slot':0,'operation_id':'unknown-board'})
lease_bytes=lease.read_bytes()
with self.connect(self.database) as con:
db.rollout_upsert(con, 'r', 'next', 0, 'e2e', 'teacher')
program = '''
import json
from harness import recovery
report = recovery.reconcile_run('r')
print(json.dumps({'ready': [e['ready'] for e in report['executions']]}))
'''
env={k:v for k,v in os.environ.items() if not k.startswith('BENCH_')}
env.update(BENCH_ROOT=str(self.root), BENCH_STATE_DIR=str(self.root/'state'),
PYTHONPATH=str(ROOT), PYTHONDONTWRITEBYTECODE='1')
child=subprocess.run([sys.executable,'-c',program],cwd=ROOT,env=env,capture_output=True,text=True,timeout=8)
self.assertEqual(child.returncode,0,child.stderr)
after=owner.snapshot()
# This host rebooted since that identity: it ended with the old boot (lost, nothing left to clean). The live
# process that reuses the PID is someone else's and is never signalled; the board lease waits for its receipt.
self.assertEqual((after['execution_state'],after['execution_outcome'],after['cleanup_state']),('TERMINAL','LOST','VERIFIED'))
self.assertGreater(after['epoch'],before['epoch'])
self.assertEqual(after['execution_id'],before['execution_id'])
with self.assertRaisesRegex(execution.ExecutionStopped,'OWNER_FENCED'):
with owner.admission(): self.fail('old epoch reopened admission')
self.assertIsNone(victim.poll())
self.assertEqual(lease.read_bytes(),lease_bytes)
self.receipt('JOINT-010-boot', collector_processes=1,cleanup='VERIFIED',
unrelated_birth_identity_alive=True,board_lease_preserved=True,old_epoch_fenced=True)
if __name__ == '__main__':
unittest.main()