data-gen-bpu/scripts/test_settle_interrupted.py
bench 0da66edde0 Resume rollouts after the collector dies (crash, kill, reboot, planned handover)
Processes are disposable; rollout state is resumable. When the collector dies,
the next one reconciles each running rollout from durable state instead of
quarantining or losing it:

- execution: a process of an earlier boot on this host is DEAD; a top-level
  reaper whose owner (the collector) died still proves its subtree empty
  (scopes record the owner that started them); reconcile() pauses a rollout
  with a proven-clean crash checkpoint as CONTROLLER_RESTARTED and marks its
  running tool calls INTERRUPTED; resume() across a reboot rebases monotonic
  clocks by wall time and credits the downtime as a pause
- stageworker.settle_interrupted: an admitted step whose executor is gone ends
  as STAGE_EXECUTOR_EXIT (free compile) and its handle is told directly; a
  board step without a clean board receipt still quarantines; nothing replays
- collection_runtime.crash_checkpoint: capture the real system prompt, freeze
  the session at the last saved message (mid-work phases only); configure_pi
  skips a generation ID the ledger already completed
- pi_session.js: checkpoint after every saved message (and compaction); on a
  recovered resume drop an unfinished trailing reply and answer open tool
  calls with an honest "interrupted by a platform restart" result
- scheduler: a paused rollout's local slot held by a dead holder is released
  once every process is proven gone
- pipeline: SIGUSR1 = planned handover (stop claiming, let board steps
  finish, shorten in-flight request holds, exit); SIGTERM still drains
- export: bench_recovery closes the cut segment; the dropped unfinished reply
  is excluded (UNFINISHED_BEFORE_RESTART)
- tests: test_controller_restart (real Pi on Linux: kill -9 mid-command,
  kill -9 mid-reply, SIGUSR1 handover, power loss + reboot), test_settle_
  interrupted; three tests of the old quarantine-everything rule now assert
  the new outcome while keeping their guarantees (no replay, unrelated PID
  untouched, board lease kept)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 09:29:26 +08:00

84 lines
4.4 KiB
Python

"""A stage cut off with its collector ends like an executor exit: free compile, told to the agent, board quarantines."""
from contextlib import ExitStack
import json
from pathlib import Path
import tempfile
import time
import unittest
from unittest.mock import patch
from harness import db, execution, paths, stagerun
from harness.stageworker import Worker
from scripts import test_execution
class SettleInterrupted(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup)
root = Path(temp.name)
self.rd = root / 'runs/r/j/rollouts/00'
(self.rd / 'handles').mkdir(parents=True)
(self.rd.parent.parent / 'job.json').write_text(json.dumps({'run_id': 'r', 'job_id': 'j'}))
stack = ExitStack(); self.addCleanup(stack.close)
original = db.connect
stack.enter_context(patch.object(db, 'connect', lambda *a, **kw: original(root / 'jobs.sqlite', **kw)))
stack.enter_context(patch.object(paths, 'STATE', root))
with db.connect() as con:
db.create_run(con, 'r', {})
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher')
self.execution = execution.Execution(self.rd)
self.execution.start(test_execution.policy())
dead = dict(host='fixture', boot_id='fixture', pid=999999, proc_start_ticks=1)
stack.enter_context(patch.object(execution, 'process_status', return_value='DEAD'))
self.dead = dead
def operation(self, hid, stage):
queue = self.rd / 'stages/_queue' / hid
queue.mkdir(parents=True)
(self.rd / 'handles' / hid).mkdir()
(self.rd / 'handles' / hid / 'request.json').write_text(json.dumps({'stage': stage}))
(queue / 'request.json').write_text(json.dumps({'stage': stage, 'config': None, 'recipe': None}))
with self.execution.mutation() as state:
op = dict(operation_id=hid, payload_sha256='sha', state='DISPATCHED', execution_id=state['execution_id'],
epoch=state['epoch'], received_at=time.time(), executor=self.dead)
execution.atomic_json(queue / 'operation.json', op)
state['operations'][hid] = op
self.execution._save(state, 'OPERATION_DISPATCHED', operation_id=hid)
execution.atomic_json(queue / 'status.json', dict(state='RUNNING', attempt=0, started_at=time.time(),
operation_id=hid, payload_sha256='sha', execution_id=op['execution_id'], epoch=op['epoch']))
with db.connect() as con:
db.stage_start(con, 'r', 'j', stage, attempt=0, rollout=0, lane='long')
return queue
def test_interrupted_compile_is_free_and_the_agent_is_told(self):
queue = self.operation('h1', '05-compile')
self.assertTrue(Worker(self.rd, execution=self.execution).settle_interrupted())
result = json.loads((queue / 'status.json').read_text())
self.assertEqual((result['state'], result['result'], result['error_class']), ('DONE', 'FAIL', 'STAGE_EXECUTOR_EXIT'))
self.assertIn('不计编译次数', result['note'])
told = json.loads((self.rd / 'handles/h1/status.json').read_text())
self.assertEqual(told['error_class'], 'STAGE_EXECUTOR_EXIT')
state = self.execution.snapshot()
self.assertEqual(state['operations']['h1']['state'], 'DONE')
self.assertNotEqual(state['cleanup_state'], 'QUARANTINED')
with db.connect() as con:
row = con.execute("SELECT state,error_class FROM stages WHERE stage='05-compile'").fetchone()
self.assertEqual(tuple(row), ('FAIL', 'STAGE_EXECUTOR_EXIT'))
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 0)
def test_board_step_without_a_clean_board_receipt_still_quarantines(self):
self.operation('h2', '07-board')
self.assertTrue(Worker(self.rd, execution=self.execution).settle_interrupted())
state = self.execution.snapshot()
self.assertEqual((state['cleanup_state'], state['stop_reason']), ('QUARANTINED', 'BOARD_CLEANUP_UNCONFIRMED'))
def test_a_live_executor_is_left_alone(self):
self.operation('h3', '05-compile')
with patch.object(execution, 'process_status', return_value='ALIVE'):
self.assertFalse(Worker(self.rd, execution=self.execution).settle_interrupted())
self.assertEqual(self.execution.snapshot()['operations']['h3']['state'], 'DISPATCHED')
if __name__ == '__main__':
unittest.main()