Processes are disposable; rollout state is resumable. When the collector dies, the next one reconciles each running rollout from durable state instead of quarantining or losing it: - execution: a process of an earlier boot on this host is DEAD; a top-level reaper whose owner (the collector) died still proves its subtree empty (scopes record the owner that started them); reconcile() pauses a rollout with a proven-clean crash checkpoint as CONTROLLER_RESTARTED and marks its running tool calls INTERRUPTED; resume() across a reboot rebases monotonic clocks by wall time and credits the downtime as a pause - stageworker.settle_interrupted: an admitted step whose executor is gone ends as STAGE_EXECUTOR_EXIT (free compile) and its handle is told directly; a board step without a clean board receipt still quarantines; nothing replays - collection_runtime.crash_checkpoint: capture the real system prompt, freeze the session at the last saved message (mid-work phases only); configure_pi skips a generation ID the ledger already completed - pi_session.js: checkpoint after every saved message (and compaction); on a recovered resume drop an unfinished trailing reply and answer open tool calls with an honest "interrupted by a platform restart" result - scheduler: a paused rollout's local slot held by a dead holder is released once every process is proven gone - pipeline: SIGUSR1 = planned handover (stop claiming, let board steps finish, shorten in-flight request holds, exit); SIGTERM still drains - export: bench_recovery closes the cut segment; the dropped unfinished reply is excluded (UNFINISHED_BEFORE_RESTART) - tests: test_controller_restart (real Pi on Linux: kill -9 mid-command, kill -9 mid-reply, SIGUSR1 handover, power loss + reboot), test_settle_ interrupted; three tests of the old quarantine-everything rule now assert the new outcome while keeping their guarantees (no replay, unrelated PID untouched, board lease kept) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
84 lines
4.4 KiB
Python
84 lines
4.4 KiB
Python
"""A stage cut off with its collector ends like an executor exit: free compile, told to the agent, board quarantines."""
|
|
from contextlib import ExitStack
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import db, execution, paths, stagerun
|
|
from harness.stageworker import Worker
|
|
from scripts import test_execution
|
|
|
|
|
|
class SettleInterrupted(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup)
|
|
root = Path(temp.name)
|
|
self.rd = root / 'runs/r/j/rollouts/00'
|
|
(self.rd / 'handles').mkdir(parents=True)
|
|
(self.rd.parent.parent / 'job.json').write_text(json.dumps({'run_id': 'r', 'job_id': 'j'}))
|
|
stack = ExitStack(); self.addCleanup(stack.close)
|
|
original = db.connect
|
|
stack.enter_context(patch.object(db, 'connect', lambda *a, **kw: original(root / 'jobs.sqlite', **kw)))
|
|
stack.enter_context(patch.object(paths, 'STATE', root))
|
|
with db.connect() as con:
|
|
db.create_run(con, 'r', {})
|
|
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher')
|
|
self.execution = execution.Execution(self.rd)
|
|
self.execution.start(test_execution.policy())
|
|
dead = dict(host='fixture', boot_id='fixture', pid=999999, proc_start_ticks=1)
|
|
stack.enter_context(patch.object(execution, 'process_status', return_value='DEAD'))
|
|
self.dead = dead
|
|
|
|
def operation(self, hid, stage):
|
|
queue = self.rd / 'stages/_queue' / hid
|
|
queue.mkdir(parents=True)
|
|
(self.rd / 'handles' / hid).mkdir()
|
|
(self.rd / 'handles' / hid / 'request.json').write_text(json.dumps({'stage': stage}))
|
|
(queue / 'request.json').write_text(json.dumps({'stage': stage, 'config': None, 'recipe': None}))
|
|
with self.execution.mutation() as state:
|
|
op = dict(operation_id=hid, payload_sha256='sha', state='DISPATCHED', execution_id=state['execution_id'],
|
|
epoch=state['epoch'], received_at=time.time(), executor=self.dead)
|
|
execution.atomic_json(queue / 'operation.json', op)
|
|
state['operations'][hid] = op
|
|
self.execution._save(state, 'OPERATION_DISPATCHED', operation_id=hid)
|
|
execution.atomic_json(queue / 'status.json', dict(state='RUNNING', attempt=0, started_at=time.time(),
|
|
operation_id=hid, payload_sha256='sha', execution_id=op['execution_id'], epoch=op['epoch']))
|
|
with db.connect() as con:
|
|
db.stage_start(con, 'r', 'j', stage, attempt=0, rollout=0, lane='long')
|
|
return queue
|
|
|
|
def test_interrupted_compile_is_free_and_the_agent_is_told(self):
|
|
queue = self.operation('h1', '05-compile')
|
|
self.assertTrue(Worker(self.rd, execution=self.execution).settle_interrupted())
|
|
result = json.loads((queue / 'status.json').read_text())
|
|
self.assertEqual((result['state'], result['result'], result['error_class']), ('DONE', 'FAIL', 'STAGE_EXECUTOR_EXIT'))
|
|
self.assertIn('不计编译次数', result['note'])
|
|
told = json.loads((self.rd / 'handles/h1/status.json').read_text())
|
|
self.assertEqual(told['error_class'], 'STAGE_EXECUTOR_EXIT')
|
|
state = self.execution.snapshot()
|
|
self.assertEqual(state['operations']['h1']['state'], 'DONE')
|
|
self.assertNotEqual(state['cleanup_state'], 'QUARANTINED')
|
|
with db.connect() as con:
|
|
row = con.execute("SELECT state,error_class FROM stages WHERE stage='05-compile'").fetchone()
|
|
self.assertEqual(tuple(row), ('FAIL', 'STAGE_EXECUTOR_EXIT'))
|
|
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 0)
|
|
|
|
def test_board_step_without_a_clean_board_receipt_still_quarantines(self):
|
|
self.operation('h2', '07-board')
|
|
self.assertTrue(Worker(self.rd, execution=self.execution).settle_interrupted())
|
|
state = self.execution.snapshot()
|
|
self.assertEqual((state['cleanup_state'], state['stop_reason']), ('QUARANTINED', 'BOARD_CLEANUP_UNCONFIRMED'))
|
|
|
|
def test_a_live_executor_is_left_alone(self):
|
|
self.operation('h3', '05-compile')
|
|
with patch.object(execution, 'process_status', return_value='ALIVE'):
|
|
self.assertFalse(Worker(self.rd, execution=self.execution).settle_interrupted())
|
|
self.assertEqual(self.execution.snapshot()['operations']['h3']['state'], 'DISPATCHED')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|