data-gen-bpu/scripts/test_controller_restart.py
bench 0da66edde0 Resume rollouts after the collector dies (crash, kill, reboot, planned handover)
Processes are disposable; rollout state is resumable. When the collector dies,
the next one reconciles each running rollout from durable state instead of
quarantining or losing it:

- execution: a process of an earlier boot on this host is DEAD; a top-level
  reaper whose owner (the collector) died still proves its subtree empty
  (scopes record the owner that started them); reconcile() pauses a rollout
  with a proven-clean crash checkpoint as CONTROLLER_RESTARTED and marks its
  running tool calls INTERRUPTED; resume() across a reboot rebases monotonic
  clocks by wall time and credits the downtime as a pause
- stageworker.settle_interrupted: an admitted step whose executor is gone ends
  as STAGE_EXECUTOR_EXIT (free compile) and its handle is told directly; a
  board step without a clean board receipt still quarantines; nothing replays
- collection_runtime.crash_checkpoint: capture the real system prompt, freeze
  the session at the last saved message (mid-work phases only); configure_pi
  skips a generation ID the ledger already completed
- pi_session.js: checkpoint after every saved message (and compaction); on a
  recovered resume drop an unfinished trailing reply and answer open tool
  calls with an honest "interrupted by a platform restart" result
- scheduler: a paused rollout's local slot held by a dead holder is released
  once every process is proven gone
- pipeline: SIGUSR1 = planned handover (stop claiming, let board steps
  finish, shorten in-flight request holds, exit); SIGTERM still drains
- export: bench_recovery closes the cut segment; the dropped unfinished reply
  is excluded (UNFINISHED_BEFORE_RESTART)
- tests: test_controller_restart (real Pi on Linux: kill -9 mid-command,
  kill -9 mid-reply, SIGUSR1 handover, power loss + reboot), test_settle_
  interrupted; three tests of the old quarantine-everything rule now assert
  the new outcome while keeping their guarantees (no replay, unrelated PID
  untouched, board lease kept)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 09:29:26 +08:00

275 lines
16 KiB
Python

"""A collector killed mid-rollout (crash, kill -9, reboot) leaves a rollout the next collector resumes.
Real collector processes, real Pi SDK, a local fake model. The first collector is SIGKILLed while the agent's bash
command runs; the second (optionally under a new boot id) must resume the rollout from its last saved message, tell
the agent its command was interrupted, and finish it. Linux only (process receipts need /proc).
Run: BENCH_TEST_NODE=<node> PYTHONPATH=. python3 -m unittest scripts.test_controller_restart
"""
from contextlib import ExitStack
from dataclasses import asdict
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from unittest.mock import patch
import yaml
ROOT = Path(__file__).resolve().parents[1]
NODE = Path(os.environ.get('BENCH_TEST_NODE', '/home/ming01.wang/toolchains/node/bin/node'))
MARK = '平台的调度程序重启了'
def drive(spec_path):
"""One collector, in its own process, against the fixture project in spec."""
spec = json.loads(Path(spec_path).read_text())
from harness import db, paths, pipeline, rollout
project, state = Path(spec['project']), Path(spec['project']) / 'state'
cfg = yaml.safe_load((project / 'config.yaml').read_text())
original_connect, original_readonly = db.connect, db.readonly
def prepare(run_id, job_id, kk, mode, policy, **_):
rd = paths.RUNS / run_id / job_id / 'rollouts' / f'{kk:02d}'
ws = rd / 'workspace'
(ws / '.pi').mkdir(parents=True)
(ws / '.agent-home/.pi/agent').mkdir(parents=True)
(rd / 'agent').mkdir(); (rd / 'handles').mkdir()
(ws / '.pi/settings.json').write_text(json.dumps({'defaultProvider': 'dr-gateway', 'defaultModel': 'fixture',
'retry': {'enabled': False, 'provider': {'maxRetries': 0}}, 'extensions': [], 'skills': []}))
(ws / '.agent-home/.pi/agent/models.json').write_text(json.dumps({'providers': {'dr-gateway': {
'baseUrl': 'http://127.0.0.1:1/v1', 'api': 'openai-completions', 'apiKey': 'BENCH_RUNNER_API_KEY',
'models': [{'id': 'fixture', 'name': 'fixture', 'reasoning': False, 'input': ['text'],
'contextWindow': 10000000, 'maxTokens': 512,
'cost': {'input': 0, 'output': 0, 'cacheRead': 0, 'cacheWrite': 0}}]}}}))
for name in ('system.md', 'append-system.md'):
(rd / 'agent' / name).write_text('Run the command, then report.')
(rd / 'agent/prompt.md').write_text('Run the command, then report.')
(rd / 'rollout.json').write_text(json.dumps({'state': 'RUNNING', 'runner': 'pi', 'runner_version': 'pi-sdk-v1',
'policy': policy, 'mode': mode, 'jail_policy': 'private-home', 'max_turns': 20, 'max_tokens': 200000,
'model': 'fixture'}))
job = {'run_id': run_id, 'job_id': job_id, 'task': 'cls', 'march': 'nash-p', 'backend': 'ptq'}
(rd.parent.parent / 'job.json').write_text(json.dumps(job))
return rd, job, (rd / 'agent/prompt.md').read_text()
with ExitStack() as stack:
for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent, 'AGENT': ROOT / 'agent',
'RUNS': project / 'runs', 'STATE': state, 'CONFIG_PATH': project / 'config.yaml'}.items():
stack.enter_context(patch.object(paths, name, value))
stack.enter_context(patch.object(paths, 'config', return_value=cfg))
stack.enter_context(patch.object(paths, 'secret', return_value='upstream-fixture'))
stack.enter_context(patch.object(db, 'connect', lambda **kw: original_connect(state / 'jobs.sqlite', **kw)))
stack.enter_context(patch.object(db, 'readonly', lambda: original_readonly(state / 'jobs.sqlite')))
stack.enter_context(patch.object(rollout, 'prepare', side_effect=prepare))
from harness import verify
for module in (rollout, verify): # recovery imports it from verify directly
stack.enter_context(patch.object(module, 'verify_rollout', return_value={
'terminal': 'UNVERIFIED', 'claim_consistent': False, 'review': False,
'execution_control': {'db_state': 'UNVERIFIED'}}))
stack.enter_context(patch.object(pipeline, 'refresh_run_dataset', return_value=None))
summary = pipeline.collect_rollouts('r', parallel=1, watch=False, log=lambda m: print(m, flush=True))
print('SUMMARY ' + json.dumps(summary), flush=True)
@unittest.skipUnless(Path('/proc').is_dir() and NODE.is_file(), 'Linux and installed Pi 0.73.1 SDK required')
class ControllerRestart(unittest.TestCase):
def setUp(self):
if os.environ.get('BENCH_TEST_KEEP'):
self.project = Path(tempfile.mkdtemp()) / 'bench'; print('project', self.project)
else:
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup)
self.project = Path(temp.name) / 'bench'
(self.project / 'state').mkdir(parents=True)
self.bodies = []
self.hang_first = threading.Event() # set: the first model request never answers (collector dies mid-reply)
outer = self
class Model(BaseHTTPRequestHandler):
def log_message(self, *_): pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
outer.bodies.append(body)
if len(outer.bodies) == 1 and outer.hang_first.is_set():
time.sleep(60); return
told = any(m.get('role') == 'tool' for m in body['messages']) # any result: report and stop
delta = ({'role': 'assistant', 'content': 'the command was interrupted; reporting that'} if told else
{'role': 'assistant', 'tool_calls': [{'index': 0, 'id': f'call-{len(outer.bodies)}', 'type': 'function',
'function': {'name': 'bash', 'arguments': json.dumps(
{'command': 'touch started && sleep 30 && touch finished'})}}]})
chunks = [dict(id='m', object='chat.completion.chunk', model='fixture',
choices=[dict(index=0, delta=delta, finish_reason=None)]),
dict(id='m', object='chat.completion.chunk', model='fixture',
choices=[dict(index=0, delta={}, finish_reason='stop' if told else 'tool_calls')],
usage={'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15})]
payload = ''.join('data: ' + json.dumps(c) + '\n\n' for c in chunks).encode() + b'data: [DONE]\n\n'
self.send_response(200); self.send_header('Content-Type', 'text/event-stream')
self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload)
server = ThreadingHTTPServer(('127.0.0.1', 0), Model)
threading.Thread(target=server.serve_forever, daemon=True).start()
self.addCleanup(server.server_close); self.addCleanup(server.shutdown)
from harness.quota import QuotaPolicy
from scripts import test_execution
cfg = yaml.safe_load((ROOT / 'config.yaml').read_text())
cfg['paths'].update(work_root=str(self.project / 'work'), node_bin=str(NODE))
cfg['pools']['agent']['workers'] = 1
cfg['rollouts'].update(jail=False, private_home=True, tools=['bash'], hard_ceiling_s=300)
cfg['endpoints']['teacher'].update(base_url=f'http://127.0.0.1:{server.server_port}/v1', model='fixture',
api_key_env='FIXTURE_KEY')
lifecycle = test_execution.policy(); lifecycle['execution_deadline_s'] = 300
policy = QuotaPolicy(max_concurrency=2, rpm=1000, tpm=10000000, max_attempts=3, window_s=.2, wait_budget_s=1,
cooldown_initial_s=.05, cooldown_local_max_s=.1, jitter_fraction=0, connect_timeout_s=.5,
first_byte_timeout_s=2, stream_idle_timeout_s=2, request_total_s=5, poll_s=.05)
cfg['collection_v1'].update(quota_ledger=str(self.project / 'state/quota.sqlite'),
quota_groups={'account': asdict(policy)}, credential_groups={'teacher': 'account'},
lifecycle=lifecycle)
(self.project / 'config.yaml').write_text(yaml.safe_dump(cfg))
from harness import db
with db.connect(self.project / 'state/jobs.sqlite') as con:
db.create_run(con, 'r', {})
db.rollout_upsert(con, 'r', 'j0', 0, 'e2e', 'teacher')
self.rd = self.project / 'runs/r/j0/rollouts/00'
def collector(self, **spec):
path = self.project / f'spec-{time.monotonic_ns()}.json'
path.write_text(json.dumps({'project': str(self.project), **spec}))
log = (self.project / f'{path.stem}.log').open('w')
self.addCleanup(log.close)
proc = subprocess.Popen([sys.executable, '-c', 'import sys; from scripts.test_controller_restart import drive; drive(sys.argv[1])',
str(path)], cwd=ROOT, env={**os.environ, 'PYTHONPATH': str(ROOT),
'BENCH_STATE_DIR': str(self.project / 'state')}, stdout=log, stderr=subprocess.STDOUT)
self.addCleanup(lambda: proc.poll() is None and proc.kill())
return proc, Path(log.name)
def crash_while_command_runs(self, power_loss=False):
first, log = self.collector()
deadline = time.monotonic() + 60
while not (self.rd / 'workspace/started').exists():
self.assertIsNone(first.poll(), log.read_text()[-3000:])
self.assertLess(time.monotonic(), deadline, log.read_text()[-3000:])
time.sleep(.2)
if power_loss:
# Everything dies at once, reapers and agent too: nobody aborts the command or writes a receipt.
from harness import execution
state = execution.Execution(self.rd).snapshot()
for process in sorted(state['processes'], key=lambda p: p.get('role') != 'supervisor'):
if execution.process_status(process) == 'ALIVE':
os.kill(process['pid'], signal.SIGKILL)
first.send_signal(signal.SIGKILL); first.wait(10)
time.sleep(3) # the reapers notice their owner is gone, kill the agent and write their receipts
def finish(self, **spec):
second, log = self.collector(**spec)
try:
second.wait(180)
except subprocess.TimeoutExpired:
second.kill()
self.fail('second collector did not finish:\n' + log.read_text()[-4000:])
return log.read_text()
def check_resumed(self, output, told):
from harness import db
with db.connect(self.project / 'state/jobs.sqlite') as con:
rows = [dict(r) for r in con.execute('SELECT job_id,kk,state FROM rollouts')]
self.assertEqual([(r['job_id'], r['kk']) for r in rows], [('j0', 0)], output[-3000:]) # same rollout, no new kk
# The agent finished normally; only the fixture verifier never writes its verdict back.
self.assertEqual(rows[0]['state'], 'VERIFYING', output[-3000:])
final = json.loads((self.rd / 'execution.json').read_text())
self.assertEqual((final['execution_outcome'], final['cleanup_state']), ('COMPLETED', 'VERIFIED'))
events = [json.loads(line) for line in (self.rd / 'execution.events.jsonl').read_text().splitlines()]
kinds = [e['type'] for e in events]
self.assertIn('EXECUTION_RESUMED', kinds)
self.assertEqual([e.get('reconciliation') for e in events if e['type'] == 'EXECUTION_RECONCILED'],
['CONTROLLER_RESTARTED'])
self.assertTrue((self.rd / 'workspace/started').exists())
self.assertFalse((self.rd / 'workspace/finished').exists()) # the command really was cut off
resent = [b for b in self.bodies if MARK in json.dumps(b, ensure_ascii=False)]
# Killed collector: its reaper SIGTERMs Pi, which records the command as aborted itself. Power loss: nobody
# does, and the resumed runner answers the open call with the interrupted note exactly once.
self.assertEqual(len(resent), int(told), [b['messages'][-1] for b in self.bodies])
agent = [json.loads(line) for line in (self.rd / 'agent/events.jsonl').read_text().splitlines()]
self.assertIn('bench_recovery', [e.get('type') for e in agent])
from export import trajectory
(self.rd / 'agent/system.md').write_text(next(e for e in agent if e.get('bench'))['bench']['system_prompt'])
(self.rd / 'agent/tools.json').write_text(json.dumps(next(e for e in agent if e.get('bench'))['bench']['tools']))
meta = json.loads((self.rd / 'rollout.json').read_text())
frozen = trajectory.freeze(self.rd)
result = trajectory.reconstruct(frozen)
self.assertEqual(result['errors'], [], meta)
self.assertTrue(trajectory.validate(frozen, result)['pass'])
roles = [m['role'] for m in result['messages']]
self.assertEqual(roles[:4], ['system', 'user', 'assistant', 'toolResult'], roles)
self.assertEqual(MARK in json.dumps(result['messages'][3], ensure_ascii=False), told)
self.assertEqual(roles[-1], 'assistant')
def test_killed_collector_rollout_resumes_from_its_last_message(self):
self.crash_while_command_runs()
self.check_resumed(self.finish(), told=False)
def test_collector_killed_mid_reply_resends_the_same_request(self):
self.hang_first.set()
first, log = self.collector()
deadline = time.monotonic() + 60
while not self.bodies:
self.assertIsNone(first.poll(), log.read_text()[-3000:]); self.assertLess(time.monotonic(), deadline)
time.sleep(.2)
first.send_signal(signal.SIGKILL); first.wait(10); time.sleep(3)
# The cut-off request may resume only once the ledger gives up on it (request_total_s): a watching collector
# keeps waiting; these one-pass collectors are simply started again.
for _ in range(10):
output = self.finish()
if json.loads((self.rd / 'execution.json').read_text()).get('execution_outcome'):
break
time.sleep(2)
self.assertEqual(self.bodies[1]['messages'], self.bodies[0]['messages'], output[-2000:]) # same request again
import sqlite3
with sqlite3.connect(self.project / 'state/quota.sqlite') as con:
first_id = con.execute('SELECT generation_id FROM generations ORDER BY created_at LIMIT 1').fetchone()[0]
sends = con.execute('SELECT COUNT(*) FROM attempts WHERE generation_id=?', (first_id,)).fetchone()[0]
self.assertEqual(sends, 2) # the cut-off request was resent under its own ID, not a new one
self.assertEqual(json.loads((self.rd / 'execution.json').read_text())['execution_outcome'], 'COMPLETED', output[-3000:])
def test_planned_handover_signal_exits_and_the_next_collector_resumes(self):
first, log = self.collector()
deadline = time.monotonic() + 60
while not (self.rd / 'workspace/started').exists():
self.assertIsNone(first.poll(), log.read_text()[-3000:]); self.assertLess(time.monotonic(), deadline)
time.sleep(.2)
first.send_signal(signal.SIGUSR1)
self.assertEqual(first.wait(60), 0, log.read_text()[-3000:])
self.assertIn('handover: exiting now', log.read_text())
time.sleep(3)
self.check_resumed(self.finish(), told=False)
def test_rollout_resumes_after_power_loss_and_reboot(self):
self.crash_while_command_runs(power_loss=True)
self.pretend_earlier_boot()
self.check_resumed(self.finish(), told=True)
def pretend_earlier_boot(self):
"""As if the VM rebooted after the kill: everything recorded belongs to an earlier boot, and the reapers
died with it before writing any receipt."""
from harness import execution
real = execution.boot_id()
def older(value):
if isinstance(value, dict):
return {k: ('earlier-boot' if k == 'boot_id' and v == real else older(v)) for k, v in value.items()}
return [older(v) for v in value] if isinstance(value, list) else value
for path in [self.rd / 'execution.json', *self.rd.glob('stages/_queue/*/operation.json'),
*(self.project / 'state/locks').glob('*.json')]:
execution.atomic_json(path, older(json.loads(path.read_text())))
for receipt in (self.rd / 'process-receipts').glob('*.json'):
receipt.unlink()
if __name__ == '__main__':
unittest.main()