"""A collector killed mid-rollout (crash, kill -9, reboot) leaves a rollout the next collector resumes. Real collector processes, real Pi SDK, a local fake model. The first collector is SIGKILLed while the agent's bash command runs; the second (optionally under a new boot id) must resume the rollout from its last saved message, tell the agent its command was interrupted, and finish it. Linux only (process receipts need /proc). Run: BENCH_TEST_NODE= PYTHONPATH=. python3 -m unittest scripts.test_controller_restart """ from contextlib import ExitStack from dataclasses import asdict from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os from pathlib import Path import signal import subprocess import sys import tempfile import threading import time import unittest from unittest.mock import patch import yaml ROOT = Path(__file__).resolve().parents[1] NODE = Path(os.environ.get('BENCH_TEST_NODE', '/home/ming01.wang/toolchains/node/bin/node')) MARK = '平台的调度程序重启了' def drive(spec_path): """One collector, in its own process, against the fixture project in spec.""" spec = json.loads(Path(spec_path).read_text()) from harness import db, paths, pipeline, rollout project, state = Path(spec['project']), Path(spec['project']) / 'state' cfg = yaml.safe_load((project / 'config.yaml').read_text()) original_connect, original_readonly = db.connect, db.readonly def prepare(run_id, job_id, kk, mode, policy, **_): rd = paths.RUNS / run_id / job_id / 'rollouts' / f'{kk:02d}' ws = rd / 'workspace' (ws / '.pi').mkdir(parents=True) (ws / '.agent-home/.pi/agent').mkdir(parents=True) (rd / 'agent').mkdir(); (rd / 'handles').mkdir() (ws / '.pi/settings.json').write_text(json.dumps({'defaultProvider': 'dr-gateway', 'defaultModel': 'fixture', 'retry': {'enabled': False, 'provider': {'maxRetries': 0}}, 'extensions': [], 'skills': []})) (ws / '.agent-home/.pi/agent/models.json').write_text(json.dumps({'providers': {'dr-gateway': { 'baseUrl': 'http://127.0.0.1:1/v1', 'api': 'openai-completions', 'apiKey': 'BENCH_RUNNER_API_KEY', 'models': [{'id': 'fixture', 'name': 'fixture', 'reasoning': False, 'input': ['text'], 'contextWindow': 10000000, 'maxTokens': 512, 'cost': {'input': 0, 'output': 0, 'cacheRead': 0, 'cacheWrite': 0}}]}}})) for name in ('system.md', 'append-system.md'): (rd / 'agent' / name).write_text('Run the command, then report.') (rd / 'agent/prompt.md').write_text('Run the command, then report.') (rd / 'rollout.json').write_text(json.dumps({'state': 'RUNNING', 'runner': 'pi', 'runner_version': 'pi-sdk-v1', 'policy': policy, 'mode': mode, 'jail_policy': 'private-home', 'max_turns': 20, 'max_tokens': 200000, 'model': 'fixture'})) job = {'run_id': run_id, 'job_id': job_id, 'task': 'cls', 'march': 'nash-p', 'backend': 'ptq'} (rd.parent.parent / 'job.json').write_text(json.dumps(job)) return rd, job, (rd / 'agent/prompt.md').read_text() with ExitStack() as stack: for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent, 'AGENT': ROOT / 'agent', 'RUNS': project / 'runs', 'STATE': state, 'CONFIG_PATH': project / 'config.yaml'}.items(): stack.enter_context(patch.object(paths, name, value)) stack.enter_context(patch.object(paths, 'config', return_value=cfg)) stack.enter_context(patch.object(paths, 'secret', return_value='upstream-fixture')) stack.enter_context(patch.object(db, 'connect', lambda **kw: original_connect(state / 'jobs.sqlite', **kw))) stack.enter_context(patch.object(db, 'readonly', lambda: original_readonly(state / 'jobs.sqlite'))) stack.enter_context(patch.object(rollout, 'prepare', side_effect=prepare)) from harness import verify for module in (rollout, verify): # recovery imports it from verify directly stack.enter_context(patch.object(module, 'verify_rollout', return_value={ 'terminal': 'UNVERIFIED', 'claim_consistent': False, 'review': False, 'execution_control': {'db_state': 'UNVERIFIED'}})) stack.enter_context(patch.object(pipeline, 'refresh_run_dataset', return_value=None)) summary = pipeline.collect_rollouts('r', parallel=1, watch=False, log=lambda m: print(m, flush=True)) print('SUMMARY ' + json.dumps(summary), flush=True) @unittest.skipUnless(Path('/proc').is_dir() and NODE.is_file(), 'Linux and installed Pi 0.73.1 SDK required') class ControllerRestart(unittest.TestCase): def setUp(self): if os.environ.get('BENCH_TEST_KEEP'): self.project = Path(tempfile.mkdtemp()) / 'bench'; print('project', self.project) else: temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup) self.project = Path(temp.name) / 'bench' (self.project / 'state').mkdir(parents=True) self.bodies = [] self.hang_first = threading.Event() # set: the first model request never answers (collector dies mid-reply) outer = self class Model(BaseHTTPRequestHandler): def log_message(self, *_): pass def do_POST(self): body = json.loads(self.rfile.read(int(self.headers['Content-Length']))) outer.bodies.append(body) if len(outer.bodies) == 1 and outer.hang_first.is_set(): time.sleep(60); return told = any(m.get('role') == 'tool' for m in body['messages']) # any result: report and stop delta = ({'role': 'assistant', 'content': 'the command was interrupted; reporting that'} if told else {'role': 'assistant', 'tool_calls': [{'index': 0, 'id': f'call-{len(outer.bodies)}', 'type': 'function', 'function': {'name': 'bash', 'arguments': json.dumps( {'command': 'touch started && sleep 30 && touch finished'})}}]}) chunks = [dict(id='m', object='chat.completion.chunk', model='fixture', choices=[dict(index=0, delta=delta, finish_reason=None)]), dict(id='m', object='chat.completion.chunk', model='fixture', choices=[dict(index=0, delta={}, finish_reason='stop' if told else 'tool_calls')], usage={'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15})] payload = ''.join('data: ' + json.dumps(c) + '\n\n' for c in chunks).encode() + b'data: [DONE]\n\n' self.send_response(200); self.send_header('Content-Type', 'text/event-stream') self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload) server = ThreadingHTTPServer(('127.0.0.1', 0), Model) threading.Thread(target=server.serve_forever, daemon=True).start() self.addCleanup(server.server_close); self.addCleanup(server.shutdown) from harness.quota import QuotaPolicy from scripts import test_execution cfg = yaml.safe_load((ROOT / 'config.yaml').read_text()) cfg['paths'].update(work_root=str(self.project / 'work'), node_bin=str(NODE)) cfg['pools']['agent']['workers'] = 1 cfg['rollouts'].update(jail=False, private_home=True, tools=['bash'], hard_ceiling_s=300) cfg['endpoints']['teacher'].update(base_url=f'http://127.0.0.1:{server.server_port}/v1', model='fixture', api_key_env='FIXTURE_KEY') lifecycle = test_execution.policy(); lifecycle['execution_deadline_s'] = 300 policy = QuotaPolicy(max_concurrency=2, rpm=1000, tpm=10000000, max_attempts=3, window_s=.2, wait_budget_s=1, cooldown_initial_s=.05, cooldown_local_max_s=.1, jitter_fraction=0, connect_timeout_s=.5, first_byte_timeout_s=2, stream_idle_timeout_s=2, request_total_s=5, poll_s=.05) cfg['collection_v1'].update(quota_ledger=str(self.project / 'state/quota.sqlite'), quota_groups={'account': asdict(policy)}, credential_groups={'teacher': 'account'}, lifecycle=lifecycle) (self.project / 'config.yaml').write_text(yaml.safe_dump(cfg)) from harness import db with db.connect(self.project / 'state/jobs.sqlite') as con: db.create_run(con, 'r', {}) db.rollout_upsert(con, 'r', 'j0', 0, 'e2e', 'teacher') self.rd = self.project / 'runs/r/j0/rollouts/00' def collector(self, **spec): path = self.project / f'spec-{time.monotonic_ns()}.json' path.write_text(json.dumps({'project': str(self.project), **spec})) log = (self.project / f'{path.stem}.log').open('w') self.addCleanup(log.close) proc = subprocess.Popen([sys.executable, '-c', 'import sys; from scripts.test_controller_restart import drive; drive(sys.argv[1])', str(path)], cwd=ROOT, env={**os.environ, 'PYTHONPATH': str(ROOT), 'BENCH_STATE_DIR': str(self.project / 'state')}, stdout=log, stderr=subprocess.STDOUT) self.addCleanup(lambda: proc.poll() is None and proc.kill()) return proc, Path(log.name) def crash_while_command_runs(self, power_loss=False): first, log = self.collector() deadline = time.monotonic() + 60 while not (self.rd / 'workspace/started').exists(): self.assertIsNone(first.poll(), log.read_text()[-3000:]) self.assertLess(time.monotonic(), deadline, log.read_text()[-3000:]) time.sleep(.2) if power_loss: # Everything dies at once, reapers and agent too: nobody aborts the command or writes a receipt. from harness import execution state = execution.Execution(self.rd).snapshot() for process in sorted(state['processes'], key=lambda p: p.get('role') != 'supervisor'): if execution.process_status(process) == 'ALIVE': os.kill(process['pid'], signal.SIGKILL) first.send_signal(signal.SIGKILL); first.wait(10) time.sleep(3) # the reapers notice their owner is gone, kill the agent and write their receipts def finish(self, **spec): second, log = self.collector(**spec) try: second.wait(180) except subprocess.TimeoutExpired: second.kill() self.fail('second collector did not finish:\n' + log.read_text()[-4000:]) return log.read_text() def check_resumed(self, output, told): from harness import db with db.connect(self.project / 'state/jobs.sqlite') as con: rows = [dict(r) for r in con.execute('SELECT job_id,kk,state FROM rollouts')] self.assertEqual([(r['job_id'], r['kk']) for r in rows], [('j0', 0)], output[-3000:]) # same rollout, no new kk # The agent finished normally; only the fixture verifier never writes its verdict back. self.assertEqual(rows[0]['state'], 'VERIFYING', output[-3000:]) final = json.loads((self.rd / 'execution.json').read_text()) self.assertEqual((final['execution_outcome'], final['cleanup_state']), ('COMPLETED', 'VERIFIED')) events = [json.loads(line) for line in (self.rd / 'execution.events.jsonl').read_text().splitlines()] kinds = [e['type'] for e in events] self.assertIn('EXECUTION_RESUMED', kinds) self.assertEqual([e.get('reconciliation') for e in events if e['type'] == 'EXECUTION_RECONCILED'], ['CONTROLLER_RESTARTED']) self.assertTrue((self.rd / 'workspace/started').exists()) self.assertFalse((self.rd / 'workspace/finished').exists()) # the command really was cut off resent = [b for b in self.bodies if MARK in json.dumps(b, ensure_ascii=False)] # Killed collector: its reaper SIGTERMs Pi, which records the command as aborted itself. Power loss: nobody # does, and the resumed runner answers the open call with the interrupted note exactly once. self.assertEqual(len(resent), int(told), [b['messages'][-1] for b in self.bodies]) agent = [json.loads(line) for line in (self.rd / 'agent/events.jsonl').read_text().splitlines()] self.assertIn('bench_recovery', [e.get('type') for e in agent]) from export import trajectory (self.rd / 'agent/system.md').write_text(next(e for e in agent if e.get('bench'))['bench']['system_prompt']) (self.rd / 'agent/tools.json').write_text(json.dumps(next(e for e in agent if e.get('bench'))['bench']['tools'])) meta = json.loads((self.rd / 'rollout.json').read_text()) frozen = trajectory.freeze(self.rd) result = trajectory.reconstruct(frozen) self.assertEqual(result['errors'], [], meta) self.assertTrue(trajectory.validate(frozen, result)['pass']) roles = [m['role'] for m in result['messages']] self.assertEqual(roles[:4], ['system', 'user', 'assistant', 'toolResult'], roles) self.assertEqual(MARK in json.dumps(result['messages'][3], ensure_ascii=False), told) self.assertEqual(roles[-1], 'assistant') def test_killed_collector_rollout_resumes_from_its_last_message(self): self.crash_while_command_runs() self.check_resumed(self.finish(), told=False) def test_collector_killed_mid_reply_resends_the_same_request(self): self.hang_first.set() first, log = self.collector() deadline = time.monotonic() + 60 while not self.bodies: self.assertIsNone(first.poll(), log.read_text()[-3000:]); self.assertLess(time.monotonic(), deadline) time.sleep(.2) first.send_signal(signal.SIGKILL); first.wait(10); time.sleep(3) # The cut-off request may resume only once the ledger gives up on it (request_total_s): a watching collector # keeps waiting; these one-pass collectors are simply started again. for _ in range(10): output = self.finish() if json.loads((self.rd / 'execution.json').read_text()).get('execution_outcome'): break time.sleep(2) self.assertEqual(self.bodies[1]['messages'], self.bodies[0]['messages'], output[-2000:]) # same request again import sqlite3 with sqlite3.connect(self.project / 'state/quota.sqlite') as con: first_id = con.execute('SELECT generation_id FROM generations ORDER BY created_at LIMIT 1').fetchone()[0] sends = con.execute('SELECT COUNT(*) FROM attempts WHERE generation_id=?', (first_id,)).fetchone()[0] self.assertEqual(sends, 2) # the cut-off request was resent under its own ID, not a new one self.assertEqual(json.loads((self.rd / 'execution.json').read_text())['execution_outcome'], 'COMPLETED', output[-3000:]) def test_planned_handover_signal_exits_and_the_next_collector_resumes(self): first, log = self.collector() deadline = time.monotonic() + 60 while not (self.rd / 'workspace/started').exists(): self.assertIsNone(first.poll(), log.read_text()[-3000:]); self.assertLess(time.monotonic(), deadline) time.sleep(.2) first.send_signal(signal.SIGUSR1) self.assertEqual(first.wait(60), 0, log.read_text()[-3000:]) self.assertIn('handover: exiting now', log.read_text()) # No pause: the next collector starts while the command's reaper is still killing its tree (10-19 staging). self.check_resumed(self.finish(), told=False) def test_rollout_resumes_after_power_loss_and_reboot(self): self.crash_while_command_runs(power_loss=True) self.pretend_earlier_boot() self.check_resumed(self.finish(), told=True) def pretend_earlier_boot(self): """As if the VM rebooted after the kill: everything recorded belongs to an earlier boot, and the reapers died with it before writing any receipt.""" from harness import execution real = execution.boot_id() def older(value): if isinstance(value, dict): return {k: ('earlier-boot' if k == 'boot_id' and v == real else older(v)) for k, v in value.items()} return [older(v) for v in value] if isinstance(value, list) else value for path in [self.rd / 'execution.json', *self.rd.glob('stages/_queue/*/operation.json'), *(self.project / 'state/locks').glob('*.json')]: execution.atomic_json(path, older(json.loads(path.read_text()))) for receipt in (self.rd / 'process-receipts').glob('*.json'): receipt.unlink() if __name__ == '__main__': unittest.main()