Processes are disposable; rollout state is resumable. When the collector dies, the next one reconciles each running rollout from durable state instead of quarantining or losing it: - execution: a process of an earlier boot on this host is DEAD; a top-level reaper whose owner (the collector) died still proves its subtree empty (scopes record the owner that started them); reconcile() pauses a rollout with a proven-clean crash checkpoint as CONTROLLER_RESTARTED and marks its running tool calls INTERRUPTED; resume() across a reboot rebases monotonic clocks by wall time and credits the downtime as a pause - stageworker.settle_interrupted: an admitted step whose executor is gone ends as STAGE_EXECUTOR_EXIT (free compile) and its handle is told directly; a board step without a clean board receipt still quarantines; nothing replays - collection_runtime.crash_checkpoint: capture the real system prompt, freeze the session at the last saved message (mid-work phases only); configure_pi skips a generation ID the ledger already completed - pi_session.js: checkpoint after every saved message (and compaction); on a recovered resume drop an unfinished trailing reply and answer open tool calls with an honest "interrupted by a platform restart" result - scheduler: a paused rollout's local slot held by a dead holder is released once every process is proven gone - pipeline: SIGUSR1 = planned handover (stop claiming, let board steps finish, shorten in-flight request holds, exit); SIGTERM still drains - export: bench_recovery closes the cut segment; the dropped unfinished reply is excluded (UNFINISHED_BEFORE_RESTART) - tests: test_controller_restart (real Pi on Linux: kill -9 mid-command, kill -9 mid-reply, SIGUSR1 handover, power loss + reboot), test_settle_ interrupted; three tests of the old quarantine-everything rule now assert the new outcome while keeping their guarantees (no replay, unrelated PID untouched, board lease kept) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
275 lines
16 KiB
Python
275 lines
16 KiB
Python
"""A collector killed mid-rollout (crash, kill -9, reboot) leaves a rollout the next collector resumes.
|
|
|
|
Real collector processes, real Pi SDK, a local fake model. The first collector is SIGKILLed while the agent's bash
|
|
command runs; the second (optionally under a new boot id) must resume the rollout from its last saved message, tell
|
|
the agent its command was interrupted, and finish it. Linux only (process receipts need /proc).
|
|
Run: BENCH_TEST_NODE=<node> PYTHONPATH=. python3 -m unittest scripts.test_controller_restart
|
|
"""
|
|
from contextlib import ExitStack
|
|
from dataclasses import asdict
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import yaml
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
NODE = Path(os.environ.get('BENCH_TEST_NODE', '/home/ming01.wang/toolchains/node/bin/node'))
|
|
MARK = '平台的调度程序重启了'
|
|
|
|
|
|
def drive(spec_path):
|
|
"""One collector, in its own process, against the fixture project in spec."""
|
|
spec = json.loads(Path(spec_path).read_text())
|
|
from harness import db, paths, pipeline, rollout
|
|
project, state = Path(spec['project']), Path(spec['project']) / 'state'
|
|
cfg = yaml.safe_load((project / 'config.yaml').read_text())
|
|
original_connect, original_readonly = db.connect, db.readonly
|
|
|
|
def prepare(run_id, job_id, kk, mode, policy, **_):
|
|
rd = paths.RUNS / run_id / job_id / 'rollouts' / f'{kk:02d}'
|
|
ws = rd / 'workspace'
|
|
(ws / '.pi').mkdir(parents=True)
|
|
(ws / '.agent-home/.pi/agent').mkdir(parents=True)
|
|
(rd / 'agent').mkdir(); (rd / 'handles').mkdir()
|
|
(ws / '.pi/settings.json').write_text(json.dumps({'defaultProvider': 'dr-gateway', 'defaultModel': 'fixture',
|
|
'retry': {'enabled': False, 'provider': {'maxRetries': 0}}, 'extensions': [], 'skills': []}))
|
|
(ws / '.agent-home/.pi/agent/models.json').write_text(json.dumps({'providers': {'dr-gateway': {
|
|
'baseUrl': 'http://127.0.0.1:1/v1', 'api': 'openai-completions', 'apiKey': 'BENCH_RUNNER_API_KEY',
|
|
'models': [{'id': 'fixture', 'name': 'fixture', 'reasoning': False, 'input': ['text'],
|
|
'contextWindow': 10000000, 'maxTokens': 512,
|
|
'cost': {'input': 0, 'output': 0, 'cacheRead': 0, 'cacheWrite': 0}}]}}}))
|
|
for name in ('system.md', 'append-system.md'):
|
|
(rd / 'agent' / name).write_text('Run the command, then report.')
|
|
(rd / 'agent/prompt.md').write_text('Run the command, then report.')
|
|
(rd / 'rollout.json').write_text(json.dumps({'state': 'RUNNING', 'runner': 'pi', 'runner_version': 'pi-sdk-v1',
|
|
'policy': policy, 'mode': mode, 'jail_policy': 'private-home', 'max_turns': 20, 'max_tokens': 200000,
|
|
'model': 'fixture'}))
|
|
job = {'run_id': run_id, 'job_id': job_id, 'task': 'cls', 'march': 'nash-p', 'backend': 'ptq'}
|
|
(rd.parent.parent / 'job.json').write_text(json.dumps(job))
|
|
return rd, job, (rd / 'agent/prompt.md').read_text()
|
|
|
|
with ExitStack() as stack:
|
|
for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent, 'AGENT': ROOT / 'agent',
|
|
'RUNS': project / 'runs', 'STATE': state, 'CONFIG_PATH': project / 'config.yaml'}.items():
|
|
stack.enter_context(patch.object(paths, name, value))
|
|
stack.enter_context(patch.object(paths, 'config', return_value=cfg))
|
|
stack.enter_context(patch.object(paths, 'secret', return_value='upstream-fixture'))
|
|
stack.enter_context(patch.object(db, 'connect', lambda **kw: original_connect(state / 'jobs.sqlite', **kw)))
|
|
stack.enter_context(patch.object(db, 'readonly', lambda: original_readonly(state / 'jobs.sqlite')))
|
|
stack.enter_context(patch.object(rollout, 'prepare', side_effect=prepare))
|
|
from harness import verify
|
|
for module in (rollout, verify): # recovery imports it from verify directly
|
|
stack.enter_context(patch.object(module, 'verify_rollout', return_value={
|
|
'terminal': 'UNVERIFIED', 'claim_consistent': False, 'review': False,
|
|
'execution_control': {'db_state': 'UNVERIFIED'}}))
|
|
stack.enter_context(patch.object(pipeline, 'refresh_run_dataset', return_value=None))
|
|
summary = pipeline.collect_rollouts('r', parallel=1, watch=False, log=lambda m: print(m, flush=True))
|
|
print('SUMMARY ' + json.dumps(summary), flush=True)
|
|
|
|
|
|
@unittest.skipUnless(Path('/proc').is_dir() and NODE.is_file(), 'Linux and installed Pi 0.73.1 SDK required')
|
|
class ControllerRestart(unittest.TestCase):
|
|
def setUp(self):
|
|
if os.environ.get('BENCH_TEST_KEEP'):
|
|
self.project = Path(tempfile.mkdtemp()) / 'bench'; print('project', self.project)
|
|
else:
|
|
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup)
|
|
self.project = Path(temp.name) / 'bench'
|
|
(self.project / 'state').mkdir(parents=True)
|
|
self.bodies = []
|
|
self.hang_first = threading.Event() # set: the first model request never answers (collector dies mid-reply)
|
|
outer = self
|
|
|
|
class Model(BaseHTTPRequestHandler):
|
|
def log_message(self, *_): pass
|
|
|
|
def do_POST(self):
|
|
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
|
outer.bodies.append(body)
|
|
if len(outer.bodies) == 1 and outer.hang_first.is_set():
|
|
time.sleep(60); return
|
|
told = any(m.get('role') == 'tool' for m in body['messages']) # any result: report and stop
|
|
delta = ({'role': 'assistant', 'content': 'the command was interrupted; reporting that'} if told else
|
|
{'role': 'assistant', 'tool_calls': [{'index': 0, 'id': f'call-{len(outer.bodies)}', 'type': 'function',
|
|
'function': {'name': 'bash', 'arguments': json.dumps(
|
|
{'command': 'touch started && sleep 30 && touch finished'})}}]})
|
|
chunks = [dict(id='m', object='chat.completion.chunk', model='fixture',
|
|
choices=[dict(index=0, delta=delta, finish_reason=None)]),
|
|
dict(id='m', object='chat.completion.chunk', model='fixture',
|
|
choices=[dict(index=0, delta={}, finish_reason='stop' if told else 'tool_calls')],
|
|
usage={'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15})]
|
|
payload = ''.join('data: ' + json.dumps(c) + '\n\n' for c in chunks).encode() + b'data: [DONE]\n\n'
|
|
self.send_response(200); self.send_header('Content-Type', 'text/event-stream')
|
|
self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload)
|
|
server = ThreadingHTTPServer(('127.0.0.1', 0), Model)
|
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
self.addCleanup(server.server_close); self.addCleanup(server.shutdown)
|
|
|
|
from harness.quota import QuotaPolicy
|
|
from scripts import test_execution
|
|
cfg = yaml.safe_load((ROOT / 'config.yaml').read_text())
|
|
cfg['paths'].update(work_root=str(self.project / 'work'), node_bin=str(NODE))
|
|
cfg['pools']['agent']['workers'] = 1
|
|
cfg['rollouts'].update(jail=False, private_home=True, tools=['bash'], hard_ceiling_s=300)
|
|
cfg['endpoints']['teacher'].update(base_url=f'http://127.0.0.1:{server.server_port}/v1', model='fixture',
|
|
api_key_env='FIXTURE_KEY')
|
|
lifecycle = test_execution.policy(); lifecycle['execution_deadline_s'] = 300
|
|
policy = QuotaPolicy(max_concurrency=2, rpm=1000, tpm=10000000, max_attempts=3, window_s=.2, wait_budget_s=1,
|
|
cooldown_initial_s=.05, cooldown_local_max_s=.1, jitter_fraction=0, connect_timeout_s=.5,
|
|
first_byte_timeout_s=2, stream_idle_timeout_s=2, request_total_s=5, poll_s=.05)
|
|
cfg['collection_v1'].update(quota_ledger=str(self.project / 'state/quota.sqlite'),
|
|
quota_groups={'account': asdict(policy)}, credential_groups={'teacher': 'account'},
|
|
lifecycle=lifecycle)
|
|
(self.project / 'config.yaml').write_text(yaml.safe_dump(cfg))
|
|
from harness import db
|
|
with db.connect(self.project / 'state/jobs.sqlite') as con:
|
|
db.create_run(con, 'r', {})
|
|
db.rollout_upsert(con, 'r', 'j0', 0, 'e2e', 'teacher')
|
|
self.rd = self.project / 'runs/r/j0/rollouts/00'
|
|
|
|
def collector(self, **spec):
|
|
path = self.project / f'spec-{time.monotonic_ns()}.json'
|
|
path.write_text(json.dumps({'project': str(self.project), **spec}))
|
|
log = (self.project / f'{path.stem}.log').open('w')
|
|
self.addCleanup(log.close)
|
|
proc = subprocess.Popen([sys.executable, '-c', 'import sys; from scripts.test_controller_restart import drive; drive(sys.argv[1])',
|
|
str(path)], cwd=ROOT, env={**os.environ, 'PYTHONPATH': str(ROOT),
|
|
'BENCH_STATE_DIR': str(self.project / 'state')}, stdout=log, stderr=subprocess.STDOUT)
|
|
self.addCleanup(lambda: proc.poll() is None and proc.kill())
|
|
return proc, Path(log.name)
|
|
|
|
def crash_while_command_runs(self, power_loss=False):
|
|
first, log = self.collector()
|
|
deadline = time.monotonic() + 60
|
|
while not (self.rd / 'workspace/started').exists():
|
|
self.assertIsNone(first.poll(), log.read_text()[-3000:])
|
|
self.assertLess(time.monotonic(), deadline, log.read_text()[-3000:])
|
|
time.sleep(.2)
|
|
if power_loss:
|
|
# Everything dies at once, reapers and agent too: nobody aborts the command or writes a receipt.
|
|
from harness import execution
|
|
state = execution.Execution(self.rd).snapshot()
|
|
for process in sorted(state['processes'], key=lambda p: p.get('role') != 'supervisor'):
|
|
if execution.process_status(process) == 'ALIVE':
|
|
os.kill(process['pid'], signal.SIGKILL)
|
|
first.send_signal(signal.SIGKILL); first.wait(10)
|
|
time.sleep(3) # the reapers notice their owner is gone, kill the agent and write their receipts
|
|
|
|
def finish(self, **spec):
|
|
second, log = self.collector(**spec)
|
|
try:
|
|
second.wait(180)
|
|
except subprocess.TimeoutExpired:
|
|
second.kill()
|
|
self.fail('second collector did not finish:\n' + log.read_text()[-4000:])
|
|
return log.read_text()
|
|
|
|
def check_resumed(self, output, told):
|
|
from harness import db
|
|
with db.connect(self.project / 'state/jobs.sqlite') as con:
|
|
rows = [dict(r) for r in con.execute('SELECT job_id,kk,state FROM rollouts')]
|
|
self.assertEqual([(r['job_id'], r['kk']) for r in rows], [('j0', 0)], output[-3000:]) # same rollout, no new kk
|
|
# The agent finished normally; only the fixture verifier never writes its verdict back.
|
|
self.assertEqual(rows[0]['state'], 'VERIFYING', output[-3000:])
|
|
final = json.loads((self.rd / 'execution.json').read_text())
|
|
self.assertEqual((final['execution_outcome'], final['cleanup_state']), ('COMPLETED', 'VERIFIED'))
|
|
events = [json.loads(line) for line in (self.rd / 'execution.events.jsonl').read_text().splitlines()]
|
|
kinds = [e['type'] for e in events]
|
|
self.assertIn('EXECUTION_RESUMED', kinds)
|
|
self.assertEqual([e.get('reconciliation') for e in events if e['type'] == 'EXECUTION_RECONCILED'],
|
|
['CONTROLLER_RESTARTED'])
|
|
self.assertTrue((self.rd / 'workspace/started').exists())
|
|
self.assertFalse((self.rd / 'workspace/finished').exists()) # the command really was cut off
|
|
resent = [b for b in self.bodies if MARK in json.dumps(b, ensure_ascii=False)]
|
|
# Killed collector: its reaper SIGTERMs Pi, which records the command as aborted itself. Power loss: nobody
|
|
# does, and the resumed runner answers the open call with the interrupted note exactly once.
|
|
self.assertEqual(len(resent), int(told), [b['messages'][-1] for b in self.bodies])
|
|
agent = [json.loads(line) for line in (self.rd / 'agent/events.jsonl').read_text().splitlines()]
|
|
self.assertIn('bench_recovery', [e.get('type') for e in agent])
|
|
from export import trajectory
|
|
(self.rd / 'agent/system.md').write_text(next(e for e in agent if e.get('bench'))['bench']['system_prompt'])
|
|
(self.rd / 'agent/tools.json').write_text(json.dumps(next(e for e in agent if e.get('bench'))['bench']['tools']))
|
|
meta = json.loads((self.rd / 'rollout.json').read_text())
|
|
frozen = trajectory.freeze(self.rd)
|
|
result = trajectory.reconstruct(frozen)
|
|
self.assertEqual(result['errors'], [], meta)
|
|
self.assertTrue(trajectory.validate(frozen, result)['pass'])
|
|
roles = [m['role'] for m in result['messages']]
|
|
self.assertEqual(roles[:4], ['system', 'user', 'assistant', 'toolResult'], roles)
|
|
self.assertEqual(MARK in json.dumps(result['messages'][3], ensure_ascii=False), told)
|
|
self.assertEqual(roles[-1], 'assistant')
|
|
|
|
def test_killed_collector_rollout_resumes_from_its_last_message(self):
|
|
self.crash_while_command_runs()
|
|
self.check_resumed(self.finish(), told=False)
|
|
|
|
def test_collector_killed_mid_reply_resends_the_same_request(self):
|
|
self.hang_first.set()
|
|
first, log = self.collector()
|
|
deadline = time.monotonic() + 60
|
|
while not self.bodies:
|
|
self.assertIsNone(first.poll(), log.read_text()[-3000:]); self.assertLess(time.monotonic(), deadline)
|
|
time.sleep(.2)
|
|
first.send_signal(signal.SIGKILL); first.wait(10); time.sleep(3)
|
|
# The cut-off request may resume only once the ledger gives up on it (request_total_s): a watching collector
|
|
# keeps waiting; these one-pass collectors are simply started again.
|
|
for _ in range(10):
|
|
output = self.finish()
|
|
if json.loads((self.rd / 'execution.json').read_text()).get('execution_outcome'):
|
|
break
|
|
time.sleep(2)
|
|
self.assertEqual(self.bodies[1]['messages'], self.bodies[0]['messages'], output[-2000:]) # same request again
|
|
import sqlite3
|
|
with sqlite3.connect(self.project / 'state/quota.sqlite') as con:
|
|
first_id = con.execute('SELECT generation_id FROM generations ORDER BY created_at LIMIT 1').fetchone()[0]
|
|
sends = con.execute('SELECT COUNT(*) FROM attempts WHERE generation_id=?', (first_id,)).fetchone()[0]
|
|
self.assertEqual(sends, 2) # the cut-off request was resent under its own ID, not a new one
|
|
self.assertEqual(json.loads((self.rd / 'execution.json').read_text())['execution_outcome'], 'COMPLETED', output[-3000:])
|
|
|
|
def test_planned_handover_signal_exits_and_the_next_collector_resumes(self):
|
|
first, log = self.collector()
|
|
deadline = time.monotonic() + 60
|
|
while not (self.rd / 'workspace/started').exists():
|
|
self.assertIsNone(first.poll(), log.read_text()[-3000:]); self.assertLess(time.monotonic(), deadline)
|
|
time.sleep(.2)
|
|
first.send_signal(signal.SIGUSR1)
|
|
self.assertEqual(first.wait(60), 0, log.read_text()[-3000:])
|
|
self.assertIn('handover: exiting now', log.read_text())
|
|
time.sleep(3)
|
|
self.check_resumed(self.finish(), told=False)
|
|
|
|
def test_rollout_resumes_after_power_loss_and_reboot(self):
|
|
self.crash_while_command_runs(power_loss=True)
|
|
self.pretend_earlier_boot()
|
|
self.check_resumed(self.finish(), told=True)
|
|
|
|
def pretend_earlier_boot(self):
|
|
"""As if the VM rebooted after the kill: everything recorded belongs to an earlier boot, and the reapers
|
|
died with it before writing any receipt."""
|
|
from harness import execution
|
|
real = execution.boot_id()
|
|
|
|
def older(value):
|
|
if isinstance(value, dict):
|
|
return {k: ('earlier-boot' if k == 'boot_id' and v == real else older(v)) for k, v in value.items()}
|
|
return [older(v) for v in value] if isinstance(value, list) else value
|
|
for path in [self.rd / 'execution.json', *self.rd.glob('stages/_queue/*/operation.json'),
|
|
*(self.project / 'state/locks').glob('*.json')]:
|
|
execution.atomic_json(path, older(json.loads(path.read_text())))
|
|
for receipt in (self.rd / 'process-receipts').glob('*.json'):
|
|
receipt.unlink()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|