data-gen-bpu/scripts/test_controller_restart.py
bench 72912a6b17 Restart recovery: keep slots for cut-off replies, wait for exiting reapers
Found on the staging collector (10-19): after kill -9 two rollouts whose reply was cut off were read as
not resumable for request_total_s while new rollouts took their slots, and a rollout handed over with a
bash command running was quarantined because the next collector looked before the reaper finished.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 11:02:59 +08:00

275 lines
17 KiB
Python

"""A collector killed mid-rollout (crash, kill -9, reboot) leaves a rollout the next collector resumes.
Real collector processes, real Pi SDK, a local fake model. The first collector is SIGKILLed while the agent's bash
command runs; the second (optionally under a new boot id) must resume the rollout from its last saved message, tell
the agent its command was interrupted, and finish it. Linux only (process receipts need /proc).
Run: BENCH_TEST_NODE=<node> PYTHONPATH=. python3 -m unittest scripts.test_controller_restart
"""
from contextlib import ExitStack
from dataclasses import asdict
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from unittest.mock import patch
import yaml
ROOT = Path(__file__).resolve().parents[1]
NODE = Path(os.environ.get('BENCH_TEST_NODE', '/home/ming01.wang/toolchains/node/bin/node'))
MARK = '平台的调度程序重启了'
def drive(spec_path):
"""One collector, in its own process, against the fixture project in spec."""
spec = json.loads(Path(spec_path).read_text())
from harness import db, paths, pipeline, rollout
project, state = Path(spec['project']), Path(spec['project']) / 'state'
cfg = yaml.safe_load((project / 'config.yaml').read_text())
original_connect, original_readonly = db.connect, db.readonly
def prepare(run_id, job_id, kk, mode, policy, **_):
rd = paths.RUNS / run_id / job_id / 'rollouts' / f'{kk:02d}'
ws = rd / 'workspace'
(ws / '.pi').mkdir(parents=True)
(ws / '.agent-home/.pi/agent').mkdir(parents=True)
(rd / 'agent').mkdir(); (rd / 'handles').mkdir()
(ws / '.pi/settings.json').write_text(json.dumps({'defaultProvider': 'dr-gateway', 'defaultModel': 'fixture',
'retry': {'enabled': False, 'provider': {'maxRetries': 0}}, 'extensions': [], 'skills': []}))
(ws / '.agent-home/.pi/agent/models.json').write_text(json.dumps({'providers': {'dr-gateway': {
'baseUrl': 'http://127.0.0.1:1/v1', 'api': 'openai-completions', 'apiKey': 'BENCH_RUNNER_API_KEY',
'models': [{'id': 'fixture', 'name': 'fixture', 'reasoning': False, 'input': ['text'],
'contextWindow': 10000000, 'maxTokens': 512,
'cost': {'input': 0, 'output': 0, 'cacheRead': 0, 'cacheWrite': 0}}]}}}))
for name in ('system.md', 'append-system.md'):
(rd / 'agent' / name).write_text('Run the command, then report.')
(rd / 'agent/prompt.md').write_text('Run the command, then report.')
(rd / 'rollout.json').write_text(json.dumps({'state': 'RUNNING', 'runner': 'pi', 'runner_version': 'pi-sdk-v1',
'policy': policy, 'mode': mode, 'jail_policy': 'private-home', 'max_turns': 20, 'max_tokens': 200000,
'model': 'fixture'}))
job = {'run_id': run_id, 'job_id': job_id, 'task': 'cls', 'march': 'nash-p', 'backend': 'ptq'}
(rd.parent.parent / 'job.json').write_text(json.dumps(job))
return rd, job, (rd / 'agent/prompt.md').read_text()
with ExitStack() as stack:
for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent, 'AGENT': ROOT / 'agent',
'RUNS': project / 'runs', 'STATE': state, 'CONFIG_PATH': project / 'config.yaml'}.items():
stack.enter_context(patch.object(paths, name, value))
stack.enter_context(patch.object(paths, 'config', return_value=cfg))
stack.enter_context(patch.object(paths, 'secret', return_value='upstream-fixture'))
stack.enter_context(patch.object(db, 'connect', lambda **kw: original_connect(state / 'jobs.sqlite', **kw)))
stack.enter_context(patch.object(db, 'readonly', lambda: original_readonly(state / 'jobs.sqlite')))
stack.enter_context(patch.object(rollout, 'prepare', side_effect=prepare))
from harness import verify
for module in (rollout, verify): # recovery imports it from verify directly
stack.enter_context(patch.object(module, 'verify_rollout', return_value={
'terminal': 'UNVERIFIED', 'claim_consistent': False, 'review': False,
'execution_control': {'db_state': 'UNVERIFIED'}}))
stack.enter_context(patch.object(pipeline, 'refresh_run_dataset', return_value=None))
summary = pipeline.collect_rollouts('r', parallel=1, watch=False, log=lambda m: print(m, flush=True))
print('SUMMARY ' + json.dumps(summary), flush=True)
@unittest.skipUnless(Path('/proc').is_dir() and NODE.is_file(), 'Linux and installed Pi 0.73.1 SDK required')
class ControllerRestart(unittest.TestCase):
def setUp(self):
if os.environ.get('BENCH_TEST_KEEP'):
self.project = Path(tempfile.mkdtemp()) / 'bench'; print('project', self.project)
else:
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup)
self.project = Path(temp.name) / 'bench'
(self.project / 'state').mkdir(parents=True)
self.bodies = []
self.hang_first = threading.Event() # set: the first model request never answers (collector dies mid-reply)
outer = self
class Model(BaseHTTPRequestHandler):
def log_message(self, *_): pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
outer.bodies.append(body)
if len(outer.bodies) == 1 and outer.hang_first.is_set():
time.sleep(60); return
told = any(m.get('role') == 'tool' for m in body['messages']) # any result: report and stop
delta = ({'role': 'assistant', 'content': 'the command was interrupted; reporting that'} if told else
{'role': 'assistant', 'tool_calls': [{'index': 0, 'id': f'call-{len(outer.bodies)}', 'type': 'function',
'function': {'name': 'bash', 'arguments': json.dumps(
{'command': 'touch started && sleep 30 && touch finished'})}}]})
chunks = [dict(id='m', object='chat.completion.chunk', model='fixture',
choices=[dict(index=0, delta=delta, finish_reason=None)]),
dict(id='m', object='chat.completion.chunk', model='fixture',
choices=[dict(index=0, delta={}, finish_reason='stop' if told else 'tool_calls')],
usage={'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15})]
payload = ''.join('data: ' + json.dumps(c) + '\n\n' for c in chunks).encode() + b'data: [DONE]\n\n'
self.send_response(200); self.send_header('Content-Type', 'text/event-stream')
self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload)
server = ThreadingHTTPServer(('127.0.0.1', 0), Model)
threading.Thread(target=server.serve_forever, daemon=True).start()
self.addCleanup(server.server_close); self.addCleanup(server.shutdown)
from harness.quota import QuotaPolicy
from scripts import test_execution
cfg = yaml.safe_load((ROOT / 'config.yaml').read_text())
cfg['paths'].update(work_root=str(self.project / 'work'), node_bin=str(NODE))
cfg['pools']['agent']['workers'] = 1
cfg['rollouts'].update(jail=False, private_home=True, tools=['bash'], hard_ceiling_s=300)
cfg['endpoints']['teacher'].update(base_url=f'http://127.0.0.1:{server.server_port}/v1', model='fixture',
api_key_env='FIXTURE_KEY')
lifecycle = test_execution.policy(); lifecycle['execution_deadline_s'] = 300
policy = QuotaPolicy(max_concurrency=2, rpm=1000, tpm=10000000, max_attempts=3, window_s=.2, wait_budget_s=1,
cooldown_initial_s=.05, cooldown_local_max_s=.1, jitter_fraction=0, connect_timeout_s=.5,
first_byte_timeout_s=2, stream_idle_timeout_s=2, request_total_s=5, poll_s=.05)
cfg['collection_v1'].update(quota_ledger=str(self.project / 'state/quota.sqlite'),
quota_groups={'account': asdict(policy)}, credential_groups={'teacher': 'account'},
lifecycle=lifecycle)
(self.project / 'config.yaml').write_text(yaml.safe_dump(cfg))
from harness import db
with db.connect(self.project / 'state/jobs.sqlite') as con:
db.create_run(con, 'r', {})
db.rollout_upsert(con, 'r', 'j0', 0, 'e2e', 'teacher')
self.rd = self.project / 'runs/r/j0/rollouts/00'
def collector(self, **spec):
path = self.project / f'spec-{time.monotonic_ns()}.json'
path.write_text(json.dumps({'project': str(self.project), **spec}))
log = (self.project / f'{path.stem}.log').open('w')
self.addCleanup(log.close)
proc = subprocess.Popen([sys.executable, '-c', 'import sys; from scripts.test_controller_restart import drive; drive(sys.argv[1])',
str(path)], cwd=ROOT, env={**os.environ, 'PYTHONPATH': str(ROOT),
'BENCH_STATE_DIR': str(self.project / 'state')}, stdout=log, stderr=subprocess.STDOUT)
self.addCleanup(lambda: proc.poll() is None and proc.kill())
return proc, Path(log.name)
def crash_while_command_runs(self, power_loss=False):
first, log = self.collector()
deadline = time.monotonic() + 60
while not (self.rd / 'workspace/started').exists():
self.assertIsNone(first.poll(), log.read_text()[-3000:])
self.assertLess(time.monotonic(), deadline, log.read_text()[-3000:])
time.sleep(.2)
if power_loss:
# Everything dies at once, reapers and agent too: nobody aborts the command or writes a receipt.
from harness import execution
state = execution.Execution(self.rd).snapshot()
for process in sorted(state['processes'], key=lambda p: p.get('role') != 'supervisor'):
if execution.process_status(process) == 'ALIVE':
os.kill(process['pid'], signal.SIGKILL)
first.send_signal(signal.SIGKILL); first.wait(10)
time.sleep(3) # the reapers notice their owner is gone, kill the agent and write their receipts
def finish(self, **spec):
second, log = self.collector(**spec)
try:
second.wait(180)
except subprocess.TimeoutExpired:
second.kill()
self.fail('second collector did not finish:\n' + log.read_text()[-4000:])
return log.read_text()
def check_resumed(self, output, told):
from harness import db
with db.connect(self.project / 'state/jobs.sqlite') as con:
rows = [dict(r) for r in con.execute('SELECT job_id,kk,state FROM rollouts')]
self.assertEqual([(r['job_id'], r['kk']) for r in rows], [('j0', 0)], output[-3000:]) # same rollout, no new kk
# The agent finished normally; only the fixture verifier never writes its verdict back.
self.assertEqual(rows[0]['state'], 'VERIFYING', output[-3000:])
final = json.loads((self.rd / 'execution.json').read_text())
self.assertEqual((final['execution_outcome'], final['cleanup_state']), ('COMPLETED', 'VERIFIED'))
events = [json.loads(line) for line in (self.rd / 'execution.events.jsonl').read_text().splitlines()]
kinds = [e['type'] for e in events]
self.assertIn('EXECUTION_RESUMED', kinds)
self.assertEqual([e.get('reconciliation') for e in events if e['type'] == 'EXECUTION_RECONCILED'],
['CONTROLLER_RESTARTED'])
self.assertTrue((self.rd / 'workspace/started').exists())
self.assertFalse((self.rd / 'workspace/finished').exists()) # the command really was cut off
resent = [b for b in self.bodies if MARK in json.dumps(b, ensure_ascii=False)]
# Killed collector: its reaper SIGTERMs Pi, which records the command as aborted itself. Power loss: nobody
# does, and the resumed runner answers the open call with the interrupted note exactly once.
self.assertEqual(len(resent), int(told), [b['messages'][-1] for b in self.bodies])
agent = [json.loads(line) for line in (self.rd / 'agent/events.jsonl').read_text().splitlines()]
self.assertIn('bench_recovery', [e.get('type') for e in agent])
from export import trajectory
(self.rd / 'agent/system.md').write_text(next(e for e in agent if e.get('bench'))['bench']['system_prompt'])
(self.rd / 'agent/tools.json').write_text(json.dumps(next(e for e in agent if e.get('bench'))['bench']['tools']))
meta = json.loads((self.rd / 'rollout.json').read_text())
frozen = trajectory.freeze(self.rd)
result = trajectory.reconstruct(frozen)
self.assertEqual(result['errors'], [], meta)
self.assertTrue(trajectory.validate(frozen, result)['pass'])
roles = [m['role'] for m in result['messages']]
self.assertEqual(roles[:4], ['system', 'user', 'assistant', 'toolResult'], roles)
self.assertEqual(MARK in json.dumps(result['messages'][3], ensure_ascii=False), told)
self.assertEqual(roles[-1], 'assistant')
def test_killed_collector_rollout_resumes_from_its_last_message(self):
self.crash_while_command_runs()
self.check_resumed(self.finish(), told=False)
def test_collector_killed_mid_reply_resends_the_same_request(self):
self.hang_first.set()
first, log = self.collector()
deadline = time.monotonic() + 60
while not self.bodies:
self.assertIsNone(first.poll(), log.read_text()[-3000:]); self.assertLess(time.monotonic(), deadline)
time.sleep(.2)
first.send_signal(signal.SIGKILL); first.wait(10); time.sleep(3)
# The cut-off request may resume only once the ledger gives up on it (request_total_s): a watching collector
# keeps waiting; these one-pass collectors are simply started again.
for _ in range(10):
output = self.finish()
if json.loads((self.rd / 'execution.json').read_text()).get('execution_outcome'):
break
time.sleep(2)
self.assertEqual(self.bodies[1]['messages'], self.bodies[0]['messages'], output[-2000:]) # same request again
import sqlite3
with sqlite3.connect(self.project / 'state/quota.sqlite') as con:
first_id = con.execute('SELECT generation_id FROM generations ORDER BY created_at LIMIT 1').fetchone()[0]
sends = con.execute('SELECT COUNT(*) FROM attempts WHERE generation_id=?', (first_id,)).fetchone()[0]
self.assertEqual(sends, 2) # the cut-off request was resent under its own ID, not a new one
self.assertEqual(json.loads((self.rd / 'execution.json').read_text())['execution_outcome'], 'COMPLETED', output[-3000:])
def test_planned_handover_signal_exits_and_the_next_collector_resumes(self):
first, log = self.collector()
deadline = time.monotonic() + 60
while not (self.rd / 'workspace/started').exists():
self.assertIsNone(first.poll(), log.read_text()[-3000:]); self.assertLess(time.monotonic(), deadline)
time.sleep(.2)
first.send_signal(signal.SIGUSR1)
self.assertEqual(first.wait(60), 0, log.read_text()[-3000:])
self.assertIn('handover: exiting now', log.read_text())
# No pause: the next collector starts while the command's reaper is still killing its tree (10-19 staging).
self.check_resumed(self.finish(), told=False)
def test_rollout_resumes_after_power_loss_and_reboot(self):
self.crash_while_command_runs(power_loss=True)
self.pretend_earlier_boot()
self.check_resumed(self.finish(), told=True)
def pretend_earlier_boot(self):
"""As if the VM rebooted after the kill: everything recorded belongs to an earlier boot, and the reapers
died with it before writing any receipt."""
from harness import execution
real = execution.boot_id()
def older(value):
if isinstance(value, dict):
return {k: ('earlier-boot' if k == 'boot_id' and v == real else older(v)) for k, v in value.items()}
return [older(v) for v in value] if isinstance(value, list) else value
for path in [self.rd / 'execution.json', *self.rd.glob('stages/_queue/*/operation.json'),
*(self.project / 'state/locks').glob('*.json')]:
execution.atomic_json(path, older(json.loads(path.read_text())))
for receipt in (self.rd / 'process-receipts').glob('*.json'):
receipt.unlink()
if __name__ == '__main__':
unittest.main()