data-gen-bpu/harness/collection_runtime.py
bench 0da66edde0 Resume rollouts after the collector dies (crash, kill, reboot, planned handover)
Processes are disposable; rollout state is resumable. When the collector dies,
the next one reconciles each running rollout from durable state instead of
quarantining or losing it:

- execution: a process of an earlier boot on this host is DEAD; a top-level
  reaper whose owner (the collector) died still proves its subtree empty
  (scopes record the owner that started them); reconcile() pauses a rollout
  with a proven-clean crash checkpoint as CONTROLLER_RESTARTED and marks its
  running tool calls INTERRUPTED; resume() across a reboot rebases monotonic
  clocks by wall time and credits the downtime as a pause
- stageworker.settle_interrupted: an admitted step whose executor is gone ends
  as STAGE_EXECUTOR_EXIT (free compile) and its handle is told directly; a
  board step without a clean board receipt still quarantines; nothing replays
- collection_runtime.crash_checkpoint: capture the real system prompt, freeze
  the session at the last saved message (mid-work phases only); configure_pi
  skips a generation ID the ledger already completed
- pi_session.js: checkpoint after every saved message (and compaction); on a
  recovered resume drop an unfinished trailing reply and answer open tool
  calls with an honest "interrupted by a platform restart" result
- scheduler: a paused rollout's local slot held by a dead holder is released
  once every process is proven gone
- pipeline: SIGUSR1 = planned handover (stop claiming, let board steps
  finish, shorten in-flight request holds, exit); SIGTERM still drains
- export: bench_recovery closes the cut segment; the dropped unfinished reply
  is excluded (UNFINISHED_BEFORE_RESTART)
- tests: test_controller_restart (real Pi on Linux: kill -9 mid-command,
  kill -9 mid-reply, SIGUSR1 handover, power loss + reboot), test_settle_
  interrupted; three tests of the old quarantine-everything rule now assert
  the new outcome while keeping their guarantees (no replay, unrelated PID
  untouched, board lease kept)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 09:29:26 +08:00

300 lines
16 KiB
Python

"""Small wiring layer for the shared broker, Pi checkpoint and lifecycle ledger."""
from __future__ import annotations
import atexit
from contextlib import ExitStack
import hashlib
import json
import logging
from pathlib import Path
import threading
from . import paths
from .execution import atomic_json, resume_policy, validate_policy
from .quota import Broker, QuotaError, QuotaPolicy, Upstream, quota_snapshot, validate_pi_settings
_lock = threading.Lock()
_broker = None
_configuration = None
def config():
value = paths.config().get('collection_v1')
if not isinstance(value, dict) or not value.get('version'):
raise ValueError('COLLECTION_CONFIG_INCOMPLETE')
ledger = Path(value.get('quota_ledger', ''))
if not ledger.is_absolute():
raise ValueError('QUOTA_CONFIG_INCOMPLETE: absolute shared ledger required')
groups = {name: QuotaPolicy(**policy) for name, policy in value.get('quota_groups', {}).items()}
if not groups:
raise ValueError('QUOTA_CONFIG_INCOMPLETE')
if not paths.config().get('rollouts', {}).get('private_home'):
raise ValueError('PRIVATE_HOME_REQUIRED')
validate_policy(value.get('lifecycle', {}))
aliases = value.get('credential_groups', {})
for alias, group in aliases.items():
if group not in groups or alias not in paths.config()['endpoints']:
raise ValueError('QUOTA_CONFIG_INCOMPLETE: unknown credential group')
if not aliases:
raise ValueError('QUOTA_CONFIG_INCOMPLETE: no credential group')
return value, groups
def preflight(policy):
"""Validate local prerequisites before a queued attempt is claimed; no HTTP."""
import subprocess
value, _ = config()
if policy not in value['credential_groups']:
raise ValueError('QUOTA_CONFIG_INCOMPLETE: ' + policy)
for alias in value['credential_groups']:
endpoint = paths.config()['endpoints'][alias]
if not paths.secret(endpoint['api_key_env']):
raise ValueError('CREDENTIAL_UNAVAILABLE: ' + alias)
broker() # Validate the persistent shared policy before claim; never send a request.
# 0.5 s (the default) timed out whenever the controller was busy (10-13: ~17 rollouts/hour lost).
quota = quota_snapshot(Path(value['quota_ledger']), timeout_s=5)
group = next((row for row in quota.get('groups', [])
if row['group_id'] == value['credential_groups'][policy]), None)
if quota['status'] != 'OBSERVED' or group is None:
raise QuotaError(quota.get('reason') or 'QUOTA_SOURCE_UNAVAILABLE')
if group['state'] not in ('OPEN', 'COOLING', 'HALF_OPEN', 'RECOVERING'):
raise QuotaError(group['reason'] or group['state'] or 'QUOTA_SOURCE_UNAVAILABLE')
node = str(paths.expand(paths.config()['paths'].get('node_bin') or 'node'))
script = "const p=require(process.argv[1]).piRoot();process.stdout.write(require(p+'/package.json').version)"
try:
found = subprocess.run([node, '-e', script, str(paths.AGENT / 'dump_tools.js')],
capture_output=True, text=True, timeout=10)
except (OSError, subprocess.TimeoutExpired) as error:
raise ValueError('PI_SDK_UNAVAILABLE') from error
if found.returncode or found.stdout.strip() != '0.73.1':
raise ValueError('UNSUPPORTED_PI_VERSION')
def broker():
global _broker, _configuration
value, groups = config()
fingerprint = json.dumps(value, sort_keys=True)
with _lock:
if _broker is not None:
if fingerprint != _configuration:
raise ValueError('RUNTIME_CONFIG_CHANGED: drain before reload')
return _broker
upstreams = {}
for alias, group in value['credential_groups'].items():
endpoint = paths.config()['endpoints'][alias]
key = paths.secret(endpoint['api_key_env'])
if not key:
raise ValueError('CREDENTIAL_UNAVAILABLE: ' + alias)
upstreams[alias] = Upstream(group, endpoint['base_url'], key, (endpoint['model'],))
instance = Broker(Path(value['quota_ledger']), groups, upstreams)
instance.start()
_broker, _configuration = instance, fingerprint
atexit.register(instance.close)
return instance
def configure_pi(rd, execution, policy, *, resume=False):
"""Only a scoped local token reaches Pi; never the upstream credential."""
rd = Path(rd)
value, groups = config()
state = execution.snapshot()
if resume:
resume_policy(state)
metadata_path = rd / 'rollout.json'
metadata = json.loads(metadata_path.read_bytes())
if (metadata.get('runner') != 'pi'
or metadata.get('runner_version') not in (None, 'legacy-pi', 'pi-sdk-v1')
or metadata.get('execution_id') not in (None, state['execution_id'])):
raise ValueError('ROLLOUT_EXECUTION_MISMATCH')
if resume and (metadata.get('runner_version') != 'pi-sdk-v1'
or metadata.get('execution_id') != state['execution_id']):
raise ValueError('RESUME_SOURCE_IDENTITY_MISSING')
metadata.update(runner_version='pi-sdk-v1', execution_id=state['execution_id'], execution_managed=True)
atomic_json(metadata_path, metadata)
endpoint = paths.config()['endpoints'][policy]
provider = 'dr-gateway' if policy == 'teacher' else 'student-vllm'
scope = broker().issue_scope(state['execution_id'], policy, endpoint['model'])
settings_path = rd / 'workspace/.pi/settings.json'
settings = json.loads(settings_path.read_bytes())
settings['retry'] = {'enabled': False, 'provider': {'maxRetries': 0, 'timeoutMs': int(scope.client_timeout_s * 1000)}}
validate_pi_settings(settings, groups[value['credential_groups'][policy]])
atomic_json(settings_path, settings)
model_path = rd / 'workspace/.agent-home/.pi/agent/models.json'
models = json.loads(model_path.read_bytes())
selected = models['providers'][provider]
selected['baseUrl'], selected['apiKey'] = scope.base_url, 'BENCH_RUNNER_API_KEY'
atomic_json(model_path, {'providers': {provider: selected}})
from .rollout import rollout_tools
runner = {'execution_id': state['execution_id'], 'model': endpoint['model'], 'provider': provider,
'managed_tool_deadline_s': state['config'].get('managed_tool_deadline_s', state['config']['execution_deadline_s']),
'execution_deadline_monotonic': state['deadline_monotonic'],
'epoch': state['epoch'], 'ordinary_tool_deadline_s': state['config'].get('ordinary_tool_deadline_s', 300),
'base_url': scope.base_url, 'client_timeout_ms': settings['retry']['provider']['timeoutMs'],
'session_dir': str(rd / 'workspace/.pi-sessions'), 'tools': rollout_tools(),
'prompt_path': str(rd / 'agent/prompt.md'), 'append_system_path': str(rd / 'agent/append-system.md'),
'resume': resume}
if resume:
checkpoint = json.loads((rd / 'agent/session-checkpoint.json').read_bytes())
data = (rd / 'agent/native-session.jsonl').read_bytes()
if hashlib.sha256(data).hexdigest() != checkpoint['sha256']:
raise ValueError('CHECKPOINT_CHANGED')
target = Path(checkpoint['session_path'])
if not target.resolve().is_relative_to((rd / 'workspace/.pi-sessions').resolve()):
raise ValueError('CHECKPOINT_PATH_INVALID')
# The original append-only session must still equal the protected copy.
if not target.is_file() or target.is_symlink() or target.read_bytes() != data:
raise ValueError('CHECKPOINT_CHANGED')
generation = checkpoint['generation']
if checkpoint.get('recovered'):
# The collector may have died after a reply reached the ledger but before the agent saved it: that ID is
# spent, the resumed agent asks again under the next one. A paused (resumable) ID is reused as usual.
while broker().generation_state(f"{state['execution_id']}:{generation}") in ('COMPLETE', 'FAILED', 'CANCELLED'):
generation += 1
runner.update(session_path=str(target), session_sha256=checkpoint['sha256'],
system_prompt=(rd / 'agent/system.md').read_text(), system_sha256=checkpoint['system_sha256'],
generation=generation, nonempty_delta=checkpoint['nonempty_delta'],
recovered=bool(checkpoint.get('recovered')))
atomic_json(rd / 'agent/pi-session.json', runner)
return scope
class ToolDeadlines:
"""Incremental parent-side fallback; JS timers cannot bypass a kernel cleanup."""
def __init__(self, rd, execution):
self.path = Path(rd) / 'agent/events.jsonl'
self.execution = execution
self.offset = self.path.stat().st_size if self.path.exists() else 0
def poll(self):
offset = self.offset
def events():
# observe_tools consumes this iterator under its existing state lock:
# a DONE arriving during lock contention must precede timeout judgment.
nonlocal offset
if self.path.exists():
with ExitStack() as readers:
stream = readers.enter_context(self.path.open('rb'))
stream.seek(offset)
while line := stream.readline():
if not line.endswith(b'\n'):
break # Partial boundaries are retried on the next poll.
try:
event = json.loads(line.decode('utf-8'))
except (UnicodeDecodeError, json.JSONDecodeError) as error:
logging.warning('TOOL_EVENT_REREAD path=%s offset=%d bytes=%d sha256=%s error=%s',
self.path, offset, len(line), hashlib.sha256(line).hexdigest(),
type(error).__name__)
# Discard buffered bytes, including any following lines.
stream.close()
stream = readers.enter_context(self.path.open('rb'))
stream.seek(offset)
line = stream.readline()
if not line.endswith(b'\n'):
raise ValueError('INCOMPLETE_TOOL_EVENT_REREAD') from error
event = json.loads(line.decode('utf-8'))
offset = stream.tell()
if event.get('type') in ('tool_execution_start', 'tool_execution_end') and event.get('bench_tool'):
yield event['bench_tool']
reason = self.execution.observe_tools(events())
self.offset = offset # Do not discard boundaries if applying them failed.
if reason:
# Fence upstream admission before watchdog asks the owned scope to stop.
broker().cancel(self.execution.snapshot()['execution_id'])
return reason
def freeze_checkpoint(rd, execution):
"""Copy stable session bytes out of the writable workspace after Pi exits."""
from .stageworker import config_bytes
rd = Path(rd)
checkpoint = json.loads(config_bytes(rd / 'workspace', str(rd / 'workspace/.bench-session.json')))
state = execution.snapshot()
if checkpoint.get('execution_id') != state['execution_id']:
raise ValueError('CHECKPOINT_EXECUTION_MISMATCH')
path = Path(checkpoint['session_path'])
if not path.resolve().is_relative_to((rd / 'workspace/.pi-sessions').resolve()):
raise ValueError('CHECKPOINT_PATH_INVALID')
# A long rollout's session passes 1 MiB (10-15..10-17: 17 rollouts, 4 ended as INFRA_ERROR); no size cap here.
data = config_bytes(rd / 'workspace', str(path), limit=None)
if hashlib.sha256(data).hexdigest() != checkpoint.get('sha256'):
raise ValueError('CHECKPOINT_CHANGED')
# Binary session snapshots require fsync, like JSON lifecycle records.
snapshot = rd / 'agent/native-session.jsonl'
import os
temp = snapshot.with_suffix('.tmp')
with temp.open('wb') as stream:
stream.write(data); stream.flush(); os.fsync(stream.fileno())
os.replace(temp, snapshot)
descriptor = os.open(snapshot.parent, os.O_RDONLY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
atomic_json(rd / 'agent/session-checkpoint.json', checkpoint)
return {**checkpoint, 'path': str(snapshot)}
def crash_checkpoint(rd, execution):
"""The collector died while this agent ran. Its runner fsyncs the session and records a checkpoint after every
message, so the last one is a clean boundary: freeze it as a resumable pause. None when that boundary cannot be
proven (no session yet, or the session moved past the last checkpoint); the rollout is then lost as before."""
rd = Path(rd)
try:
capture_context(rd) # the exact system prompt and tools the agent ran with, as after a normal exit
checkpoint = freeze_checkpoint(rd, execution)
except (OSError, ValueError, KeyError, TypeError):
return None
# Mid-work only. An agent that had already finished (or hit its budget or a tool deadline) is not continued:
# its collector died during finalization, and the rollout ends as before.
if checkpoint.get('phase') not in ('WAITING_LLM', 'RUNNING_TOOL', 'PAUSED_INFRA') or checkpoint.get('complete'):
return None
checkpoint.update(phase='PAUSED_INFRA', resumable=True, recovered=True)
atomic_json(rd / 'agent/session-checkpoint.json', {k: v for k, v in checkpoint.items() if k != 'path'})
return checkpoint
def capture_context(rd):
"""The first SDK header records the actual system/tools used for sampling."""
rd = Path(rd)
with (rd / 'agent/events.jsonl').open() as stream:
for line in stream:
event = json.loads(line)
context = event.get('bench') if event.get('type') == 'session' else None
if context:
if context.get('runner_version') != 'pi-sdk-v1':
raise ValueError('UNSUPPORTED_CONTEXT_SNAPSHOT')
system = context.get('system_prompt')
tools = context.get('tools')
if not isinstance(system, str) or not system or not isinstance(tools, list) or not tools:
raise ValueError('CONTEXT_SNAPSHOT_MISSING')
(rd / 'agent/system.md').write_text(system)
atomic_json(rd / 'agent/tools.json', tools)
return
raise ValueError('CONTEXT_SNAPSHOT_MISSING')
def budget_used(rd):
"""Managed Pi model-behavior budget; HTTP retries are separate broker cost."""
turns = tokens = 0
from export.trajectory import _empty_content
path = Path(rd) / 'agent/events.jsonl'
if path.exists():
with path.open(errors='replace') as stream: # events.jsonl reaches hundreds of MB; never read it whole
lines = [line for line in stream if '"turn_end"' in line]
for line in lines:
try:
event = json.loads(line)
except ValueError:
continue
message = event.get('message', {})
if event.get('type') != 'turn_end':
continue
if (message.get('role') == 'assistant' and message.get('stopReason') == 'error'
and _empty_content(message) and not any(k in message for k in ('tool_calls', 'toolCalls', 'function_call', 'toolCallId'))):
continue
turns += 1
usage = message.get('usage') or {}
tokens += int(usage.get('input', usage.get('input_tokens', 0)) or 0)
tokens += int(usage.get('output', usage.get('output_tokens', 0)) or 0)
return {'turns': turns, 'tokens': tokens}