data-gen-bpu/harness/collection_runtime.py
bench 8a1ce8e6d9 额度账本读取时限 0.5 秒放宽到 5 秒;回复被截断且已确认结束的请求可以续跑
机器忙时读账本超过 0.5 秒:领题前检查失败会作废题目(10-13 下午约每小时 17 条),
续跑前检查失败会让暂停的轨迹退回去再等(vit.l.16 等反复退回好几个小时)。
INCOMPLETE_GENERATION 和 REQUEST_OUTCOME_UNKNOWN 一样,都是丢了一次回复,
只要账本记录了请求已结束(下面的未结束请求检查仍然生效),就可以从存档点重发。
这些代码都在调度程序进程里,重启后才生效。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 20:26:31 +08:00

273 lines
14 KiB
Python

"""Small wiring layer for the shared broker, Pi checkpoint and lifecycle ledger."""
from __future__ import annotations
import atexit
from contextlib import ExitStack
import hashlib
import json
import logging
from pathlib import Path
import threading
from . import paths
from .execution import atomic_json, resume_policy, validate_policy
from .quota import Broker, QuotaError, QuotaPolicy, Upstream, quota_snapshot, validate_pi_settings
_lock = threading.Lock()
_broker = None
_configuration = None
def config():
value = paths.config().get('collection_v1')
if not isinstance(value, dict) or not value.get('version'):
raise ValueError('COLLECTION_CONFIG_INCOMPLETE')
ledger = Path(value.get('quota_ledger', ''))
if not ledger.is_absolute():
raise ValueError('QUOTA_CONFIG_INCOMPLETE: absolute shared ledger required')
groups = {name: QuotaPolicy(**policy) for name, policy in value.get('quota_groups', {}).items()}
if not groups:
raise ValueError('QUOTA_CONFIG_INCOMPLETE')
if not paths.config().get('rollouts', {}).get('private_home'):
raise ValueError('PRIVATE_HOME_REQUIRED')
validate_policy(value.get('lifecycle', {}))
aliases = value.get('credential_groups', {})
for alias, group in aliases.items():
if group not in groups or alias not in paths.config()['endpoints']:
raise ValueError('QUOTA_CONFIG_INCOMPLETE: unknown credential group')
if not aliases:
raise ValueError('QUOTA_CONFIG_INCOMPLETE: no credential group')
return value, groups
def preflight(policy):
"""Validate local prerequisites before a queued attempt is claimed; no HTTP."""
import subprocess
value, _ = config()
if policy not in value['credential_groups']:
raise ValueError('QUOTA_CONFIG_INCOMPLETE: ' + policy)
for alias in value['credential_groups']:
endpoint = paths.config()['endpoints'][alias]
if not paths.secret(endpoint['api_key_env']):
raise ValueError('CREDENTIAL_UNAVAILABLE: ' + alias)
broker() # Validate the persistent shared policy before claim; never send a request.
# 0.5 s (the default) timed out whenever the controller was busy (10-13: ~17 rollouts/hour lost).
quota = quota_snapshot(Path(value['quota_ledger']), timeout_s=5)
group = next((row for row in quota.get('groups', [])
if row['group_id'] == value['credential_groups'][policy]), None)
if quota['status'] != 'OBSERVED' or group is None:
raise QuotaError(quota.get('reason') or 'QUOTA_SOURCE_UNAVAILABLE')
if group['state'] not in ('OPEN', 'COOLING', 'HALF_OPEN', 'RECOVERING'):
raise QuotaError(group['reason'] or group['state'] or 'QUOTA_SOURCE_UNAVAILABLE')
node = str(paths.expand(paths.config()['paths'].get('node_bin') or 'node'))
script = "const p=require(process.argv[1]).piRoot();process.stdout.write(require(p+'/package.json').version)"
try:
found = subprocess.run([node, '-e', script, str(paths.AGENT / 'dump_tools.js')],
capture_output=True, text=True, timeout=10)
except (OSError, subprocess.TimeoutExpired) as error:
raise ValueError('PI_SDK_UNAVAILABLE') from error
if found.returncode or found.stdout.strip() != '0.73.1':
raise ValueError('UNSUPPORTED_PI_VERSION')
def broker():
global _broker, _configuration
value, groups = config()
fingerprint = json.dumps(value, sort_keys=True)
with _lock:
if _broker is not None:
if fingerprint != _configuration:
raise ValueError('RUNTIME_CONFIG_CHANGED: drain before reload')
return _broker
upstreams = {}
for alias, group in value['credential_groups'].items():
endpoint = paths.config()['endpoints'][alias]
key = paths.secret(endpoint['api_key_env'])
if not key:
raise ValueError('CREDENTIAL_UNAVAILABLE: ' + alias)
upstreams[alias] = Upstream(group, endpoint['base_url'], key, (endpoint['model'],))
instance = Broker(Path(value['quota_ledger']), groups, upstreams)
instance.start()
_broker, _configuration = instance, fingerprint
atexit.register(instance.close)
return instance
def configure_pi(rd, execution, policy, *, resume=False):
"""Only a scoped local token reaches Pi; never the upstream credential."""
rd = Path(rd)
value, groups = config()
state = execution.snapshot()
if resume:
resume_policy(state)
metadata_path = rd / 'rollout.json'
metadata = json.loads(metadata_path.read_bytes())
if (metadata.get('runner') != 'pi'
or metadata.get('runner_version') not in (None, 'legacy-pi', 'pi-sdk-v1')
or metadata.get('execution_id') not in (None, state['execution_id'])):
raise ValueError('ROLLOUT_EXECUTION_MISMATCH')
if resume and (metadata.get('runner_version') != 'pi-sdk-v1'
or metadata.get('execution_id') != state['execution_id']):
raise ValueError('RESUME_SOURCE_IDENTITY_MISSING')
metadata.update(runner_version='pi-sdk-v1', execution_id=state['execution_id'], execution_managed=True)
atomic_json(metadata_path, metadata)
endpoint = paths.config()['endpoints'][policy]
provider = 'dr-gateway' if policy == 'teacher' else 'student-vllm'
scope = broker().issue_scope(state['execution_id'], policy, endpoint['model'])
settings_path = rd / 'workspace/.pi/settings.json'
settings = json.loads(settings_path.read_bytes())
settings['retry'] = {'enabled': False, 'provider': {'maxRetries': 0, 'timeoutMs': int(scope.client_timeout_s * 1000)}}
validate_pi_settings(settings, groups[value['credential_groups'][policy]])
atomic_json(settings_path, settings)
model_path = rd / 'workspace/.agent-home/.pi/agent/models.json'
models = json.loads(model_path.read_bytes())
selected = models['providers'][provider]
selected['baseUrl'], selected['apiKey'] = scope.base_url, 'BENCH_RUNNER_API_KEY'
atomic_json(model_path, {'providers': {provider: selected}})
from .rollout import rollout_tools
runner = {'execution_id': state['execution_id'], 'model': endpoint['model'], 'provider': provider,
'managed_tool_deadline_s': state['config'].get('managed_tool_deadline_s', state['config']['execution_deadline_s']),
'execution_deadline_monotonic': state['deadline_monotonic'],
'epoch': state['epoch'], 'ordinary_tool_deadline_s': state['config'].get('ordinary_tool_deadline_s', 300),
'base_url': scope.base_url, 'client_timeout_ms': settings['retry']['provider']['timeoutMs'],
'session_dir': str(rd / 'workspace/.pi-sessions'), 'tools': rollout_tools(),
'prompt_path': str(rd / 'agent/prompt.md'), 'append_system_path': str(rd / 'agent/append-system.md'),
'resume': resume}
if resume:
checkpoint = json.loads((rd / 'agent/session-checkpoint.json').read_bytes())
data = (rd / 'agent/native-session.jsonl').read_bytes()
if hashlib.sha256(data).hexdigest() != checkpoint['sha256']:
raise ValueError('CHECKPOINT_CHANGED')
target = Path(checkpoint['session_path'])
if not target.resolve().is_relative_to((rd / 'workspace/.pi-sessions').resolve()):
raise ValueError('CHECKPOINT_PATH_INVALID')
# The original append-only session must still equal the protected copy.
if not target.is_file() or target.is_symlink() or target.read_bytes() != data:
raise ValueError('CHECKPOINT_CHANGED')
runner.update(session_path=str(target), session_sha256=checkpoint['sha256'],
system_prompt=(rd / 'agent/system.md').read_text(), system_sha256=checkpoint['system_sha256'],
generation=checkpoint['generation'], nonempty_delta=checkpoint['nonempty_delta'])
atomic_json(rd / 'agent/pi-session.json', runner)
return scope
class ToolDeadlines:
"""Incremental parent-side fallback; JS timers cannot bypass a kernel cleanup."""
def __init__(self, rd, execution):
self.path = Path(rd) / 'agent/events.jsonl'
self.execution = execution
self.offset = self.path.stat().st_size if self.path.exists() else 0
def poll(self):
offset = self.offset
def events():
# observe_tools consumes this iterator under its existing state lock:
# a DONE arriving during lock contention must precede timeout judgment.
nonlocal offset
if self.path.exists():
with ExitStack() as readers:
stream = readers.enter_context(self.path.open('rb'))
stream.seek(offset)
while line := stream.readline():
if not line.endswith(b'\n'):
break # Partial boundaries are retried on the next poll.
try:
event = json.loads(line.decode('utf-8'))
except (UnicodeDecodeError, json.JSONDecodeError) as error:
logging.warning('TOOL_EVENT_REREAD path=%s offset=%d bytes=%d sha256=%s error=%s',
self.path, offset, len(line), hashlib.sha256(line).hexdigest(),
type(error).__name__)
# Discard buffered bytes, including any following lines.
stream.close()
stream = readers.enter_context(self.path.open('rb'))
stream.seek(offset)
line = stream.readline()
if not line.endswith(b'\n'):
raise ValueError('INCOMPLETE_TOOL_EVENT_REREAD') from error
event = json.loads(line.decode('utf-8'))
offset = stream.tell()
if event.get('type') in ('tool_execution_start', 'tool_execution_end') and event.get('bench_tool'):
yield event['bench_tool']
reason = self.execution.observe_tools(events())
self.offset = offset # Do not discard boundaries if applying them failed.
if reason:
# Fence upstream admission before watchdog asks the owned scope to stop.
broker().cancel(self.execution.snapshot()['execution_id'])
return reason
def freeze_checkpoint(rd, execution):
"""Copy stable session bytes out of the writable workspace after Pi exits."""
from .stageworker import config_bytes
rd = Path(rd)
checkpoint = json.loads(config_bytes(rd / 'workspace', str(rd / 'workspace/.bench-session.json')))
state = execution.snapshot()
if checkpoint.get('execution_id') != state['execution_id']:
raise ValueError('CHECKPOINT_EXECUTION_MISMATCH')
path = Path(checkpoint['session_path'])
if not path.resolve().is_relative_to((rd / 'workspace/.pi-sessions').resolve()):
raise ValueError('CHECKPOINT_PATH_INVALID')
data = config_bytes(rd / 'workspace', str(path))
if hashlib.sha256(data).hexdigest() != checkpoint.get('sha256'):
raise ValueError('CHECKPOINT_CHANGED')
# Binary session snapshots require fsync, like JSON lifecycle records.
snapshot = rd / 'agent/native-session.jsonl'
import os
temp = snapshot.with_suffix('.tmp')
with temp.open('wb') as stream:
stream.write(data); stream.flush(); os.fsync(stream.fileno())
os.replace(temp, snapshot)
descriptor = os.open(snapshot.parent, os.O_RDONLY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
atomic_json(rd / 'agent/session-checkpoint.json', checkpoint)
return {**checkpoint, 'path': str(snapshot)}
def capture_context(rd):
"""The first SDK header records the actual system/tools used for sampling."""
rd = Path(rd)
with (rd / 'agent/events.jsonl').open() as stream:
for line in stream:
event = json.loads(line)
context = event.get('bench') if event.get('type') == 'session' else None
if context:
if context.get('runner_version') != 'pi-sdk-v1':
raise ValueError('UNSUPPORTED_CONTEXT_SNAPSHOT')
system = context.get('system_prompt')
tools = context.get('tools')
if not isinstance(system, str) or not system or not isinstance(tools, list) or not tools:
raise ValueError('CONTEXT_SNAPSHOT_MISSING')
(rd / 'agent/system.md').write_text(system)
atomic_json(rd / 'agent/tools.json', tools)
return
raise ValueError('CONTEXT_SNAPSHOT_MISSING')
def budget_used(rd):
"""Managed Pi model-behavior budget; HTTP retries are separate broker cost."""
turns = tokens = 0
from export.trajectory import _empty_content
path = Path(rd) / 'agent/events.jsonl'
if path.exists():
with path.open(errors='replace') as stream: # events.jsonl reaches hundreds of MB; never read it whole
lines = [line for line in stream if '"turn_end"' in line]
for line in lines:
try:
event = json.loads(line)
except ValueError:
continue
message = event.get('message', {})
if event.get('type') != 'turn_end':
continue
if (message.get('role') == 'assistant' and message.get('stopReason') == 'error'
and _empty_content(message) and not any(k in message for k in ('tool_calls', 'toolCalls', 'function_call', 'toolCallId'))):
continue
turns += 1
usage = message.get('usage') or {}
tokens += int(usage.get('input', usage.get('input_tokens', 0)) or 0)
tokens += int(usage.get('output', usage.get('output_tokens', 0)) or 0)
return {'turns': turns, 'tokens': tokens}