机器忙时读账本超过 0.5 秒:领题前检查失败会作废题目(10-13 下午约每小时 17 条), 续跑前检查失败会让暂停的轨迹退回去再等(vit.l.16 等反复退回好几个小时)。 INCOMPLETE_GENERATION 和 REQUEST_OUTCOME_UNKNOWN 一样,都是丢了一次回复, 只要账本记录了请求已结束(下面的未结束请求检查仍然生效),就可以从存档点重发。 这些代码都在调度程序进程里,重启后才生效。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
273 lines
14 KiB
Python
273 lines
14 KiB
Python
"""Small wiring layer for the shared broker, Pi checkpoint and lifecycle ledger."""
|
|
from __future__ import annotations
|
|
|
|
import atexit
|
|
from contextlib import ExitStack
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
import threading
|
|
|
|
from . import paths
|
|
from .execution import atomic_json, resume_policy, validate_policy
|
|
from .quota import Broker, QuotaError, QuotaPolicy, Upstream, quota_snapshot, validate_pi_settings
|
|
|
|
_lock = threading.Lock()
|
|
_broker = None
|
|
_configuration = None
|
|
|
|
|
|
def config():
|
|
value = paths.config().get('collection_v1')
|
|
if not isinstance(value, dict) or not value.get('version'):
|
|
raise ValueError('COLLECTION_CONFIG_INCOMPLETE')
|
|
ledger = Path(value.get('quota_ledger', ''))
|
|
if not ledger.is_absolute():
|
|
raise ValueError('QUOTA_CONFIG_INCOMPLETE: absolute shared ledger required')
|
|
groups = {name: QuotaPolicy(**policy) for name, policy in value.get('quota_groups', {}).items()}
|
|
if not groups:
|
|
raise ValueError('QUOTA_CONFIG_INCOMPLETE')
|
|
if not paths.config().get('rollouts', {}).get('private_home'):
|
|
raise ValueError('PRIVATE_HOME_REQUIRED')
|
|
validate_policy(value.get('lifecycle', {}))
|
|
aliases = value.get('credential_groups', {})
|
|
for alias, group in aliases.items():
|
|
if group not in groups or alias not in paths.config()['endpoints']:
|
|
raise ValueError('QUOTA_CONFIG_INCOMPLETE: unknown credential group')
|
|
if not aliases:
|
|
raise ValueError('QUOTA_CONFIG_INCOMPLETE: no credential group')
|
|
return value, groups
|
|
|
|
|
|
def preflight(policy):
|
|
"""Validate local prerequisites before a queued attempt is claimed; no HTTP."""
|
|
import subprocess
|
|
value, _ = config()
|
|
if policy not in value['credential_groups']:
|
|
raise ValueError('QUOTA_CONFIG_INCOMPLETE: ' + policy)
|
|
for alias in value['credential_groups']:
|
|
endpoint = paths.config()['endpoints'][alias]
|
|
if not paths.secret(endpoint['api_key_env']):
|
|
raise ValueError('CREDENTIAL_UNAVAILABLE: ' + alias)
|
|
broker() # Validate the persistent shared policy before claim; never send a request.
|
|
# 0.5 s (the default) timed out whenever the controller was busy (10-13: ~17 rollouts/hour lost).
|
|
quota = quota_snapshot(Path(value['quota_ledger']), timeout_s=5)
|
|
group = next((row for row in quota.get('groups', [])
|
|
if row['group_id'] == value['credential_groups'][policy]), None)
|
|
if quota['status'] != 'OBSERVED' or group is None:
|
|
raise QuotaError(quota.get('reason') or 'QUOTA_SOURCE_UNAVAILABLE')
|
|
if group['state'] not in ('OPEN', 'COOLING', 'HALF_OPEN', 'RECOVERING'):
|
|
raise QuotaError(group['reason'] or group['state'] or 'QUOTA_SOURCE_UNAVAILABLE')
|
|
node = str(paths.expand(paths.config()['paths'].get('node_bin') or 'node'))
|
|
script = "const p=require(process.argv[1]).piRoot();process.stdout.write(require(p+'/package.json').version)"
|
|
try:
|
|
found = subprocess.run([node, '-e', script, str(paths.AGENT / 'dump_tools.js')],
|
|
capture_output=True, text=True, timeout=10)
|
|
except (OSError, subprocess.TimeoutExpired) as error:
|
|
raise ValueError('PI_SDK_UNAVAILABLE') from error
|
|
if found.returncode or found.stdout.strip() != '0.73.1':
|
|
raise ValueError('UNSUPPORTED_PI_VERSION')
|
|
|
|
|
|
def broker():
|
|
global _broker, _configuration
|
|
value, groups = config()
|
|
fingerprint = json.dumps(value, sort_keys=True)
|
|
with _lock:
|
|
if _broker is not None:
|
|
if fingerprint != _configuration:
|
|
raise ValueError('RUNTIME_CONFIG_CHANGED: drain before reload')
|
|
return _broker
|
|
upstreams = {}
|
|
for alias, group in value['credential_groups'].items():
|
|
endpoint = paths.config()['endpoints'][alias]
|
|
key = paths.secret(endpoint['api_key_env'])
|
|
if not key:
|
|
raise ValueError('CREDENTIAL_UNAVAILABLE: ' + alias)
|
|
upstreams[alias] = Upstream(group, endpoint['base_url'], key, (endpoint['model'],))
|
|
instance = Broker(Path(value['quota_ledger']), groups, upstreams)
|
|
instance.start()
|
|
_broker, _configuration = instance, fingerprint
|
|
atexit.register(instance.close)
|
|
return instance
|
|
|
|
|
|
def configure_pi(rd, execution, policy, *, resume=False):
|
|
"""Only a scoped local token reaches Pi; never the upstream credential."""
|
|
rd = Path(rd)
|
|
value, groups = config()
|
|
state = execution.snapshot()
|
|
if resume:
|
|
resume_policy(state)
|
|
metadata_path = rd / 'rollout.json'
|
|
metadata = json.loads(metadata_path.read_bytes())
|
|
if (metadata.get('runner') != 'pi'
|
|
or metadata.get('runner_version') not in (None, 'legacy-pi', 'pi-sdk-v1')
|
|
or metadata.get('execution_id') not in (None, state['execution_id'])):
|
|
raise ValueError('ROLLOUT_EXECUTION_MISMATCH')
|
|
if resume and (metadata.get('runner_version') != 'pi-sdk-v1'
|
|
or metadata.get('execution_id') != state['execution_id']):
|
|
raise ValueError('RESUME_SOURCE_IDENTITY_MISSING')
|
|
metadata.update(runner_version='pi-sdk-v1', execution_id=state['execution_id'], execution_managed=True)
|
|
atomic_json(metadata_path, metadata)
|
|
endpoint = paths.config()['endpoints'][policy]
|
|
provider = 'dr-gateway' if policy == 'teacher' else 'student-vllm'
|
|
scope = broker().issue_scope(state['execution_id'], policy, endpoint['model'])
|
|
settings_path = rd / 'workspace/.pi/settings.json'
|
|
settings = json.loads(settings_path.read_bytes())
|
|
settings['retry'] = {'enabled': False, 'provider': {'maxRetries': 0, 'timeoutMs': int(scope.client_timeout_s * 1000)}}
|
|
validate_pi_settings(settings, groups[value['credential_groups'][policy]])
|
|
atomic_json(settings_path, settings)
|
|
model_path = rd / 'workspace/.agent-home/.pi/agent/models.json'
|
|
models = json.loads(model_path.read_bytes())
|
|
selected = models['providers'][provider]
|
|
selected['baseUrl'], selected['apiKey'] = scope.base_url, 'BENCH_RUNNER_API_KEY'
|
|
atomic_json(model_path, {'providers': {provider: selected}})
|
|
from .rollout import rollout_tools
|
|
runner = {'execution_id': state['execution_id'], 'model': endpoint['model'], 'provider': provider,
|
|
'managed_tool_deadline_s': state['config'].get('managed_tool_deadline_s', state['config']['execution_deadline_s']),
|
|
'execution_deadline_monotonic': state['deadline_monotonic'],
|
|
'epoch': state['epoch'], 'ordinary_tool_deadline_s': state['config'].get('ordinary_tool_deadline_s', 300),
|
|
'base_url': scope.base_url, 'client_timeout_ms': settings['retry']['provider']['timeoutMs'],
|
|
'session_dir': str(rd / 'workspace/.pi-sessions'), 'tools': rollout_tools(),
|
|
'prompt_path': str(rd / 'agent/prompt.md'), 'append_system_path': str(rd / 'agent/append-system.md'),
|
|
'resume': resume}
|
|
if resume:
|
|
checkpoint = json.loads((rd / 'agent/session-checkpoint.json').read_bytes())
|
|
data = (rd / 'agent/native-session.jsonl').read_bytes()
|
|
if hashlib.sha256(data).hexdigest() != checkpoint['sha256']:
|
|
raise ValueError('CHECKPOINT_CHANGED')
|
|
target = Path(checkpoint['session_path'])
|
|
if not target.resolve().is_relative_to((rd / 'workspace/.pi-sessions').resolve()):
|
|
raise ValueError('CHECKPOINT_PATH_INVALID')
|
|
# The original append-only session must still equal the protected copy.
|
|
if not target.is_file() or target.is_symlink() or target.read_bytes() != data:
|
|
raise ValueError('CHECKPOINT_CHANGED')
|
|
runner.update(session_path=str(target), session_sha256=checkpoint['sha256'],
|
|
system_prompt=(rd / 'agent/system.md').read_text(), system_sha256=checkpoint['system_sha256'],
|
|
generation=checkpoint['generation'], nonempty_delta=checkpoint['nonempty_delta'])
|
|
atomic_json(rd / 'agent/pi-session.json', runner)
|
|
return scope
|
|
|
|
|
|
class ToolDeadlines:
|
|
"""Incremental parent-side fallback; JS timers cannot bypass a kernel cleanup."""
|
|
def __init__(self, rd, execution):
|
|
self.path = Path(rd) / 'agent/events.jsonl'
|
|
self.execution = execution
|
|
self.offset = self.path.stat().st_size if self.path.exists() else 0
|
|
|
|
def poll(self):
|
|
offset = self.offset
|
|
def events():
|
|
# observe_tools consumes this iterator under its existing state lock:
|
|
# a DONE arriving during lock contention must precede timeout judgment.
|
|
nonlocal offset
|
|
if self.path.exists():
|
|
with ExitStack() as readers:
|
|
stream = readers.enter_context(self.path.open('rb'))
|
|
stream.seek(offset)
|
|
while line := stream.readline():
|
|
if not line.endswith(b'\n'):
|
|
break # Partial boundaries are retried on the next poll.
|
|
try:
|
|
event = json.loads(line.decode('utf-8'))
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
|
logging.warning('TOOL_EVENT_REREAD path=%s offset=%d bytes=%d sha256=%s error=%s',
|
|
self.path, offset, len(line), hashlib.sha256(line).hexdigest(),
|
|
type(error).__name__)
|
|
# Discard buffered bytes, including any following lines.
|
|
stream.close()
|
|
stream = readers.enter_context(self.path.open('rb'))
|
|
stream.seek(offset)
|
|
line = stream.readline()
|
|
if not line.endswith(b'\n'):
|
|
raise ValueError('INCOMPLETE_TOOL_EVENT_REREAD') from error
|
|
event = json.loads(line.decode('utf-8'))
|
|
offset = stream.tell()
|
|
if event.get('type') in ('tool_execution_start', 'tool_execution_end') and event.get('bench_tool'):
|
|
yield event['bench_tool']
|
|
reason = self.execution.observe_tools(events())
|
|
self.offset = offset # Do not discard boundaries if applying them failed.
|
|
if reason:
|
|
# Fence upstream admission before watchdog asks the owned scope to stop.
|
|
broker().cancel(self.execution.snapshot()['execution_id'])
|
|
return reason
|
|
|
|
|
|
def freeze_checkpoint(rd, execution):
|
|
"""Copy stable session bytes out of the writable workspace after Pi exits."""
|
|
from .stageworker import config_bytes
|
|
rd = Path(rd)
|
|
checkpoint = json.loads(config_bytes(rd / 'workspace', str(rd / 'workspace/.bench-session.json')))
|
|
state = execution.snapshot()
|
|
if checkpoint.get('execution_id') != state['execution_id']:
|
|
raise ValueError('CHECKPOINT_EXECUTION_MISMATCH')
|
|
path = Path(checkpoint['session_path'])
|
|
if not path.resolve().is_relative_to((rd / 'workspace/.pi-sessions').resolve()):
|
|
raise ValueError('CHECKPOINT_PATH_INVALID')
|
|
data = config_bytes(rd / 'workspace', str(path))
|
|
if hashlib.sha256(data).hexdigest() != checkpoint.get('sha256'):
|
|
raise ValueError('CHECKPOINT_CHANGED')
|
|
# Binary session snapshots require fsync, like JSON lifecycle records.
|
|
snapshot = rd / 'agent/native-session.jsonl'
|
|
import os
|
|
temp = snapshot.with_suffix('.tmp')
|
|
with temp.open('wb') as stream:
|
|
stream.write(data); stream.flush(); os.fsync(stream.fileno())
|
|
os.replace(temp, snapshot)
|
|
descriptor = os.open(snapshot.parent, os.O_RDONLY)
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
atomic_json(rd / 'agent/session-checkpoint.json', checkpoint)
|
|
return {**checkpoint, 'path': str(snapshot)}
|
|
|
|
|
|
def capture_context(rd):
|
|
"""The first SDK header records the actual system/tools used for sampling."""
|
|
rd = Path(rd)
|
|
with (rd / 'agent/events.jsonl').open() as stream:
|
|
for line in stream:
|
|
event = json.loads(line)
|
|
context = event.get('bench') if event.get('type') == 'session' else None
|
|
if context:
|
|
if context.get('runner_version') != 'pi-sdk-v1':
|
|
raise ValueError('UNSUPPORTED_CONTEXT_SNAPSHOT')
|
|
system = context.get('system_prompt')
|
|
tools = context.get('tools')
|
|
if not isinstance(system, str) or not system or not isinstance(tools, list) or not tools:
|
|
raise ValueError('CONTEXT_SNAPSHOT_MISSING')
|
|
(rd / 'agent/system.md').write_text(system)
|
|
atomic_json(rd / 'agent/tools.json', tools)
|
|
return
|
|
raise ValueError('CONTEXT_SNAPSHOT_MISSING')
|
|
|
|
|
|
def budget_used(rd):
|
|
"""Managed Pi model-behavior budget; HTTP retries are separate broker cost."""
|
|
turns = tokens = 0
|
|
from export.trajectory import _empty_content
|
|
path = Path(rd) / 'agent/events.jsonl'
|
|
if path.exists():
|
|
with path.open(errors='replace') as stream: # events.jsonl reaches hundreds of MB; never read it whole
|
|
lines = [line for line in stream if '"turn_end"' in line]
|
|
for line in lines:
|
|
try:
|
|
event = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
message = event.get('message', {})
|
|
if event.get('type') != 'turn_end':
|
|
continue
|
|
if (message.get('role') == 'assistant' and message.get('stopReason') == 'error'
|
|
and _empty_content(message) and not any(k in message for k in ('tool_calls', 'toolCalls', 'function_call', 'toolCallId'))):
|
|
continue
|
|
turns += 1
|
|
usage = message.get('usage') or {}
|
|
tokens += int(usage.get('input', usage.get('input_tokens', 0)) or 0)
|
|
tokens += int(usage.get('output', usage.get('output_tokens', 0)) or 0)
|
|
return {'turns': turns, 'tokens': tokens}
|