Processes are disposable; rollout state is resumable. When the collector dies, the next one reconciles each running rollout from durable state instead of quarantining or losing it: - execution: a process of an earlier boot on this host is DEAD; a top-level reaper whose owner (the collector) died still proves its subtree empty (scopes record the owner that started them); reconcile() pauses a rollout with a proven-clean crash checkpoint as CONTROLLER_RESTARTED and marks its running tool calls INTERRUPTED; resume() across a reboot rebases monotonic clocks by wall time and credits the downtime as a pause - stageworker.settle_interrupted: an admitted step whose executor is gone ends as STAGE_EXECUTOR_EXIT (free compile) and its handle is told directly; a board step without a clean board receipt still quarantines; nothing replays - collection_runtime.crash_checkpoint: capture the real system prompt, freeze the session at the last saved message (mid-work phases only); configure_pi skips a generation ID the ledger already completed - pi_session.js: checkpoint after every saved message (and compaction); on a recovered resume drop an unfinished trailing reply and answer open tool calls with an honest "interrupted by a platform restart" result - scheduler: a paused rollout's local slot held by a dead holder is released once every process is proven gone - pipeline: SIGUSR1 = planned handover (stop claiming, let board steps finish, shorten in-flight request holds, exit); SIGTERM still drains - export: bench_recovery closes the cut segment; the dropped unfinished reply is excluded (UNFINISHED_BEFORE_RESTART) - tests: test_controller_restart (real Pi on Linux: kill -9 mid-command, kill -9 mid-reply, SIGUSR1 handover, power loss + reboot), test_settle_ interrupted; three tests of the old quarantine-everything rule now assert the new outcome while keeping their guarantees (no replay, unrelated PID untouched, board lease kept) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
360 lines
20 KiB
JavaScript
360 lines
20 KiB
JavaScript
#!/usr/bin/env node
|
|
/** Pi 0.73.1 SDK runner: durable session boundaries and prompt-free continuation. */
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
const crypto = require('node:crypto');
|
|
const {pathToFileURL} = require('node:url');
|
|
const {piRoot, toFunction} = require('./dump_tools.js');
|
|
const {createBudget} = require('./runner-budget.js');
|
|
const {installBrokerTransport} = require('./broker-transport.js');
|
|
|
|
const hash = value => crypto.createHash('sha256').update(value).digest('hex');
|
|
// The answer a tool call gets when the collector died while it ran (agent/pi_session.js recovered resume).
|
|
const INTERRUPTED = '【平台】执行这个工具调用期间,平台的调度程序重启了,这次调用被中断、没有拿到结果(命令可能已经执行了一部分)。' +
|
|
'正式阶段和开发命令被打断的不计编译次数,它们的 handle 会显示被打断。需要的话先检查当前状态,再决定是否重新执行。';
|
|
|
|
/** Close the tool calls a dead collector left open: every call of the last assistant message without a result. */
|
|
function interruptedResults(messages) {
|
|
const last = messages.findLastIndex(message => message.role === 'assistant');
|
|
if (last < 0) return [];
|
|
const answered = new Set(messages.slice(last + 1).filter(m => m.role === 'toolResult').map(m => m.toolCallId));
|
|
return (Array.isArray(messages[last].content) ? messages[last].content : [])
|
|
.filter(block => block.type === 'toolCall' && !answered.has(block.id))
|
|
.map(block => ({role: 'toolResult', toolCallId: block.id, toolName: block.name,
|
|
content: [{type: 'text', text: INTERRUPTED}], isError: true, timestamp: Date.now()}));
|
|
}
|
|
const encode = value => JSON.stringify(value) + '\n';
|
|
|
|
function syncFile(file) {
|
|
const fd = fs.openSync(file, 'r');
|
|
try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); }
|
|
}
|
|
|
|
function atomic(file, value) {
|
|
const temp = file + '.tmp';
|
|
const fd = fs.openSync(temp, 'w', 0o600);
|
|
try { fs.writeFileSync(fd, encode(value)); fs.fsyncSync(fd); } finally { fs.closeSync(fd); }
|
|
fs.renameSync(temp, file);
|
|
const directory = fs.openSync(path.dirname(file), 'r');
|
|
try { fs.fsyncSync(directory); } finally { fs.closeSync(directory); }
|
|
}
|
|
|
|
function emptyProviderError(message) {
|
|
if (message?.role !== 'assistant' || message.stopReason !== 'error') return false;
|
|
if (['tool_calls', 'toolCalls', 'function_call', 'toolCallId'].some(key => Object.hasOwn(message, key))) return false;
|
|
if (!Object.hasOwn(message, 'content')) return false;
|
|
const value = message.content;
|
|
return value === null || value === '' || (Array.isArray(value) && value.every(block =>
|
|
['text', 'output_text'].includes(block?.type) && block.text === '' &&
|
|
Object.keys(block).every(key => ['type', 'text'].includes(key))));
|
|
}
|
|
|
|
function closed(messages) {
|
|
const calls = new Set(), results = new Set();
|
|
for (const message of messages) {
|
|
for (const block of Array.isArray(message.content) ? message.content : []) {
|
|
if (block.type !== 'toolCall') continue;
|
|
if (!block.id || calls.has(block.id)) return false;
|
|
calls.add(block.id);
|
|
}
|
|
if (message.role === 'toolResult') {
|
|
if (!calls.has(message.toolCallId) || results.has(message.toolCallId)) return false;
|
|
results.add(message.toolCallId);
|
|
}
|
|
}
|
|
return calls.size === results.size;
|
|
}
|
|
|
|
function continuationBoundary(messages) {
|
|
// Supported Pi custom context is real input; keep its position and payload.
|
|
// Require a completed user/tool boundary below any trailing custom messages.
|
|
let index = messages.length - 1;
|
|
while (messages[index]?.role === 'custom') {
|
|
if (typeof messages[index].customType !== 'string' || !messages[index].customType ||
|
|
!Object.hasOwn(messages[index], 'content')) return false;
|
|
index -= 1;
|
|
}
|
|
return ['user', 'toolResult'].includes(messages[index]?.role);
|
|
}
|
|
|
|
async function main(configFile) {
|
|
const config = JSON.parse(fs.readFileSync(configFile, 'utf8'));
|
|
const toolCap = config.ordinary_tool_deadline_s ?? 300;
|
|
if (!Number.isFinite(toolCap) || toolCap <= 0) throw new Error('INVALID_TOOL_DEADLINE');
|
|
const managedCap = config.managed_tool_deadline_s ?? toolCap;
|
|
if (!Number.isFinite(managedCap) || managedCap <= 0) throw new Error('INVALID_MANAGED_TOOL_DEADLINE');
|
|
if (config.execution_deadline_monotonic !== undefined &&
|
|
(!Number.isFinite(config.execution_deadline_monotonic) || config.execution_deadline_monotonic <= 0)) {
|
|
throw new Error('INVALID_EXECUTION_DEADLINE');
|
|
}
|
|
const budget = createBudget(config);
|
|
const root = piRoot();
|
|
let generation = config.generation || 0, activeGeneration = null, nonemptyDelta = false;
|
|
const restoreTransport = installBrokerTransport(config.base_url, config.client_timeout_ms, root, error => {
|
|
process.stderr.write(encode({type: 'bench_transport_error', execution_id: config.execution_id,
|
|
generation_id: activeGeneration, ...error}));
|
|
});
|
|
const version = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')).version;
|
|
if (version !== '0.73.1') throw new Error('UNSUPPORTED_PI_VERSION');
|
|
const pi = await import(pathToFileURL(path.join(root, 'dist/index.js')).href);
|
|
const cwd = process.cwd(), agentDir = process.env.PI_CODING_AGENT_DIR;
|
|
if (!agentDir || !config.execution_id) throw new Error('PRIVATE_SCOPE_REQUIRED');
|
|
const settings = pi.SettingsManager.create(cwd, agentDir);
|
|
if (config.resume && hash(fs.readFileSync(config.session_path)) !== config.session_sha256) {
|
|
throw new Error('CHECKPOINT_SESSION_CHANGED');
|
|
}
|
|
const effective = settings.getRetrySettings(), providerRetry = settings.getProviderRetrySettings();
|
|
if (effective.enabled !== false || providerRetry.maxRetries !== 0 || providerRetry.timeoutMs < config.client_timeout_ms) {
|
|
throw new Error('SDK_RETRY_CONFIGURATION_UNSAFE');
|
|
}
|
|
const resourceLoader = new pi.DefaultResourceLoader({cwd, agentDir, settingsManager: settings,
|
|
...(config.resume ? {systemPrompt: config.system_prompt, appendSystemPrompt: []} :
|
|
{appendSystemPrompt: [fs.readFileSync(config.append_system_path, 'utf8')]})});
|
|
await resourceLoader.reload();
|
|
if (resourceLoader.getExtensions().errors.length) throw new Error('EXTENSION_LOAD_FAILED');
|
|
const manager = config.resume ? pi.SessionManager.open(config.session_path, config.session_dir, cwd) :
|
|
pi.SessionManager.create(cwd, config.session_dir);
|
|
const result = await pi.createAgentSession({cwd, agentDir, settingsManager: settings,
|
|
resourceLoader, sessionManager: manager, tools: config.tools});
|
|
const session = result.session;
|
|
if (config.resume && session.state.messages.some(message => message.customType === 'bench-budget')) {
|
|
budget.markAnnouncementSeen();
|
|
}
|
|
if (result.modelFallbackMessage || session.model?.id !== config.model || session.model?.provider !== config.provider ||
|
|
session.model?.baseUrl !== config.base_url) throw new Error('MODEL_OR_BROKER_MISMATCH');
|
|
await session.bindExtensions({onError() { throw new Error('EXTENSION_RUNTIME_FAILED'); }});
|
|
if (config.resume) {
|
|
if (hash(config.system_prompt) !== config.system_sha256) throw new Error('CHECKPOINT_SYSTEM_CHANGED');
|
|
// Pi normally rebuilds its dynamic date/cwd preamble when opening a session.
|
|
// Continuation uses the exact protected prompt from the original execution.
|
|
session.agent.state.systemPrompt = config.system_prompt;
|
|
}
|
|
const file = path.join(cwd, '.bench-session.json');
|
|
let lastEvent = null, lastPhase = 'STARTING';
|
|
let toolStop = null;
|
|
const activeTools = new Map();
|
|
const checkpoint = (phase, complete = false, resumable = false) => {
|
|
lastPhase = phase;
|
|
const sessionPath = manager.getSessionFile();
|
|
let sessionHash = null;
|
|
if (sessionPath && fs.existsSync(sessionPath)) {
|
|
syncFile(sessionPath);
|
|
sessionHash = hash(fs.readFileSync(sessionPath));
|
|
}
|
|
atomic(file, {schema: 'bench.pi-checkpoint/1', execution_id: config.execution_id, phase, complete, resumable, budget: budget.snapshot(),
|
|
session_path: sessionPath, sha256: sessionHash, generation, generation_id: activeGeneration,
|
|
nonempty_delta: nonemptyDelta, last_event: lastEvent, model: config.model, provider: config.provider,
|
|
system_sha256: hash(session.systemPrompt), closure: closed(session.state.messages), tool_stop: toolStop});
|
|
};
|
|
// Pi's append is synchronous but does not fsync. Persist the session before
|
|
// the pre-tool hook can admit a side effect; leave original errors untouched.
|
|
for (const name of ['appendMessage', 'appendCustomMessageEntry', 'appendCompaction']) {
|
|
const append = manager[name].bind(manager);
|
|
manager[name] = (...args) => {
|
|
const id = append(...args);
|
|
const sessionPath = manager.getSessionFile();
|
|
if (sessionPath && fs.existsSync(sessionPath)) syncFile(sessionPath);
|
|
// Every saved message is a boundary a crashed collector can resume from (collection_runtime.crash_checkpoint).
|
|
checkpoint(lastPhase);
|
|
return id;
|
|
};
|
|
}
|
|
const output = event => {
|
|
fs.writeSync(1, encode(event));
|
|
try { fs.fsyncSync(1); } catch (error) { if (!['EINVAL', 'ENOTSUP'].includes(error.code)) throw error; }
|
|
};
|
|
// Marks the end of a segment the dead collector never closed (export/trajectory.py).
|
|
if (config.recovered) output({type: 'bench_recovery', execution_id: config.execution_id, epoch: config.epoch ?? 1});
|
|
output({...manager.getHeader(), bench: {runner_version: 'pi-sdk-v1', execution_id: config.execution_id,
|
|
system_prompt: session.systemPrompt, tools: session.agent.state.tools.map(toFunction)}});
|
|
const monotonic = () => Number(process.hrtime.bigint()) / 1e9;
|
|
const expireTool = call => {
|
|
if (toolStop) return;
|
|
toolStop = {...call, reason: 'TOOL_DEADLINE_EXCEEDED'};
|
|
checkpoint('TOOL_DEADLINE_EXCEEDED');
|
|
// Stop the entire attempt. The outer kernel scope, not SDK abort, proves cleanup.
|
|
session.agent.abort();
|
|
};
|
|
const beforeTool = session.agent.beforeToolCall;
|
|
session.agent.beforeToolCall = async (context, signal) => {
|
|
if (toolStop || signal?.aborted) return {block: true, reason: toolStop?.reason || 'ATTEMPT_ABORTED'};
|
|
const budgetBlock = budget.toolBlock(context.assistantMessage);
|
|
if (budgetBlock) return budgetBlock;
|
|
const blocked = beforeTool ? await beforeTool(context, signal) : undefined;
|
|
if (blocked) return blocked;
|
|
// Pi 0.73.1 skips its queue barrier when no extension handles tool_call.
|
|
// The pinned SDK queue must finish append+fsync before any side effect.
|
|
if (!session._agentEventQueue?.then) throw new Error('UNSUPPORTED_PI_EVENT_QUEUE');
|
|
await session._agentEventQueue;
|
|
if (toolStop || signal?.aborted) return {block: true, reason: toolStop?.reason || 'ATTEMPT_ABORTED'};
|
|
const active = activeTools.get(context.toolCall.id);
|
|
if (!active) throw new Error('TOOL_START_NOT_PERSISTED');
|
|
// Bash has no default timeout, so a long command would reach the system cap
|
|
// below and end the whole attempt. Give it a native timeout a minute earlier
|
|
// so only that command fails. args is Pi's private copy; the persisted call
|
|
// keeps the model's own arguments.
|
|
if (context.toolCall.name === 'bash') {
|
|
const room = Math.floor(active.call.deadline_monotonic - monotonic()) - 60;
|
|
if (room > 0 && !(context.args.timeout > 0 && context.args.timeout <= room)) context.args.timeout = room;
|
|
}
|
|
// A shorter native bash timeout remains a normal, recoverable tool error.
|
|
// This independent system cap ends the attempt and cannot be extended by args.
|
|
active.timer = setTimeout(() => expireTool(active.call),
|
|
Math.max(1, Math.ceil((active.call.deadline_monotonic - monotonic()) * 1000)));
|
|
checkpoint('RUNNING_TOOL');
|
|
if (monotonic() >= active.call.deadline_monotonic) {
|
|
expireTool(active.call);
|
|
return {block: true, reason: toolStop.reason};
|
|
}
|
|
return undefined;
|
|
};
|
|
const afterTool = session.agent.afterToolCall;
|
|
session.agent.afterToolCall = async (context, signal) => {
|
|
const active = activeTools.get(context.toolCall.id);
|
|
if (active) {
|
|
clearTimeout(active.timer);
|
|
if (monotonic() >= active.call.deadline_monotonic) expireTool(active.call);
|
|
}
|
|
const result = afterTool ? await afterTool(context, signal) : undefined;
|
|
if (!toolStop) return result;
|
|
// Keep native content (including partial output) and the original call ID.
|
|
return {...result, isError: true, terminate: true,
|
|
details: {...(result?.details ?? context.result.details), bench_stop_reason: toolStop.reason}};
|
|
};
|
|
const stream = session.agent.streamFn;
|
|
session.agent.streamFn = (model, context, options) => {
|
|
if (toolStop) throw new Error(toolStop.reason);
|
|
budget.beforeGeneration();
|
|
activeGeneration = config.execution_id + ':' + generation;
|
|
nonemptyDelta = false;
|
|
checkpoint('WAITING_LLM');
|
|
return stream(model, context, {...options,
|
|
headers: {...options?.headers, 'X-Bench-Generation-Id': activeGeneration}});
|
|
};
|
|
// Version-bound SDK hook: the native loop closes tool results, emits turn_end
|
|
// then agent_end itself. Never abort/exit merely because the last turn ended.
|
|
if (typeof session.agent.createLoopConfig !== 'function') throw new Error('UNSUPPORTED_PI_LOOP_CONFIG');
|
|
const createLoopConfig = session.agent.createLoopConfig.bind(session.agent);
|
|
session.agent.createLoopConfig = (...args) => {
|
|
const loop = createLoopConfig(...args);
|
|
const steering = loop.getSteeringMessages;
|
|
return {...loop,
|
|
getSteeringMessages: async () => [...(await steering?.() || []), ...budget.announcement()],
|
|
shouldStopAfterTurn: async context => {
|
|
budget.observe(context.message, emptyProviderError);
|
|
const rollout = process.env.BENCH_ROLLOUT_DIR;
|
|
if (rollout) {
|
|
const target = path.join(rollout, process.env.BENCH_JAILED === '1' ? 'workspace/budget.json' : 'rollout.json');
|
|
let record = {};
|
|
try { record = JSON.parse(fs.readFileSync(target, 'utf8')); } catch {}
|
|
atomic(target, {...record, ...budget.snapshot()});
|
|
}
|
|
return budget.shouldStop() || Boolean(await loop.shouldStopAfterTurn?.(context));
|
|
},
|
|
};
|
|
};
|
|
let compactedRetry = false;
|
|
session.subscribe(event => {
|
|
lastEvent = event.type;
|
|
let compaction;
|
|
if (event.type === 'compaction_end' && event.willRetry) {
|
|
// Pi compacted after a context overflow and resends: the failed generation's ID is spent. kept_messages is
|
|
// how many earlier messages the model still sees after the summary (export/trajectory.py builds that view).
|
|
generation += 1; compactedRetry = true;
|
|
compaction = {kept_messages: session.agent.state.messages
|
|
.filter(m => m.role !== 'compactionSummary' && !emptyProviderError(m)).length};
|
|
}
|
|
if (event.type === 'message_update') {
|
|
const delta = event.assistantMessageEvent;
|
|
if (delta?.type?.endsWith('_delta') && delta.delta) nonemptyDelta = true;
|
|
}
|
|
if (event.type === 'message_end' && event.message?.role === 'assistant' &&
|
|
!['error', 'aborted'].includes(event.message.stopReason)) generation += 1;
|
|
let boundary;
|
|
if (event.type === 'tool_execution_start') {
|
|
const started = monotonic();
|
|
const requestedCap = event.toolName === 'terminal_exec' ? managedCap : toolCap;
|
|
const cap = Math.max(0.001, Math.min(requestedCap,
|
|
config.execution_deadline_monotonic === undefined ? Infinity : config.execution_deadline_monotonic - started));
|
|
const call = {tool_call_id: event.toolCallId, tool_name: event.toolName,
|
|
started_monotonic: started, deadline_monotonic: started + cap, cap_s: cap};
|
|
activeTools.set(call.tool_call_id, {call});
|
|
boundary = {state: 'RUNNING', ...call};
|
|
} else if (event.type === 'tool_execution_end') {
|
|
const active = activeTools.get(event.toolCallId);
|
|
if (active) {
|
|
clearTimeout(active.timer);
|
|
boundary = {state: toolStop?.tool_call_id === event.toolCallId ? 'TIMED_OUT' : 'DONE', ...active.call};
|
|
activeTools.delete(event.toolCallId);
|
|
}
|
|
}
|
|
// message_update repeats the whole message twice (message, assistantMessageEvent.partial) on every
|
|
// token, ~800MB of events.jsonl per rollout; readers use only its type and delta, message_end has the text.
|
|
const written = event.type === 'message_update'
|
|
? {...event, message: undefined, assistantMessageEvent: {...event.assistantMessageEvent, partial: undefined}}
|
|
: event;
|
|
// Metadata decorates real native boundaries; it never adds conversation text.
|
|
output(boundary ? {...written, bench_tool: {...boundary, execution_id: config.execution_id,
|
|
epoch: config.epoch ?? 1, observed_monotonic: monotonic()}} : compaction ? {...written, bench_compaction: compaction} : written);
|
|
});
|
|
const stop = () => session.agent.abort();
|
|
process.on('SIGTERM', stop); process.on('SIGINT', stop);
|
|
try {
|
|
if (config.resume) {
|
|
if (config.system_sha256 !== hash(session.systemPrompt)) throw new Error('CHECKPOINT_SYSTEM_CHANGED');
|
|
// A paused generation that already streamed must not be replayed. A crash checkpoint is taken after a saved
|
|
// message (complete by construction); whatever was streaming then was never saved, so nothing is replayed.
|
|
if (config.nonempty_delta && !config.recovered) throw new Error('UNSAFE_CONTINUATION');
|
|
// Every earlier continuation required the same complete, empty-error
|
|
// checkpoint. Native errors remain append-only; exclude those proved
|
|
// fragments from working context even after several pauses/tool rounds.
|
|
const messages = session.state.messages.filter(message => !emptyProviderError(message));
|
|
if (config.recovered) {
|
|
// A collector killed mid-reply (or its reaper's SIGTERM aborting Pi) leaves an unfinished assistant message;
|
|
// the model never completed it, so it is dropped like an empty provider error, not continued.
|
|
while (messages.at(-1)?.role === 'assistant' && ['aborted', 'error'].includes(messages.at(-1).stopReason) &&
|
|
!(Array.isArray(messages.at(-1).content) ? messages.at(-1).content : []).some(block => block.type === 'toolCall')) {
|
|
messages.pop();
|
|
}
|
|
for (const result of interruptedResults(messages)) {
|
|
manager.appendMessage(result);
|
|
messages.push(result);
|
|
output({type: 'message_end', message: result});
|
|
}
|
|
}
|
|
if (!closed(messages) || !continuationBoundary(messages)) throw new Error('UNSAFE_CONTINUATION');
|
|
session.agent.state.messages = messages;
|
|
await session.agent.continue();
|
|
} else {
|
|
await session.prompt(fs.readFileSync(config.prompt_path, 'utf8'));
|
|
}
|
|
await session._agentEventQueue;
|
|
// Pi resumes 100 ms after an overflow compaction (agent-session.js _runAutoCompaction): wait for that run too.
|
|
while (compactedRetry) {
|
|
compactedRetry = false;
|
|
await new Promise(resolve => setTimeout(resolve, 300));
|
|
await session.agent.waitForIdle();
|
|
await session._agentEventQueue;
|
|
}
|
|
const last = session.state.messages.at(-1);
|
|
const complete = last?.role === 'assistant' && last.stopReason === 'stop' && closed(session.state.messages);
|
|
const safeResume = !nonemptyDelta && emptyProviderError(last) && closed(session.state.messages) &&
|
|
continuationBoundary(session.state.messages.slice(0, -1));
|
|
const budgetStop = budget.shouldStop() && !complete;
|
|
checkpoint(toolStop ? 'TOOL_DEADLINE_EXCEEDED' : complete ? 'COMPLETED' : budgetStop ? 'BUDGET_EXHAUSTED' : 'PAUSED_INFRA',
|
|
!toolStop && complete, !toolStop && !budgetStop && safeResume);
|
|
process.exitCode = toolStop ? 78 : complete ? 0 : budgetStop ? 75 : 76;
|
|
} finally {
|
|
for (const active of activeTools.values()) clearTimeout(active.timer);
|
|
process.off('SIGTERM', stop); process.off('SIGINT', stop);
|
|
session.dispose();
|
|
restoreTransport();
|
|
}
|
|
}
|
|
|
|
module.exports = {emptyProviderError, closed, continuationBoundary, main};
|
|
if (require.main === module) main(process.argv[2]).catch(error => {
|
|
process.stderr.write(encode({type: 'bench_runner_error', code: error.message?.match(/^[A-Z_]+$/)?.[0] || 'SDK_RUNNER_FAILED'}));
|
|
process.exitCode = 77;
|
|
});
|