data-gen-bpu/agent/pi_session.js
bench 0da66edde0 Resume rollouts after the collector dies (crash, kill, reboot, planned handover)
Processes are disposable; rollout state is resumable. When the collector dies,
the next one reconciles each running rollout from durable state instead of
quarantining or losing it:

- execution: a process of an earlier boot on this host is DEAD; a top-level
  reaper whose owner (the collector) died still proves its subtree empty
  (scopes record the owner that started them); reconcile() pauses a rollout
  with a proven-clean crash checkpoint as CONTROLLER_RESTARTED and marks its
  running tool calls INTERRUPTED; resume() across a reboot rebases monotonic
  clocks by wall time and credits the downtime as a pause
- stageworker.settle_interrupted: an admitted step whose executor is gone ends
  as STAGE_EXECUTOR_EXIT (free compile) and its handle is told directly; a
  board step without a clean board receipt still quarantines; nothing replays
- collection_runtime.crash_checkpoint: capture the real system prompt, freeze
  the session at the last saved message (mid-work phases only); configure_pi
  skips a generation ID the ledger already completed
- pi_session.js: checkpoint after every saved message (and compaction); on a
  recovered resume drop an unfinished trailing reply and answer open tool
  calls with an honest "interrupted by a platform restart" result
- scheduler: a paused rollout's local slot held by a dead holder is released
  once every process is proven gone
- pipeline: SIGUSR1 = planned handover (stop claiming, let board steps
  finish, shorten in-flight request holds, exit); SIGTERM still drains
- export: bench_recovery closes the cut segment; the dropped unfinished reply
  is excluded (UNFINISHED_BEFORE_RESTART)
- tests: test_controller_restart (real Pi on Linux: kill -9 mid-command,
  kill -9 mid-reply, SIGUSR1 handover, power loss + reboot), test_settle_
  interrupted; three tests of the old quarantine-everything rule now assert
  the new outcome while keeping their guarantees (no replay, unrelated PID
  untouched, board lease kept)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 09:29:26 +08:00

360 lines
20 KiB
JavaScript

#!/usr/bin/env node
/** Pi 0.73.1 SDK runner: durable session boundaries and prompt-free continuation. */
const fs = require('node:fs');
const path = require('node:path');
const crypto = require('node:crypto');
const {pathToFileURL} = require('node:url');
const {piRoot, toFunction} = require('./dump_tools.js');
const {createBudget} = require('./runner-budget.js');
const {installBrokerTransport} = require('./broker-transport.js');
const hash = value => crypto.createHash('sha256').update(value).digest('hex');
// The answer a tool call gets when the collector died while it ran (agent/pi_session.js recovered resume).
const INTERRUPTED = '【平台】执行这个工具调用期间,平台的调度程序重启了,这次调用被中断、没有拿到结果(命令可能已经执行了一部分)。' +
'正式阶段和开发命令被打断的不计编译次数,它们的 handle 会显示被打断。需要的话先检查当前状态,再决定是否重新执行。';
/** Close the tool calls a dead collector left open: every call of the last assistant message without a result. */
function interruptedResults(messages) {
const last = messages.findLastIndex(message => message.role === 'assistant');
if (last < 0) return [];
const answered = new Set(messages.slice(last + 1).filter(m => m.role === 'toolResult').map(m => m.toolCallId));
return (Array.isArray(messages[last].content) ? messages[last].content : [])
.filter(block => block.type === 'toolCall' && !answered.has(block.id))
.map(block => ({role: 'toolResult', toolCallId: block.id, toolName: block.name,
content: [{type: 'text', text: INTERRUPTED}], isError: true, timestamp: Date.now()}));
}
const encode = value => JSON.stringify(value) + '\n';
function syncFile(file) {
const fd = fs.openSync(file, 'r');
try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); }
}
function atomic(file, value) {
const temp = file + '.tmp';
const fd = fs.openSync(temp, 'w', 0o600);
try { fs.writeFileSync(fd, encode(value)); fs.fsyncSync(fd); } finally { fs.closeSync(fd); }
fs.renameSync(temp, file);
const directory = fs.openSync(path.dirname(file), 'r');
try { fs.fsyncSync(directory); } finally { fs.closeSync(directory); }
}
function emptyProviderError(message) {
if (message?.role !== 'assistant' || message.stopReason !== 'error') return false;
if (['tool_calls', 'toolCalls', 'function_call', 'toolCallId'].some(key => Object.hasOwn(message, key))) return false;
if (!Object.hasOwn(message, 'content')) return false;
const value = message.content;
return value === null || value === '' || (Array.isArray(value) && value.every(block =>
['text', 'output_text'].includes(block?.type) && block.text === '' &&
Object.keys(block).every(key => ['type', 'text'].includes(key))));
}
function closed(messages) {
const calls = new Set(), results = new Set();
for (const message of messages) {
for (const block of Array.isArray(message.content) ? message.content : []) {
if (block.type !== 'toolCall') continue;
if (!block.id || calls.has(block.id)) return false;
calls.add(block.id);
}
if (message.role === 'toolResult') {
if (!calls.has(message.toolCallId) || results.has(message.toolCallId)) return false;
results.add(message.toolCallId);
}
}
return calls.size === results.size;
}
function continuationBoundary(messages) {
// Supported Pi custom context is real input; keep its position and payload.
// Require a completed user/tool boundary below any trailing custom messages.
let index = messages.length - 1;
while (messages[index]?.role === 'custom') {
if (typeof messages[index].customType !== 'string' || !messages[index].customType ||
!Object.hasOwn(messages[index], 'content')) return false;
index -= 1;
}
return ['user', 'toolResult'].includes(messages[index]?.role);
}
async function main(configFile) {
const config = JSON.parse(fs.readFileSync(configFile, 'utf8'));
const toolCap = config.ordinary_tool_deadline_s ?? 300;
if (!Number.isFinite(toolCap) || toolCap <= 0) throw new Error('INVALID_TOOL_DEADLINE');
const managedCap = config.managed_tool_deadline_s ?? toolCap;
if (!Number.isFinite(managedCap) || managedCap <= 0) throw new Error('INVALID_MANAGED_TOOL_DEADLINE');
if (config.execution_deadline_monotonic !== undefined &&
(!Number.isFinite(config.execution_deadline_monotonic) || config.execution_deadline_monotonic <= 0)) {
throw new Error('INVALID_EXECUTION_DEADLINE');
}
const budget = createBudget(config);
const root = piRoot();
let generation = config.generation || 0, activeGeneration = null, nonemptyDelta = false;
const restoreTransport = installBrokerTransport(config.base_url, config.client_timeout_ms, root, error => {
process.stderr.write(encode({type: 'bench_transport_error', execution_id: config.execution_id,
generation_id: activeGeneration, ...error}));
});
const version = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')).version;
if (version !== '0.73.1') throw new Error('UNSUPPORTED_PI_VERSION');
const pi = await import(pathToFileURL(path.join(root, 'dist/index.js')).href);
const cwd = process.cwd(), agentDir = process.env.PI_CODING_AGENT_DIR;
if (!agentDir || !config.execution_id) throw new Error('PRIVATE_SCOPE_REQUIRED');
const settings = pi.SettingsManager.create(cwd, agentDir);
if (config.resume && hash(fs.readFileSync(config.session_path)) !== config.session_sha256) {
throw new Error('CHECKPOINT_SESSION_CHANGED');
}
const effective = settings.getRetrySettings(), providerRetry = settings.getProviderRetrySettings();
if (effective.enabled !== false || providerRetry.maxRetries !== 0 || providerRetry.timeoutMs < config.client_timeout_ms) {
throw new Error('SDK_RETRY_CONFIGURATION_UNSAFE');
}
const resourceLoader = new pi.DefaultResourceLoader({cwd, agentDir, settingsManager: settings,
...(config.resume ? {systemPrompt: config.system_prompt, appendSystemPrompt: []} :
{appendSystemPrompt: [fs.readFileSync(config.append_system_path, 'utf8')]})});
await resourceLoader.reload();
if (resourceLoader.getExtensions().errors.length) throw new Error('EXTENSION_LOAD_FAILED');
const manager = config.resume ? pi.SessionManager.open(config.session_path, config.session_dir, cwd) :
pi.SessionManager.create(cwd, config.session_dir);
const result = await pi.createAgentSession({cwd, agentDir, settingsManager: settings,
resourceLoader, sessionManager: manager, tools: config.tools});
const session = result.session;
if (config.resume && session.state.messages.some(message => message.customType === 'bench-budget')) {
budget.markAnnouncementSeen();
}
if (result.modelFallbackMessage || session.model?.id !== config.model || session.model?.provider !== config.provider ||
session.model?.baseUrl !== config.base_url) throw new Error('MODEL_OR_BROKER_MISMATCH');
await session.bindExtensions({onError() { throw new Error('EXTENSION_RUNTIME_FAILED'); }});
if (config.resume) {
if (hash(config.system_prompt) !== config.system_sha256) throw new Error('CHECKPOINT_SYSTEM_CHANGED');
// Pi normally rebuilds its dynamic date/cwd preamble when opening a session.
// Continuation uses the exact protected prompt from the original execution.
session.agent.state.systemPrompt = config.system_prompt;
}
const file = path.join(cwd, '.bench-session.json');
let lastEvent = null, lastPhase = 'STARTING';
let toolStop = null;
const activeTools = new Map();
const checkpoint = (phase, complete = false, resumable = false) => {
lastPhase = phase;
const sessionPath = manager.getSessionFile();
let sessionHash = null;
if (sessionPath && fs.existsSync(sessionPath)) {
syncFile(sessionPath);
sessionHash = hash(fs.readFileSync(sessionPath));
}
atomic(file, {schema: 'bench.pi-checkpoint/1', execution_id: config.execution_id, phase, complete, resumable, budget: budget.snapshot(),
session_path: sessionPath, sha256: sessionHash, generation, generation_id: activeGeneration,
nonempty_delta: nonemptyDelta, last_event: lastEvent, model: config.model, provider: config.provider,
system_sha256: hash(session.systemPrompt), closure: closed(session.state.messages), tool_stop: toolStop});
};
// Pi's append is synchronous but does not fsync. Persist the session before
// the pre-tool hook can admit a side effect; leave original errors untouched.
for (const name of ['appendMessage', 'appendCustomMessageEntry', 'appendCompaction']) {
const append = manager[name].bind(manager);
manager[name] = (...args) => {
const id = append(...args);
const sessionPath = manager.getSessionFile();
if (sessionPath && fs.existsSync(sessionPath)) syncFile(sessionPath);
// Every saved message is a boundary a crashed collector can resume from (collection_runtime.crash_checkpoint).
checkpoint(lastPhase);
return id;
};
}
const output = event => {
fs.writeSync(1, encode(event));
try { fs.fsyncSync(1); } catch (error) { if (!['EINVAL', 'ENOTSUP'].includes(error.code)) throw error; }
};
// Marks the end of a segment the dead collector never closed (export/trajectory.py).
if (config.recovered) output({type: 'bench_recovery', execution_id: config.execution_id, epoch: config.epoch ?? 1});
output({...manager.getHeader(), bench: {runner_version: 'pi-sdk-v1', execution_id: config.execution_id,
system_prompt: session.systemPrompt, tools: session.agent.state.tools.map(toFunction)}});
const monotonic = () => Number(process.hrtime.bigint()) / 1e9;
const expireTool = call => {
if (toolStop) return;
toolStop = {...call, reason: 'TOOL_DEADLINE_EXCEEDED'};
checkpoint('TOOL_DEADLINE_EXCEEDED');
// Stop the entire attempt. The outer kernel scope, not SDK abort, proves cleanup.
session.agent.abort();
};
const beforeTool = session.agent.beforeToolCall;
session.agent.beforeToolCall = async (context, signal) => {
if (toolStop || signal?.aborted) return {block: true, reason: toolStop?.reason || 'ATTEMPT_ABORTED'};
const budgetBlock = budget.toolBlock(context.assistantMessage);
if (budgetBlock) return budgetBlock;
const blocked = beforeTool ? await beforeTool(context, signal) : undefined;
if (blocked) return blocked;
// Pi 0.73.1 skips its queue barrier when no extension handles tool_call.
// The pinned SDK queue must finish append+fsync before any side effect.
if (!session._agentEventQueue?.then) throw new Error('UNSUPPORTED_PI_EVENT_QUEUE');
await session._agentEventQueue;
if (toolStop || signal?.aborted) return {block: true, reason: toolStop?.reason || 'ATTEMPT_ABORTED'};
const active = activeTools.get(context.toolCall.id);
if (!active) throw new Error('TOOL_START_NOT_PERSISTED');
// Bash has no default timeout, so a long command would reach the system cap
// below and end the whole attempt. Give it a native timeout a minute earlier
// so only that command fails. args is Pi's private copy; the persisted call
// keeps the model's own arguments.
if (context.toolCall.name === 'bash') {
const room = Math.floor(active.call.deadline_monotonic - monotonic()) - 60;
if (room > 0 && !(context.args.timeout > 0 && context.args.timeout <= room)) context.args.timeout = room;
}
// A shorter native bash timeout remains a normal, recoverable tool error.
// This independent system cap ends the attempt and cannot be extended by args.
active.timer = setTimeout(() => expireTool(active.call),
Math.max(1, Math.ceil((active.call.deadline_monotonic - monotonic()) * 1000)));
checkpoint('RUNNING_TOOL');
if (monotonic() >= active.call.deadline_monotonic) {
expireTool(active.call);
return {block: true, reason: toolStop.reason};
}
return undefined;
};
const afterTool = session.agent.afterToolCall;
session.agent.afterToolCall = async (context, signal) => {
const active = activeTools.get(context.toolCall.id);
if (active) {
clearTimeout(active.timer);
if (monotonic() >= active.call.deadline_monotonic) expireTool(active.call);
}
const result = afterTool ? await afterTool(context, signal) : undefined;
if (!toolStop) return result;
// Keep native content (including partial output) and the original call ID.
return {...result, isError: true, terminate: true,
details: {...(result?.details ?? context.result.details), bench_stop_reason: toolStop.reason}};
};
const stream = session.agent.streamFn;
session.agent.streamFn = (model, context, options) => {
if (toolStop) throw new Error(toolStop.reason);
budget.beforeGeneration();
activeGeneration = config.execution_id + ':' + generation;
nonemptyDelta = false;
checkpoint('WAITING_LLM');
return stream(model, context, {...options,
headers: {...options?.headers, 'X-Bench-Generation-Id': activeGeneration}});
};
// Version-bound SDK hook: the native loop closes tool results, emits turn_end
// then agent_end itself. Never abort/exit merely because the last turn ended.
if (typeof session.agent.createLoopConfig !== 'function') throw new Error('UNSUPPORTED_PI_LOOP_CONFIG');
const createLoopConfig = session.agent.createLoopConfig.bind(session.agent);
session.agent.createLoopConfig = (...args) => {
const loop = createLoopConfig(...args);
const steering = loop.getSteeringMessages;
return {...loop,
getSteeringMessages: async () => [...(await steering?.() || []), ...budget.announcement()],
shouldStopAfterTurn: async context => {
budget.observe(context.message, emptyProviderError);
const rollout = process.env.BENCH_ROLLOUT_DIR;
if (rollout) {
const target = path.join(rollout, process.env.BENCH_JAILED === '1' ? 'workspace/budget.json' : 'rollout.json');
let record = {};
try { record = JSON.parse(fs.readFileSync(target, 'utf8')); } catch {}
atomic(target, {...record, ...budget.snapshot()});
}
return budget.shouldStop() || Boolean(await loop.shouldStopAfterTurn?.(context));
},
};
};
let compactedRetry = false;
session.subscribe(event => {
lastEvent = event.type;
let compaction;
if (event.type === 'compaction_end' && event.willRetry) {
// Pi compacted after a context overflow and resends: the failed generation's ID is spent. kept_messages is
// how many earlier messages the model still sees after the summary (export/trajectory.py builds that view).
generation += 1; compactedRetry = true;
compaction = {kept_messages: session.agent.state.messages
.filter(m => m.role !== 'compactionSummary' && !emptyProviderError(m)).length};
}
if (event.type === 'message_update') {
const delta = event.assistantMessageEvent;
if (delta?.type?.endsWith('_delta') && delta.delta) nonemptyDelta = true;
}
if (event.type === 'message_end' && event.message?.role === 'assistant' &&
!['error', 'aborted'].includes(event.message.stopReason)) generation += 1;
let boundary;
if (event.type === 'tool_execution_start') {
const started = monotonic();
const requestedCap = event.toolName === 'terminal_exec' ? managedCap : toolCap;
const cap = Math.max(0.001, Math.min(requestedCap,
config.execution_deadline_monotonic === undefined ? Infinity : config.execution_deadline_monotonic - started));
const call = {tool_call_id: event.toolCallId, tool_name: event.toolName,
started_monotonic: started, deadline_monotonic: started + cap, cap_s: cap};
activeTools.set(call.tool_call_id, {call});
boundary = {state: 'RUNNING', ...call};
} else if (event.type === 'tool_execution_end') {
const active = activeTools.get(event.toolCallId);
if (active) {
clearTimeout(active.timer);
boundary = {state: toolStop?.tool_call_id === event.toolCallId ? 'TIMED_OUT' : 'DONE', ...active.call};
activeTools.delete(event.toolCallId);
}
}
// message_update repeats the whole message twice (message, assistantMessageEvent.partial) on every
// token, ~800MB of events.jsonl per rollout; readers use only its type and delta, message_end has the text.
const written = event.type === 'message_update'
? {...event, message: undefined, assistantMessageEvent: {...event.assistantMessageEvent, partial: undefined}}
: event;
// Metadata decorates real native boundaries; it never adds conversation text.
output(boundary ? {...written, bench_tool: {...boundary, execution_id: config.execution_id,
epoch: config.epoch ?? 1, observed_monotonic: monotonic()}} : compaction ? {...written, bench_compaction: compaction} : written);
});
const stop = () => session.agent.abort();
process.on('SIGTERM', stop); process.on('SIGINT', stop);
try {
if (config.resume) {
if (config.system_sha256 !== hash(session.systemPrompt)) throw new Error('CHECKPOINT_SYSTEM_CHANGED');
// A paused generation that already streamed must not be replayed. A crash checkpoint is taken after a saved
// message (complete by construction); whatever was streaming then was never saved, so nothing is replayed.
if (config.nonempty_delta && !config.recovered) throw new Error('UNSAFE_CONTINUATION');
// Every earlier continuation required the same complete, empty-error
// checkpoint. Native errors remain append-only; exclude those proved
// fragments from working context even after several pauses/tool rounds.
const messages = session.state.messages.filter(message => !emptyProviderError(message));
if (config.recovered) {
// A collector killed mid-reply (or its reaper's SIGTERM aborting Pi) leaves an unfinished assistant message;
// the model never completed it, so it is dropped like an empty provider error, not continued.
while (messages.at(-1)?.role === 'assistant' && ['aborted', 'error'].includes(messages.at(-1).stopReason) &&
!(Array.isArray(messages.at(-1).content) ? messages.at(-1).content : []).some(block => block.type === 'toolCall')) {
messages.pop();
}
for (const result of interruptedResults(messages)) {
manager.appendMessage(result);
messages.push(result);
output({type: 'message_end', message: result});
}
}
if (!closed(messages) || !continuationBoundary(messages)) throw new Error('UNSAFE_CONTINUATION');
session.agent.state.messages = messages;
await session.agent.continue();
} else {
await session.prompt(fs.readFileSync(config.prompt_path, 'utf8'));
}
await session._agentEventQueue;
// Pi resumes 100 ms after an overflow compaction (agent-session.js _runAutoCompaction): wait for that run too.
while (compactedRetry) {
compactedRetry = false;
await new Promise(resolve => setTimeout(resolve, 300));
await session.agent.waitForIdle();
await session._agentEventQueue;
}
const last = session.state.messages.at(-1);
const complete = last?.role === 'assistant' && last.stopReason === 'stop' && closed(session.state.messages);
const safeResume = !nonemptyDelta && emptyProviderError(last) && closed(session.state.messages) &&
continuationBoundary(session.state.messages.slice(0, -1));
const budgetStop = budget.shouldStop() && !complete;
checkpoint(toolStop ? 'TOOL_DEADLINE_EXCEEDED' : complete ? 'COMPLETED' : budgetStop ? 'BUDGET_EXHAUSTED' : 'PAUSED_INFRA',
!toolStop && complete, !toolStop && !budgetStop && safeResume);
process.exitCode = toolStop ? 78 : complete ? 0 : budgetStop ? 75 : 76;
} finally {
for (const active of activeTools.values()) clearTimeout(active.timer);
process.off('SIGTERM', stop); process.off('SIGINT', stop);
session.dispose();
restoreTransport();
}
}
module.exports = {emptyProviderError, closed, continuationBoundary, main};
if (require.main === module) main(process.argv[2]).catch(error => {
process.stderr.write(encode({type: 'bench_runner_error', code: error.message?.match(/^[A-Z_]+$/)?.[0] || 'SDK_RUNNER_FAILED'}));
process.exitCode = 77;
});