import path from 'node:path'; import { rm, writeFile } from 'node:fs/promises'; import type { Run } from '@wishforge/contracts'; import { BASELINE_BUNDLE, BASELINE_VERSION, bundleDigest, bundlePrompt, questionsIn } from '@wishforge/agent'; import { executeCommand, outputTail, readWishforgeManifest, proxyEnvironment, requireBranchName, RuntimeError, safeIdentifier, TIMEOUT_EXIT_CODE, validateRepositoryUrl, type CommandExecutor, type CommandResult, type ProviderAdapter, type PortableCheckpoint, } from '@wishforge/runtime'; import { client, gitFor, outcomeForError, sandboxMismatch, stoppedOutcome, tenantRoot, waiting, type ActivityOutcome, type RunContext, type RunInput, type WorkerConfig, } from './common.js'; import { ControlPlaneError, type ControlPlaneClient } from './client.js'; import { LeaseKeeper, activityContext } from './lease.js'; import { installPlan } from './project-contract.js'; import { providerRegistry } from './providers.js'; /** Where a run continues when it already has a checkpoint or a release; no executor lease is needed for that. */ function resumePoint(context: RunContext): ActivityOutcome | undefined { const { run, delivery } = context; if (delivery?.release) return waiting( delivery.release.status === 'awaiting_approval' ? 'approval' : 'deployment', `Release ${delivery.release.id} is ${delivery.release.status}; it has to finish before anything else runs.`, ); if ( ['build', 'release', 'accept'].includes(run.stage) && delivery?.verificationEvidenceId && delivery.buildEvidenceId && delivery.environmentId ) return { status: 'release', verificationEvidenceId: delivery.verificationEvidenceId, buildEvidenceId: delivery.buildEvidenceId, environmentId: delivery.environmentId, }; if (context.checkpoint && ['verify', 'integrate', 'build'].includes(run.stage)) return { status: 'verify' }; if (['release', 'accept'].includes(run.stage)) return waiting('deployment', 'The build evidence for this release is missing; reconcile the release first.'); } /** Lease the run, or explain why it cannot run now. */ export async function leaseRun( api: ControlPlaneClient, config: WorkerConfig, input: RunInput, ): Promise<{ run: Run; keeper: LeaseKeeper } | ActivityOutcome> { try { const run = await api.request(`/internal/runs/${input.runId}/lease`, { workerId: config.workerId, generation: input.generation, }); if (!run.lease) throw new RuntimeError('LEASE_MISSING', 'The control plane did not grant a lease'); return { run, keeper: new LeaseKeeper(api, input.runId, run.lease.token).start() }; } catch (error) { if (!(error instanceof ControlPlaneError)) throw error; if (error.domainCode === 'CONCURRENCY_LIMIT') return waiting('worker_unavailable', 'All execution slots of this tenant are in use; continue when one is free.'); if (['EXECUTION_BUDGET_EXHAUSTED', 'TOKEN_BUDGET_EXHAUSTED'].includes(error.domainCode)) return waiting('budget', 'The run used up its token or execution-time budget.'); if (error.domainCode === 'RUN_NOT_RUNNABLE') { const { run } = await api.request(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 }); return stoppedOutcome(run) ?? { status: 'waiting' }; } throw error; } } function createProvider(config: WorkerConfig, context: RunContext, isolatedHome: string): ProviderAdapter | undefined { if (config.providerFactory) return config.providerFactory(context, isolatedHome); return providerRegistry[context.run.provider]?.(config, context, isolatedHome); } async function install( execute: CommandExecutor, commands: string[][], cwd: string, home: string, proxy?: string, ): Promise<{ argv: string[]; result: CommandResult } | undefined> { for (const argv of commands) { let result: CommandResult; try { result = await execute({ executable: argv[0]!, args: argv.slice(1), cwd, env: { PATH: process.env.PATH, HOME: home, LANG: 'C.UTF-8', CI: 'true', ...proxyEnvironment(proxy) }, timeoutMs: 15 * 60_000, }); } catch (error) { if (!(error instanceof RuntimeError) || !['TIMEOUT', 'OUTPUT_LIMIT'].includes(error.code)) throw error; result = { exitCode: error.code === 'TIMEOUT' ? TIMEOUT_EXIT_CODE : 1, stdout: '', stderr: error.message }; } if (result.exitCode !== 0) return { argv, result }; } } /** * Executor activity: prepares an isolated workspace from the target branch or the latest * checkpoint, installs dependencies, runs the agent and hands a checkpoint to verification. */ export async function executeRun(config: WorkerConfig, input: RunInput): Promise { const api = client(config, input); const observed = await api.request(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 }); const stopped = stoppedOutcome(observed.run); if (stopped) return stopped; if (observed.run.status === 'waiting') return { status: 'waiting' }; const resume = resumePoint(observed); if (resume) return resume; const unbound = sandboxMismatch(config, input); if (unbound) return waiting('sandbox_provisioner', unbound); const leased = await leaseRun(api, config, input); if ('status' in leased) return leased; const { keeper } = leased; const leaseToken = keeper.token; const activity = activityContext(); const isolatedHome = path.join( config.isolatedProviderHomeRoot, safeIdentifier(input.tenantId), safeIdentifier(input.runId), ); const execute: CommandExecutor = command => (config.execute ?? executeCommand)({ ...command, signal: keeper.signal }); let provider: ProviderAdapter | undefined; let providerStarted = false; const abort = () => { if (providerStarted) void provider!.cancel(input.runId).catch(() => undefined); }; keeper.signal.addEventListener('abort', abort, { once: true }); let workspace: string | undefined; let checkpointPersisted = false; let recoveryBaseCommit: string | undefined; let recoveryCheckpoint: PortableCheckpoint | undefined; const transition = (stage: Run['stage'], message: string, extras: Record = {}) => api.request(`/internal/runs/${input.runId}/transition`, { leaseToken, stage, status: 'running', message, ...extras, }); const settleFinalUsage = async (measurements: Array<{ operationId: string; tokens: number }>) => { for (const measurement of measurements) { await api.request( `/internal/runs/${input.runId}/meter`, { leaseToken, ...measurement, computeSeconds: 0 }, { retries: 2 }, ); // A final total may consume the remaining budget. Source is already durable; do not // continue to verification with the now-expired lease. Replays use the same operation ID. await keeper.ensure(); } }; try { const context = await api.request(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 }); let run = context.run; try { requireBranchName(context.project.targetBranch); if (context.project.repositoryUrl) validateRepositoryUrl(context.project.repositoryUrl, { allowLocalHttp: config.allowLocalGitHttp }); } catch (error) { return waiting('git_storage', error instanceof Error ? error.message : 'The project repository is not usable'); } const previousCompletion = context.checkpoint?.providerCompletion; if (previousCompletion) { await settleFinalUsage(previousCompletion.finalUsage); if ( previousCompletion.status === 'completed' && previousCompletion.revision === run.revision && previousCompletion.repairCount === run.repairCount && run.stage === 'implement' ) { const questions = run.repairCount ? [] : questionsIn(previousCompletion.summary); if (questions.length && context.checkpoint!.sourceCommit === context.checkpoint!.baseCommit) return waiting('clarification', `The executor needs answers before implementing:\n${questions.join('\n')}`); await transition( 'verify', 'Saved implementation and final usage reconciled; independent verification is next', { sourceCommit: context.checkpoint!.sourceCommit, ...(previousCompletion.summary ? { summary: previousCompletion.summary } : {}), }, ); return { status: 'verify' }; } } provider = createProvider(config, context, isolatedHome); if (!provider) return waiting('credentials', `The ${run.provider} executor is not available on this worker.`); const capabilities = provider.capabilities(); if (!capabilities.coding) return waiting('credentials', capabilities.reason ?? `The ${run.provider} executor is not configured.`); const bundle = context.bundleConfig ?? (run.bundleVersion === BASELINE_VERSION ? BASELINE_BUNDLE : undefined); if (!bundle || (run.bundleVersion !== BASELINE_VERSION && bundleDigest(bundle) !== run.bundleVersion)) throw new RuntimeError( 'BUNDLE_UNAVAILABLE', 'The pinned agent configuration is missing or differs from its version', ); if (run.stage === 'clarify') run = await transition('plan', 'Acceptance criteria recorded; preparing the workspace'); run = await transition( 'implement', run.repairCount ? `Repair round ${run.repairCount}: preparing the workspace` : 'Preparing an isolated workspace', ); const git = gitFor(config, context, execute); const attempt = `${input.runId}-a${leaseToken}`; const prepared = context.checkpoint ? await git.restore({ checkpoint: context.checkpoint, repository: context.project.repositoryUrl, root: tenantRoot(config, input), runId: attempt, }) : await git.create({ repository: context.project.repositoryUrl, targetBranch: context.project.targetBranch, sourceBranch: run.sourceBranch, baseCommit: run.sourceCommit, root: tenantRoot(config, input), runId: attempt, }); workspace = prepared.workspace; let baseCommit = prepared.baseCommit; recoveryBaseCommit = baseCommit; let conflicts: string[] = []; let lastFailure = run.repairCount ? run.lastFailure : undefined; // After a merge conflict the repair round starts by merging the current target branch; // the agent resolves what Git could not, and the checkpoint is relative to that target. if (run.repairCount && run.lastFailure?.kind === 'integration_conflict') { const merged = await git.mergeTarget(workspace, context.project.targetBranch); baseCommit = merged.targetCommit; recoveryBaseCommit = baseCommit; conflicts = merged.conflicts; // The prompt names the conflicts of this merge; the target may have moved since integration. lastFailure = { ...run.lastFailure, targetCommit: merged.targetCommit, outputTail: conflicts.join('\n') || '(Git merged cleanly this time; check that both changes still work together.)', }; } const manifest = await readWishforgeManifest({ workspace, sourceCommit: await git.head(workspace) }, execute).catch( () => undefined, ); const plan = installPlan(context.installCommands, manifest?.manifest); // Conflicted manifests cannot be installed; the agent resolves them first and the verifier installs. if (plan.commands.length && !conflicts.length) { const failed = await install( execute, plan.commands, path.join(workspace, plan.directory), isolatedHome, config.installProxy, ); if (failed) return waiting( 'infrastructure', `Installing dependencies failed: ${failed.argv.join(' ')} exited ${failed.result.exitCode}.\n${outputTail(failed.result, 1500)}`, ); await git.excludeUntracked(workspace); } await keeper.ensure(); const prompt = bundlePrompt(bundle, { goal: run.goal, acceptanceCriteria: run.acceptanceCriteria, repairCount: run.repairCount, lastFailure, lessons: context.lessons, sourceBranch: run.sourceBranch, }); await provider.start({ runId: input.runId, tenantId: input.tenantId, workspace, prompt: config.claudeAuthHome && run.provider === 'claude-code' ? `${prompt}\n\nLocal execution contract: only Read/Edit/Write/Glob/Grep file tools are available. Implement the requested source changes using those tools. A separate trusted verifier runs installation, tests, Git commit/push and deployment afterwards; do not ask for Bash, Git, deployment credentials or permission to run those commands. Do not claim tests were run by you. Preserve unrelated code and follow the repository versioning instructions.` : prompt, baseCommit, model: bundle.model, }); providerStarted = true; // Cancellation can arrive while start awaits filesystem/process preparation, // before the abort listener knows there is a provider attempt to stop. if (keeper.signal.aborted || keeper.error) abort(); await keeper.ensure(); let summary: string | undefined; const finalUsage: Array<{ operationId: string; tokens: number }> = []; for await (const event of provider.events(input.runId)) { activity?.heartbeat({ runId: input.runId, sequence: event.sequence, leaseToken }); if (event.type === 'output' && typeof event.data.text === 'string') summary = event.data.text; // Only normalized events leave the sandbox; reasoning and raw process output are not exported. await api.request(`/internal/runs/${input.runId}/events`, { leaseToken, event }, { retries: 2 }); if (event.type === 'usage' && context.providerCredential?.metering !== 'proxy') { const tokens = Number(event.data.input_tokens ?? 0) + Number(event.data.output_tokens ?? 0) + Number(event.data.cache_creation_input_tokens ?? 0) + Number(event.data.cache_read_input_tokens ?? 0); if (Number.isSafeInteger(tokens) && tokens >= 0) { const measurement = { operationId: `provider-${input.runId}-${leaseToken}-${event.sequence}`, tokens }; if (event.data.final === true) finalUsage.push(measurement); else await api.request( `/internal/runs/${input.runId}/meter`, { leaseToken, ...measurement, computeSeconds: 0, }, { retries: 2 }, ); } } } if (keeper.error) throw keeper.error; const providerRun = await provider.inspect(input.runId); // The work so far is kept even when the executor failed, so continuing does not start over. if (providerRun.status === 'running') throw new RuntimeError( 'WORKSPACE_NOT_QUIESCENT', 'The provider stream ended before the workspace stopped changing', ); const checkpoint: PortableCheckpoint = { ...(await provider.exportCheckpoint(input.runId)), providerCompletion: { status: providerRun.status, revision: run.revision, repairCount: run.repairCount, ...(summary ? { summary: summary.slice(0, 4000) } : {}), finalUsage, }, }; recoveryCheckpoint = checkpoint; await keeper.ensure(); await api.request(`/internal/runs/${input.runId}/checkpoint`, { leaseToken, checkpoint, workspace }); checkpointPersisted = true; await settleFinalUsage(finalUsage); if (providerRun.status !== 'completed') return waiting( 'provider_error', `The ${run.provider} executor ${providerRun.status === 'canceled' ? 'stopped' : 'failed'}: ${providerRun.error ?? 'no details'}. Its work was saved; continue to retry.`, ); // An agent that asked instead of changing anything waits for the person's answer; answering // creates a new requirement revision that starts again from planning. const questions = run.repairCount ? [] : questionsIn(summary); if (questions.length && checkpoint.sourceCommit === checkpoint.baseCommit) return waiting( 'clarification', `The ${run.provider} executor needs answers before implementing:\n${questions.map((question, i) => `${i + 1}. ${question}`).join('\n')}`, ); await transition('verify', 'Implementation checkpoint created; independent verification is next', { sourceCommit: checkpoint.sourceCommit, ...(summary ? { summary: summary.slice(0, 4000) } : {}), }); return { status: 'verify' }; } catch (error) { abort(); return await outcomeForError( keeper.error ?? error, async () => (await api.request(`/internal/runs/${input.runId}/context`)).run, ); } finally { keeper.close(); keeper.signal.removeEventListener('abort', abort); await api.request(`/internal/runs/${input.runId}/lease/release`, { leaseToken }).catch(() => undefined); if (workspace && checkpointPersisted) await rm(workspace, { recursive: true, force: true }).catch(() => undefined); else if (workspace) { // Lease loss, cancellation and ambiguous uploads must never erase the only copy. // Keep a private sidecar outside the agent tree so operators can recover this exact attempt. const providerRun = providerStarted ? await provider?.inspect(input.runId).catch(() => undefined) : undefined; await writeFile( `${workspace}.recovery.json`, JSON.stringify( { schemaVersion: 1, tenantId: input.tenantId, runId: input.runId, generation: input.generation, leaseToken, workspace, baseCommit: recoveryBaseCommit, provider: providerRun?.provider, providerStatus: providerRun?.status, sessionId: providerRun?.sessionId, ...(recoveryCheckpoint ? { checkpoint: recoveryCheckpoint } : {}), retainedAt: new Date().toISOString(), reason: 'Checkpoint persistence was not confirmed; workspace retained', }, null, 2, ), { mode: 0o600 }, ).catch(() => undefined); } } }