Add authenticated LAN access, native Claude execution, checkpoint and budget recovery, and signed POC build/deploy/rollback contracts. Include local service management, regression coverage, and evidence from the completed POC delivery.
415 lines
18 KiB
TypeScript
415 lines
18 KiB
TypeScript
import path from 'node:path';
|
|
import { rm, writeFile } from 'node:fs/promises';
|
|
import type { Run } from '@wishforge/contracts';
|
|
import { BASELINE_BUNDLE, BASELINE_VERSION, bundleDigest, bundlePrompt, questionsIn } from '@wishforge/agent';
|
|
import {
|
|
executeCommand,
|
|
outputTail,
|
|
readWishforgeManifest,
|
|
proxyEnvironment,
|
|
requireBranchName,
|
|
RuntimeError,
|
|
safeIdentifier,
|
|
TIMEOUT_EXIT_CODE,
|
|
validateRepositoryUrl,
|
|
type CommandExecutor,
|
|
type CommandResult,
|
|
type ProviderAdapter,
|
|
type PortableCheckpoint,
|
|
} from '@wishforge/runtime';
|
|
import {
|
|
client,
|
|
gitFor,
|
|
outcomeForError,
|
|
sandboxMismatch,
|
|
stoppedOutcome,
|
|
tenantRoot,
|
|
waiting,
|
|
type ActivityOutcome,
|
|
type RunContext,
|
|
type RunInput,
|
|
type WorkerConfig,
|
|
} from './common.js';
|
|
import { ControlPlaneError, type ControlPlaneClient } from './client.js';
|
|
import { LeaseKeeper, activityContext } from './lease.js';
|
|
import { installPlan } from './project-contract.js';
|
|
import { providerRegistry } from './providers.js';
|
|
|
|
/** Where a run continues when it already has a checkpoint or a release; no executor lease is needed for that. */
|
|
function resumePoint(context: RunContext): ActivityOutcome | undefined {
|
|
const { run, delivery } = context;
|
|
if (delivery?.release)
|
|
return waiting(
|
|
delivery.release.status === 'awaiting_approval' ? 'approval' : 'deployment',
|
|
`Release ${delivery.release.id} is ${delivery.release.status}; it has to finish before anything else runs.`,
|
|
);
|
|
if (
|
|
['build', 'release', 'accept'].includes(run.stage) &&
|
|
delivery?.verificationEvidenceId &&
|
|
delivery.buildEvidenceId &&
|
|
delivery.environmentId
|
|
)
|
|
return {
|
|
status: 'release',
|
|
verificationEvidenceId: delivery.verificationEvidenceId,
|
|
buildEvidenceId: delivery.buildEvidenceId,
|
|
environmentId: delivery.environmentId,
|
|
};
|
|
if (context.checkpoint && ['verify', 'integrate', 'build'].includes(run.stage)) return { status: 'verify' };
|
|
if (['release', 'accept'].includes(run.stage))
|
|
return waiting('deployment', 'The build evidence for this release is missing; reconcile the release first.');
|
|
}
|
|
|
|
/** Lease the run, or explain why it cannot run now. */
|
|
export async function leaseRun(
|
|
api: ControlPlaneClient,
|
|
config: WorkerConfig,
|
|
input: RunInput,
|
|
): Promise<{ run: Run; keeper: LeaseKeeper } | ActivityOutcome> {
|
|
try {
|
|
const run = await api.request<Run>(`/internal/runs/${input.runId}/lease`, {
|
|
workerId: config.workerId,
|
|
generation: input.generation,
|
|
});
|
|
if (!run.lease) throw new RuntimeError('LEASE_MISSING', 'The control plane did not grant a lease');
|
|
return { run, keeper: new LeaseKeeper(api, input.runId, run.lease.token).start() };
|
|
} catch (error) {
|
|
if (!(error instanceof ControlPlaneError)) throw error;
|
|
if (error.domainCode === 'CONCURRENCY_LIMIT')
|
|
return waiting('worker_unavailable', 'All execution slots of this tenant are in use; continue when one is free.');
|
|
if (['EXECUTION_BUDGET_EXHAUSTED', 'TOKEN_BUDGET_EXHAUSTED'].includes(error.domainCode))
|
|
return waiting('budget', 'The run used up its token or execution-time budget.');
|
|
if (error.domainCode === 'RUN_NOT_RUNNABLE') {
|
|
const { run } = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
|
|
return stoppedOutcome(run) ?? { status: 'waiting' };
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function createProvider(config: WorkerConfig, context: RunContext, isolatedHome: string): ProviderAdapter | undefined {
|
|
if (config.providerFactory) return config.providerFactory(context, isolatedHome);
|
|
return providerRegistry[context.run.provider]?.(config, context, isolatedHome);
|
|
}
|
|
|
|
async function install(
|
|
execute: CommandExecutor,
|
|
commands: string[][],
|
|
cwd: string,
|
|
home: string,
|
|
proxy?: string,
|
|
): Promise<{ argv: string[]; result: CommandResult } | undefined> {
|
|
for (const argv of commands) {
|
|
let result: CommandResult;
|
|
try {
|
|
result = await execute({
|
|
executable: argv[0]!,
|
|
args: argv.slice(1),
|
|
cwd,
|
|
env: { PATH: process.env.PATH, HOME: home, LANG: 'C.UTF-8', CI: 'true', ...proxyEnvironment(proxy) },
|
|
timeoutMs: 15 * 60_000,
|
|
});
|
|
} catch (error) {
|
|
if (!(error instanceof RuntimeError) || !['TIMEOUT', 'OUTPUT_LIMIT'].includes(error.code)) throw error;
|
|
result = { exitCode: error.code === 'TIMEOUT' ? TIMEOUT_EXIT_CODE : 1, stdout: '', stderr: error.message };
|
|
}
|
|
if (result.exitCode !== 0) return { argv, result };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Executor activity: prepares an isolated workspace from the target branch or the latest
|
|
* checkpoint, installs dependencies, runs the agent and hands a checkpoint to verification.
|
|
*/
|
|
export async function executeRun(config: WorkerConfig, input: RunInput): Promise<ActivityOutcome> {
|
|
const api = client(config, input);
|
|
const observed = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
|
|
const stopped = stoppedOutcome(observed.run);
|
|
if (stopped) return stopped;
|
|
if (observed.run.status === 'waiting') return { status: 'waiting' };
|
|
const resume = resumePoint(observed);
|
|
if (resume) return resume;
|
|
const unbound = sandboxMismatch(config, input);
|
|
if (unbound) return waiting('sandbox_provisioner', unbound);
|
|
const leased = await leaseRun(api, config, input);
|
|
if ('status' in leased) return leased;
|
|
const { keeper } = leased;
|
|
const leaseToken = keeper.token;
|
|
const activity = activityContext();
|
|
const isolatedHome = path.join(
|
|
config.isolatedProviderHomeRoot,
|
|
safeIdentifier(input.tenantId),
|
|
safeIdentifier(input.runId),
|
|
);
|
|
const execute: CommandExecutor = command => (config.execute ?? executeCommand)({ ...command, signal: keeper.signal });
|
|
let provider: ProviderAdapter | undefined;
|
|
let providerStarted = false;
|
|
const abort = () => {
|
|
if (providerStarted) void provider!.cancel(input.runId).catch(() => undefined);
|
|
};
|
|
keeper.signal.addEventListener('abort', abort, { once: true });
|
|
let workspace: string | undefined;
|
|
let checkpointPersisted = false;
|
|
let recoveryBaseCommit: string | undefined;
|
|
let recoveryCheckpoint: PortableCheckpoint | undefined;
|
|
const transition = (stage: Run['stage'], message: string, extras: Record<string, unknown> = {}) =>
|
|
api.request<Run>(`/internal/runs/${input.runId}/transition`, {
|
|
leaseToken,
|
|
stage,
|
|
status: 'running',
|
|
message,
|
|
...extras,
|
|
});
|
|
const settleFinalUsage = async (measurements: Array<{ operationId: string; tokens: number }>) => {
|
|
for (const measurement of measurements) {
|
|
await api.request(
|
|
`/internal/runs/${input.runId}/meter`,
|
|
{ leaseToken, ...measurement, computeSeconds: 0 },
|
|
{ retries: 2 },
|
|
);
|
|
// A final total may consume the remaining budget. Source is already durable; do not
|
|
// continue to verification with the now-expired lease. Replays use the same operation ID.
|
|
await keeper.ensure();
|
|
}
|
|
};
|
|
try {
|
|
const context = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
|
|
let run = context.run;
|
|
try {
|
|
requireBranchName(context.project.targetBranch);
|
|
if (context.project.repositoryUrl)
|
|
validateRepositoryUrl(context.project.repositoryUrl, { allowLocalHttp: config.allowLocalGitHttp });
|
|
} catch (error) {
|
|
return waiting('git_storage', error instanceof Error ? error.message : 'The project repository is not usable');
|
|
}
|
|
const previousCompletion = context.checkpoint?.providerCompletion;
|
|
if (previousCompletion) {
|
|
await settleFinalUsage(previousCompletion.finalUsage);
|
|
if (
|
|
previousCompletion.status === 'completed' &&
|
|
previousCompletion.revision === run.revision &&
|
|
previousCompletion.repairCount === run.repairCount &&
|
|
run.stage === 'implement'
|
|
) {
|
|
const questions = run.repairCount ? [] : questionsIn(previousCompletion.summary);
|
|
if (questions.length && context.checkpoint!.sourceCommit === context.checkpoint!.baseCommit)
|
|
return waiting('clarification', `The executor needs answers before implementing:\n${questions.join('\n')}`);
|
|
await transition(
|
|
'verify',
|
|
'Saved implementation and final usage reconciled; independent verification is next',
|
|
{
|
|
sourceCommit: context.checkpoint!.sourceCommit,
|
|
...(previousCompletion.summary ? { summary: previousCompletion.summary } : {}),
|
|
},
|
|
);
|
|
return { status: 'verify' };
|
|
}
|
|
}
|
|
provider = createProvider(config, context, isolatedHome);
|
|
if (!provider) return waiting('credentials', `The ${run.provider} executor is not available on this worker.`);
|
|
const capabilities = provider.capabilities();
|
|
if (!capabilities.coding)
|
|
return waiting('credentials', capabilities.reason ?? `The ${run.provider} executor is not configured.`);
|
|
const bundle = context.bundleConfig ?? (run.bundleVersion === BASELINE_VERSION ? BASELINE_BUNDLE : undefined);
|
|
if (!bundle || (run.bundleVersion !== BASELINE_VERSION && bundleDigest(bundle) !== run.bundleVersion))
|
|
throw new RuntimeError(
|
|
'BUNDLE_UNAVAILABLE',
|
|
'The pinned agent configuration is missing or differs from its version',
|
|
);
|
|
if (run.stage === 'clarify')
|
|
run = await transition('plan', 'Acceptance criteria recorded; preparing the workspace');
|
|
run = await transition(
|
|
'implement',
|
|
run.repairCount ? `Repair round ${run.repairCount}: preparing the workspace` : 'Preparing an isolated workspace',
|
|
);
|
|
const git = gitFor(config, context, execute);
|
|
const attempt = `${input.runId}-a${leaseToken}`;
|
|
const prepared = context.checkpoint
|
|
? await git.restore({
|
|
checkpoint: context.checkpoint,
|
|
repository: context.project.repositoryUrl,
|
|
root: tenantRoot(config, input),
|
|
runId: attempt,
|
|
})
|
|
: await git.create({
|
|
repository: context.project.repositoryUrl,
|
|
targetBranch: context.project.targetBranch,
|
|
sourceBranch: run.sourceBranch,
|
|
baseCommit: run.sourceCommit,
|
|
root: tenantRoot(config, input),
|
|
runId: attempt,
|
|
});
|
|
workspace = prepared.workspace;
|
|
let baseCommit = prepared.baseCommit;
|
|
recoveryBaseCommit = baseCommit;
|
|
let conflicts: string[] = [];
|
|
let lastFailure = run.repairCount ? run.lastFailure : undefined;
|
|
// After a merge conflict the repair round starts by merging the current target branch;
|
|
// the agent resolves what Git could not, and the checkpoint is relative to that target.
|
|
if (run.repairCount && run.lastFailure?.kind === 'integration_conflict') {
|
|
const merged = await git.mergeTarget(workspace, context.project.targetBranch);
|
|
baseCommit = merged.targetCommit;
|
|
recoveryBaseCommit = baseCommit;
|
|
conflicts = merged.conflicts;
|
|
// The prompt names the conflicts of this merge; the target may have moved since integration.
|
|
lastFailure = {
|
|
...run.lastFailure,
|
|
targetCommit: merged.targetCommit,
|
|
outputTail:
|
|
conflicts.join('\n') || '(Git merged cleanly this time; check that both changes still work together.)',
|
|
};
|
|
}
|
|
const manifest = await readWishforgeManifest({ workspace, sourceCommit: await git.head(workspace) }, execute).catch(
|
|
() => undefined,
|
|
);
|
|
const plan = installPlan(context.installCommands, manifest?.manifest);
|
|
// Conflicted manifests cannot be installed; the agent resolves them first and the verifier installs.
|
|
if (plan.commands.length && !conflicts.length) {
|
|
const failed = await install(
|
|
execute,
|
|
plan.commands,
|
|
path.join(workspace, plan.directory),
|
|
isolatedHome,
|
|
config.installProxy,
|
|
);
|
|
if (failed)
|
|
return waiting(
|
|
'infrastructure',
|
|
`Installing dependencies failed: ${failed.argv.join(' ')} exited ${failed.result.exitCode}.\n${outputTail(failed.result, 1500)}`,
|
|
);
|
|
await git.excludeUntracked(workspace);
|
|
}
|
|
await keeper.ensure();
|
|
const prompt = bundlePrompt(bundle, {
|
|
goal: run.goal,
|
|
acceptanceCriteria: run.acceptanceCriteria,
|
|
repairCount: run.repairCount,
|
|
lastFailure,
|
|
lessons: context.lessons,
|
|
sourceBranch: run.sourceBranch,
|
|
});
|
|
await provider.start({
|
|
runId: input.runId,
|
|
tenantId: input.tenantId,
|
|
workspace,
|
|
prompt:
|
|
config.claudeAuthHome && run.provider === 'claude-code'
|
|
? `${prompt}\n\nLocal execution contract: only Read/Edit/Write/Glob/Grep file tools are available. Implement the requested source changes using those tools. A separate trusted verifier runs installation, tests, Git commit/push and deployment afterwards; do not ask for Bash, Git, deployment credentials or permission to run those commands. Do not claim tests were run by you. Preserve unrelated code and follow the repository versioning instructions.`
|
|
: prompt,
|
|
baseCommit,
|
|
model: bundle.model,
|
|
});
|
|
providerStarted = true;
|
|
if (keeper.error) abort();
|
|
let summary: string | undefined;
|
|
const finalUsage: Array<{ operationId: string; tokens: number }> = [];
|
|
for await (const event of provider.events(input.runId)) {
|
|
activity?.heartbeat({ runId: input.runId, sequence: event.sequence, leaseToken });
|
|
if (event.type === 'output' && typeof event.data.text === 'string') summary = event.data.text;
|
|
// Only normalized events leave the sandbox; reasoning and raw process output are not exported.
|
|
await api.request(`/internal/runs/${input.runId}/events`, { leaseToken, event }, { retries: 2 });
|
|
if (event.type === 'usage' && context.providerCredential?.metering !== 'proxy') {
|
|
const tokens =
|
|
Number(event.data.input_tokens ?? 0) +
|
|
Number(event.data.output_tokens ?? 0) +
|
|
Number(event.data.cache_creation_input_tokens ?? 0) +
|
|
Number(event.data.cache_read_input_tokens ?? 0);
|
|
if (Number.isSafeInteger(tokens) && tokens >= 0) {
|
|
const measurement = { operationId: `provider-${input.runId}-${leaseToken}-${event.sequence}`, tokens };
|
|
if (event.data.final === true) finalUsage.push(measurement);
|
|
else
|
|
await api.request(
|
|
`/internal/runs/${input.runId}/meter`,
|
|
{
|
|
leaseToken,
|
|
...measurement,
|
|
computeSeconds: 0,
|
|
},
|
|
{ retries: 2 },
|
|
);
|
|
}
|
|
}
|
|
}
|
|
if (keeper.error) throw keeper.error;
|
|
const providerRun = await provider.inspect(input.runId);
|
|
// The work so far is kept even when the executor failed, so continuing does not start over.
|
|
if (providerRun.status === 'running')
|
|
throw new RuntimeError(
|
|
'WORKSPACE_NOT_QUIESCENT',
|
|
'The provider stream ended before the workspace stopped changing',
|
|
);
|
|
const checkpoint: PortableCheckpoint = {
|
|
...(await provider.exportCheckpoint(input.runId)),
|
|
providerCompletion: {
|
|
status: providerRun.status,
|
|
revision: run.revision,
|
|
repairCount: run.repairCount,
|
|
...(summary ? { summary: summary.slice(0, 4000) } : {}),
|
|
finalUsage,
|
|
},
|
|
};
|
|
recoveryCheckpoint = checkpoint;
|
|
await keeper.ensure();
|
|
await api.request(`/internal/runs/${input.runId}/checkpoint`, { leaseToken, checkpoint, workspace });
|
|
checkpointPersisted = true;
|
|
await settleFinalUsage(finalUsage);
|
|
if (providerRun.status !== 'completed')
|
|
return waiting(
|
|
'provider_error',
|
|
`The ${run.provider} executor ${providerRun.status === 'canceled' ? 'stopped' : 'failed'}: ${providerRun.error ?? 'no details'}. Its work was saved; continue to retry.`,
|
|
);
|
|
// An agent that asked instead of changing anything waits for the person's answer; answering
|
|
// creates a new requirement revision that starts again from planning.
|
|
const questions = run.repairCount ? [] : questionsIn(summary);
|
|
if (questions.length && checkpoint.sourceCommit === checkpoint.baseCommit)
|
|
return waiting(
|
|
'clarification',
|
|
`The ${run.provider} executor needs answers before implementing:\n${questions.map((question, i) => `${i + 1}. ${question}`).join('\n')}`,
|
|
);
|
|
await transition('verify', 'Implementation checkpoint created; independent verification is next', {
|
|
sourceCommit: checkpoint.sourceCommit,
|
|
...(summary ? { summary: summary.slice(0, 4000) } : {}),
|
|
});
|
|
return { status: 'verify' };
|
|
} catch (error) {
|
|
abort();
|
|
return await outcomeForError(
|
|
keeper.error ?? error,
|
|
async () => (await api.request<RunContext>(`/internal/runs/${input.runId}/context`)).run,
|
|
);
|
|
} finally {
|
|
keeper.close();
|
|
keeper.signal.removeEventListener('abort', abort);
|
|
await api.request(`/internal/runs/${input.runId}/lease/release`, { leaseToken }).catch(() => undefined);
|
|
if (workspace && checkpointPersisted) await rm(workspace, { recursive: true, force: true }).catch(() => undefined);
|
|
else if (workspace) {
|
|
// Lease loss, cancellation and ambiguous uploads must never erase the only copy.
|
|
// Keep a private sidecar outside the agent tree so operators can recover this exact attempt.
|
|
const providerRun = providerStarted ? await provider?.inspect(input.runId).catch(() => undefined) : undefined;
|
|
await writeFile(
|
|
`${workspace}.recovery.json`,
|
|
JSON.stringify(
|
|
{
|
|
schemaVersion: 1,
|
|
tenantId: input.tenantId,
|
|
runId: input.runId,
|
|
generation: input.generation,
|
|
leaseToken,
|
|
workspace,
|
|
baseCommit: recoveryBaseCommit,
|
|
provider: providerRun?.provider,
|
|
providerStatus: providerRun?.status,
|
|
sessionId: providerRun?.sessionId,
|
|
...(recoveryCheckpoint ? { checkpoint: recoveryCheckpoint } : {}),
|
|
retainedAt: new Date().toISOString(),
|
|
reason: 'Checkpoint persistence was not confirmed; workspace retained',
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
{ mode: 0o600 },
|
|
).catch(() => undefined);
|
|
}
|
|
}
|
|
}
|