- Skills: per-project know-how the platform proposes from each succeeded run (in the background, or on request from a run), people confirm, edit, disable, delete or write; active ones enter later prompts - Export one skill as SKILL.md, or all as a zip with a directory per skill, as .claude/skills/ expects - A wish's report: every run, its stage timeline, the agent's plan, replies, commands, changed files and model usage, the change, the checks, failures, reviews and releases, and the full event log, as Markdown, or laid out for printing to PDF Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
487 lines
22 KiB
TypeScript
487 lines
22 KiB
TypeScript
import path from 'node:path';
|
|
import { rm, writeFile } from 'node:fs/promises';
|
|
import { countedTokens, type Run } from '@wishforge/contracts';
|
|
import { BASELINE_BUNDLE, BASELINE_VERSION, bundleDigest, bundlePrompt, questionsIn } from '@wishforge/agent';
|
|
import {
|
|
executeCommand,
|
|
outputTail,
|
|
readWishforgeManifest,
|
|
proxyEnvironment,
|
|
requireBranchName,
|
|
RuntimeError,
|
|
safeIdentifier,
|
|
stoppedResult,
|
|
validateRepositoryUrl,
|
|
type CommandExecutor,
|
|
type CommandResult,
|
|
type ProviderAdapter,
|
|
type PortableCheckpoint,
|
|
} from '@wishforge/runtime';
|
|
import {
|
|
client,
|
|
gitFor,
|
|
outcomeForError,
|
|
sandboxMismatch,
|
|
stoppedOutcome,
|
|
tenantRoot,
|
|
waiting,
|
|
type ActivityOutcome,
|
|
type RunContext,
|
|
type RunInput,
|
|
type WorkerConfig,
|
|
} from './common.js';
|
|
import { ControlPlaneError, type ControlPlaneClient } from './client.js';
|
|
import { LeaseKeeper, activityContext } from './lease.js';
|
|
import { installPlan } from './project-contract.js';
|
|
import { closeHostChecks, hostCheckExecutor, type HostCheckExecutor } from './host-checks.js';
|
|
import { providerRegistry } from './providers.js';
|
|
|
|
/** Where a run continues when it already has a checkpoint or a release; no executor lease is needed for that. */
|
|
function resumePoint(context: RunContext): ActivityOutcome | undefined {
|
|
const { run, delivery } = context;
|
|
if (delivery?.release)
|
|
return waiting(
|
|
delivery.release.status === 'awaiting_approval' ? 'approval' : 'deployment',
|
|
`Release ${delivery.release.id} is ${delivery.release.status}; it has to finish before anything else runs.`,
|
|
);
|
|
if (
|
|
['build', 'release', 'accept'].includes(run.stage) &&
|
|
delivery?.verificationEvidenceId &&
|
|
delivery.buildEvidenceId &&
|
|
delivery.environmentId
|
|
)
|
|
return {
|
|
status: 'release',
|
|
verificationEvidenceId: delivery.verificationEvidenceId,
|
|
buildEvidenceId: delivery.buildEvidenceId,
|
|
environmentId: delivery.environmentId,
|
|
};
|
|
if (context.checkpoint && ['verify', 'integrate', 'build'].includes(run.stage)) return { status: 'verify' };
|
|
if (['release', 'accept'].includes(run.stage))
|
|
return waiting('deployment', 'The build evidence for this release is missing; reconcile the release first.');
|
|
}
|
|
|
|
/** Lease the run, or explain why it cannot run now. */
|
|
export async function leaseRun(
|
|
api: ControlPlaneClient,
|
|
config: WorkerConfig,
|
|
input: RunInput,
|
|
): Promise<{ run: Run; keeper: LeaseKeeper } | ActivityOutcome> {
|
|
try {
|
|
const run = await api.request<Run>(`/internal/runs/${input.runId}/lease`, {
|
|
workerId: config.workerId,
|
|
generation: input.generation,
|
|
});
|
|
if (!run.lease) throw new RuntimeError('LEASE_MISSING', 'The control plane did not grant a lease');
|
|
return { run, keeper: new LeaseKeeper(api, input.runId, run.lease.token, undefined, config.heartbeatMs).start() };
|
|
} catch (error) {
|
|
if (!(error instanceof ControlPlaneError)) throw error;
|
|
if (error.domainCode === 'CONCURRENCY_LIMIT')
|
|
return waiting('worker_unavailable', 'All execution slots of this tenant are in use; continue when one is free.');
|
|
if (['EXECUTION_BUDGET_EXHAUSTED', 'TOKEN_BUDGET_EXHAUSTED'].includes(error.domainCode))
|
|
return waiting('budget', 'The run used up its token or execution-time budget.');
|
|
if (error.domainCode === 'RUN_NOT_RUNNABLE') {
|
|
const { run } = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
|
|
return stoppedOutcome(run) ?? { status: 'waiting' };
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function createProvider(config: WorkerConfig, context: RunContext, isolatedHome: string): ProviderAdapter | undefined {
|
|
if (config.providerFactory) return config.providerFactory(context, isolatedHome);
|
|
return providerRegistry[context.run.provider]?.(config, context, isolatedHome);
|
|
}
|
|
|
|
async function install(
|
|
execute: CommandExecutor,
|
|
commands: string[][],
|
|
cwd: string,
|
|
sandboxRoot: string,
|
|
home: string,
|
|
proxy?: string,
|
|
): Promise<{ argv: string[]; result: CommandResult } | undefined> {
|
|
for (const argv of commands) {
|
|
let result: CommandResult;
|
|
try {
|
|
result = await execute({
|
|
executable: argv[0]!,
|
|
args: argv.slice(1),
|
|
cwd,
|
|
sandboxRoot,
|
|
env: { PATH: process.env.PATH, HOME: home, LANG: 'C.UTF-8', CI: 'true', ...proxyEnvironment(proxy) },
|
|
timeoutMs: 15 * 60_000,
|
|
network: true,
|
|
});
|
|
} catch (error) {
|
|
if (!(error instanceof RuntimeError) || !['TIMEOUT', 'OUTPUT_LIMIT'].includes(error.code)) throw error;
|
|
result = stoppedResult(error, { timeoutMs: 15 * 60_000 });
|
|
}
|
|
if (result.exitCode !== 0) return { argv, result };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Executor activity: prepares an isolated workspace from the target branch or the latest
|
|
* checkpoint, installs dependencies, runs the agent and hands a checkpoint to verification.
|
|
*/
|
|
export async function executeRun(config: WorkerConfig, input: RunInput): Promise<ActivityOutcome> {
|
|
const api = client(config, input);
|
|
const observed = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
|
|
const stopped = stoppedOutcome(observed.run);
|
|
if (stopped) return stopped;
|
|
if (observed.run.status === 'waiting') return { status: 'waiting' };
|
|
const resume = resumePoint(observed);
|
|
if (resume) return resume;
|
|
const unbound = sandboxMismatch(config, input);
|
|
if (unbound) return waiting('sandbox_provisioner', unbound);
|
|
const leased = await leaseRun(api, config, input);
|
|
if ('status' in leased) return leased;
|
|
const { keeper } = leased;
|
|
const leaseToken = keeper.token;
|
|
const activity = activityContext();
|
|
const isolatedHome = path.join(
|
|
config.isolatedProviderHomeRoot,
|
|
safeIdentifier(input.tenantId),
|
|
safeIdentifier(input.runId),
|
|
);
|
|
const execute: CommandExecutor = command => (config.execute ?? executeCommand)({ ...command, signal: keeper.signal });
|
|
let checks: HostCheckExecutor | undefined;
|
|
let provider: ProviderAdapter | undefined;
|
|
let providerStarted = false;
|
|
const abort = () => {
|
|
if (providerStarted) void provider!.cancel(input.runId).catch(() => undefined);
|
|
};
|
|
keeper.signal.addEventListener('abort', abort, { once: true });
|
|
keeper.pauseRequested.addEventListener('abort', abort, { once: true });
|
|
/** A person paused the run: tell the control plane the work so far is saved, if there was any. */
|
|
const pause = async (saved: boolean): Promise<ActivityOutcome> => {
|
|
await api.request(`/internal/runs/${input.runId}/pause`, { leaseToken, saved });
|
|
return { status: 'paused' };
|
|
};
|
|
let workspace: string | undefined;
|
|
let checkpointPersisted = false;
|
|
let recoveryBaseCommit: string | undefined;
|
|
let recoveryCheckpoint: PortableCheckpoint | undefined;
|
|
const transition = (stage: Run['stage'], message: string, extras: Record<string, unknown> = {}) =>
|
|
api.request<Run>(`/internal/runs/${input.runId}/transition`, {
|
|
leaseToken,
|
|
stage,
|
|
status: 'running',
|
|
message,
|
|
...extras,
|
|
});
|
|
const settleFinalUsage = async (measurements: Array<{ operationId: string; tokens: number }>) => {
|
|
for (const measurement of measurements) {
|
|
await api.request(
|
|
`/internal/runs/${input.runId}/meter`,
|
|
{ leaseToken, ...measurement, computeSeconds: 0 },
|
|
{ retries: 2 },
|
|
);
|
|
// A final total may consume the remaining budget. Source is already durable; do not
|
|
// continue to verification with the now-expired lease. Replays use the same operation ID.
|
|
await keeper.ensure();
|
|
}
|
|
};
|
|
try {
|
|
const context = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
|
|
let run = context.run;
|
|
try {
|
|
requireBranchName(context.project.targetBranch);
|
|
if (context.project.repositoryUrl)
|
|
validateRepositoryUrl(context.project.repositoryUrl, { allowLocalHttp: config.allowLocalGitHttp });
|
|
} catch (error) {
|
|
return waiting('git_storage', error instanceof Error ? error.message : 'The project repository is not usable');
|
|
}
|
|
const previousCompletion = context.checkpoint?.providerCompletion;
|
|
if (previousCompletion) {
|
|
await settleFinalUsage(previousCompletion.finalUsage);
|
|
if (
|
|
previousCompletion.status === 'completed' &&
|
|
previousCompletion.revision === run.revision &&
|
|
previousCompletion.repairCount === run.repairCount &&
|
|
run.stage === 'implement'
|
|
) {
|
|
const questions = run.repairCount ? [] : questionsIn(previousCompletion.summary);
|
|
if (questions.length && context.checkpoint!.sourceCommit === context.checkpoint!.baseCommit)
|
|
return waiting('clarification', `The executor needs answers before implementing:\n${questions.join('\n')}`);
|
|
await transition(
|
|
'verify',
|
|
'Saved implementation and final usage reconciled; independent verification is next',
|
|
{
|
|
sourceCommit: context.checkpoint!.sourceCommit,
|
|
...(previousCompletion.summary ? { summary: previousCompletion.summary } : {}),
|
|
},
|
|
);
|
|
return { status: 'verify' };
|
|
}
|
|
}
|
|
provider = createProvider(config, context, isolatedHome);
|
|
if (!provider) return waiting('credentials', `The ${run.provider} executor is not available on this worker.`);
|
|
const capabilities = provider.capabilities();
|
|
if (!capabilities.coding)
|
|
return waiting('credentials', capabilities.reason ?? `The ${run.provider} executor is not configured.`);
|
|
const bundle = context.bundleConfig ?? (run.bundleVersion === BASELINE_VERSION ? BASELINE_BUNDLE : undefined);
|
|
if (!bundle || (run.bundleVersion !== BASELINE_VERSION && bundleDigest(bundle) !== run.bundleVersion))
|
|
throw new RuntimeError(
|
|
'BUNDLE_UNAVAILABLE',
|
|
'The pinned agent configuration is missing or differs from its version',
|
|
);
|
|
if (run.stage === 'clarify')
|
|
run = await transition('plan', 'Acceptance criteria recorded; preparing the workspace');
|
|
run = await transition(
|
|
'implement',
|
|
run.repairCount ? `Repair round ${run.repairCount}: preparing the workspace` : 'Preparing an isolated workspace',
|
|
);
|
|
const git = gitFor(config, context, execute);
|
|
const attempt = `${input.runId}-a${leaseToken}`;
|
|
const prepared = context.checkpoint
|
|
? await git.restore({
|
|
checkpoint: context.checkpoint,
|
|
repository: context.project.repositoryUrl,
|
|
root: tenantRoot(config, input),
|
|
runId: attempt,
|
|
})
|
|
: await git.create({
|
|
repository: context.project.repositoryUrl,
|
|
targetBranch: context.project.targetBranch,
|
|
sourceBranch: run.sourceBranch,
|
|
baseCommit: run.sourceCommit,
|
|
root: tenantRoot(config, input),
|
|
runId: attempt,
|
|
});
|
|
workspace = prepared.workspace;
|
|
let baseCommit = prepared.baseCommit;
|
|
recoveryBaseCommit = baseCommit;
|
|
let conflicts: string[] = [];
|
|
let lastFailure = run.repairCount ? run.lastFailure : undefined;
|
|
// After a merge conflict the repair round starts by merging the current target branch;
|
|
// the agent resolves what Git could not, and the checkpoint is relative to that target.
|
|
if (run.repairCount && run.lastFailure?.kind === 'integration_conflict') {
|
|
const merged = await git.mergeTarget(workspace, context.project.targetBranch);
|
|
baseCommit = merged.targetCommit;
|
|
recoveryBaseCommit = baseCommit;
|
|
conflicts = merged.conflicts;
|
|
// The prompt names the conflicts of this merge; the target may have moved since integration.
|
|
lastFailure = {
|
|
...run.lastFailure,
|
|
targetCommit: merged.targetCommit,
|
|
outputTail:
|
|
conflicts.join('\n') || '(Git merged cleanly this time; check that both changes still work together.)',
|
|
};
|
|
}
|
|
// A frontend-only wish changed backend code: the platform puts those files back before the
|
|
// agent works again, which with file tools alone could neither recover nor delete them.
|
|
if (run.repairCount && run.lastFailure?.kind === 'scope')
|
|
await git.restorePaths(workspace, baseCommit, run.lastFailure.outputTail.split('\n').filter(Boolean));
|
|
// A revert's first round reverses the change itself: no agent, unless later commits changed
|
|
// the same lines, which it then resolves.
|
|
let revert: { integratedCommit: string; conflicts: string[] } | undefined;
|
|
if (run.revertOf && !context.checkpoint && !run.repairCount) {
|
|
const reverted = await git.revertChange(workspace, run.revertOf.targetCommit, run.revertOf.integratedCommit);
|
|
if (!reverted.conflicts.length) {
|
|
const summary = `撤销了运行 ${run.revertOf.runId} 合入的提交 ${run.revertOf.integratedCommit.slice(0, 12)} 带来的改动。`;
|
|
const checkpoint: PortableCheckpoint = {
|
|
...(await git.checkpoint(workspace, baseCommit, input.runId, run.provider)),
|
|
providerCompletion: {
|
|
status: 'completed',
|
|
revision: run.revision,
|
|
repairCount: run.repairCount,
|
|
summary,
|
|
finalUsage: [],
|
|
},
|
|
};
|
|
await keeper.ensure();
|
|
await api.request(`/internal/runs/${input.runId}/checkpoint`, { leaseToken, checkpoint, workspace });
|
|
checkpointPersisted = true;
|
|
if (keeper.pauseRequested.aborted) return await pause(true);
|
|
await transition('verify', 'Reverted the change without the agent; independent verification is next', {
|
|
sourceCommit: checkpoint.sourceCommit,
|
|
summary,
|
|
});
|
|
return { status: 'verify' };
|
|
}
|
|
revert = { integratedCommit: run.revertOf.integratedCommit, conflicts: reverted.conflicts };
|
|
conflicts = reverted.conflicts;
|
|
}
|
|
const manifest = await readWishforgeManifest({ workspace, sourceCommit: await git.head(workspace) }, execute).catch(
|
|
() => undefined,
|
|
);
|
|
const plan = installPlan(context.installCommands, manifest?.manifest);
|
|
// Conflicted manifests cannot be installed; the agent resolves them first and the verifier installs.
|
|
if (plan.commands.length && !conflicts.length) {
|
|
checks = hostCheckExecutor(config, config.execute ?? executeCommand, keeper.signal);
|
|
const failed = await install(
|
|
checks,
|
|
plan.commands,
|
|
path.join(workspace, plan.directory),
|
|
workspace,
|
|
isolatedHome,
|
|
config.installProxy,
|
|
);
|
|
// A repair round for an installation the agent's manifest changes broke starts the agent
|
|
// anyway, with this output; stopping here would leave it no way to fix them.
|
|
if (failed && lastFailure?.kind === 'install')
|
|
lastFailure = {
|
|
...lastFailure,
|
|
argv: failed.argv,
|
|
exitCode: failed.result.exitCode,
|
|
outputTail: outputTail(failed.result, 4000),
|
|
};
|
|
else if (failed)
|
|
return waiting(
|
|
'infrastructure',
|
|
`Installing dependencies failed: ${failed.argv.join(' ')} exited ${failed.result.exitCode}.\n${outputTail(failed.result, 1500)}`,
|
|
);
|
|
await git.excludeUntracked(workspace);
|
|
}
|
|
await keeper.ensure();
|
|
if (keeper.pauseRequested.aborted) return await pause(false);
|
|
const prompt = bundlePrompt(bundle, {
|
|
goal: run.goal,
|
|
acceptanceCriteria: run.acceptanceCriteria,
|
|
repairCount: run.repairCount,
|
|
lastFailure,
|
|
lessons: context.lessons,
|
|
skills: context.skills,
|
|
sourceBranch: run.sourceBranch,
|
|
...(revert ? { revert } : {}),
|
|
...(run.frontendOnly && context.project.codeReview?.paths.length
|
|
? {
|
|
frontendOnly: {
|
|
backend: context.project.codeReview.paths,
|
|
except: context.project.codeReview.exceptPaths,
|
|
},
|
|
}
|
|
: {}),
|
|
});
|
|
await provider.start({
|
|
runId: input.runId,
|
|
tenantId: input.tenantId,
|
|
workspace,
|
|
// An executor whose CLI restricts its tools adds what it may use (see the adapters).
|
|
prompt,
|
|
baseCommit,
|
|
// The model the run pinned at submission (its project's choice), else the agent configuration's.
|
|
model: run.providerModel ?? bundle.model,
|
|
});
|
|
providerStarted = true;
|
|
// Cancellation can arrive while start awaits filesystem/process preparation,
|
|
// before the abort listener knows there is a provider attempt to stop.
|
|
if (keeper.signal.aborted || keeper.error || keeper.pauseRequested.aborted) abort();
|
|
await keeper.ensure();
|
|
let summary: string | undefined;
|
|
const finalUsage: Array<{ operationId: string; tokens: number }> = [];
|
|
for await (const event of provider.events(input.runId)) {
|
|
activity?.heartbeat({ runId: input.runId, sequence: event.sequence, leaseToken });
|
|
if (event.type === 'output' && typeof event.data.text === 'string') summary = event.data.text;
|
|
// Only normalized events leave the sandbox; reasoning and raw process output are not exported.
|
|
await api.request(`/internal/runs/${input.runId}/events`, { leaseToken, event }, { retries: 2 });
|
|
if (event.type === 'usage' && context.providerCredential?.metering !== 'proxy') {
|
|
// Cache reads count a tenth, as they cost (countedTokens).
|
|
const tokens = countedTokens(event.data);
|
|
if (Number.isSafeInteger(tokens) && tokens >= 0) {
|
|
const measurement = { operationId: `provider-${input.runId}-${leaseToken}-${event.sequence}`, tokens };
|
|
if (event.data.final === true) finalUsage.push(measurement);
|
|
else
|
|
await api.request(
|
|
`/internal/runs/${input.runId}/meter`,
|
|
{
|
|
leaseToken,
|
|
...measurement,
|
|
computeSeconds: 0,
|
|
},
|
|
{ retries: 2 },
|
|
);
|
|
}
|
|
}
|
|
}
|
|
if (keeper.error) throw keeper.error;
|
|
const providerRun = await provider.inspect(input.runId);
|
|
// The work so far is kept even when the executor failed, so continuing does not start over.
|
|
if (providerRun.status === 'running')
|
|
throw new RuntimeError(
|
|
'WORKSPACE_NOT_QUIESCENT',
|
|
'The provider stream ended before the workspace stopped changing',
|
|
);
|
|
const checkpoint: PortableCheckpoint = {
|
|
...(await provider.exportCheckpoint(input.runId)),
|
|
providerCompletion: {
|
|
status: providerRun.status,
|
|
revision: run.revision,
|
|
repairCount: run.repairCount,
|
|
...(summary ? { summary: summary.slice(0, 4000) } : {}),
|
|
finalUsage,
|
|
},
|
|
};
|
|
recoveryCheckpoint = checkpoint;
|
|
await keeper.ensure();
|
|
await api.request(`/internal/runs/${input.runId}/checkpoint`, { leaseToken, checkpoint, workspace });
|
|
checkpointPersisted = true;
|
|
await settleFinalUsage(finalUsage);
|
|
// Paused: the agent was stopped, or had just finished; continuing starts from this checkpoint.
|
|
if (keeper.pauseRequested.aborted) return await pause(true);
|
|
if (providerRun.status !== 'completed')
|
|
return waiting(
|
|
'provider_error',
|
|
`The ${run.provider} executor ${providerRun.status === 'canceled' ? 'stopped' : 'failed'}: ${providerRun.error ?? 'no details'}. Its work was saved; continue to retry.`,
|
|
);
|
|
// An agent that asked instead of changing anything waits for the person's answer; answering
|
|
// creates a new requirement revision that starts again from planning.
|
|
const questions = run.repairCount ? [] : questionsIn(summary);
|
|
if (questions.length && checkpoint.sourceCommit === checkpoint.baseCommit)
|
|
return waiting(
|
|
'clarification',
|
|
`The ${run.provider} executor needs answers before implementing:\n${questions.map((question, i) => `${i + 1}. ${question}`).join('\n')}`,
|
|
);
|
|
await transition('verify', 'Implementation checkpoint created; independent verification is next', {
|
|
sourceCommit: checkpoint.sourceCommit,
|
|
...(summary ? { summary: summary.slice(0, 4000) } : {}),
|
|
});
|
|
return { status: 'verify' };
|
|
} catch (error) {
|
|
abort();
|
|
return await outcomeForError(
|
|
keeper.error ?? error,
|
|
async () => (await api.request<RunContext>(`/internal/runs/${input.runId}/context`)).run,
|
|
);
|
|
} finally {
|
|
keeper.close();
|
|
keeper.signal.removeEventListener('abort', abort);
|
|
keeper.pauseRequested.removeEventListener('abort', abort);
|
|
const checksClosed = await closeHostChecks(checks);
|
|
await api.request(`/internal/runs/${input.runId}/lease/release`, { leaseToken }).catch(() => undefined);
|
|
// Before the agent ran, the workspace holds nothing a checkpoint or the repository cannot recreate.
|
|
if (workspace && (checkpointPersisted || !providerStarted)) {
|
|
if (checksClosed) await rm(workspace, { recursive: true, force: true }).catch(() => undefined);
|
|
} else if (workspace) {
|
|
// Lease loss, cancellation and ambiguous uploads must never erase the only copy.
|
|
// Keep a private sidecar outside the agent tree so operators can recover this exact attempt.
|
|
const providerRun = providerStarted ? await provider?.inspect(input.runId).catch(() => undefined) : undefined;
|
|
await writeFile(
|
|
`${workspace}.recovery.json`,
|
|
JSON.stringify(
|
|
{
|
|
schemaVersion: 1,
|
|
tenantId: input.tenantId,
|
|
runId: input.runId,
|
|
generation: input.generation,
|
|
leaseToken,
|
|
workspace,
|
|
baseCommit: recoveryBaseCommit,
|
|
provider: providerRun?.provider,
|
|
providerStatus: providerRun?.status,
|
|
sessionId: providerRun?.sessionId,
|
|
...(recoveryCheckpoint ? { checkpoint: recoveryCheckpoint } : {}),
|
|
retainedAt: new Date().toISOString(),
|
|
reason: 'Checkpoint persistence was not confirmed; workspace retained',
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
{ mode: 0o600 },
|
|
).catch(() => undefined);
|
|
}
|
|
}
|
|
}
|