eust-w 72085cf752 ✨ Let a project offer models and a wish choose one
- Projects list the models their wishes may run with, the first by
  default; owners and admins set them in the project settings
- A wish may choose one of its project's models; the choice is checked
  and pinned to the run, which shows it, and the executor starts the
  agent with it (--model), else with the agent configuration's
- Model names may be spelled as the model gateway does
  (deepseek-pro@latest[1m]), in projects and agent configurations alike

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 03:08:35 +08:00

489 lines
22 KiB
TypeScript

import path from 'node:path';
import { rm, writeFile } from 'node:fs/promises';
import type { Run } from '@wishforge/contracts';
import { BASELINE_BUNDLE, BASELINE_VERSION, bundleDigest, bundlePrompt, questionsIn } from '@wishforge/agent';
import {
executeCommand,
outputTail,
readWishforgeManifest,
proxyEnvironment,
requireBranchName,
RuntimeError,
safeIdentifier,
stoppedResult,
validateRepositoryUrl,
type CommandExecutor,
type CommandResult,
type ProviderAdapter,
type PortableCheckpoint,
} from '@wishforge/runtime';
import {
client,
gitFor,
outcomeForError,
sandboxMismatch,
stoppedOutcome,
tenantRoot,
waiting,
type ActivityOutcome,
type RunContext,
type RunInput,
type WorkerConfig,
} from './common.js';
import { ControlPlaneError, type ControlPlaneClient } from './client.js';
import { LeaseKeeper, activityContext } from './lease.js';
import { installPlan } from './project-contract.js';
import { closeHostChecks, hostCheckExecutor, type HostCheckExecutor } from './host-checks.js';
import { providerRegistry } from './providers.js';
/** Where a run continues when it already has a checkpoint or a release; no executor lease is needed for that. */
function resumePoint(context: RunContext): ActivityOutcome | undefined {
const { run, delivery } = context;
if (delivery?.release)
return waiting(
delivery.release.status === 'awaiting_approval' ? 'approval' : 'deployment',
`Release ${delivery.release.id} is ${delivery.release.status}; it has to finish before anything else runs.`,
);
if (
['build', 'release', 'accept'].includes(run.stage) &&
delivery?.verificationEvidenceId &&
delivery.buildEvidenceId &&
delivery.environmentId
)
return {
status: 'release',
verificationEvidenceId: delivery.verificationEvidenceId,
buildEvidenceId: delivery.buildEvidenceId,
environmentId: delivery.environmentId,
};
if (context.checkpoint && ['verify', 'integrate', 'build'].includes(run.stage)) return { status: 'verify' };
if (['release', 'accept'].includes(run.stage))
return waiting('deployment', 'The build evidence for this release is missing; reconcile the release first.');
}
/** Lease the run, or explain why it cannot run now. */
export async function leaseRun(
api: ControlPlaneClient,
config: WorkerConfig,
input: RunInput,
): Promise<{ run: Run; keeper: LeaseKeeper } | ActivityOutcome> {
try {
const run = await api.request<Run>(`/internal/runs/${input.runId}/lease`, {
workerId: config.workerId,
generation: input.generation,
});
if (!run.lease) throw new RuntimeError('LEASE_MISSING', 'The control plane did not grant a lease');
return { run, keeper: new LeaseKeeper(api, input.runId, run.lease.token, undefined, config.heartbeatMs).start() };
} catch (error) {
if (!(error instanceof ControlPlaneError)) throw error;
if (error.domainCode === 'CONCURRENCY_LIMIT')
return waiting('worker_unavailable', 'All execution slots of this tenant are in use; continue when one is free.');
if (['EXECUTION_BUDGET_EXHAUSTED', 'TOKEN_BUDGET_EXHAUSTED'].includes(error.domainCode))
return waiting('budget', 'The run used up its token or execution-time budget.');
if (error.domainCode === 'RUN_NOT_RUNNABLE') {
const { run } = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
return stoppedOutcome(run) ?? { status: 'waiting' };
}
throw error;
}
}
function createProvider(config: WorkerConfig, context: RunContext, isolatedHome: string): ProviderAdapter | undefined {
if (config.providerFactory) return config.providerFactory(context, isolatedHome);
return providerRegistry[context.run.provider]?.(config, context, isolatedHome);
}
async function install(
execute: CommandExecutor,
commands: string[][],
cwd: string,
sandboxRoot: string,
home: string,
proxy?: string,
): Promise<{ argv: string[]; result: CommandResult } | undefined> {
for (const argv of commands) {
let result: CommandResult;
try {
result = await execute({
executable: argv[0]!,
args: argv.slice(1),
cwd,
sandboxRoot,
env: { PATH: process.env.PATH, HOME: home, LANG: 'C.UTF-8', CI: 'true', ...proxyEnvironment(proxy) },
timeoutMs: 15 * 60_000,
network: true,
});
} catch (error) {
if (!(error instanceof RuntimeError) || !['TIMEOUT', 'OUTPUT_LIMIT'].includes(error.code)) throw error;
result = stoppedResult(error, { timeoutMs: 15 * 60_000 });
}
if (result.exitCode !== 0) return { argv, result };
}
}
/**
* Executor activity: prepares an isolated workspace from the target branch or the latest
* checkpoint, installs dependencies, runs the agent and hands a checkpoint to verification.
*/
export async function executeRun(config: WorkerConfig, input: RunInput): Promise<ActivityOutcome> {
const api = client(config, input);
const observed = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
const stopped = stoppedOutcome(observed.run);
if (stopped) return stopped;
if (observed.run.status === 'waiting') return { status: 'waiting' };
const resume = resumePoint(observed);
if (resume) return resume;
const unbound = sandboxMismatch(config, input);
if (unbound) return waiting('sandbox_provisioner', unbound);
const leased = await leaseRun(api, config, input);
if ('status' in leased) return leased;
const { keeper } = leased;
const leaseToken = keeper.token;
const activity = activityContext();
const isolatedHome = path.join(
config.isolatedProviderHomeRoot,
safeIdentifier(input.tenantId),
safeIdentifier(input.runId),
);
const execute: CommandExecutor = command => (config.execute ?? executeCommand)({ ...command, signal: keeper.signal });
let checks: HostCheckExecutor | undefined;
let provider: ProviderAdapter | undefined;
let providerStarted = false;
const abort = () => {
if (providerStarted) void provider!.cancel(input.runId).catch(() => undefined);
};
keeper.signal.addEventListener('abort', abort, { once: true });
keeper.pauseRequested.addEventListener('abort', abort, { once: true });
/** A person paused the run: tell the control plane the work so far is saved, if there was any. */
const pause = async (saved: boolean): Promise<ActivityOutcome> => {
await api.request(`/internal/runs/${input.runId}/pause`, { leaseToken, saved });
return { status: 'paused' };
};
let workspace: string | undefined;
let checkpointPersisted = false;
let recoveryBaseCommit: string | undefined;
let recoveryCheckpoint: PortableCheckpoint | undefined;
const transition = (stage: Run['stage'], message: string, extras: Record<string, unknown> = {}) =>
api.request<Run>(`/internal/runs/${input.runId}/transition`, {
leaseToken,
stage,
status: 'running',
message,
...extras,
});
const settleFinalUsage = async (measurements: Array<{ operationId: string; tokens: number }>) => {
for (const measurement of measurements) {
await api.request(
`/internal/runs/${input.runId}/meter`,
{ leaseToken, ...measurement, computeSeconds: 0 },
{ retries: 2 },
);
// A final total may consume the remaining budget. Source is already durable; do not
// continue to verification with the now-expired lease. Replays use the same operation ID.
await keeper.ensure();
}
};
try {
const context = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
let run = context.run;
try {
requireBranchName(context.project.targetBranch);
if (context.project.repositoryUrl)
validateRepositoryUrl(context.project.repositoryUrl, { allowLocalHttp: config.allowLocalGitHttp });
} catch (error) {
return waiting('git_storage', error instanceof Error ? error.message : 'The project repository is not usable');
}
const previousCompletion = context.checkpoint?.providerCompletion;
if (previousCompletion) {
await settleFinalUsage(previousCompletion.finalUsage);
if (
previousCompletion.status === 'completed' &&
previousCompletion.revision === run.revision &&
previousCompletion.repairCount === run.repairCount &&
run.stage === 'implement'
) {
const questions = run.repairCount ? [] : questionsIn(previousCompletion.summary);
if (questions.length && context.checkpoint!.sourceCommit === context.checkpoint!.baseCommit)
return waiting('clarification', `The executor needs answers before implementing:\n${questions.join('\n')}`);
await transition(
'verify',
'Saved implementation and final usage reconciled; independent verification is next',
{
sourceCommit: context.checkpoint!.sourceCommit,
...(previousCompletion.summary ? { summary: previousCompletion.summary } : {}),
},
);
return { status: 'verify' };
}
}
provider = createProvider(config, context, isolatedHome);
if (!provider) return waiting('credentials', `The ${run.provider} executor is not available on this worker.`);
const capabilities = provider.capabilities();
if (!capabilities.coding)
return waiting('credentials', capabilities.reason ?? `The ${run.provider} executor is not configured.`);
const bundle = context.bundleConfig ?? (run.bundleVersion === BASELINE_VERSION ? BASELINE_BUNDLE : undefined);
if (!bundle || (run.bundleVersion !== BASELINE_VERSION && bundleDigest(bundle) !== run.bundleVersion))
throw new RuntimeError(
'BUNDLE_UNAVAILABLE',
'The pinned agent configuration is missing or differs from its version',
);
if (run.stage === 'clarify')
run = await transition('plan', 'Acceptance criteria recorded; preparing the workspace');
run = await transition(
'implement',
run.repairCount ? `Repair round ${run.repairCount}: preparing the workspace` : 'Preparing an isolated workspace',
);
const git = gitFor(config, context, execute);
const attempt = `${input.runId}-a${leaseToken}`;
const prepared = context.checkpoint
? await git.restore({
checkpoint: context.checkpoint,
repository: context.project.repositoryUrl,
root: tenantRoot(config, input),
runId: attempt,
})
: await git.create({
repository: context.project.repositoryUrl,
targetBranch: context.project.targetBranch,
sourceBranch: run.sourceBranch,
baseCommit: run.sourceCommit,
root: tenantRoot(config, input),
runId: attempt,
});
workspace = prepared.workspace;
let baseCommit = prepared.baseCommit;
recoveryBaseCommit = baseCommit;
let conflicts: string[] = [];
let lastFailure = run.repairCount ? run.lastFailure : undefined;
// After a merge conflict the repair round starts by merging the current target branch;
// the agent resolves what Git could not, and the checkpoint is relative to that target.
if (run.repairCount && run.lastFailure?.kind === 'integration_conflict') {
const merged = await git.mergeTarget(workspace, context.project.targetBranch);
baseCommit = merged.targetCommit;
recoveryBaseCommit = baseCommit;
conflicts = merged.conflicts;
// The prompt names the conflicts of this merge; the target may have moved since integration.
lastFailure = {
...run.lastFailure,
targetCommit: merged.targetCommit,
outputTail:
conflicts.join('\n') || '(Git merged cleanly this time; check that both changes still work together.)',
};
}
// A frontend-only wish changed backend code: the platform puts those files back before the
// agent works again, which with file tools alone could neither recover nor delete them.
if (run.repairCount && run.lastFailure?.kind === 'scope')
await git.restorePaths(workspace, baseCommit, run.lastFailure.outputTail.split('\n').filter(Boolean));
// A revert's first round reverses the change itself: no agent, unless later commits changed
// the same lines, which it then resolves.
let revert: { integratedCommit: string; conflicts: string[] } | undefined;
if (run.revertOf && !context.checkpoint && !run.repairCount) {
const reverted = await git.revertChange(workspace, run.revertOf.targetCommit, run.revertOf.integratedCommit);
if (!reverted.conflicts.length) {
const summary = `撤销了运行 ${run.revertOf.runId} 合入的提交 ${run.revertOf.integratedCommit.slice(0, 12)} 带来的改动。`;
const checkpoint: PortableCheckpoint = {
...(await git.checkpoint(workspace, baseCommit, input.runId, run.provider)),
providerCompletion: {
status: 'completed',
revision: run.revision,
repairCount: run.repairCount,
summary,
finalUsage: [],
},
};
await keeper.ensure();
await api.request(`/internal/runs/${input.runId}/checkpoint`, { leaseToken, checkpoint, workspace });
checkpointPersisted = true;
if (keeper.pauseRequested.aborted) return await pause(true);
await transition('verify', 'Reverted the change without the agent; independent verification is next', {
sourceCommit: checkpoint.sourceCommit,
summary,
});
return { status: 'verify' };
}
revert = { integratedCommit: run.revertOf.integratedCommit, conflicts: reverted.conflicts };
conflicts = reverted.conflicts;
}
const manifest = await readWishforgeManifest({ workspace, sourceCommit: await git.head(workspace) }, execute).catch(
() => undefined,
);
const plan = installPlan(context.installCommands, manifest?.manifest);
// Conflicted manifests cannot be installed; the agent resolves them first and the verifier installs.
if (plan.commands.length && !conflicts.length) {
checks = hostCheckExecutor(config, config.execute ?? executeCommand, keeper.signal);
const failed = await install(
checks,
plan.commands,
path.join(workspace, plan.directory),
workspace,
isolatedHome,
config.installProxy,
);
// A repair round for an installation the agent's manifest changes broke starts the agent
// anyway, with this output; stopping here would leave it no way to fix them.
if (failed && lastFailure?.kind === 'install')
lastFailure = {
...lastFailure,
argv: failed.argv,
exitCode: failed.result.exitCode,
outputTail: outputTail(failed.result, 4000),
};
else if (failed)
return waiting(
'infrastructure',
`Installing dependencies failed: ${failed.argv.join(' ')} exited ${failed.result.exitCode}.\n${outputTail(failed.result, 1500)}`,
);
await git.excludeUntracked(workspace);
}
await keeper.ensure();
if (keeper.pauseRequested.aborted) return await pause(false);
const prompt = bundlePrompt(bundle, {
goal: run.goal,
acceptanceCriteria: run.acceptanceCriteria,
repairCount: run.repairCount,
lastFailure,
lessons: context.lessons,
sourceBranch: run.sourceBranch,
...(revert ? { revert } : {}),
...(run.frontendOnly && context.project.codeReview?.paths.length
? {
frontendOnly: {
backend: context.project.codeReview.paths,
except: context.project.codeReview.exceptPaths,
},
}
: {}),
});
await provider.start({
runId: input.runId,
tenantId: input.tenantId,
workspace,
// An executor whose CLI restricts its tools adds what it may use (see the adapters).
prompt,
baseCommit,
// The model the run pinned at submission (its project's choice), else the agent configuration's.
model: run.providerModel ?? bundle.model,
});
providerStarted = true;
// Cancellation can arrive while start awaits filesystem/process preparation,
// before the abort listener knows there is a provider attempt to stop.
if (keeper.signal.aborted || keeper.error || keeper.pauseRequested.aborted) abort();
await keeper.ensure();
let summary: string | undefined;
const finalUsage: Array<{ operationId: string; tokens: number }> = [];
for await (const event of provider.events(input.runId)) {
activity?.heartbeat({ runId: input.runId, sequence: event.sequence, leaseToken });
if (event.type === 'output' && typeof event.data.text === 'string') summary = event.data.text;
// Only normalized events leave the sandbox; reasoning and raw process output are not exported.
await api.request(`/internal/runs/${input.runId}/events`, { leaseToken, event }, { retries: 2 });
if (event.type === 'usage' && context.providerCredential?.metering !== 'proxy') {
const tokens =
Number(event.data.input_tokens ?? 0) +
Number(event.data.output_tokens ?? 0) +
Number(event.data.cache_creation_input_tokens ?? 0) +
Number(event.data.cache_read_input_tokens ?? 0);
if (Number.isSafeInteger(tokens) && tokens >= 0) {
const measurement = { operationId: `provider-${input.runId}-${leaseToken}-${event.sequence}`, tokens };
if (event.data.final === true) finalUsage.push(measurement);
else
await api.request(
`/internal/runs/${input.runId}/meter`,
{
leaseToken,
...measurement,
computeSeconds: 0,
},
{ retries: 2 },
);
}
}
}
if (keeper.error) throw keeper.error;
const providerRun = await provider.inspect(input.runId);
// The work so far is kept even when the executor failed, so continuing does not start over.
if (providerRun.status === 'running')
throw new RuntimeError(
'WORKSPACE_NOT_QUIESCENT',
'The provider stream ended before the workspace stopped changing',
);
const checkpoint: PortableCheckpoint = {
...(await provider.exportCheckpoint(input.runId)),
providerCompletion: {
status: providerRun.status,
revision: run.revision,
repairCount: run.repairCount,
...(summary ? { summary: summary.slice(0, 4000) } : {}),
finalUsage,
},
};
recoveryCheckpoint = checkpoint;
await keeper.ensure();
await api.request(`/internal/runs/${input.runId}/checkpoint`, { leaseToken, checkpoint, workspace });
checkpointPersisted = true;
await settleFinalUsage(finalUsage);
// Paused: the agent was stopped, or had just finished; continuing starts from this checkpoint.
if (keeper.pauseRequested.aborted) return await pause(true);
if (providerRun.status !== 'completed')
return waiting(
'provider_error',
`The ${run.provider} executor ${providerRun.status === 'canceled' ? 'stopped' : 'failed'}: ${providerRun.error ?? 'no details'}. Its work was saved; continue to retry.`,
);
// An agent that asked instead of changing anything waits for the person's answer; answering
// creates a new requirement revision that starts again from planning.
const questions = run.repairCount ? [] : questionsIn(summary);
if (questions.length && checkpoint.sourceCommit === checkpoint.baseCommit)
return waiting(
'clarification',
`The ${run.provider} executor needs answers before implementing:\n${questions.map((question, i) => `${i + 1}. ${question}`).join('\n')}`,
);
await transition('verify', 'Implementation checkpoint created; independent verification is next', {
sourceCommit: checkpoint.sourceCommit,
...(summary ? { summary: summary.slice(0, 4000) } : {}),
});
return { status: 'verify' };
} catch (error) {
abort();
return await outcomeForError(
keeper.error ?? error,
async () => (await api.request<RunContext>(`/internal/runs/${input.runId}/context`)).run,
);
} finally {
keeper.close();
keeper.signal.removeEventListener('abort', abort);
keeper.pauseRequested.removeEventListener('abort', abort);
const checksClosed = await closeHostChecks(checks);
await api.request(`/internal/runs/${input.runId}/lease/release`, { leaseToken }).catch(() => undefined);
// Before the agent ran, the workspace holds nothing a checkpoint or the repository cannot recreate.
if (workspace && (checkpointPersisted || !providerStarted)) {
if (checksClosed) await rm(workspace, { recursive: true, force: true }).catch(() => undefined);
} else if (workspace) {
// Lease loss, cancellation and ambiguous uploads must never erase the only copy.
// Keep a private sidecar outside the agent tree so operators can recover this exact attempt.
const providerRun = providerStarted ? await provider?.inspect(input.runId).catch(() => undefined) : undefined;
await writeFile(
`${workspace}.recovery.json`,
JSON.stringify(
{
schemaVersion: 1,
tenantId: input.tenantId,
runId: input.runId,
generation: input.generation,
leaseToken,
workspace,
baseCommit: recoveryBaseCommit,
provider: providerRun?.provider,
providerStatus: providerRun?.status,
sessionId: providerRun?.sessionId,
...(recoveryCheckpoint ? { checkpoint: recoveryCheckpoint } : {}),
retainedAt: new Date().toISOString(),
reason: 'Checkpoint persistence was not confirmed; workspace retained',
},
null,
2,
),
{ mode: 0o600 },
).catch(() => undefined);
}
}
}