eust-w 6fc7eaf00c ✨ Improve local WishForge reliability and LAN access
Detect actual service and Runner readiness, restore LAN access after network changes, protect production web builds with rollback, honor cancellation during executor startup, and show truthful cached-data and credential states.
2026-10-25 03:29:51 +08:00

418 lines
18 KiB
TypeScript

import path from 'node:path';
import { rm, writeFile } from 'node:fs/promises';
import type { Run } from '@wishforge/contracts';
import { BASELINE_BUNDLE, BASELINE_VERSION, bundleDigest, bundlePrompt, questionsIn } from '@wishforge/agent';
import {
executeCommand,
outputTail,
readWishforgeManifest,
proxyEnvironment,
requireBranchName,
RuntimeError,
safeIdentifier,
TIMEOUT_EXIT_CODE,
validateRepositoryUrl,
type CommandExecutor,
type CommandResult,
type ProviderAdapter,
type PortableCheckpoint,
} from '@wishforge/runtime';
import {
client,
gitFor,
outcomeForError,
sandboxMismatch,
stoppedOutcome,
tenantRoot,
waiting,
type ActivityOutcome,
type RunContext,
type RunInput,
type WorkerConfig,
} from './common.js';
import { ControlPlaneError, type ControlPlaneClient } from './client.js';
import { LeaseKeeper, activityContext } from './lease.js';
import { installPlan } from './project-contract.js';
import { providerRegistry } from './providers.js';
/** Where a run continues when it already has a checkpoint or a release; no executor lease is needed for that. */
function resumePoint(context: RunContext): ActivityOutcome | undefined {
const { run, delivery } = context;
if (delivery?.release)
return waiting(
delivery.release.status === 'awaiting_approval' ? 'approval' : 'deployment',
`Release ${delivery.release.id} is ${delivery.release.status}; it has to finish before anything else runs.`,
);
if (
['build', 'release', 'accept'].includes(run.stage) &&
delivery?.verificationEvidenceId &&
delivery.buildEvidenceId &&
delivery.environmentId
)
return {
status: 'release',
verificationEvidenceId: delivery.verificationEvidenceId,
buildEvidenceId: delivery.buildEvidenceId,
environmentId: delivery.environmentId,
};
if (context.checkpoint && ['verify', 'integrate', 'build'].includes(run.stage)) return { status: 'verify' };
if (['release', 'accept'].includes(run.stage))
return waiting('deployment', 'The build evidence for this release is missing; reconcile the release first.');
}
/** Lease the run, or explain why it cannot run now. */
export async function leaseRun(
api: ControlPlaneClient,
config: WorkerConfig,
input: RunInput,
): Promise<{ run: Run; keeper: LeaseKeeper } | ActivityOutcome> {
try {
const run = await api.request<Run>(`/internal/runs/${input.runId}/lease`, {
workerId: config.workerId,
generation: input.generation,
});
if (!run.lease) throw new RuntimeError('LEASE_MISSING', 'The control plane did not grant a lease');
return { run, keeper: new LeaseKeeper(api, input.runId, run.lease.token).start() };
} catch (error) {
if (!(error instanceof ControlPlaneError)) throw error;
if (error.domainCode === 'CONCURRENCY_LIMIT')
return waiting('worker_unavailable', 'All execution slots of this tenant are in use; continue when one is free.');
if (['EXECUTION_BUDGET_EXHAUSTED', 'TOKEN_BUDGET_EXHAUSTED'].includes(error.domainCode))
return waiting('budget', 'The run used up its token or execution-time budget.');
if (error.domainCode === 'RUN_NOT_RUNNABLE') {
const { run } = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
return stoppedOutcome(run) ?? { status: 'waiting' };
}
throw error;
}
}
function createProvider(config: WorkerConfig, context: RunContext, isolatedHome: string): ProviderAdapter | undefined {
if (config.providerFactory) return config.providerFactory(context, isolatedHome);
return providerRegistry[context.run.provider]?.(config, context, isolatedHome);
}
async function install(
execute: CommandExecutor,
commands: string[][],
cwd: string,
home: string,
proxy?: string,
): Promise<{ argv: string[]; result: CommandResult } | undefined> {
for (const argv of commands) {
let result: CommandResult;
try {
result = await execute({
executable: argv[0]!,
args: argv.slice(1),
cwd,
env: { PATH: process.env.PATH, HOME: home, LANG: 'C.UTF-8', CI: 'true', ...proxyEnvironment(proxy) },
timeoutMs: 15 * 60_000,
});
} catch (error) {
if (!(error instanceof RuntimeError) || !['TIMEOUT', 'OUTPUT_LIMIT'].includes(error.code)) throw error;
result = { exitCode: error.code === 'TIMEOUT' ? TIMEOUT_EXIT_CODE : 1, stdout: '', stderr: error.message };
}
if (result.exitCode !== 0) return { argv, result };
}
}
/**
* Executor activity: prepares an isolated workspace from the target branch or the latest
* checkpoint, installs dependencies, runs the agent and hands a checkpoint to verification.
*/
export async function executeRun(config: WorkerConfig, input: RunInput): Promise<ActivityOutcome> {
const api = client(config, input);
const observed = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
const stopped = stoppedOutcome(observed.run);
if (stopped) return stopped;
if (observed.run.status === 'waiting') return { status: 'waiting' };
const resume = resumePoint(observed);
if (resume) return resume;
const unbound = sandboxMismatch(config, input);
if (unbound) return waiting('sandbox_provisioner', unbound);
const leased = await leaseRun(api, config, input);
if ('status' in leased) return leased;
const { keeper } = leased;
const leaseToken = keeper.token;
const activity = activityContext();
const isolatedHome = path.join(
config.isolatedProviderHomeRoot,
safeIdentifier(input.tenantId),
safeIdentifier(input.runId),
);
const execute: CommandExecutor = command => (config.execute ?? executeCommand)({ ...command, signal: keeper.signal });
let provider: ProviderAdapter | undefined;
let providerStarted = false;
const abort = () => {
if (providerStarted) void provider!.cancel(input.runId).catch(() => undefined);
};
keeper.signal.addEventListener('abort', abort, { once: true });
let workspace: string | undefined;
let checkpointPersisted = false;
let recoveryBaseCommit: string | undefined;
let recoveryCheckpoint: PortableCheckpoint | undefined;
const transition = (stage: Run['stage'], message: string, extras: Record<string, unknown> = {}) =>
api.request<Run>(`/internal/runs/${input.runId}/transition`, {
leaseToken,
stage,
status: 'running',
message,
...extras,
});
const settleFinalUsage = async (measurements: Array<{ operationId: string; tokens: number }>) => {
for (const measurement of measurements) {
await api.request(
`/internal/runs/${input.runId}/meter`,
{ leaseToken, ...measurement, computeSeconds: 0 },
{ retries: 2 },
);
// A final total may consume the remaining budget. Source is already durable; do not
// continue to verification with the now-expired lease. Replays use the same operation ID.
await keeper.ensure();
}
};
try {
const context = await api.request<RunContext>(`/internal/runs/${input.runId}/context`, undefined, { retries: 2 });
let run = context.run;
try {
requireBranchName(context.project.targetBranch);
if (context.project.repositoryUrl)
validateRepositoryUrl(context.project.repositoryUrl, { allowLocalHttp: config.allowLocalGitHttp });
} catch (error) {
return waiting('git_storage', error instanceof Error ? error.message : 'The project repository is not usable');
}
const previousCompletion = context.checkpoint?.providerCompletion;
if (previousCompletion) {
await settleFinalUsage(previousCompletion.finalUsage);
if (
previousCompletion.status === 'completed' &&
previousCompletion.revision === run.revision &&
previousCompletion.repairCount === run.repairCount &&
run.stage === 'implement'
) {
const questions = run.repairCount ? [] : questionsIn(previousCompletion.summary);
if (questions.length && context.checkpoint!.sourceCommit === context.checkpoint!.baseCommit)
return waiting('clarification', `The executor needs answers before implementing:\n${questions.join('\n')}`);
await transition(
'verify',
'Saved implementation and final usage reconciled; independent verification is next',
{
sourceCommit: context.checkpoint!.sourceCommit,
...(previousCompletion.summary ? { summary: previousCompletion.summary } : {}),
},
);
return { status: 'verify' };
}
}
provider = createProvider(config, context, isolatedHome);
if (!provider) return waiting('credentials', `The ${run.provider} executor is not available on this worker.`);
const capabilities = provider.capabilities();
if (!capabilities.coding)
return waiting('credentials', capabilities.reason ?? `The ${run.provider} executor is not configured.`);
const bundle = context.bundleConfig ?? (run.bundleVersion === BASELINE_VERSION ? BASELINE_BUNDLE : undefined);
if (!bundle || (run.bundleVersion !== BASELINE_VERSION && bundleDigest(bundle) !== run.bundleVersion))
throw new RuntimeError(
'BUNDLE_UNAVAILABLE',
'The pinned agent configuration is missing or differs from its version',
);
if (run.stage === 'clarify')
run = await transition('plan', 'Acceptance criteria recorded; preparing the workspace');
run = await transition(
'implement',
run.repairCount ? `Repair round ${run.repairCount}: preparing the workspace` : 'Preparing an isolated workspace',
);
const git = gitFor(config, context, execute);
const attempt = `${input.runId}-a${leaseToken}`;
const prepared = context.checkpoint
? await git.restore({
checkpoint: context.checkpoint,
repository: context.project.repositoryUrl,
root: tenantRoot(config, input),
runId: attempt,
})
: await git.create({
repository: context.project.repositoryUrl,
targetBranch: context.project.targetBranch,
sourceBranch: run.sourceBranch,
baseCommit: run.sourceCommit,
root: tenantRoot(config, input),
runId: attempt,
});
workspace = prepared.workspace;
let baseCommit = prepared.baseCommit;
recoveryBaseCommit = baseCommit;
let conflicts: string[] = [];
let lastFailure = run.repairCount ? run.lastFailure : undefined;
// After a merge conflict the repair round starts by merging the current target branch;
// the agent resolves what Git could not, and the checkpoint is relative to that target.
if (run.repairCount && run.lastFailure?.kind === 'integration_conflict') {
const merged = await git.mergeTarget(workspace, context.project.targetBranch);
baseCommit = merged.targetCommit;
recoveryBaseCommit = baseCommit;
conflicts = merged.conflicts;
// The prompt names the conflicts of this merge; the target may have moved since integration.
lastFailure = {
...run.lastFailure,
targetCommit: merged.targetCommit,
outputTail:
conflicts.join('\n') || '(Git merged cleanly this time; check that both changes still work together.)',
};
}
const manifest = await readWishforgeManifest({ workspace, sourceCommit: await git.head(workspace) }, execute).catch(
() => undefined,
);
const plan = installPlan(context.installCommands, manifest?.manifest);
// Conflicted manifests cannot be installed; the agent resolves them first and the verifier installs.
if (plan.commands.length && !conflicts.length) {
const failed = await install(
execute,
plan.commands,
path.join(workspace, plan.directory),
isolatedHome,
config.installProxy,
);
if (failed)
return waiting(
'infrastructure',
`Installing dependencies failed: ${failed.argv.join(' ')} exited ${failed.result.exitCode}.\n${outputTail(failed.result, 1500)}`,
);
await git.excludeUntracked(workspace);
}
await keeper.ensure();
const prompt = bundlePrompt(bundle, {
goal: run.goal,
acceptanceCriteria: run.acceptanceCriteria,
repairCount: run.repairCount,
lastFailure,
lessons: context.lessons,
sourceBranch: run.sourceBranch,
});
await provider.start({
runId: input.runId,
tenantId: input.tenantId,
workspace,
prompt:
config.claudeAuthHome && run.provider === 'claude-code'
? `${prompt}\n\nLocal execution contract: only Read/Edit/Write/Glob/Grep file tools are available. Implement the requested source changes using those tools. A separate trusted verifier runs installation, tests, Git commit/push and deployment afterwards; do not ask for Bash, Git, deployment credentials or permission to run those commands. Do not claim tests were run by you. Preserve unrelated code and follow the repository versioning instructions.`
: prompt,
baseCommit,
model: bundle.model,
});
providerStarted = true;
// Cancellation can arrive while start awaits filesystem/process preparation,
// before the abort listener knows there is a provider attempt to stop.
if (keeper.signal.aborted || keeper.error) abort();
await keeper.ensure();
let summary: string | undefined;
const finalUsage: Array<{ operationId: string; tokens: number }> = [];
for await (const event of provider.events(input.runId)) {
activity?.heartbeat({ runId: input.runId, sequence: event.sequence, leaseToken });
if (event.type === 'output' && typeof event.data.text === 'string') summary = event.data.text;
// Only normalized events leave the sandbox; reasoning and raw process output are not exported.
await api.request(`/internal/runs/${input.runId}/events`, { leaseToken, event }, { retries: 2 });
if (event.type === 'usage' && context.providerCredential?.metering !== 'proxy') {
const tokens =
Number(event.data.input_tokens ?? 0) +
Number(event.data.output_tokens ?? 0) +
Number(event.data.cache_creation_input_tokens ?? 0) +
Number(event.data.cache_read_input_tokens ?? 0);
if (Number.isSafeInteger(tokens) && tokens >= 0) {
const measurement = { operationId: `provider-${input.runId}-${leaseToken}-${event.sequence}`, tokens };
if (event.data.final === true) finalUsage.push(measurement);
else
await api.request(
`/internal/runs/${input.runId}/meter`,
{
leaseToken,
...measurement,
computeSeconds: 0,
},
{ retries: 2 },
);
}
}
}
if (keeper.error) throw keeper.error;
const providerRun = await provider.inspect(input.runId);
// The work so far is kept even when the executor failed, so continuing does not start over.
if (providerRun.status === 'running')
throw new RuntimeError(
'WORKSPACE_NOT_QUIESCENT',
'The provider stream ended before the workspace stopped changing',
);
const checkpoint: PortableCheckpoint = {
...(await provider.exportCheckpoint(input.runId)),
providerCompletion: {
status: providerRun.status,
revision: run.revision,
repairCount: run.repairCount,
...(summary ? { summary: summary.slice(0, 4000) } : {}),
finalUsage,
},
};
recoveryCheckpoint = checkpoint;
await keeper.ensure();
await api.request(`/internal/runs/${input.runId}/checkpoint`, { leaseToken, checkpoint, workspace });
checkpointPersisted = true;
await settleFinalUsage(finalUsage);
if (providerRun.status !== 'completed')
return waiting(
'provider_error',
`The ${run.provider} executor ${providerRun.status === 'canceled' ? 'stopped' : 'failed'}: ${providerRun.error ?? 'no details'}. Its work was saved; continue to retry.`,
);
// An agent that asked instead of changing anything waits for the person's answer; answering
// creates a new requirement revision that starts again from planning.
const questions = run.repairCount ? [] : questionsIn(summary);
if (questions.length && checkpoint.sourceCommit === checkpoint.baseCommit)
return waiting(
'clarification',
`The ${run.provider} executor needs answers before implementing:\n${questions.map((question, i) => `${i + 1}. ${question}`).join('\n')}`,
);
await transition('verify', 'Implementation checkpoint created; independent verification is next', {
sourceCommit: checkpoint.sourceCommit,
...(summary ? { summary: summary.slice(0, 4000) } : {}),
});
return { status: 'verify' };
} catch (error) {
abort();
return await outcomeForError(
keeper.error ?? error,
async () => (await api.request<RunContext>(`/internal/runs/${input.runId}/context`)).run,
);
} finally {
keeper.close();
keeper.signal.removeEventListener('abort', abort);
await api.request(`/internal/runs/${input.runId}/lease/release`, { leaseToken }).catch(() => undefined);
if (workspace && checkpointPersisted) await rm(workspace, { recursive: true, force: true }).catch(() => undefined);
else if (workspace) {
// Lease loss, cancellation and ambiguous uploads must never erase the only copy.
// Keep a private sidecar outside the agent tree so operators can recover this exact attempt.
const providerRun = providerStarted ? await provider?.inspect(input.runId).catch(() => undefined) : undefined;
await writeFile(
`${workspace}.recovery.json`,
JSON.stringify(
{
schemaVersion: 1,
tenantId: input.tenantId,
runId: input.runId,
generation: input.generation,
leaseToken,
workspace,
baseCommit: recoveryBaseCommit,
provider: providerRun?.provider,
providerStatus: providerRun?.status,
sessionId: providerRun?.sessionId,
...(recoveryCheckpoint ? { checkpoint: recoveryCheckpoint } : {}),
retainedAt: new Date().toISOString(),
reason: 'Checkpoint persistence was not confirmed; workspace retained',
},
null,
2,
),
{ mode: 0o600 },
).catch(() => undefined);
}
}
}