- Drop the WorkBuddy Cloud executor stub: it always reported "unverified" and could never run. WorkBuddy remains an MCP client for submitting wishes. Providers are now codex and codebuddy; an unknown provider waits with an explicit reason instead of a fake adapter. - Remove internal routes nobody calls (/internal/releases/:id/begin, /accept, /failure and /internal/workspaces/:id/provisioned) with their domain methods, the never-consumed release.ready and release.rollback outbox events, an unreachable idempotency branch, dead helpers in runtime/deploy.ts and an unused RunnerBroker dependency. - Enable noUnusedLocals in the base and web TypeScript configs; the runtime package now extends the base config. - Add tests/tsconfig.json and include it in `npm run typecheck`. This surfaced tests still calling the deleted begin/accept path (now exercised through the Runner protocol) and five latent type errors. - Web: provider capability labels match the API's keys and the wish form disables unavailable providers. - Delete the invalid example Runner Job manifest; stop tracking the generated apps/web/next-env.d.ts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
945 lines
41 KiB
TypeScript
945 lines
41 KiB
TypeScript
import path from 'node:path';
|
|
import { realpath, mkdir } from 'node:fs/promises';
|
|
import { Context } from '@temporalio/activity';
|
|
import { ApplicationFailure } from '@temporalio/common';
|
|
import type { Project, Run, VerificationProfile } from '@wishforge/contracts';
|
|
import {
|
|
buildDockerImage,
|
|
executeCommand,
|
|
type CommandExecutor,
|
|
CodeBuddyAdapter,
|
|
CodexAdapter,
|
|
GitWorkspace,
|
|
isolatedVerifierExecutor,
|
|
RuntimeError,
|
|
safeIdentifier,
|
|
requireBranchName,
|
|
validateRepositoryUrl,
|
|
verifyWorkspace,
|
|
validateVerificationProfile,
|
|
type PortableCheckpoint,
|
|
type ProviderAdapter,
|
|
type VerificationReport,
|
|
} from '@wishforge/runtime';
|
|
import { ControlPlaneClient } from './client.js';
|
|
import { projectContract } from './project-contract.js';
|
|
import { loadGitHubDeliveryState, advanceGitHubDelivery, restoreMergedGitHubSource } from './github-delivery.js';
|
|
import { BASELINE_BUNDLE, bundleDigest, bundlePrompt, type AgentBundleConfig } from '@wishforge/agent';
|
|
|
|
export interface RunInput {
|
|
tenantId: string;
|
|
runId: string;
|
|
generation: string;
|
|
}
|
|
export interface DeliveryConfig {
|
|
integrationRepository?: string;
|
|
imageRepository: string;
|
|
architecture: 'linux/amd64' | 'linux/arm64';
|
|
environmentId: string;
|
|
dockerfile?: string;
|
|
dockerContext?: string;
|
|
sourceContext?: string;
|
|
containerPort?: number;
|
|
healthPath?: string;
|
|
}
|
|
export interface RunContext {
|
|
run: Run;
|
|
project: Project & { deliveryConfig?: DeliveryConfig };
|
|
workspace?: string;
|
|
checkpoint?: PortableCheckpoint;
|
|
providerCredential?: {
|
|
apiKey?: string;
|
|
baseUrl?: string;
|
|
model?: string;
|
|
metering?: 'proxy';
|
|
env?: Record<string, string>;
|
|
};
|
|
gitCredential?: { token: string; username?: 'x-access-token' };
|
|
verificationCommands?: string[][];
|
|
verificationProfile?: VerificationProfile;
|
|
verificationProfileDigest?: string;
|
|
verificationProfileRequired?: boolean;
|
|
bundleConfig?: AgentBundleConfig;
|
|
delivery?: {
|
|
release?: { id: string; status: string };
|
|
verificationEvidenceId?: string;
|
|
buildEvidenceId?: string;
|
|
environmentId?: string;
|
|
};
|
|
}
|
|
export interface ActivityOutcome {
|
|
status: 'waiting' | 'failed' | 'canceled' | 'paused' | 'verify' | 'release' | 'repair' | 'complete';
|
|
reason?: string;
|
|
verificationEvidenceId?: string;
|
|
buildEvidenceId?: string;
|
|
environmentId?: string;
|
|
}
|
|
export interface WorkerConfig {
|
|
controlPlaneUrl: string;
|
|
workerToken?: string;
|
|
verifierToken?: string;
|
|
deployerToken?: string;
|
|
workerId: string;
|
|
workspaceRoot: string;
|
|
isolatedProviderHomeRoot: string;
|
|
sandboxEnforced: boolean;
|
|
sandboxRunId?: string;
|
|
sandboxTenantId?: string;
|
|
sandboxGeneration?: string;
|
|
verifierBinary?: string;
|
|
codeBuddyBinary?: string;
|
|
dockerConfig?: string;
|
|
allowLocalGitHttp?: boolean;
|
|
resourceLimits?: { cpuMilli: number; memoryMiB: number; storageMiB?: number };
|
|
fetcher?: typeof fetch;
|
|
providerFactory?: (context: RunContext, isolatedHome: string) => ProviderAdapter;
|
|
}
|
|
|
|
function client(config: WorkerConfig, input: RunInput, verifier = false) {
|
|
safeIdentifier(input.tenantId);
|
|
safeIdentifier(input.runId);
|
|
safeIdentifier(input.generation);
|
|
return new ControlPlaneClient(
|
|
config.controlPlaneUrl,
|
|
input.tenantId,
|
|
(verifier ? config.verifierToken : config.workerToken) ?? '',
|
|
config.fetcher,
|
|
);
|
|
}
|
|
export function createActivities(config: WorkerConfig) {
|
|
return {
|
|
async inspectRunState(input: RunInput): Promise<{ status: Run['status']; generation?: string }> {
|
|
return client(config, input).request(`/internal/runs/${input.runId}/state`);
|
|
},
|
|
async executeRun(input: RunInput): Promise<ActivityOutcome> {
|
|
const api = client(config, input);
|
|
const observed = await api.request<RunContext>(`/internal/runs/${input.runId}/context`);
|
|
if (observed.run.status === 'succeeded') return { status: 'complete' };
|
|
if (observed.run.status === 'failed' || observed.run.status === 'canceled' || observed.run.status === 'paused')
|
|
return { status: observed.run.status };
|
|
const leased = await api.request<Run | { run: Run }>(`/internal/runs/${input.runId}/lease`, {
|
|
workerId: config.workerId,
|
|
generation: input.generation,
|
|
});
|
|
let run = 'run' in leased ? (leased.run as Run) : (leased as Run);
|
|
if (!run.lease) throw new RuntimeError('LEASE_MISSING', 'Control plane did not grant a lease');
|
|
const leaseToken = run.lease.token;
|
|
const transition = async (
|
|
stage: Run['stage'],
|
|
status: Run['status'],
|
|
message: string,
|
|
extras: Record<string, unknown> = {},
|
|
) => {
|
|
run = await api.request<Run>(`/internal/runs/${input.runId}/transition`, {
|
|
leaseToken,
|
|
expectedVersion: run.version,
|
|
stage,
|
|
status,
|
|
message,
|
|
...extras,
|
|
});
|
|
};
|
|
const releaseLease = async () => {
|
|
await api.request(`/internal/runs/${input.runId}/lease/release`, { leaseToken });
|
|
};
|
|
const wait = async (reason: string): Promise<ActivityOutcome> => {
|
|
await transition(run.stage, 'waiting', reason, { waitingReason: reason });
|
|
await releaseLease();
|
|
return { status: 'waiting', reason };
|
|
};
|
|
const context = await api.request<RunContext>(`/internal/runs/${input.runId}/context`);
|
|
run = context.run;
|
|
if (['canceled', 'paused', 'failed', 'succeeded'].includes(run.status))
|
|
return {
|
|
status: run.status === 'canceled' ? 'canceled' : run.status === 'paused' ? 'paused' : 'waiting',
|
|
reason: `Control plane status: ${run.status}`,
|
|
};
|
|
if (
|
|
!config.sandboxEnforced ||
|
|
config.sandboxRunId !== input.runId ||
|
|
config.sandboxTenantId !== input.tenantId ||
|
|
config.sandboxGeneration !== input.generation
|
|
)
|
|
return wait('unconfigured: an independently provisioned VM/Kata sandbox must bind this exact tenant and run');
|
|
const preparedDelivery = context.delivery;
|
|
if (preparedDelivery?.release || ['release', 'accept'].includes(run.stage))
|
|
return wait('deployment: existing release must be reconciled before further execution');
|
|
if (
|
|
['build', 'preview'].includes(run.stage) &&
|
|
preparedDelivery?.verificationEvidenceId &&
|
|
preparedDelivery.buildEvidenceId &&
|
|
preparedDelivery.environmentId
|
|
) {
|
|
if (run.stage === 'build')
|
|
await transition(
|
|
'preview',
|
|
'running',
|
|
'Recovered immutable build evidence; preparing its existing release contract',
|
|
);
|
|
await releaseLease();
|
|
return {
|
|
status: 'release',
|
|
verificationEvidenceId: preparedDelivery.verificationEvidenceId,
|
|
buildEvidenceId: preparedDelivery.buildEvidenceId,
|
|
environmentId: preparedDelivery.environmentId,
|
|
};
|
|
}
|
|
if (run.stage === 'preview')
|
|
return wait(
|
|
'delivery_evidence_missing: immutable build and verification evidence must be restored before publishing',
|
|
);
|
|
if (context.checkpoint && ['verify', 'integrate', 'build'].includes(run.stage)) {
|
|
await releaseLease();
|
|
return { status: 'verify' };
|
|
}
|
|
try {
|
|
requireBranchName(context.project.targetBranch);
|
|
if (context.project.repositoryUrl)
|
|
validateRepositoryUrl(context.project.repositoryUrl, { allowLocalHttp: config.allowLocalGitHttp });
|
|
} catch (error) {
|
|
return wait(
|
|
`${error instanceof RuntimeError ? error.code : 'REPOSITORY_UNAVAILABLE'}: ${error instanceof Error ? error.message : 'Repository preflight failed'}`,
|
|
);
|
|
}
|
|
const isolatedHome = path.join(
|
|
config.isolatedProviderHomeRoot,
|
|
safeIdentifier(input.tenantId),
|
|
safeIdentifier(input.runId),
|
|
);
|
|
const provider =
|
|
config.providerFactory?.(context, isolatedHome) ??
|
|
(run.provider === 'codex'
|
|
? new CodexAdapter({
|
|
apiKey: context.providerCredential?.apiKey,
|
|
baseUrl: context.providerCredential?.baseUrl,
|
|
model: context.providerCredential?.model,
|
|
isolatedHome,
|
|
})
|
|
: run.provider === 'codebuddy'
|
|
? new CodeBuddyAdapter({
|
|
binary: config.codeBuddyBinary,
|
|
isolatedHome,
|
|
credentialEnv:
|
|
context.providerCredential?.env ??
|
|
(context.providerCredential?.apiKey
|
|
? { CODEBUDDY_API_KEY: context.providerCredential.apiKey }
|
|
: undefined),
|
|
sandboxEnforced: config.sandboxEnforced,
|
|
})
|
|
: undefined);
|
|
if (!provider) return wait(`unsupported_provider: ${run.provider} is not available on this worker`);
|
|
if (!provider.capabilities().coding)
|
|
return wait(`${provider.capabilities().availability}: ${provider.capabilities().reason}`);
|
|
if (run.stage === 'clarify')
|
|
await transition('plan', 'running', 'Acceptance criteria recorded; preparing isolated implementation');
|
|
if (run.stage === 'plan' || run.stage === 'implement')
|
|
await transition('implement', 'running', 'Starting provider in an isolated run branch');
|
|
let providerStarted = false;
|
|
let heartbeatError: Error | undefined;
|
|
let timer: ReturnType<typeof setInterval> | undefined;
|
|
const abort = () => {
|
|
if (providerStarted) void provider.cancel(input.runId);
|
|
};
|
|
let activity: Context | undefined;
|
|
try {
|
|
activity = Context.current();
|
|
activity.cancellationSignal.addEventListener('abort', abort, { once: true });
|
|
} catch {
|
|
/* Unit tests inject dependencies without a Temporal runtime. */
|
|
}
|
|
try {
|
|
let inHeartbeat = false;
|
|
timer = setInterval(() => {
|
|
if (inHeartbeat) return;
|
|
inHeartbeat = true;
|
|
void (async () => {
|
|
activity?.heartbeat({ runId: input.runId, leaseToken });
|
|
const observed = await api.request<Run>(`/internal/runs/${input.runId}/heartbeat`, { leaseToken });
|
|
if (observed.status !== 'running') {
|
|
heartbeatError = new RuntimeError(
|
|
'RUN_STOPPED',
|
|
`Run was ${observed.status}: ${observed.waitingReason ?? 'execution stopped'}`,
|
|
);
|
|
abort();
|
|
} else run = observed;
|
|
})()
|
|
.catch(error => {
|
|
heartbeatError = error instanceof Error ? error : new Error('Lease heartbeat failed');
|
|
abort();
|
|
})
|
|
.finally(() => {
|
|
inHeartbeat = false;
|
|
});
|
|
}, 10_000);
|
|
const execute: CommandExecutor = command =>
|
|
executeCommand({ ...command, signal: activity?.cancellationSignal });
|
|
const git = new GitWorkspace(execute, {
|
|
allowLocalHttp: config.allowLocalGitHttp,
|
|
credential:
|
|
context.gitCredential && context.project.repositoryUrl
|
|
? { ...context.gitCredential, repositoryUrl: context.project.repositoryUrl }
|
|
: undefined,
|
|
});
|
|
const prepared = context.checkpoint
|
|
? await git.restore({
|
|
checkpoint: context.checkpoint,
|
|
repository: context.project.repositoryUrl,
|
|
root: path.join(config.workspaceRoot, input.tenantId),
|
|
runId: `${input.runId}-attempt-${leaseToken}`,
|
|
})
|
|
: await git.create({
|
|
repository: context.project.repositoryUrl,
|
|
targetBranch: context.project.targetBranch,
|
|
baseCommit: run.sourceCommit,
|
|
root: path.join(config.workspaceRoot, input.tenantId),
|
|
runId: input.runId,
|
|
});
|
|
if (heartbeatError) throw heartbeatError;
|
|
const bundle = context.bundleConfig ?? (run.bundleVersion === 'baseline-v1' ? BASELINE_BUNDLE : undefined);
|
|
if (!bundle || (run.bundleVersion !== 'baseline-v1' && bundleDigest(bundle) !== run.bundleVersion))
|
|
throw new RuntimeError(
|
|
'BUNDLE_UNAVAILABLE',
|
|
'Pinned AgentBundle content is missing or differs from the run version',
|
|
);
|
|
const prompt = bundlePrompt(bundle, {
|
|
goal: run.goal,
|
|
acceptanceCriteria: run.acceptanceCriteria,
|
|
repairCount: run.repairCount,
|
|
error: run.error,
|
|
});
|
|
await provider.start({
|
|
runId: input.runId,
|
|
tenantId: input.tenantId,
|
|
workspace: prepared.workspace,
|
|
prompt,
|
|
baseCommit: prepared.baseCommit,
|
|
model: bundle.model,
|
|
});
|
|
providerStarted = true;
|
|
if (heartbeatError) {
|
|
abort();
|
|
throw heartbeatError;
|
|
}
|
|
const deadline = setTimeout(abort, Math.max(1, run.budget.maxMinutes) * 60_000);
|
|
try {
|
|
for await (const event of provider.events(input.runId)) {
|
|
activity?.heartbeat({ runId: input.runId, sequence: event.sequence, leaseToken });
|
|
// Persist only normalized events; reasoning and raw process diagnostics are not exported.
|
|
await api.request(`/internal/runs/${input.runId}/events`, { leaseToken, event });
|
|
if (event.type === 'usage' && context.providerCredential?.metering !== 'proxy') {
|
|
const tokens = Number(event.data.input_tokens ?? 0) + Number(event.data.output_tokens ?? 0);
|
|
if (Number.isSafeInteger(tokens) && tokens >= 0)
|
|
await api.request(`/internal/runs/${input.runId}/meter`, {
|
|
leaseToken,
|
|
operationId: `provider-${input.runId}-${leaseToken}-${event.sequence}`,
|
|
tokens,
|
|
computeSeconds: 0,
|
|
});
|
|
}
|
|
}
|
|
} finally {
|
|
clearTimeout(deadline);
|
|
}
|
|
if (heartbeatError) throw heartbeatError;
|
|
const providerRun = await provider.inspect(input.runId);
|
|
const checkpoint = await provider.exportCheckpoint(input.runId);
|
|
await api.request(`/internal/runs/${input.runId}/checkpoint`, {
|
|
leaseToken,
|
|
checkpoint,
|
|
workspace: prepared.workspace,
|
|
});
|
|
// Checkpoint persistence can advance the entity version. Always refresh before CAS.
|
|
run = (await api.request<RunContext>(`/internal/runs/${input.runId}/context`)).run;
|
|
if (providerRun.status !== 'completed') {
|
|
await transition(
|
|
'implement',
|
|
providerRun.status === 'canceled' ? 'waiting' : 'failed',
|
|
providerRun.error ?? `Provider ${providerRun.status}`,
|
|
{ sourceCommit: checkpoint.sourceCommit },
|
|
);
|
|
return { status: providerRun.status === 'canceled' ? 'waiting' : 'failed', reason: providerRun.error };
|
|
}
|
|
await transition(
|
|
'verify',
|
|
'running',
|
|
'Implementation checkpoint created; independent verification is required',
|
|
{ sourceCommit: checkpoint.sourceCommit },
|
|
);
|
|
return { status: 'verify' };
|
|
} catch (error) {
|
|
abort();
|
|
if (heartbeatError || (error instanceof RuntimeError && error.code.startsWith('CONTROL_PLANE_'))) throw error;
|
|
const message = error instanceof Error ? error.message : 'Provider execution failed';
|
|
run = (await api.request<RunContext>(`/internal/runs/${input.runId}/context`)).run;
|
|
await transition(run.stage, 'waiting', message, {
|
|
waitingReason: error instanceof RuntimeError ? error.code : 'EXECUTION_FAILED',
|
|
});
|
|
return { status: 'waiting', reason: message };
|
|
} finally {
|
|
if (timer) clearInterval(timer);
|
|
activity?.cancellationSignal.removeEventListener('abort', abort);
|
|
await releaseLease().catch(() => undefined);
|
|
}
|
|
},
|
|
async verifyRun(input: RunInput): Promise<ActivityOutcome> {
|
|
const api = client(config, input, true);
|
|
const leased = await api.request<Run>(`/internal/runs/${input.runId}/lease`, {
|
|
workerId: config.workerId,
|
|
generation: input.generation,
|
|
});
|
|
if (!leased.lease) throw new RuntimeError('LEASE_MISSING', 'Verifier did not receive a lease');
|
|
const leaseToken = leased.lease.token;
|
|
let leaseError: Error | undefined;
|
|
const verifierAbort = new AbortController();
|
|
let activity: Context | undefined;
|
|
try {
|
|
activity = Context.current();
|
|
} catch {
|
|
/* Contract unit tests have no Temporal activity. */
|
|
}
|
|
const verifierSignal = activity
|
|
? AbortSignal.any([activity.cancellationSignal, verifierAbort.signal])
|
|
: verifierAbort.signal;
|
|
const stopVerifier = (error: Error) => {
|
|
leaseError = error;
|
|
verifierAbort.abort(error);
|
|
};
|
|
const timer = setInterval(() => {
|
|
activity?.heartbeat({ runId: input.runId, leaseToken });
|
|
void api
|
|
.request<Run>(`/internal/runs/${input.runId}/heartbeat`, { leaseToken })
|
|
.then(observed => {
|
|
if (observed.status !== 'running')
|
|
stopVerifier(new RuntimeError('RUN_STOPPED', 'Execution budget or run state stopped verification'));
|
|
})
|
|
.catch(error => {
|
|
stopVerifier(error instanceof Error ? error : new Error('Verifier lease failed'));
|
|
});
|
|
}, 10_000);
|
|
const ensureLease = async () => {
|
|
activity?.heartbeat({ runId: input.runId, leaseToken });
|
|
if (leaseError) throw leaseError;
|
|
const observed = await api.request<Run>(`/internal/runs/${input.runId}/heartbeat`, { leaseToken });
|
|
if (observed.status !== 'running') throw new RuntimeError('RUN_STOPPED', 'Run stopped');
|
|
};
|
|
const transition = async (stage: Run['stage'], message: string, extras: Record<string, unknown> = {}) => {
|
|
await ensureLease();
|
|
const context = await api.request<RunContext>(`/internal/runs/${input.runId}/context`);
|
|
await api.request(`/internal/runs/${input.runId}/transition`, {
|
|
leaseToken,
|
|
expectedVersion: context.run.version,
|
|
stage,
|
|
status: 'running',
|
|
message,
|
|
...extras,
|
|
});
|
|
};
|
|
let integrationFence: number | undefined;
|
|
let integrationTimer: ReturnType<typeof setInterval> | undefined;
|
|
let integrationRenewing = false;
|
|
// The control plane re-checks the project integration lease on every integrate/build transition.
|
|
const assertIntegrationGate = async () => {
|
|
if (integrationFence === undefined)
|
|
throw new RuntimeError('INTEGRATION_LEASE_REQUIRED', 'Project integration lease is required');
|
|
await ensureLease();
|
|
};
|
|
try {
|
|
const context = await api.request<RunContext>(`/internal/runs/${input.runId}/context`);
|
|
if (
|
|
!config.sandboxEnforced ||
|
|
config.sandboxRunId !== input.runId ||
|
|
config.sandboxTenantId !== input.tenantId ||
|
|
config.sandboxGeneration !== input.generation
|
|
)
|
|
throw new RuntimeError('UNCONFIGURED', 'Verifier requires an isolated sandbox bound to this tenant and run');
|
|
const commands = [
|
|
...(context.verificationProfile?.commands ?? []),
|
|
...(context.verificationCommands ?? context.project.verificationCommands),
|
|
];
|
|
if (context.verificationProfileRequired && !context.verificationProfile)
|
|
return {
|
|
status: 'waiting',
|
|
reason: 'verification_profile: production requires an operator-installed independent acceptance profile',
|
|
};
|
|
if (context.verificationProfile) await validateVerificationProfile(context.verificationProfile);
|
|
const profileBinding = context.verificationProfileDigest
|
|
? { verificationProfileDigest: context.verificationProfileDigest }
|
|
: {};
|
|
if (!context.checkpoint)
|
|
return {
|
|
status: 'waiting',
|
|
reason: 'Checkpoint/workspace is unavailable; restore the portable checkpoint first',
|
|
};
|
|
if (!commands.length)
|
|
return { status: 'waiting', reason: 'No protected project verification commands are configured' };
|
|
const execute: CommandExecutor = command => executeCommand({ ...command, signal: verifierSignal });
|
|
const git = new GitWorkspace(execute, {
|
|
allowLocalHttp: config.allowLocalGitHttp,
|
|
credential:
|
|
context.gitCredential && context.project.repositoryUrl
|
|
? { ...context.gitCredential, repositoryUrl: context.project.repositoryUrl }
|
|
: undefined,
|
|
});
|
|
const { workspace } = await git.restore({
|
|
checkpoint: context.checkpoint,
|
|
repository: context.project.repositoryUrl,
|
|
root: path.join(config.workspaceRoot, input.tenantId),
|
|
runId: `${input.runId}-verifier-${leaseToken}`,
|
|
});
|
|
let contract = await projectContract(
|
|
{
|
|
workspace,
|
|
sourceCommit: context.checkpoint.sourceCommit,
|
|
commands,
|
|
delivery: context.project.deliveryConfig,
|
|
resourceLimits: config.resourceLimits,
|
|
},
|
|
execute,
|
|
);
|
|
const report: VerificationReport = await verifyWorkspace(
|
|
{ workspace, sourceCommit: context.checkpoint.sourceCommit, commands: contract.commands },
|
|
execute,
|
|
isolatedVerifierExecutor(workspace, config.verifierBinary, execute),
|
|
);
|
|
await ensureLease();
|
|
let verification = await api.request<{ id: string }>('/internal/evidence', {
|
|
runId: input.runId,
|
|
revision: context.run.revision,
|
|
leaseToken,
|
|
type: 'verification',
|
|
...profileBinding,
|
|
...report,
|
|
});
|
|
if (!report.passed) {
|
|
if (context.run.githubOperationId) {
|
|
const state = await loadGitHubDeliveryState(api, input.runId, leaseToken);
|
|
if (state.operation?.mergedCommitSha)
|
|
return {
|
|
status: 'waiting',
|
|
reason:
|
|
'github_merged_verification_failed: merged source failed acceptance; create a new revision/review for the repair instead of reusing the merged PR',
|
|
};
|
|
}
|
|
return {
|
|
status: context.run.repairCount < context.run.budget.maxRepairRounds ? 'repair' : 'waiting',
|
|
reason: 'Verification failed; protected command evidence is recorded',
|
|
};
|
|
}
|
|
const delivery = context.project.deliveryConfig;
|
|
if (!delivery?.imageRepository || !delivery.environmentId || !delivery.architecture)
|
|
return {
|
|
status: 'waiting',
|
|
reason:
|
|
'Verification passed; protected deliveryConfig requires imageRepository, architecture and environmentId',
|
|
};
|
|
integrationFence = (
|
|
await api.request<{ fence: number }>(`/internal/runs/${input.runId}/integration-lease/acquire`, {
|
|
leaseToken,
|
|
})
|
|
).fence;
|
|
integrationTimer = setInterval(() => {
|
|
if (integrationRenewing) return;
|
|
integrationRenewing = true;
|
|
void (async () => {
|
|
await api.request(`/internal/runs/${input.runId}/integration-lease/renew`, {
|
|
leaseToken,
|
|
fence: integrationFence,
|
|
});
|
|
})()
|
|
.catch(error => stopVerifier(error instanceof Error ? error : new Error('Integration ownership changed')))
|
|
.finally(() => {
|
|
integrationRenewing = false;
|
|
});
|
|
}, 10_000);
|
|
const refreshGitCredential = async () => {
|
|
const fresh = await api.request<RunContext>(`/internal/runs/${input.runId}/context`);
|
|
git.setCredential(
|
|
fresh.gitCredential && context.project.repositoryUrl
|
|
? { ...fresh.gitCredential, repositoryUrl: context.project.repositoryUrl }
|
|
: undefined,
|
|
);
|
|
};
|
|
const integrationRoot = path.join(config.workspaceRoot, safeIdentifier(input.tenantId), 'integrations');
|
|
await mkdir(integrationRoot, { recursive: true });
|
|
const allowedRoot = await realpath(integrationRoot);
|
|
const integrationRepository =
|
|
delivery.integrationRepository ?? path.join(allowedRoot, `${safeIdentifier(context.project.id)}.git`);
|
|
const existingReal = await realpath(integrationRepository).catch(error => {
|
|
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return path.resolve(integrationRepository);
|
|
throw error;
|
|
});
|
|
if (!existingReal.startsWith(`${allowedRoot}${path.sep}`))
|
|
throw new RuntimeError(
|
|
'INTEGRATION_SCOPE',
|
|
'Integration repository must stay inside this tenant integrations directory',
|
|
);
|
|
const targetRef = `refs/heads/${context.project.targetBranch}`;
|
|
await refreshGitCredential();
|
|
await git.initializeIntegration({
|
|
repository: integrationRepository,
|
|
originalRepository: context.project.repositoryUrl,
|
|
workspace,
|
|
sourceCommit: context.checkpoint.sourceCommit,
|
|
baseCommit: context.checkpoint.baseCommit,
|
|
targetRef,
|
|
});
|
|
const expectedTargetCommit = await git.head(integrationRepository, targetRef);
|
|
await assertIntegrationGate();
|
|
let candidateWorkspace = workspace;
|
|
let checkpoint = context.checkpoint;
|
|
const githubState = await loadGitHubDeliveryState(api, input.runId, leaseToken);
|
|
if (githubState.required && !githubState.enabled)
|
|
return {
|
|
status: 'waiting',
|
|
reason: `github_unconfigured: ${githubState.reason ?? 'Configure a repository-scoped GitHub App; direct target-branch push is disabled'}`,
|
|
};
|
|
let restoredGitHubMerge = false;
|
|
// Resolve an already-recorded review before generating a different candidate
|
|
// SHA, which would invalidate the review's exact-version approval.
|
|
if (githubState.required && githubState.operation) {
|
|
const outcome = await advanceGitHubDelivery({
|
|
api,
|
|
git,
|
|
state: githubState,
|
|
runId: input.runId,
|
|
revision: context.run.revision,
|
|
leaseToken,
|
|
repository: integrationRepository,
|
|
candidateCommit: checkpoint.sourceCommit,
|
|
verifiedCommit: report.sourceCommit,
|
|
expectedBaseCommit: githubState.operation.expectedBaseSha,
|
|
assertIntegrationGate,
|
|
refreshGitCredential,
|
|
});
|
|
if (outcome.kind === 'waiting') return { status: 'waiting', reason: outcome.reason };
|
|
if (outcome.kind !== 'merged' || !context.project.repositoryUrl)
|
|
throw new RuntimeError('GITHUB_REVIEW_REQUIRED', 'A confirmed GitHub merge is required before the build');
|
|
const restored = await restoreMergedGitHubSource({
|
|
git,
|
|
repositoryUrl: context.project.repositoryUrl,
|
|
targetBranch: context.project.targetBranch,
|
|
root: path.join(config.workspaceRoot, input.tenantId),
|
|
runId: input.runId,
|
|
provider: context.run.provider,
|
|
merged: outcome,
|
|
assertIntegrationGate,
|
|
refreshGitCredential,
|
|
});
|
|
candidateWorkspace = restored.workspace;
|
|
checkpoint = restored.checkpoint;
|
|
restoredGitHubMerge = true;
|
|
}
|
|
if (context.run.stage === 'verify' && !restoredGitHubMerge)
|
|
await transition('integrate', 'Source verified; creating candidate against current integration target');
|
|
if (restoredGitHubMerge || context.run.stage === 'verify' || context.run.stage === 'integrate') {
|
|
if (!restoredGitHubMerge) {
|
|
candidateWorkspace = path.join(
|
|
config.workspaceRoot,
|
|
input.tenantId,
|
|
`${input.runId}-candidate-${Date.now()}`,
|
|
);
|
|
await git.createCandidate({
|
|
repository: integrationRepository,
|
|
path: candidateWorkspace,
|
|
targetRef,
|
|
expectedTargetCommit,
|
|
sourceCommit: checkpoint.sourceCommit,
|
|
});
|
|
checkpoint = await git.checkpoint(
|
|
candidateWorkspace,
|
|
expectedTargetCommit,
|
|
input.runId,
|
|
context.run.provider,
|
|
);
|
|
}
|
|
await ensureLease();
|
|
await api.request(`/internal/runs/${input.runId}/checkpoint`, {
|
|
leaseToken,
|
|
checkpoint,
|
|
workspace: candidateWorkspace,
|
|
});
|
|
contract = await projectContract(
|
|
{
|
|
workspace: candidateWorkspace,
|
|
sourceCommit: checkpoint.sourceCommit,
|
|
commands,
|
|
delivery: context.project.deliveryConfig,
|
|
resourceLimits: config.resourceLimits,
|
|
},
|
|
execute,
|
|
);
|
|
const candidateReport = await verifyWorkspace(
|
|
{ workspace: candidateWorkspace, sourceCommit: checkpoint.sourceCommit, commands: contract.commands },
|
|
execute,
|
|
isolatedVerifierExecutor(candidateWorkspace, config.verifierBinary, execute),
|
|
);
|
|
await ensureLease();
|
|
verification = await api.request<{ id: string }>('/internal/evidence', {
|
|
runId: input.runId,
|
|
revision: context.run.revision,
|
|
leaseToken,
|
|
type: 'verification',
|
|
...profileBinding,
|
|
...candidateReport,
|
|
});
|
|
if (!candidateReport.passed)
|
|
return {
|
|
status:
|
|
!restoredGitHubMerge && context.run.repairCount < context.run.budget.maxRepairRounds
|
|
? 'repair'
|
|
: 'waiting',
|
|
reason: restoredGitHubMerge
|
|
? 'github_merged_verification_failed: actual merged source failed acceptance; build was blocked and a new repair review is required'
|
|
: 'Integrated candidate failed independent verification; target branch was not advanced',
|
|
};
|
|
if (githubState.required) {
|
|
if (!restoredGitHubMerge) {
|
|
const outcome = await advanceGitHubDelivery({
|
|
api,
|
|
git,
|
|
state: githubState,
|
|
runId: input.runId,
|
|
revision: context.run.revision,
|
|
leaseToken,
|
|
repository: integrationRepository,
|
|
candidateCommit: checkpoint.sourceCommit,
|
|
verifiedCommit: candidateReport.sourceCommit,
|
|
expectedBaseCommit: checkpoint.baseCommit,
|
|
assertIntegrationGate,
|
|
refreshGitCredential,
|
|
});
|
|
if (outcome.kind === 'waiting') return { status: 'waiting', reason: outcome.reason };
|
|
if (outcome.kind !== 'merged' || !context.project.repositoryUrl)
|
|
throw new RuntimeError('GITHUB_REVIEW_REQUIRED', 'GitHub has not confirmed a merge');
|
|
const restored = await restoreMergedGitHubSource({
|
|
git,
|
|
repositoryUrl: context.project.repositoryUrl,
|
|
targetBranch: context.project.targetBranch,
|
|
root: path.join(config.workspaceRoot, input.tenantId),
|
|
runId: input.runId,
|
|
provider: context.run.provider,
|
|
merged: outcome,
|
|
assertIntegrationGate,
|
|
refreshGitCredential,
|
|
});
|
|
candidateWorkspace = restored.workspace;
|
|
checkpoint = restored.checkpoint;
|
|
restoredGitHubMerge = true;
|
|
await api.request(`/internal/runs/${input.runId}/checkpoint`, {
|
|
leaseToken,
|
|
checkpoint,
|
|
workspace: candidateWorkspace,
|
|
});
|
|
// A successful pre-merge check cannot attest GitHub's actual merge SHA.
|
|
contract = await projectContract(
|
|
{
|
|
workspace: candidateWorkspace,
|
|
sourceCommit: checkpoint.sourceCommit,
|
|
commands,
|
|
delivery,
|
|
resourceLimits: config.resourceLimits,
|
|
},
|
|
execute,
|
|
);
|
|
await assertIntegrationGate();
|
|
const mergedReport = await verifyWorkspace(
|
|
{ workspace: candidateWorkspace, sourceCommit: checkpoint.sourceCommit, commands: contract.commands },
|
|
execute,
|
|
isolatedVerifierExecutor(candidateWorkspace, config.verifierBinary, execute),
|
|
);
|
|
await ensureLease();
|
|
verification = await api.request<{ id: string }>('/internal/evidence', {
|
|
runId: input.runId,
|
|
revision: context.run.revision,
|
|
leaseToken,
|
|
type: 'verification',
|
|
...profileBinding,
|
|
...mergedReport,
|
|
});
|
|
if (!mergedReport.passed)
|
|
return {
|
|
status: 'waiting',
|
|
reason:
|
|
'github_merged_verification_failed: actual merge failed independent acceptance; no image or release was created, and a new repair review is required',
|
|
};
|
|
}
|
|
} else {
|
|
if (context.project.repositoryUrl) {
|
|
await refreshGitCredential();
|
|
await assertIntegrationGate();
|
|
await git.pushVerifiedCandidate({
|
|
repository: integrationRepository,
|
|
targetRef,
|
|
expectedTargetCommit,
|
|
candidateCommit: checkpoint.sourceCommit,
|
|
verifiedCommit: candidateReport.sourceCommit,
|
|
});
|
|
}
|
|
await assertIntegrationGate();
|
|
await git.promoteVerifiedCandidate({
|
|
repository: integrationRepository,
|
|
targetRef,
|
|
expectedTargetCommit,
|
|
candidateCommit: checkpoint.sourceCommit,
|
|
verifiedCommit: candidateReport.sourceCommit,
|
|
});
|
|
}
|
|
if (
|
|
restoredGitHubMerge &&
|
|
(await api.request<RunContext>(`/internal/runs/${input.runId}/context`)).run.stage === 'verify'
|
|
)
|
|
await transition('integrate', 'Actual GitHub merge commit passed independent profile and manifest checks', {
|
|
sourceCommit: checkpoint.sourceCommit,
|
|
});
|
|
await transition(
|
|
'build',
|
|
restoredGitHubMerge
|
|
? 'Actual GitHub merge commit independently verified; preparing immutable build'
|
|
: 'Exact integrated commit verified and atomically promoted',
|
|
{ sourceCommit: checkpoint.sourceCommit },
|
|
);
|
|
}
|
|
const changedPaths = await git.changedPaths(candidateWorkspace, checkpoint.baseCommit, checkpoint.sourceCommit);
|
|
await api.request(`/internal/runs/${input.runId}/classification`, {
|
|
leaseToken,
|
|
sourceCommit: checkpoint.sourceCommit,
|
|
changedPaths,
|
|
migration: contract.migrationRequired || changedPaths.some(file => /(^|\/)migrations?(\/|$)/i.test(file)),
|
|
permissionsExpanded: changedPaths.some(file => /(^|\/)(security|auth|infra)(\/|$)/i.test(file)),
|
|
});
|
|
await ensureLease();
|
|
await assertIntegrationGate();
|
|
const image = await buildDockerImage(
|
|
{
|
|
workspace: candidateWorkspace,
|
|
sourceCommit: checkpoint.sourceCommit,
|
|
imageRepository: delivery.imageRepository,
|
|
architecture: delivery.architecture,
|
|
dockerfile: delivery.dockerfile,
|
|
sourceContext: delivery.sourceContext,
|
|
dockerContext: delivery.dockerContext,
|
|
dockerConfig: config.dockerConfig,
|
|
},
|
|
execute,
|
|
);
|
|
await ensureLease();
|
|
await assertIntegrationGate();
|
|
const build = await api.request<{ id: string }>('/internal/evidence', {
|
|
runId: input.runId,
|
|
revision: context.run.revision,
|
|
sourceCommit: checkpoint.sourceCommit,
|
|
leaseToken,
|
|
type: 'build',
|
|
passed: true,
|
|
...profileBinding,
|
|
imageDigest: image.imageDigest,
|
|
artifactUri: `oci://${image.imageDigest}`,
|
|
});
|
|
const refreshed = await api.request<RunContext>(`/internal/runs/${input.runId}/context`);
|
|
if (refreshed.run.stage === 'build')
|
|
await transition('preview', 'Immutable image pushed; target release validation is pending');
|
|
return {
|
|
status: 'release',
|
|
verificationEvidenceId: verification.id,
|
|
buildEvidenceId: build.id,
|
|
environmentId: delivery.environmentId,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
status: 'waiting',
|
|
reason: error instanceof Error ? error.message : 'Verification or delivery preparation failed',
|
|
};
|
|
} finally {
|
|
clearInterval(timer);
|
|
if (integrationTimer) clearInterval(integrationTimer);
|
|
if (integrationFence !== undefined)
|
|
await api
|
|
.request(`/internal/runs/${input.runId}/integration-lease/release`, { leaseToken, fence: integrationFence })
|
|
.catch(() => undefined);
|
|
await api.request(`/internal/runs/${input.runId}/lease/release`, { leaseToken }).catch(() => undefined);
|
|
}
|
|
},
|
|
async prepareRepair(input: RunInput & { reason: string }): Promise<void> {
|
|
const api = client(config, input);
|
|
const run = await api.request<Run>(`/internal/runs/${input.runId}/lease`, {
|
|
workerId: config.workerId,
|
|
generation: input.generation,
|
|
});
|
|
if (!run.lease) throw new RuntimeError('LEASE_MISSING', 'Repair coordinator did not receive a lease');
|
|
try {
|
|
await api.request(`/internal/runs/${input.runId}/transition`, {
|
|
leaseToken: run.lease.token,
|
|
expectedVersion: run.version,
|
|
stage: 'implement',
|
|
status: 'running',
|
|
message: input.reason,
|
|
repair: true,
|
|
});
|
|
} finally {
|
|
await api
|
|
.request(`/internal/runs/${input.runId}/lease/release`, { leaseToken: run.lease.token })
|
|
.catch(() => undefined);
|
|
}
|
|
},
|
|
async requestRelease(
|
|
input: RunInput & { verificationEvidenceId: string; buildEvidenceId: string; environmentId: string },
|
|
): Promise<ActivityOutcome> {
|
|
const api = new ControlPlaneClient(
|
|
config.controlPlaneUrl,
|
|
input.tenantId,
|
|
config.deployerToken ?? '',
|
|
config.fetcher,
|
|
);
|
|
const leased = await api.request<Run>(`/internal/runs/${input.runId}/lease`, {
|
|
workerId: config.workerId,
|
|
generation: input.generation,
|
|
});
|
|
if (!leased.lease) throw new RuntimeError('LEASE_MISSING', 'Deployer did not receive a lease');
|
|
const leaseToken = leased.lease.token;
|
|
try {
|
|
const release = await api.request<{ id: string; status: string }>(
|
|
'/internal/releases',
|
|
{
|
|
runId: input.runId,
|
|
environmentId: input.environmentId,
|
|
verificationEvidenceId: input.verificationEvidenceId,
|
|
buildEvidenceId: input.buildEvidenceId,
|
|
generation: input.generation,
|
|
},
|
|
{ idempotencyKey: `release:${input.runId}:${input.buildEvidenceId}:${input.environmentId}` },
|
|
);
|
|
const context = await api.request<RunContext>(`/internal/runs/${input.runId}/context`);
|
|
await api.request(`/internal/runs/${input.runId}/transition`, {
|
|
leaseToken,
|
|
expectedVersion: context.run.version,
|
|
stage: 'release',
|
|
status: 'waiting',
|
|
message: `Release ${release.id} is ${release.status}; deployment acceptance remains pending`,
|
|
waitingReason: release.status === 'awaiting_approval' ? 'approval' : 'deployment',
|
|
});
|
|
return { status: 'waiting', reason: `Release ${release.id}: ${release.status}` };
|
|
} finally {
|
|
await api.request(`/internal/runs/${input.runId}/lease/release`, { leaseToken }).catch(() => undefined);
|
|
}
|
|
},
|
|
async recordWaiting(
|
|
input: RunInput & { reason: string },
|
|
): Promise<{ status: 'waiting' | 'complete' | 'canceled' | 'paused' | 'superseded' }> {
|
|
const api = client(config, input);
|
|
const { run } = await api.request<RunContext>(`/internal/runs/${input.runId}/context`);
|
|
if (run.executionGeneration !== input.generation) return { status: 'superseded' };
|
|
if (run.status === 'succeeded' || run.status === 'failed') return { status: 'complete' };
|
|
if (run.status === 'canceled' || run.status === 'paused') return { status: run.status };
|
|
if (run.status === 'waiting' && (run.waitingReason === input.reason || run.waitingReason === 'budget'))
|
|
return { status: 'waiting' };
|
|
if (run.lease && Date.parse(run.lease.expiresAt) > Date.now())
|
|
throw ApplicationFailure.create({
|
|
message: 'Wait for the previous executor lease to expire',
|
|
type: 'EXECUTION_LEASE_HELD',
|
|
});
|
|
await api.request(`/internal/runs/${input.runId}/wait`, { reason: input.reason, generation: input.generation });
|
|
return { status: 'waiting' };
|
|
},
|
|
};
|
|
}
|