Verification ran git inside the tree under test, kept no command output and treated every infrastructure fault as a test failure; repairs went through an executor lease the coordinator should not hold; and a project without delivery could never succeed. This reworks the worker execution path and the control-plane contracts it depends on. - Verification exports the exact commit to a directory without .git, installs dependencies (network allowed) and runs the protected commands under bubblewrap in kata mode or directly in host mode; evidence records exit code, duration, digest, a redacted output tail and the isolation used. Production rejects host evidence. A failed installation waits as infrastructure. - Waiting reasons are an enum with a redacted detail; dispatcher, GitHub and model-proxy states map onto it, and the console explains each one with a way out. - Repairs are requested by the coordinator through /internal/runs/:id/repair; the control plane derives the failure summary from the verifier's evidence and enforces the repair budget. Continuing after the budget is spent grants one more round; a new requirement revision starts a fresh budget. - Execution identities may only move a run within their own stages and only to running or waiting; success comes from integration evidence (no delivery configured) or deployment acceptance. The unused preview stage is gone. - Integration fast-forwards when the target has not moved, merges and re-verifies otherwise, retries when a push finds the target moved, and turns merge conflicts into a repair round that merges the current target for the agent to resolve. - Successful runs that needed a repair leave a project lesson; active lessons are injected into later prompts, and admins can list, add, disable and delete them. - Executor progress (messages, commands, file changes, plan updates) is stored as normalized events; previously agent messages were recorded as the literal word "output". Provider failures keep the work as a checkpoint and wait as provider_error instead of failing the run. - Workers declare WF_SANDBOX_MODE=host|kata (host refused in production), heartbeat with retries until the lease could have expired, and retry safe control-plane reads with backoff. - tests/local-loop.test.ts runs the loop end to end with real Git, the in-process control plane behind its internal routes and a scripted agent. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
397 lines
15 KiB
TypeScript
397 lines
15 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
import { Client, Connection } from '@temporalio/client';
|
|
import { Context } from '@temporalio/activity';
|
|
import { ApplicationFailure } from '@temporalio/common';
|
|
import { msToTs } from '@temporalio/common/lib/time.js';
|
|
import { bundleWorkflowCode, NativeConnection, Runtime, Worker } from '@temporalio/worker';
|
|
import type { RunInput, ActivityOutcome } from '../../apps/worker/src/activities.js';
|
|
|
|
const enabled = process.env.WF_TEMPORAL_INTEGRATION === '1';
|
|
const namespace = 'wishforge-contract-tests';
|
|
/** Actual Temporal history/replay with fixture activities, not provider/cluster acceptance. */
|
|
describe.skipIf(!enabled)('Temporal durable runWorkflow recovery (fixture activities)', () => {
|
|
let connection: Connection,
|
|
native: NativeConnection,
|
|
client: Client,
|
|
bundle: Awaited<ReturnType<typeof bundleWorkflowCode>>;
|
|
const pending: Array<{ workers: Worker[]; runs: Promise<void>[] }> = [];
|
|
const workflowIds: string[] = [];
|
|
async function until(check: () => boolean | Promise<boolean>, timeout = 15000) {
|
|
const deadline = Date.now() + timeout;
|
|
while (!(await check())) {
|
|
if (Date.now() > deadline) throw new Error('Temporal fixture condition timed out');
|
|
await new Promise(resolve => setTimeout(resolve, 50));
|
|
}
|
|
}
|
|
async function group(
|
|
input: RunInput,
|
|
activities: {
|
|
inspectRunState?: () => Promise<{ status: string }>;
|
|
executeRun: (i: RunInput) => Promise<ActivityOutcome>;
|
|
verifyRun?: (i: RunInput) => Promise<ActivityOutcome>;
|
|
requestRelease?: (i: RunInput) => Promise<ActivityOutcome>;
|
|
recordWaiting: (i: RunInput & { reason: string }) => Promise<{ status: 'waiting' }>;
|
|
},
|
|
) {
|
|
const options = { connection: native, namespace, shutdownGraceTime: '1 second', maxCachedWorkflows: 0 };
|
|
const workers = await Promise.all([
|
|
Worker.create({
|
|
...options,
|
|
taskQueue: 'wishforge-runs',
|
|
workflowBundle: bundle,
|
|
activities: {
|
|
inspectRunState: activities.inspectRunState ?? (async () => ({ status: 'running' })),
|
|
recordWaiting: activities.recordWaiting,
|
|
prepareRepair: async () => {
|
|
throw new Error('Unexpected fixture repair');
|
|
},
|
|
},
|
|
}),
|
|
Worker.create({
|
|
...options,
|
|
taskQueue: `wishforge-executor-${input.tenantId}-${input.runId}-${input.generation}`,
|
|
activities: { executeRun: activities.executeRun },
|
|
}),
|
|
Worker.create({
|
|
...options,
|
|
taskQueue: `wishforge-verifier-${input.tenantId}-${input.runId}-${input.generation}`,
|
|
activities: {
|
|
verifyRun:
|
|
activities.verifyRun ??
|
|
(async () => {
|
|
throw new Error('Unexpected verification');
|
|
}),
|
|
},
|
|
}),
|
|
Worker.create({
|
|
...options,
|
|
taskQueue: 'wishforge-deliveries',
|
|
activities: {
|
|
requestRelease:
|
|
activities.requestRelease ??
|
|
(async () => {
|
|
throw new Error('Unexpected release');
|
|
}),
|
|
},
|
|
}),
|
|
]);
|
|
const g = { workers, runs: workers.map(w => w.run()) };
|
|
pending.push(g);
|
|
return g;
|
|
}
|
|
async function stop(g: { workers: Worker[]; runs: Promise<void>[] }) {
|
|
for (const worker of g.workers) worker.shutdown();
|
|
await Promise.all(g.runs);
|
|
pending.splice(pending.indexOf(g), 1);
|
|
}
|
|
beforeAll(async () => {
|
|
Runtime.install({
|
|
logger: { log: () => {}, trace: () => {}, debug: () => {}, info: () => {}, warn: () => {}, error: () => {} },
|
|
});
|
|
const address = process.env.TEMPORAL_ADDRESS ?? '127.0.0.1:7233';
|
|
connection = await Connection.connect({ address });
|
|
try {
|
|
await connection.workflowService.registerNamespace({
|
|
namespace,
|
|
description: 'WishForge contract-test fixtures only; no production credentials or model calls',
|
|
workflowExecutionRetentionPeriod: msToTs('1 day'),
|
|
});
|
|
} catch (error) {
|
|
if (!String(error).includes('already exists')) throw error;
|
|
}
|
|
native = await NativeConnection.connect({ address });
|
|
client = new Client({ connection, namespace });
|
|
bundle = await bundleWorkflowCode({
|
|
workflowsPath: fileURLToPath(new URL('../../apps/worker/src/workflows.ts', import.meta.url)),
|
|
});
|
|
}, 60000);
|
|
afterAll(async () => {
|
|
for (const id of workflowIds)
|
|
await client?.workflow
|
|
.getHandle(id)
|
|
.terminate('Fixture cleanup')
|
|
.catch(() => undefined);
|
|
for (const g of [...pending]) await stop(g);
|
|
await native?.close();
|
|
await connection?.close();
|
|
}, 30000);
|
|
|
|
it('persists waiting across worker shutdown and resumes with a new generation without repeating a release', async () => {
|
|
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() };
|
|
const workflowId = `contract:${input.runId}`;
|
|
workflowIds.push(workflowId);
|
|
let executeCalls = 0,
|
|
verifyCalls = 0,
|
|
releases = 0,
|
|
accepted = false;
|
|
const waiting: string[] = [];
|
|
const activities = {
|
|
inspectRunState: async () => ({ status: accepted ? 'succeeded' : 'running' }),
|
|
executeRun: async (): Promise<ActivityOutcome> => {
|
|
executeCalls++;
|
|
if (accepted) return { status: 'complete' };
|
|
return executeCalls === 1
|
|
? { status: 'waiting', reason: 'credentials', detail: 'fixture only' }
|
|
: { status: 'verify' };
|
|
},
|
|
verifyRun: async (): Promise<ActivityOutcome> => {
|
|
verifyCalls++;
|
|
return {
|
|
status: 'release',
|
|
verificationEvidenceId: 'fixture-verification',
|
|
buildEvidenceId: 'fixture-build',
|
|
environmentId: 'fixture-environment',
|
|
};
|
|
},
|
|
requestRelease: async (): Promise<ActivityOutcome> => {
|
|
releases++;
|
|
return { status: 'waiting', reason: 'deployment', detail: 'fixture acceptance pending' };
|
|
},
|
|
recordWaiting: async (i: RunInput & { reason: string }) => {
|
|
waiting.push(i.reason);
|
|
return { status: 'waiting' as const };
|
|
},
|
|
};
|
|
let workers = await group(input, activities);
|
|
const handle = await client.workflow.start('runWorkflow', {
|
|
workflowId,
|
|
taskQueue: 'wishforge-runs',
|
|
args: [input],
|
|
});
|
|
await until(() => waiting.length === 1);
|
|
expect((await handle.describe()).status.name).toBe('RUNNING');
|
|
await stop(workers);
|
|
const resumed = { ...input, generation: randomUUID() };
|
|
await handle.signal('resume', { generation: resumed.generation });
|
|
workers = await group(resumed, activities);
|
|
await until(() => releases === 1 && waiting.length === 2);
|
|
expect(verifyCalls).toBe(1);
|
|
expect((await handle.describe()).status.name).toBe('RUNNING');
|
|
await stop(workers);
|
|
accepted = true;
|
|
await handle.signal('resume');
|
|
workers = await group(resumed, activities);
|
|
await handle.result();
|
|
expect(executeCalls).toBe(2);
|
|
expect(verifyCalls).toBe(1);
|
|
expect(releases).toBe(1);
|
|
const history = await handle.fetchHistory();
|
|
const scheduled = history.events
|
|
?.filter(e => e.activityTaskScheduledEventAttributes)
|
|
.map(e => e.activityTaskScheduledEventAttributes!);
|
|
expect(scheduled?.filter(e => e.activityType?.name === 'executeRun').map(e => e.taskQueue?.name)).toEqual([
|
|
`wishforge-executor-${input.tenantId}-${input.runId}-${input.generation}`,
|
|
`wishforge-executor-${input.tenantId}-${input.runId}-${resumed.generation}`,
|
|
]);
|
|
expect(
|
|
scheduled
|
|
?.filter(e => ['executeRun', 'verifyRun'].includes(e.activityType?.name ?? ''))
|
|
.every(e => Number(e.scheduleToStartTimeout?.seconds) === 300),
|
|
).toBe(true);
|
|
await stop(workers);
|
|
}, 60000);
|
|
|
|
it('records ambiguous activity failure, retries only safe waiting synchronization, and does not invent delivery', async () => {
|
|
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
|
|
workflowId = `contract:${input.runId}`;
|
|
workflowIds.push(workflowId);
|
|
let executes = 0,
|
|
records = 0,
|
|
releases = 0;
|
|
const reasons: string[] = [];
|
|
const workers = await group(input, {
|
|
executeRun: async () => {
|
|
executes++;
|
|
throw ApplicationFailure.nonRetryable('Fixture lost provider response', 'AMBIGUOUS_PROVIDER_FIXTURE');
|
|
},
|
|
requestRelease: async () => {
|
|
releases++;
|
|
return { status: 'complete' };
|
|
},
|
|
recordWaiting: async i => {
|
|
records++;
|
|
if (records === 1)
|
|
throw ApplicationFailure.create({ message: 'Fixture old lease remains valid', type: 'EXECUTION_LEASE_HELD' });
|
|
reasons.push(i.reason);
|
|
return { status: 'waiting' };
|
|
},
|
|
});
|
|
const handle = await client.workflow.start('runWorkflow', {
|
|
workflowId,
|
|
taskQueue: 'wishforge-runs',
|
|
args: [input],
|
|
});
|
|
await until(() => reasons.length === 1);
|
|
expect(records).toBe(2);
|
|
expect(executes).toBe(1);
|
|
expect(releases).toBe(0);
|
|
expect(reasons[0]).toMatch(/^ambiguous_execution:/);
|
|
expect((await handle.describe()).status.name).toBe('RUNNING');
|
|
await new Promise(resolve => setTimeout(resolve, 300));
|
|
expect(executes).toBe(1);
|
|
await handle.cancel();
|
|
await expect(handle.result()).rejects.toThrow();
|
|
expect((await handle.describe()).status.name).toBe('CANCELLED');
|
|
await stop(workers);
|
|
}, 45000);
|
|
|
|
it('cancels an obsolete in-flight generation without recording its result over the resumed run', async () => {
|
|
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
|
|
workflowId = `contract:${input.runId}`;
|
|
workflowIds.push(workflowId);
|
|
let oldStarted = false,
|
|
newExecutions = 0,
|
|
waitingWrites = 0;
|
|
const old = await group(input, {
|
|
executeRun: async () => {
|
|
oldStarted = true;
|
|
const activity = Context.current();
|
|
const timer = setInterval(() => activity.heartbeat(), 100);
|
|
try {
|
|
await activity.cancelled;
|
|
} finally {
|
|
clearInterval(timer);
|
|
}
|
|
return { status: 'complete' };
|
|
},
|
|
recordWaiting: async () => {
|
|
waitingWrites++;
|
|
return { status: 'waiting' };
|
|
},
|
|
});
|
|
const handle = await client.workflow.start('runWorkflow', {
|
|
workflowId,
|
|
taskQueue: 'wishforge-runs',
|
|
args: [input],
|
|
});
|
|
await until(() => oldStarted);
|
|
const next = { ...input, generation: randomUUID() };
|
|
const freshWorker = await Worker.create({
|
|
connection: native,
|
|
namespace,
|
|
taskQueue: `wishforge-executor-${next.tenantId}-${next.runId}-${next.generation}`,
|
|
shutdownGraceTime: '1 second',
|
|
activities: {
|
|
executeRun: async () => {
|
|
newExecutions++;
|
|
return { status: 'complete' };
|
|
},
|
|
},
|
|
});
|
|
const fresh = { workers: [freshWorker], runs: [freshWorker.run()] };
|
|
pending.push(fresh);
|
|
await handle.signal('resume', { generation: next.generation });
|
|
await handle.result();
|
|
expect(newExecutions).toBe(1);
|
|
expect(waitingWrites).toBe(0);
|
|
await stop(old);
|
|
await stop(fresh);
|
|
}, 45000);
|
|
|
|
it.skipIf(process.env.WF_TEMPORAL_TIMEOUT_ACCEPTANCE !== '1')(
|
|
'records worker_unavailable after a real five-minute unstaffed-queue timeout',
|
|
async () => {
|
|
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
|
|
workflowId = `contract:${input.runId}`;
|
|
workflowIds.push(workflowId);
|
|
const reasons: string[] = [];
|
|
const worker = await Worker.create({
|
|
connection: native,
|
|
namespace,
|
|
taskQueue: 'wishforge-runs',
|
|
workflowBundle: bundle,
|
|
shutdownGraceTime: '1 second',
|
|
activities: {
|
|
inspectRunState: async () => ({ status: 'running' }),
|
|
recordWaiting: async (i: RunInput & { reason: string }) => {
|
|
reasons.push(i.reason);
|
|
return { status: 'waiting' };
|
|
},
|
|
prepareRepair: async () => {
|
|
throw new Error('Unexpected repair');
|
|
},
|
|
},
|
|
});
|
|
const workers = { workers: [worker], runs: [worker.run()] };
|
|
pending.push(workers);
|
|
const handle = await client.workflow.start('runWorkflow', {
|
|
workflowId,
|
|
taskQueue: 'wishforge-runs',
|
|
args: [input],
|
|
});
|
|
await until(() => reasons.length === 1, 330000);
|
|
expect(reasons[0]).toMatch(/^worker_unavailable:/);
|
|
expect((await handle.describe()).status.name).toBe('RUNNING');
|
|
const history = await handle.fetchHistory();
|
|
expect(
|
|
history.events?.filter(e => e.activityTaskScheduledEventAttributes?.activityType?.name === 'executeRun'),
|
|
).toHaveLength(1);
|
|
expect(history.events?.filter(e => e.activityTaskTimedOutEventAttributes)).toHaveLength(1);
|
|
await handle.cancel();
|
|
await expect(handle.result()).rejects.toThrow();
|
|
await stop(workers);
|
|
},
|
|
360000,
|
|
);
|
|
|
|
it('ignores stale generation signals and resumes after verifier failure without a second builder', async () => {
|
|
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
|
|
workflowId = `contract:${input.runId}`;
|
|
workflowIds.push(workflowId);
|
|
let builders = 0,
|
|
checks = 0,
|
|
releases = 0,
|
|
checkpoint = false;
|
|
const reasons: string[] = [];
|
|
const activities = {
|
|
executeRun: async (): Promise<ActivityOutcome> => {
|
|
if (!checkpoint) {
|
|
builders++;
|
|
checkpoint = true;
|
|
}
|
|
return { status: 'verify' };
|
|
},
|
|
verifyRun: async (): Promise<ActivityOutcome> => {
|
|
checks++;
|
|
if (checks === 1) throw new Error('Fixture verifier process exited');
|
|
return {
|
|
status: 'release',
|
|
verificationEvidenceId: 'fixture-v',
|
|
buildEvidenceId: 'fixture-b',
|
|
environmentId: 'fixture-e',
|
|
};
|
|
},
|
|
requestRelease: async (): Promise<ActivityOutcome> => {
|
|
releases++;
|
|
return { status: 'complete' };
|
|
},
|
|
recordWaiting: async (i: RunInput & { reason: string }) => {
|
|
reasons.push(i.reason);
|
|
return { status: 'waiting' as const };
|
|
},
|
|
};
|
|
let workers = await group(input, activities);
|
|
const handle = await client.workflow.start('runWorkflow', {
|
|
workflowId,
|
|
taskQueue: 'wishforge-runs',
|
|
args: [input],
|
|
});
|
|
await until(() => reasons.length === 1);
|
|
expect(reasons[0]).toContain('verification');
|
|
expect(builders).toBe(1);
|
|
expect(releases).toBe(0);
|
|
await stop(workers);
|
|
const resumed = { ...input, generation: randomUUID() };
|
|
await handle.signal('resume', { generation: resumed.generation });
|
|
await handle.signal('resume', { generation: input.generation });
|
|
workers = await group(resumed, activities);
|
|
await handle.result();
|
|
expect(builders).toBe(1);
|
|
expect(checks).toBe(2);
|
|
expect(releases).toBe(1);
|
|
await stop(workers);
|
|
}, 60000);
|
|
});
|