wishforge-ide/tests/integration/temporal.test.ts
eust-w cce214f8fb feat: close the main loop from wish to integrated commit
Verification ran git inside the tree under test, kept no command output and
treated every infrastructure fault as a test failure; repairs went through an
executor lease the coordinator should not hold; and a project without delivery
could never succeed. This reworks the worker execution path and the control-plane
contracts it depends on.

- Verification exports the exact commit to a directory without .git, installs
  dependencies (network allowed) and runs the protected commands under
  bubblewrap in kata mode or directly in host mode; evidence records exit code,
  duration, digest, a redacted output tail and the isolation used. Production
  rejects host evidence. A failed installation waits as infrastructure.
- Waiting reasons are an enum with a redacted detail; dispatcher, GitHub and
  model-proxy states map onto it, and the console explains each one with a way out.
- Repairs are requested by the coordinator through /internal/runs/:id/repair;
  the control plane derives the failure summary from the verifier's evidence and
  enforces the repair budget. Continuing after the budget is spent grants one more
  round; a new requirement revision starts a fresh budget.
- Execution identities may only move a run within their own stages and only to
  running or waiting; success comes from integration evidence (no delivery
  configured) or deployment acceptance. The unused preview stage is gone.
- Integration fast-forwards when the target has not moved, merges and re-verifies
  otherwise, retries when a push finds the target moved, and turns merge conflicts
  into a repair round that merges the current target for the agent to resolve.
- Successful runs that needed a repair leave a project lesson; active lessons are
  injected into later prompts, and admins can list, add, disable and delete them.
- Executor progress (messages, commands, file changes, plan updates) is stored as
  normalized events; previously agent messages were recorded as the literal word
  "output". Provider failures keep the work as a checkpoint and wait as
  provider_error instead of failing the run.
- Workers declare WF_SANDBOX_MODE=host|kata (host refused in production),
  heartbeat with retries until the lease could have expired, and retry safe
  control-plane reads with backoff.
- tests/local-loop.test.ts runs the loop end to end with real Git, the in-process
  control plane behind its internal routes and a scripted agent.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 15:15:54 +08:00

397 lines
15 KiB
TypeScript

import { randomUUID } from 'node:crypto';
import { fileURLToPath } from 'node:url';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { Client, Connection } from '@temporalio/client';
import { Context } from '@temporalio/activity';
import { ApplicationFailure } from '@temporalio/common';
import { msToTs } from '@temporalio/common/lib/time.js';
import { bundleWorkflowCode, NativeConnection, Runtime, Worker } from '@temporalio/worker';
import type { RunInput, ActivityOutcome } from '../../apps/worker/src/activities.js';
const enabled = process.env.WF_TEMPORAL_INTEGRATION === '1';
const namespace = 'wishforge-contract-tests';
/** Actual Temporal history/replay with fixture activities, not provider/cluster acceptance. */
describe.skipIf(!enabled)('Temporal durable runWorkflow recovery (fixture activities)', () => {
let connection: Connection,
native: NativeConnection,
client: Client,
bundle: Awaited<ReturnType<typeof bundleWorkflowCode>>;
const pending: Array<{ workers: Worker[]; runs: Promise<void>[] }> = [];
const workflowIds: string[] = [];
async function until(check: () => boolean | Promise<boolean>, timeout = 15000) {
const deadline = Date.now() + timeout;
while (!(await check())) {
if (Date.now() > deadline) throw new Error('Temporal fixture condition timed out');
await new Promise(resolve => setTimeout(resolve, 50));
}
}
async function group(
input: RunInput,
activities: {
inspectRunState?: () => Promise<{ status: string }>;
executeRun: (i: RunInput) => Promise<ActivityOutcome>;
verifyRun?: (i: RunInput) => Promise<ActivityOutcome>;
requestRelease?: (i: RunInput) => Promise<ActivityOutcome>;
recordWaiting: (i: RunInput & { reason: string }) => Promise<{ status: 'waiting' }>;
},
) {
const options = { connection: native, namespace, shutdownGraceTime: '1 second', maxCachedWorkflows: 0 };
const workers = await Promise.all([
Worker.create({
...options,
taskQueue: 'wishforge-runs',
workflowBundle: bundle,
activities: {
inspectRunState: activities.inspectRunState ?? (async () => ({ status: 'running' })),
recordWaiting: activities.recordWaiting,
prepareRepair: async () => {
throw new Error('Unexpected fixture repair');
},
},
}),
Worker.create({
...options,
taskQueue: `wishforge-executor-${input.tenantId}-${input.runId}-${input.generation}`,
activities: { executeRun: activities.executeRun },
}),
Worker.create({
...options,
taskQueue: `wishforge-verifier-${input.tenantId}-${input.runId}-${input.generation}`,
activities: {
verifyRun:
activities.verifyRun ??
(async () => {
throw new Error('Unexpected verification');
}),
},
}),
Worker.create({
...options,
taskQueue: 'wishforge-deliveries',
activities: {
requestRelease:
activities.requestRelease ??
(async () => {
throw new Error('Unexpected release');
}),
},
}),
]);
const g = { workers, runs: workers.map(w => w.run()) };
pending.push(g);
return g;
}
async function stop(g: { workers: Worker[]; runs: Promise<void>[] }) {
for (const worker of g.workers) worker.shutdown();
await Promise.all(g.runs);
pending.splice(pending.indexOf(g), 1);
}
beforeAll(async () => {
Runtime.install({
logger: { log: () => {}, trace: () => {}, debug: () => {}, info: () => {}, warn: () => {}, error: () => {} },
});
const address = process.env.TEMPORAL_ADDRESS ?? '127.0.0.1:7233';
connection = await Connection.connect({ address });
try {
await connection.workflowService.registerNamespace({
namespace,
description: 'WishForge contract-test fixtures only; no production credentials or model calls',
workflowExecutionRetentionPeriod: msToTs('1 day'),
});
} catch (error) {
if (!String(error).includes('already exists')) throw error;
}
native = await NativeConnection.connect({ address });
client = new Client({ connection, namespace });
bundle = await bundleWorkflowCode({
workflowsPath: fileURLToPath(new URL('../../apps/worker/src/workflows.ts', import.meta.url)),
});
}, 60000);
afterAll(async () => {
for (const id of workflowIds)
await client?.workflow
.getHandle(id)
.terminate('Fixture cleanup')
.catch(() => undefined);
for (const g of [...pending]) await stop(g);
await native?.close();
await connection?.close();
}, 30000);
it('persists waiting across worker shutdown and resumes with a new generation without repeating a release', async () => {
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() };
const workflowId = `contract:${input.runId}`;
workflowIds.push(workflowId);
let executeCalls = 0,
verifyCalls = 0,
releases = 0,
accepted = false;
const waiting: string[] = [];
const activities = {
inspectRunState: async () => ({ status: accepted ? 'succeeded' : 'running' }),
executeRun: async (): Promise<ActivityOutcome> => {
executeCalls++;
if (accepted) return { status: 'complete' };
return executeCalls === 1
? { status: 'waiting', reason: 'credentials', detail: 'fixture only' }
: { status: 'verify' };
},
verifyRun: async (): Promise<ActivityOutcome> => {
verifyCalls++;
return {
status: 'release',
verificationEvidenceId: 'fixture-verification',
buildEvidenceId: 'fixture-build',
environmentId: 'fixture-environment',
};
},
requestRelease: async (): Promise<ActivityOutcome> => {
releases++;
return { status: 'waiting', reason: 'deployment', detail: 'fixture acceptance pending' };
},
recordWaiting: async (i: RunInput & { reason: string }) => {
waiting.push(i.reason);
return { status: 'waiting' as const };
},
};
let workers = await group(input, activities);
const handle = await client.workflow.start('runWorkflow', {
workflowId,
taskQueue: 'wishforge-runs',
args: [input],
});
await until(() => waiting.length === 1);
expect((await handle.describe()).status.name).toBe('RUNNING');
await stop(workers);
const resumed = { ...input, generation: randomUUID() };
await handle.signal('resume', { generation: resumed.generation });
workers = await group(resumed, activities);
await until(() => releases === 1 && waiting.length === 2);
expect(verifyCalls).toBe(1);
expect((await handle.describe()).status.name).toBe('RUNNING');
await stop(workers);
accepted = true;
await handle.signal('resume');
workers = await group(resumed, activities);
await handle.result();
expect(executeCalls).toBe(2);
expect(verifyCalls).toBe(1);
expect(releases).toBe(1);
const history = await handle.fetchHistory();
const scheduled = history.events
?.filter(e => e.activityTaskScheduledEventAttributes)
.map(e => e.activityTaskScheduledEventAttributes!);
expect(scheduled?.filter(e => e.activityType?.name === 'executeRun').map(e => e.taskQueue?.name)).toEqual([
`wishforge-executor-${input.tenantId}-${input.runId}-${input.generation}`,
`wishforge-executor-${input.tenantId}-${input.runId}-${resumed.generation}`,
]);
expect(
scheduled
?.filter(e => ['executeRun', 'verifyRun'].includes(e.activityType?.name ?? ''))
.every(e => Number(e.scheduleToStartTimeout?.seconds) === 300),
).toBe(true);
await stop(workers);
}, 60000);
it('records ambiguous activity failure, retries only safe waiting synchronization, and does not invent delivery', async () => {
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
workflowId = `contract:${input.runId}`;
workflowIds.push(workflowId);
let executes = 0,
records = 0,
releases = 0;
const reasons: string[] = [];
const workers = await group(input, {
executeRun: async () => {
executes++;
throw ApplicationFailure.nonRetryable('Fixture lost provider response', 'AMBIGUOUS_PROVIDER_FIXTURE');
},
requestRelease: async () => {
releases++;
return { status: 'complete' };
},
recordWaiting: async i => {
records++;
if (records === 1)
throw ApplicationFailure.create({ message: 'Fixture old lease remains valid', type: 'EXECUTION_LEASE_HELD' });
reasons.push(i.reason);
return { status: 'waiting' };
},
});
const handle = await client.workflow.start('runWorkflow', {
workflowId,
taskQueue: 'wishforge-runs',
args: [input],
});
await until(() => reasons.length === 1);
expect(records).toBe(2);
expect(executes).toBe(1);
expect(releases).toBe(0);
expect(reasons[0]).toMatch(/^ambiguous_execution:/);
expect((await handle.describe()).status.name).toBe('RUNNING');
await new Promise(resolve => setTimeout(resolve, 300));
expect(executes).toBe(1);
await handle.cancel();
await expect(handle.result()).rejects.toThrow();
expect((await handle.describe()).status.name).toBe('CANCELLED');
await stop(workers);
}, 45000);
it('cancels an obsolete in-flight generation without recording its result over the resumed run', async () => {
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
workflowId = `contract:${input.runId}`;
workflowIds.push(workflowId);
let oldStarted = false,
newExecutions = 0,
waitingWrites = 0;
const old = await group(input, {
executeRun: async () => {
oldStarted = true;
const activity = Context.current();
const timer = setInterval(() => activity.heartbeat(), 100);
try {
await activity.cancelled;
} finally {
clearInterval(timer);
}
return { status: 'complete' };
},
recordWaiting: async () => {
waitingWrites++;
return { status: 'waiting' };
},
});
const handle = await client.workflow.start('runWorkflow', {
workflowId,
taskQueue: 'wishforge-runs',
args: [input],
});
await until(() => oldStarted);
const next = { ...input, generation: randomUUID() };
const freshWorker = await Worker.create({
connection: native,
namespace,
taskQueue: `wishforge-executor-${next.tenantId}-${next.runId}-${next.generation}`,
shutdownGraceTime: '1 second',
activities: {
executeRun: async () => {
newExecutions++;
return { status: 'complete' };
},
},
});
const fresh = { workers: [freshWorker], runs: [freshWorker.run()] };
pending.push(fresh);
await handle.signal('resume', { generation: next.generation });
await handle.result();
expect(newExecutions).toBe(1);
expect(waitingWrites).toBe(0);
await stop(old);
await stop(fresh);
}, 45000);
it.skipIf(process.env.WF_TEMPORAL_TIMEOUT_ACCEPTANCE !== '1')(
'records worker_unavailable after a real five-minute unstaffed-queue timeout',
async () => {
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
workflowId = `contract:${input.runId}`;
workflowIds.push(workflowId);
const reasons: string[] = [];
const worker = await Worker.create({
connection: native,
namespace,
taskQueue: 'wishforge-runs',
workflowBundle: bundle,
shutdownGraceTime: '1 second',
activities: {
inspectRunState: async () => ({ status: 'running' }),
recordWaiting: async (i: RunInput & { reason: string }) => {
reasons.push(i.reason);
return { status: 'waiting' };
},
prepareRepair: async () => {
throw new Error('Unexpected repair');
},
},
});
const workers = { workers: [worker], runs: [worker.run()] };
pending.push(workers);
const handle = await client.workflow.start('runWorkflow', {
workflowId,
taskQueue: 'wishforge-runs',
args: [input],
});
await until(() => reasons.length === 1, 330000);
expect(reasons[0]).toMatch(/^worker_unavailable:/);
expect((await handle.describe()).status.name).toBe('RUNNING');
const history = await handle.fetchHistory();
expect(
history.events?.filter(e => e.activityTaskScheduledEventAttributes?.activityType?.name === 'executeRun'),
).toHaveLength(1);
expect(history.events?.filter(e => e.activityTaskTimedOutEventAttributes)).toHaveLength(1);
await handle.cancel();
await expect(handle.result()).rejects.toThrow();
await stop(workers);
},
360000,
);
it('ignores stale generation signals and resumes after verifier failure without a second builder', async () => {
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
workflowId = `contract:${input.runId}`;
workflowIds.push(workflowId);
let builders = 0,
checks = 0,
releases = 0,
checkpoint = false;
const reasons: string[] = [];
const activities = {
executeRun: async (): Promise<ActivityOutcome> => {
if (!checkpoint) {
builders++;
checkpoint = true;
}
return { status: 'verify' };
},
verifyRun: async (): Promise<ActivityOutcome> => {
checks++;
if (checks === 1) throw new Error('Fixture verifier process exited');
return {
status: 'release',
verificationEvidenceId: 'fixture-v',
buildEvidenceId: 'fixture-b',
environmentId: 'fixture-e',
};
},
requestRelease: async (): Promise<ActivityOutcome> => {
releases++;
return { status: 'complete' };
},
recordWaiting: async (i: RunInput & { reason: string }) => {
reasons.push(i.reason);
return { status: 'waiting' as const };
},
};
let workers = await group(input, activities);
const handle = await client.workflow.start('runWorkflow', {
workflowId,
taskQueue: 'wishforge-runs',
args: [input],
});
await until(() => reasons.length === 1);
expect(reasons[0]).toContain('verification');
expect(builders).toBe(1);
expect(releases).toBe(0);
await stop(workers);
const resumed = { ...input, generation: randomUUID() };
await handle.signal('resume', { generation: resumed.generation });
await handle.signal('resume', { generation: input.generation });
workers = await group(resumed, activities);
await handle.result();
expect(builders).toBe(1);
expect(checks).toBe(2);
expect(releases).toBe(1);
await stop(workers);
}, 60000);
});