- Drop the WorkBuddy Cloud executor stub: it always reported "unverified" and could never run. WorkBuddy remains an MCP client for submitting wishes. Providers are now codex and codebuddy; an unknown provider waits with an explicit reason instead of a fake adapter. - Remove internal routes nobody calls (/internal/releases/:id/begin, /accept, /failure and /internal/workspaces/:id/provisioned) with their domain methods, the never-consumed release.ready and release.rollback outbox events, an unreachable idempotency branch, dead helpers in runtime/deploy.ts and an unused RunnerBroker dependency. - Enable noUnusedLocals in the base and web TypeScript configs; the runtime package now extends the base config. - Add tests/tsconfig.json and include it in `npm run typecheck`. This surfaced tests still calling the deleted begin/accept path (now exercised through the Runner protocol) and five latent type errors. - Web: provider capability labels match the API's keys and the wish form disables unavailable providers. - Delete the invalid example Runner Job manifest; stop tracking the generated apps/web/next-env.d.ts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
395 lines
15 KiB
TypeScript
395 lines
15 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
import { Client, Connection } from '@temporalio/client';
|
|
import { Context } from '@temporalio/activity';
|
|
import { ApplicationFailure } from '@temporalio/common';
|
|
import { msToTs } from '@temporalio/common/lib/time.js';
|
|
import { bundleWorkflowCode, NativeConnection, Runtime, Worker } from '@temporalio/worker';
|
|
import type { RunInput, ActivityOutcome } from '../../apps/worker/src/activities.js';
|
|
|
|
const enabled = process.env.WF_TEMPORAL_INTEGRATION === '1';
|
|
const namespace = 'wishforge-contract-tests';
|
|
/** Actual Temporal history/replay with fixture activities, not provider/cluster acceptance. */
|
|
describe.skipIf(!enabled)('Temporal durable runWorkflow recovery (fixture activities)', () => {
|
|
let connection: Connection,
|
|
native: NativeConnection,
|
|
client: Client,
|
|
bundle: Awaited<ReturnType<typeof bundleWorkflowCode>>;
|
|
const pending: Array<{ workers: Worker[]; runs: Promise<void>[] }> = [];
|
|
const workflowIds: string[] = [];
|
|
async function until(check: () => boolean | Promise<boolean>, timeout = 15000) {
|
|
const deadline = Date.now() + timeout;
|
|
while (!(await check())) {
|
|
if (Date.now() > deadline) throw new Error('Temporal fixture condition timed out');
|
|
await new Promise(resolve => setTimeout(resolve, 50));
|
|
}
|
|
}
|
|
async function group(
|
|
input: RunInput,
|
|
activities: {
|
|
inspectRunState?: () => Promise<{ status: string }>;
|
|
executeRun: (i: RunInput) => Promise<ActivityOutcome>;
|
|
verifyRun?: (i: RunInput) => Promise<ActivityOutcome>;
|
|
requestRelease?: (i: RunInput) => Promise<ActivityOutcome>;
|
|
recordWaiting: (i: RunInput & { reason: string }) => Promise<{ status: 'waiting' }>;
|
|
},
|
|
) {
|
|
const options = { connection: native, namespace, shutdownGraceTime: '1 second', maxCachedWorkflows: 0 };
|
|
const workers = await Promise.all([
|
|
Worker.create({
|
|
...options,
|
|
taskQueue: 'wishforge-runs',
|
|
workflowBundle: bundle,
|
|
activities: {
|
|
inspectRunState: activities.inspectRunState ?? (async () => ({ status: 'running' })),
|
|
recordWaiting: activities.recordWaiting,
|
|
prepareRepair: async () => {
|
|
throw new Error('Unexpected fixture repair');
|
|
},
|
|
},
|
|
}),
|
|
Worker.create({
|
|
...options,
|
|
taskQueue: `wishforge-executor-${input.tenantId}-${input.runId}-${input.generation}`,
|
|
activities: { executeRun: activities.executeRun },
|
|
}),
|
|
Worker.create({
|
|
...options,
|
|
taskQueue: `wishforge-verifier-${input.tenantId}-${input.runId}-${input.generation}`,
|
|
activities: {
|
|
verifyRun:
|
|
activities.verifyRun ??
|
|
(async () => {
|
|
throw new Error('Unexpected verification');
|
|
}),
|
|
},
|
|
}),
|
|
Worker.create({
|
|
...options,
|
|
taskQueue: 'wishforge-deliveries',
|
|
activities: {
|
|
requestRelease:
|
|
activities.requestRelease ??
|
|
(async () => {
|
|
throw new Error('Unexpected release');
|
|
}),
|
|
},
|
|
}),
|
|
]);
|
|
const g = { workers, runs: workers.map(w => w.run()) };
|
|
pending.push(g);
|
|
return g;
|
|
}
|
|
async function stop(g: { workers: Worker[]; runs: Promise<void>[] }) {
|
|
for (const worker of g.workers) worker.shutdown();
|
|
await Promise.all(g.runs);
|
|
pending.splice(pending.indexOf(g), 1);
|
|
}
|
|
beforeAll(async () => {
|
|
Runtime.install({
|
|
logger: { log: () => {}, trace: () => {}, debug: () => {}, info: () => {}, warn: () => {}, error: () => {} },
|
|
});
|
|
const address = process.env.TEMPORAL_ADDRESS ?? '127.0.0.1:7233';
|
|
connection = await Connection.connect({ address });
|
|
try {
|
|
await connection.workflowService.registerNamespace({
|
|
namespace,
|
|
description: 'WishForge contract-test fixtures only; no production credentials or model calls',
|
|
workflowExecutionRetentionPeriod: msToTs('1 day'),
|
|
});
|
|
} catch (error) {
|
|
if (!String(error).includes('already exists')) throw error;
|
|
}
|
|
native = await NativeConnection.connect({ address });
|
|
client = new Client({ connection, namespace });
|
|
bundle = await bundleWorkflowCode({
|
|
workflowsPath: fileURLToPath(new URL('../../apps/worker/src/workflows.ts', import.meta.url)),
|
|
});
|
|
}, 60000);
|
|
afterAll(async () => {
|
|
for (const id of workflowIds)
|
|
await client?.workflow
|
|
.getHandle(id)
|
|
.terminate('Fixture cleanup')
|
|
.catch(() => undefined);
|
|
for (const g of [...pending]) await stop(g);
|
|
await native?.close();
|
|
await connection?.close();
|
|
}, 30000);
|
|
|
|
it('persists waiting across worker shutdown and resumes with a new generation without repeating a release', async () => {
|
|
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() };
|
|
const workflowId = `contract:${input.runId}`;
|
|
workflowIds.push(workflowId);
|
|
let executeCalls = 0,
|
|
verifyCalls = 0,
|
|
releases = 0,
|
|
accepted = false;
|
|
const waiting: string[] = [];
|
|
const activities = {
|
|
inspectRunState: async () => ({ status: accepted ? 'succeeded' : 'running' }),
|
|
executeRun: async (): Promise<ActivityOutcome> => {
|
|
executeCalls++;
|
|
if (accepted) return { status: 'complete' };
|
|
return executeCalls === 1 ? { status: 'waiting', reason: 'credentials: fixture only' } : { status: 'verify' };
|
|
},
|
|
verifyRun: async (): Promise<ActivityOutcome> => {
|
|
verifyCalls++;
|
|
return {
|
|
status: 'release',
|
|
verificationEvidenceId: 'fixture-verification',
|
|
buildEvidenceId: 'fixture-build',
|
|
environmentId: 'fixture-environment',
|
|
};
|
|
},
|
|
requestRelease: async (): Promise<ActivityOutcome> => {
|
|
releases++;
|
|
return { status: 'waiting', reason: 'deployment: fixture acceptance pending' };
|
|
},
|
|
recordWaiting: async (i: RunInput & { reason: string }) => {
|
|
waiting.push(i.reason);
|
|
return { status: 'waiting' as const };
|
|
},
|
|
};
|
|
let workers = await group(input, activities);
|
|
const handle = await client.workflow.start('runWorkflow', {
|
|
workflowId,
|
|
taskQueue: 'wishforge-runs',
|
|
args: [input],
|
|
});
|
|
await until(() => waiting.length === 1);
|
|
expect((await handle.describe()).status.name).toBe('RUNNING');
|
|
await stop(workers);
|
|
const resumed = { ...input, generation: randomUUID() };
|
|
await handle.signal('resume', { generation: resumed.generation });
|
|
workers = await group(resumed, activities);
|
|
await until(() => releases === 1 && waiting.length === 2);
|
|
expect(verifyCalls).toBe(1);
|
|
expect((await handle.describe()).status.name).toBe('RUNNING');
|
|
await stop(workers);
|
|
accepted = true;
|
|
await handle.signal('resume');
|
|
workers = await group(resumed, activities);
|
|
await handle.result();
|
|
expect(executeCalls).toBe(2);
|
|
expect(verifyCalls).toBe(1);
|
|
expect(releases).toBe(1);
|
|
const history = await handle.fetchHistory();
|
|
const scheduled = history.events
|
|
?.filter(e => e.activityTaskScheduledEventAttributes)
|
|
.map(e => e.activityTaskScheduledEventAttributes!);
|
|
expect(scheduled?.filter(e => e.activityType?.name === 'executeRun').map(e => e.taskQueue?.name)).toEqual([
|
|
`wishforge-executor-${input.tenantId}-${input.runId}-${input.generation}`,
|
|
`wishforge-executor-${input.tenantId}-${input.runId}-${resumed.generation}`,
|
|
]);
|
|
expect(
|
|
scheduled
|
|
?.filter(e => ['executeRun', 'verifyRun'].includes(e.activityType?.name ?? ''))
|
|
.every(e => Number(e.scheduleToStartTimeout?.seconds) === 300),
|
|
).toBe(true);
|
|
await stop(workers);
|
|
}, 60000);
|
|
|
|
it('records ambiguous activity failure, retries only safe waiting synchronization, and does not invent delivery', async () => {
|
|
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
|
|
workflowId = `contract:${input.runId}`;
|
|
workflowIds.push(workflowId);
|
|
let executes = 0,
|
|
records = 0,
|
|
releases = 0;
|
|
const reasons: string[] = [];
|
|
const workers = await group(input, {
|
|
executeRun: async () => {
|
|
executes++;
|
|
throw ApplicationFailure.nonRetryable('Fixture lost provider response', 'AMBIGUOUS_PROVIDER_FIXTURE');
|
|
},
|
|
requestRelease: async () => {
|
|
releases++;
|
|
return { status: 'complete' };
|
|
},
|
|
recordWaiting: async i => {
|
|
records++;
|
|
if (records === 1)
|
|
throw ApplicationFailure.create({ message: 'Fixture old lease remains valid', type: 'EXECUTION_LEASE_HELD' });
|
|
reasons.push(i.reason);
|
|
return { status: 'waiting' };
|
|
},
|
|
});
|
|
const handle = await client.workflow.start('runWorkflow', {
|
|
workflowId,
|
|
taskQueue: 'wishforge-runs',
|
|
args: [input],
|
|
});
|
|
await until(() => reasons.length === 1);
|
|
expect(records).toBe(2);
|
|
expect(executes).toBe(1);
|
|
expect(releases).toBe(0);
|
|
expect(reasons[0]).toMatch(/^ambiguous_execution:/);
|
|
expect((await handle.describe()).status.name).toBe('RUNNING');
|
|
await new Promise(resolve => setTimeout(resolve, 300));
|
|
expect(executes).toBe(1);
|
|
await handle.cancel();
|
|
await expect(handle.result()).rejects.toThrow();
|
|
expect((await handle.describe()).status.name).toBe('CANCELLED');
|
|
await stop(workers);
|
|
}, 45000);
|
|
|
|
it('cancels an obsolete in-flight generation without recording its result over the resumed run', async () => {
|
|
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
|
|
workflowId = `contract:${input.runId}`;
|
|
workflowIds.push(workflowId);
|
|
let oldStarted = false,
|
|
newExecutions = 0,
|
|
waitingWrites = 0;
|
|
const old = await group(input, {
|
|
executeRun: async () => {
|
|
oldStarted = true;
|
|
const activity = Context.current();
|
|
const timer = setInterval(() => activity.heartbeat(), 100);
|
|
try {
|
|
await activity.cancelled;
|
|
} finally {
|
|
clearInterval(timer);
|
|
}
|
|
return { status: 'complete' };
|
|
},
|
|
recordWaiting: async () => {
|
|
waitingWrites++;
|
|
return { status: 'waiting' };
|
|
},
|
|
});
|
|
const handle = await client.workflow.start('runWorkflow', {
|
|
workflowId,
|
|
taskQueue: 'wishforge-runs',
|
|
args: [input],
|
|
});
|
|
await until(() => oldStarted);
|
|
const next = { ...input, generation: randomUUID() };
|
|
const freshWorker = await Worker.create({
|
|
connection: native,
|
|
namespace,
|
|
taskQueue: `wishforge-executor-${next.tenantId}-${next.runId}-${next.generation}`,
|
|
shutdownGraceTime: '1 second',
|
|
activities: {
|
|
executeRun: async () => {
|
|
newExecutions++;
|
|
return { status: 'complete' };
|
|
},
|
|
},
|
|
});
|
|
const fresh = { workers: [freshWorker], runs: [freshWorker.run()] };
|
|
pending.push(fresh);
|
|
await handle.signal('resume', { generation: next.generation });
|
|
await handle.result();
|
|
expect(newExecutions).toBe(1);
|
|
expect(waitingWrites).toBe(0);
|
|
await stop(old);
|
|
await stop(fresh);
|
|
}, 45000);
|
|
|
|
it.skipIf(process.env.WF_TEMPORAL_TIMEOUT_ACCEPTANCE !== '1')(
|
|
'records worker_unavailable after a real five-minute unstaffed-queue timeout',
|
|
async () => {
|
|
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
|
|
workflowId = `contract:${input.runId}`;
|
|
workflowIds.push(workflowId);
|
|
const reasons: string[] = [];
|
|
const worker = await Worker.create({
|
|
connection: native,
|
|
namespace,
|
|
taskQueue: 'wishforge-runs',
|
|
workflowBundle: bundle,
|
|
shutdownGraceTime: '1 second',
|
|
activities: {
|
|
inspectRunState: async () => ({ status: 'running' }),
|
|
recordWaiting: async (i: RunInput & { reason: string }) => {
|
|
reasons.push(i.reason);
|
|
return { status: 'waiting' };
|
|
},
|
|
prepareRepair: async () => {
|
|
throw new Error('Unexpected repair');
|
|
},
|
|
},
|
|
});
|
|
const workers = { workers: [worker], runs: [worker.run()] };
|
|
pending.push(workers);
|
|
const handle = await client.workflow.start('runWorkflow', {
|
|
workflowId,
|
|
taskQueue: 'wishforge-runs',
|
|
args: [input],
|
|
});
|
|
await until(() => reasons.length === 1, 330000);
|
|
expect(reasons[0]).toMatch(/^worker_unavailable:/);
|
|
expect((await handle.describe()).status.name).toBe('RUNNING');
|
|
const history = await handle.fetchHistory();
|
|
expect(
|
|
history.events?.filter(e => e.activityTaskScheduledEventAttributes?.activityType?.name === 'executeRun'),
|
|
).toHaveLength(1);
|
|
expect(history.events?.filter(e => e.activityTaskTimedOutEventAttributes)).toHaveLength(1);
|
|
await handle.cancel();
|
|
await expect(handle.result()).rejects.toThrow();
|
|
await stop(workers);
|
|
},
|
|
360000,
|
|
);
|
|
|
|
it('ignores stale generation signals and resumes after verifier failure without a second builder', async () => {
|
|
const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() },
|
|
workflowId = `contract:${input.runId}`;
|
|
workflowIds.push(workflowId);
|
|
let builders = 0,
|
|
checks = 0,
|
|
releases = 0,
|
|
checkpoint = false;
|
|
const reasons: string[] = [];
|
|
const activities = {
|
|
executeRun: async (): Promise<ActivityOutcome> => {
|
|
if (!checkpoint) {
|
|
builders++;
|
|
checkpoint = true;
|
|
}
|
|
return { status: 'verify' };
|
|
},
|
|
verifyRun: async (): Promise<ActivityOutcome> => {
|
|
checks++;
|
|
if (checks === 1) throw new Error('Fixture verifier process exited');
|
|
return {
|
|
status: 'release',
|
|
verificationEvidenceId: 'fixture-v',
|
|
buildEvidenceId: 'fixture-b',
|
|
environmentId: 'fixture-e',
|
|
};
|
|
},
|
|
requestRelease: async (): Promise<ActivityOutcome> => {
|
|
releases++;
|
|
return { status: 'complete' };
|
|
},
|
|
recordWaiting: async (i: RunInput & { reason: string }) => {
|
|
reasons.push(i.reason);
|
|
return { status: 'waiting' as const };
|
|
},
|
|
};
|
|
let workers = await group(input, activities);
|
|
const handle = await client.workflow.start('runWorkflow', {
|
|
workflowId,
|
|
taskQueue: 'wishforge-runs',
|
|
args: [input],
|
|
});
|
|
await until(() => reasons.length === 1);
|
|
expect(reasons[0]).toContain('verification');
|
|
expect(builders).toBe(1);
|
|
expect(releases).toBe(0);
|
|
await stop(workers);
|
|
const resumed = { ...input, generation: randomUUID() };
|
|
await handle.signal('resume', { generation: resumed.generation });
|
|
await handle.signal('resume', { generation: input.generation });
|
|
workers = await group(resumed, activities);
|
|
await handle.result();
|
|
expect(builders).toBe(1);
|
|
expect(checks).toBe(2);
|
|
expect(releases).toBe(1);
|
|
await stop(workers);
|
|
}, 60000);
|
|
});
|