import { randomUUID } from 'node:crypto'; import { fileURLToPath } from 'node:url'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { Client, Connection } from '@temporalio/client'; import { Context } from '@temporalio/activity'; import { ApplicationFailure } from '@temporalio/common'; import { msToTs } from '@temporalio/common/lib/time.js'; import { bundleWorkflowCode, NativeConnection, Runtime, Worker } from '@temporalio/worker'; import type { RunInput, ActivityOutcome } from '../../apps/worker/src/activities.js'; const enabled = process.env.WF_TEMPORAL_INTEGRATION === '1'; const namespace = 'wishforge-contract-tests'; /** Actual Temporal history/replay with fixture activities, not provider/cluster acceptance. */ describe.skipIf(!enabled)('Temporal durable runWorkflow recovery (fixture activities)', () => { let connection: Connection, native: NativeConnection, client: Client, bundle: Awaited>; const pending: Array<{ workers: Worker[]; runs: Promise[] }> = []; const workflowIds: string[] = []; async function until(check: () => boolean | Promise, timeout = 15000) { const deadline = Date.now() + timeout; while (!(await check())) { if (Date.now() > deadline) throw new Error('Temporal fixture condition timed out'); await new Promise(resolve => setTimeout(resolve, 50)); } } async function group( input: RunInput, activities: { inspectRunState?: () => Promise<{ status: string }>; executeRun: (i: RunInput) => Promise; verifyRun?: (i: RunInput) => Promise; requestRelease?: (i: RunInput) => Promise; recordWaiting: (i: RunInput & { reason: string }) => Promise<{ status: 'waiting' }>; }, ) { const options = { connection: native, namespace, shutdownGraceTime: '1 second' as const, maxCachedWorkflows: 0 }; const workers = await Promise.all([ Worker.create({ ...options, taskQueue: 'wishforge-runs', workflowBundle: bundle, activities: { inspectRunState: activities.inspectRunState ?? (async () => ({ status: 'running' })), recordWaiting: activities.recordWaiting, prepareRepair: async () => { throw new Error('Unexpected fixture repair'); }, }, }), Worker.create({ ...options, taskQueue: `wishforge-executor-${input.tenantId}-${input.runId}-${input.generation}`, activities: { executeRun: activities.executeRun }, }), Worker.create({ ...options, taskQueue: `wishforge-verifier-${input.tenantId}-${input.runId}-${input.generation}`, activities: { verifyRun: activities.verifyRun ?? (async () => { throw new Error('Unexpected verification'); }), }, }), Worker.create({ ...options, taskQueue: 'wishforge-deliveries', activities: { requestRelease: activities.requestRelease ?? (async () => { throw new Error('Unexpected release'); }), }, }), ]); const g = { workers, runs: workers.map(w => w.run()) }; pending.push(g); return g; } async function stop(g: { workers: Worker[]; runs: Promise[] }) { for (const worker of g.workers) worker.shutdown(); await Promise.all(g.runs); pending.splice(pending.indexOf(g), 1); } beforeAll(async () => { Runtime.install({ logger: { log: () => {}, trace: () => {}, debug: () => {}, info: () => {}, warn: () => {}, error: () => {} }, }); const address = process.env.TEMPORAL_ADDRESS ?? '127.0.0.1:7233'; connection = await Connection.connect({ address }); try { await connection.workflowService.registerNamespace({ namespace, description: 'WishForge contract-test fixtures only; no production credentials or model calls', workflowExecutionRetentionPeriod: msToTs('1 day'), }); } catch (error) { if (!String(error).includes('already exists')) throw error; } native = await NativeConnection.connect({ address }); client = new Client({ connection, namespace }); bundle = await bundleWorkflowCode({ workflowsPath: fileURLToPath(new URL('../../apps/worker/src/workflows.ts', import.meta.url)), }); }, 60000); afterAll(async () => { for (const id of workflowIds) await client?.workflow .getHandle(id) .terminate('Fixture cleanup') .catch(() => undefined); for (const g of [...pending]) await stop(g); await native?.close(); await connection?.close(); }, 30000); it('persists waiting across worker shutdown and resumes with a new generation without repeating a release', async () => { const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() }; const workflowId = `contract:${input.runId}`; workflowIds.push(workflowId); let executeCalls = 0, verifyCalls = 0, releases = 0, accepted = false; const waiting: string[] = []; const activities = { inspectRunState: async () => ({ status: accepted ? 'succeeded' : 'running' }), executeRun: async (): Promise => { executeCalls++; if (accepted) return { status: 'complete' }; return executeCalls === 1 ? { status: 'waiting', reason: 'credentials', detail: 'fixture only' } : { status: 'verify' }; }, verifyRun: async (): Promise => { verifyCalls++; return { status: 'release', verificationEvidenceId: 'fixture-verification', buildEvidenceId: 'fixture-build', environmentId: 'fixture-environment', }; }, requestRelease: async (): Promise => { releases++; return { status: 'waiting', reason: 'deployment', detail: 'fixture acceptance pending' }; }, recordWaiting: async (i: RunInput & { reason: string }) => { waiting.push(i.reason); return { status: 'waiting' as const }; }, }; let workers = await group(input, activities); const handle = await client.workflow.start('runWorkflow', { workflowId, taskQueue: 'wishforge-runs', args: [input], }); await until(() => waiting.length === 1); expect((await handle.describe()).status.name).toBe('RUNNING'); await stop(workers); const resumed = { ...input, generation: randomUUID() }; await handle.signal('resume', { generation: resumed.generation }); workers = await group(resumed, activities); await until(() => releases === 1 && waiting.length === 2); expect(verifyCalls).toBe(1); expect((await handle.describe()).status.name).toBe('RUNNING'); await stop(workers); accepted = true; await handle.signal('resume'); workers = await group(resumed, activities); await handle.result(); expect(executeCalls).toBe(2); expect(verifyCalls).toBe(1); expect(releases).toBe(1); const history = await handle.fetchHistory(); const scheduled = history.events ?.filter(e => e.activityTaskScheduledEventAttributes) .map(e => e.activityTaskScheduledEventAttributes!); expect(scheduled?.filter(e => e.activityType?.name === 'executeRun').map(e => e.taskQueue?.name)).toEqual([ `wishforge-executor-${input.tenantId}-${input.runId}-${input.generation}`, `wishforge-executor-${input.tenantId}-${input.runId}-${resumed.generation}`, ]); expect( scheduled ?.filter(e => ['executeRun', 'verifyRun'].includes(e.activityType?.name ?? '')) .every(e => Number(e.scheduleToStartTimeout?.seconds) === 300), ).toBe(true); await stop(workers); }, 60000); it('records ambiguous activity failure, retries only safe waiting synchronization, and does not invent delivery', async () => { const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() }, workflowId = `contract:${input.runId}`; workflowIds.push(workflowId); let executes = 0, records = 0, releases = 0; const reasons: string[] = []; const workers = await group(input, { executeRun: async () => { executes++; throw ApplicationFailure.nonRetryable('Fixture lost provider response', 'AMBIGUOUS_PROVIDER_FIXTURE'); }, requestRelease: async () => { releases++; return { status: 'complete' }; }, recordWaiting: async i => { records++; if (records === 1) throw ApplicationFailure.create({ message: 'Fixture old lease remains valid', type: 'EXECUTION_LEASE_HELD' }); reasons.push(i.reason); return { status: 'waiting' }; }, }); const handle = await client.workflow.start('runWorkflow', { workflowId, taskQueue: 'wishforge-runs', args: [input], }); await until(() => reasons.length === 1); expect(records).toBe(2); expect(executes).toBe(1); expect(releases).toBe(0); expect(reasons[0]).toMatch(/^ambiguous_execution:/); expect((await handle.describe()).status.name).toBe('RUNNING'); await new Promise(resolve => setTimeout(resolve, 300)); expect(executes).toBe(1); await handle.cancel(); await expect(handle.result()).rejects.toThrow(); expect((await handle.describe()).status.name).toBe('CANCELLED'); await stop(workers); }, 45000); it('cancels an obsolete in-flight generation without recording its result over the resumed run', async () => { const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() }, workflowId = `contract:${input.runId}`; workflowIds.push(workflowId); let oldStarted = false, newExecutions = 0, waitingWrites = 0; const old = await group(input, { executeRun: async () => { oldStarted = true; const activity = Context.current(); const timer = setInterval(() => activity.heartbeat(), 100); try { await activity.cancelled; } finally { clearInterval(timer); } return { status: 'complete' }; }, recordWaiting: async () => { waitingWrites++; return { status: 'waiting' }; }, }); const handle = await client.workflow.start('runWorkflow', { workflowId, taskQueue: 'wishforge-runs', args: [input], }); await until(() => oldStarted); const next = { ...input, generation: randomUUID() }; const freshWorker = await Worker.create({ connection: native, namespace, taskQueue: `wishforge-executor-${next.tenantId}-${next.runId}-${next.generation}`, shutdownGraceTime: '1 second' as const, activities: { executeRun: async () => { newExecutions++; return { status: 'complete' }; }, }, }); const fresh = { workers: [freshWorker], runs: [freshWorker.run()] }; pending.push(fresh); await handle.signal('resume', { generation: next.generation }); await handle.result(); expect(newExecutions).toBe(1); expect(waitingWrites).toBe(0); await stop(old); await stop(fresh); }, 45000); it.skipIf(process.env.WF_TEMPORAL_TIMEOUT_ACCEPTANCE !== '1')( 'records worker_unavailable after a real five-minute unstaffed-queue timeout', async () => { const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() }, workflowId = `contract:${input.runId}`; workflowIds.push(workflowId); const reasons: string[] = []; const worker = await Worker.create({ connection: native, namespace, taskQueue: 'wishforge-runs', workflowBundle: bundle, shutdownGraceTime: '1 second' as const, activities: { inspectRunState: async () => ({ status: 'running' }), recordWaiting: async (i: RunInput & { reason: string }) => { reasons.push(i.reason); return { status: 'waiting' }; }, prepareRepair: async () => { throw new Error('Unexpected repair'); }, }, }); const workers = { workers: [worker], runs: [worker.run()] }; pending.push(workers); const handle = await client.workflow.start('runWorkflow', { workflowId, taskQueue: 'wishforge-runs', args: [input], }); await until(() => reasons.length === 1, 330000); expect(reasons[0]).toMatch(/^worker_unavailable:/); expect((await handle.describe()).status.name).toBe('RUNNING'); const history = await handle.fetchHistory(); expect( history.events?.filter(e => e.activityTaskScheduledEventAttributes?.activityType?.name === 'executeRun'), ).toHaveLength(1); expect(history.events?.filter(e => e.activityTaskTimedOutEventAttributes)).toHaveLength(1); await handle.cancel(); await expect(handle.result()).rejects.toThrow(); await stop(workers); }, 360000, ); it('ignores stale generation signals and resumes after verifier failure without a second builder', async () => { const input = { tenantId: 'fixture', runId: randomUUID(), generation: randomUUID() }, workflowId = `contract:${input.runId}`; workflowIds.push(workflowId); let builders = 0, checks = 0, releases = 0, checkpoint = false; const reasons: string[] = []; const activities = { executeRun: async (): Promise => { if (!checkpoint) { builders++; checkpoint = true; } return { status: 'verify' }; }, verifyRun: async (): Promise => { checks++; if (checks === 1) throw new Error('Fixture verifier process exited'); return { status: 'release', verificationEvidenceId: 'fixture-v', buildEvidenceId: 'fixture-b', environmentId: 'fixture-e', }; }, requestRelease: async (): Promise => { releases++; return { status: 'complete' }; }, recordWaiting: async (i: RunInput & { reason: string }) => { reasons.push(i.reason); return { status: 'waiting' as const }; }, }; let workers = await group(input, activities); const handle = await client.workflow.start('runWorkflow', { workflowId, taskQueue: 'wishforge-runs', args: [input], }); await until(() => reasons.length === 1); expect(reasons[0]).toContain('verification'); expect(builders).toBe(1); expect(releases).toBe(0); await stop(workers); const resumed = { ...input, generation: randomUUID() }; await handle.signal('resume', { generation: resumed.generation }); await handle.signal('resume', { generation: input.generation }); workers = await group(resumed, activities); await handle.result(); expect(builders).toBe(1); expect(checks).toBe(2); expect(releases).toBe(1); await stop(workers); }, 60000); });