/** * Every way a run can stop has a documented way out (requirements FR-C5, FR-C6): each * waiting reason is explained in the console with an action, continuing re-queues the run * under a new execution generation, and stopping a run frees what it held. */ import { beforeEach, describe, expect, it } from 'vitest'; import { waitingReasons, type Actor, type Approval, type CodeReview, type Release, type Run, type WaitingReason, } from '@wishforge/contracts'; import { createHash } from 'node:crypto'; import { ControlPlane, PAUSE_GRACE_MS, record, setWaiting } from '../apps/api/src/domain/control-plane.js'; import { Connections } from '../apps/api/src/domain/connections.js'; import { waitingGuides } from '../apps/web/src/lib/waiting.js'; import { MemoryStore } from './helpers/memory-store.js'; const owner: Actor = { tenantId: 'alpha', userId: 'owner', role: 'owner', kind: 'human' }; let db: MemoryStore, control: ControlPlane, runId: string; beforeEach(async () => { db = new MemoryStore(); db.seed('alpha'); const env = { NODE_ENV: 'test', WF_VAULT_KEY: 'a'.repeat(64) }; control = new ControlPlane(db, env); await new Connections(db, env).create(owner, { provider: 'codex', label: 'test', apiKey: 'fixture-key-only' }); const project = await control.createProject(owner, { name: 'Lifecycle', repositoryUrl: 'https://example.org/l.git' }); runId = ( await control.submit( owner, project.id, { goal: 'Lifecycle fixture', acceptanceCriteria: ['It works'], budget: { maxTokens: 10_000 } }, 'lifecycle-wish', ) ).run_id; }); async function wait(reason: WaitingReason) { await db.transaction('alpha', async tx => { const run = (await tx.get('run', runId))!; setWaiting(run, reason, `Fixture: ${reason}`); await tx.put('run', run); }); } describe('waiting reasons and their ways out', () => { it('explains every waiting reason in Chinese with something the person can do', () => { for (const reason of waitingReasons) { const guide = waitingGuides[reason]; expect(guide, reason).toBeDefined(); expect(guide.title).toMatch(/[一-鿿]/); expect(guide.action).toMatch(/[一-鿿]/); // Either continuing helps, or the guide links to where the person acts, or they answer below. expect(guide.resumable || guide.link !== undefined || reason === 'clarification', reason).toBe(true); } expect(Object.keys(waitingGuides).sort()).toEqual([...waitingReasons].sort()); }); it.each(waitingReasons.filter(reason => waitingGuides[reason].resumable))( 'continuing a run waiting for %s queues a new execution generation', async reason => { const before = await control.run(owner, runId); await wait(reason); const resumed = await control.command(owner, runId, { action: 'resume' }, `resume-${reason}`); expect(resumed).toMatchObject({ status: 'queued' }); expect(resumed.waitingReason).toBeUndefined(); expect(resumed.pendingStartId).not.toBe(before.pendingStartId); const starts = (await db.transaction('alpha', tx => tx.outbox())).filter(item => item.type === 'run.start'); expect(starts.map(item => item.id)).toContain(resumed.pendingStartId); }, ); it('answers a clarification with a new requirement revision that starts from planning', async () => { await wait('clarification'); const answered = await control.command( owner, runId, { action: 'answer', message: 'Use the existing login page' }, 'answer-clarification', ); expect(answered).toMatchObject({ status: 'queued', stage: 'plan', revision: 2, repairCount: 0 }); }); it('frees the reservation when paused and keeps nothing after cancelling', async () => { expect((await control.usage(owner)).reservedTokens).toBe(10_000); await control.command(owner, runId, { action: 'pause' }, 'pause-lifecycle'); expect((await control.usage(owner)).reservedTokens).toBe(0); expect((await control.run(owner, runId)).waitingReason).toBeUndefined(); await control.command(owner, runId, { action: 'resume' }, 'resume-lifecycle'); expect((await control.usage(owner)).reservedTokens).toBe(10_000); const canceled = await control.command(owner, runId, { action: 'cancel' }, 'cancel-lifecycle'); expect(canceled).toMatchObject({ status: 'canceled', reservedTokens: 0 }); await expect(control.command(owner, runId, { action: 'resume' }, 'resume-canceled')).rejects.toMatchObject({ code: 'TERMINAL_RUN', }); }); it('invalidates pending releases, approvals and code reviews when a run is canceled', async () => { const run = await control.run(owner, runId); const releaseId = 'release-pending-1'; const approvalId = 'approval-pending-1'; const reviewId = 'review-pending-1'; await db.transaction('alpha', async tx => { await tx.put('release', { ...record('alpha'), id: releaseId, runId, projectId: run.projectId, environmentId: 'env-1', revision: run.revision, status: 'awaiting_approval', sourceCommit: 'a'.repeat(40), imageDigest: `sha256:${'b'.repeat(64)}`, architecture: 'linux/amd64', configVersion: '1', secretVersionRefs: [], evidenceIds: [], bindingDigest: 'd'.repeat(64), policyVersion: run.policyVersion, approvalId, riskReasons: ['environment_policy'], } satisfies Release); await tx.put('approval', { ...record('alpha'), id: approvalId, releaseId, bindingDigest: 'd'.repeat(64), status: 'pending', expiresAt: new Date(Date.now() + 60_000).toISOString(), } satisfies Approval); await tx.put('code_review', { ...record('alpha'), id: reviewId, runId, projectId: run.projectId, revision: run.revision, sourceCommit: 'a'.repeat(40), baseCommit: 'c'.repeat(40), paths: ['internal/server.go'], status: 'pending', } satisfies CodeReview); }); await control.command(owner, runId, { action: 'cancel' }, 'cancel-with-pending-release'); await db.transaction('alpha', async tx => { expect(await tx.get('release', releaseId)).toMatchObject({ status: 'failed', failureReason: '运行已取消', }); expect(await tx.get('approval', approvalId)).toMatchObject({ status: 'invalidated' }); expect(await tx.get('code_review', reviewId)).toMatchObject({ status: 'invalidated' }); }); }); }); describe('pausing a run that is executing', () => { const executor = (): Actor => ({ ...owner, userId: 'executor', kind: 'worker', runId }); const signals = () => db.state.get('alpha')!.outbox.filter(item => item.type === 'run.signal'); const edit = (change: (run: Run) => void) => db.transaction('alpha', async tx => { const run = (await tx.get('run', runId))!; change(run); await tx.put('run', run); }); /** Leases the run as its executor and asks for a pause; returns the executor's lease token. */ async function pausing(key: string) { if ((await control.run(owner, runId)).status === 'paused') await control.command(owner, runId, { action: 'resume' }, `resume-${key}`); const leased = await control.lease(executor(), runId, 'executor'); await control.command(owner, runId, { action: 'pause' }, `pause-${key}`); return leased; } it('lets the executor save the work before the run pauses', async () => { const leased = await pausing('saves'); const token = leased.lease!.token; expect(await control.run(owner, runId)).toMatchObject({ status: 'running', pauseRequestedAt: expect.any(String), lease: { token }, }); // The workflow is stopped only once the pause is complete; stopping it now would abort the save. expect(signals()).toEqual([]); // The agent being stopped gets no model credential, but the executor still uploads its work. expect( await new Connections(db, { NODE_ENV: 'test', WF_VAULT_KEY: 'a'.repeat(64) }).forRun(executor(), leased), ).toBe(undefined); const bundle = Buffer.from('fixture bundle'); const saved = await control.checkpoint( executor(), runId, { gitBundleBase64: bundle.toString('base64'), gitBundleSha256: createHash('sha256').update(bundle).digest('hex'), }, token, ); await expect(control.command(owner, runId, { action: 'resume' }, 'resume-while-pausing')).rejects.toMatchObject({ code: 'RUN_PAUSING', status: 409, }); expect((await control.command(owner, runId, { action: 'pause' }, 'pause-again')).status).toBe('running'); const paused = await control.completePause(executor(), runId, { leaseToken: token, saved: true }); expect(paused).toMatchObject({ status: 'paused', checkpointId: saved.id, lease: { token: token + 1 } }); expect(paused.pauseRequestedAt).toBeUndefined(); expect(signals()).toEqual([expect.objectContaining({ payload: { runId, action: 'pause' } })]); expect(db.state.get('alpha')!.events.at(-1)).toMatchObject({ type: 'run.paused', message: expect.stringContaining('已保存为检查点'), }); await expect(control.completePause(executor(), runId, { leaseToken: token, saved: true })).rejects.toMatchObject({ code: 'STALE_LEASE', }); expect((await control.command(owner, runId, { action: 'resume' }, 'resume-after-pause')).status).toBe('queued'); }); it('pauses anyway when the executor is late or gone', async () => { const coordinator: Actor = { ...owner, userId: 'coordinator', kind: 'worker' }; const overdue = (run: Run) => { run.pauseRequestedAt = new Date(Date.now() - PAUSE_GRACE_MS - 1000).toISOString(); }; const expired = (run: Run) => { run.lease!.expiresAt = new Date(Date.now() - 1000).toISOString(); }; // A late executor learns from its heartbeat. let leased = await pausing('late'); await edit(overdue); expect(await control.heartbeat(executor(), runId, leased.lease!.token)).toMatchObject({ status: 'paused' }); // A stopped executor: the next lease attempt pauses the run instead. leased = await pausing('gone'); await edit(expired); await expect(control.lease(executor(), runId, 'another-executor')).rejects.toMatchObject({ code: 'RUN_NOT_RUNNABLE', }); expect((await control.run(owner, runId)).status).toBe('paused'); // The workflow records why execution stopped: the run pauses instead of waiting. await pausing('stopped'); await edit(expired); expect( await control.waitRun(coordinator, runId, { reason: 'infrastructure', detail: 'Executor lost' }), ).toMatchObject({ status: 'paused', }); // Nobody asks at all: the background sweep completes it. await pausing('forgotten'); await edit(overdue); await control.finishOverduePauses('alpha'); expect((await control.run(owner, runId)).status).toBe('paused'); // Each completed pause stops the workflow once; the continues in between signal it too. expect(signals().filter(item => item.payload.action === 'pause')).toHaveLength(4); }); it('pauses at once when no executor holds the run, and cancelling overrides a pause in progress', async () => { expect((await control.command(owner, runId, { action: 'pause' }, 'pause-queued')).status).toBe('paused'); await control.command(owner, runId, { action: 'resume' }, 'resume-queued'); const verifier: Actor = { ...owner, userId: 'verifier', kind: 'verifier', runId }; await control.lease(verifier, runId, 'verifier'); expect((await control.command(owner, runId, { action: 'pause' }, 'pause-verifier')).status).toBe('paused'); await pausing('then-cancel'); const canceled = await control.command(owner, runId, { action: 'cancel' }, 'cancel-while-pausing'); expect(canceled.status).toBe('canceled'); expect(canceled.pauseRequestedAt).toBeUndefined(); }); });