wishforge-ide/tests/run-lifecycle.test.ts
eust-w 8247cc209a 🐛 Invalidate pending releases and approvals when a run is canceled
Canceling a run at the release stage left its awaiting_approval release
and pending approval counted as waiting on the dashboard. Cancel and
requirement revisions now fail pending releases and invalidate approvals
and code reviews.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-11-02 15:01:39 +08:00

267 lines
12 KiB
TypeScript

/**
* Every way a run can stop has a documented way out (requirements FR-C5, FR-C6): each
* waiting reason is explained in the console with an action, continuing re-queues the run
* under a new execution generation, and stopping a run frees what it held.
*/
import { beforeEach, describe, expect, it } from 'vitest';
import {
waitingReasons,
type Actor,
type Approval,
type CodeReview,
type Release,
type Run,
type WaitingReason,
} from '@wishforge/contracts';
import { createHash } from 'node:crypto';
import { ControlPlane, PAUSE_GRACE_MS, record, setWaiting } from '../apps/api/src/domain/control-plane.js';
import { Connections } from '../apps/api/src/domain/connections.js';
import { waitingGuides } from '../apps/web/src/lib/waiting.js';
import { MemoryStore } from './helpers/memory-store.js';
const owner: Actor = { tenantId: 'alpha', userId: 'owner', role: 'owner', kind: 'human' };
let db: MemoryStore, control: ControlPlane, runId: string;
beforeEach(async () => {
db = new MemoryStore();
db.seed('alpha');
const env = { NODE_ENV: 'test', WF_VAULT_KEY: 'a'.repeat(64) };
control = new ControlPlane(db, env);
await new Connections(db, env).create(owner, { provider: 'codex', label: 'test', apiKey: 'fixture-key-only' });
const project = await control.createProject(owner, { name: 'Lifecycle', repositoryUrl: 'https://example.org/l.git' });
runId = (
await control.submit(
owner,
project.id,
{ goal: 'Lifecycle fixture', acceptanceCriteria: ['It works'], budget: { maxTokens: 10_000 } },
'lifecycle-wish',
)
).run_id;
});
async function wait(reason: WaitingReason) {
await db.transaction('alpha', async tx => {
const run = (await tx.get<Run>('run', runId))!;
setWaiting(run, reason, `Fixture: ${reason}`);
await tx.put('run', run);
});
}
describe('waiting reasons and their ways out', () => {
it('explains every waiting reason in Chinese with something the person can do', () => {
for (const reason of waitingReasons) {
const guide = waitingGuides[reason];
expect(guide, reason).toBeDefined();
expect(guide.title).toMatch(/[一-鿿]/);
expect(guide.action).toMatch(/[一-鿿]/);
// Either continuing helps, or the guide links to where the person acts, or they answer below.
expect(guide.resumable || guide.link !== undefined || reason === 'clarification', reason).toBe(true);
}
expect(Object.keys(waitingGuides).sort()).toEqual([...waitingReasons].sort());
});
it.each(waitingReasons.filter(reason => waitingGuides[reason].resumable))(
'continuing a run waiting for %s queues a new execution generation',
async reason => {
const before = await control.run(owner, runId);
await wait(reason);
const resumed = await control.command(owner, runId, { action: 'resume' }, `resume-${reason}`);
expect(resumed).toMatchObject({ status: 'queued' });
expect(resumed.waitingReason).toBeUndefined();
expect(resumed.pendingStartId).not.toBe(before.pendingStartId);
const starts = (await db.transaction('alpha', tx => tx.outbox())).filter(item => item.type === 'run.start');
expect(starts.map(item => item.id)).toContain(resumed.pendingStartId);
},
);
it('answers a clarification with a new requirement revision that starts from planning', async () => {
await wait('clarification');
const answered = await control.command(
owner,
runId,
{ action: 'answer', message: 'Use the existing login page' },
'answer-clarification',
);
expect(answered).toMatchObject({ status: 'queued', stage: 'plan', revision: 2, repairCount: 0 });
});
it('frees the reservation when paused and keeps nothing after cancelling', async () => {
expect((await control.usage(owner)).reservedTokens).toBe(10_000);
await control.command(owner, runId, { action: 'pause' }, 'pause-lifecycle');
expect((await control.usage(owner)).reservedTokens).toBe(0);
expect((await control.run(owner, runId)).waitingReason).toBeUndefined();
await control.command(owner, runId, { action: 'resume' }, 'resume-lifecycle');
expect((await control.usage(owner)).reservedTokens).toBe(10_000);
const canceled = await control.command(owner, runId, { action: 'cancel' }, 'cancel-lifecycle');
expect(canceled).toMatchObject({ status: 'canceled', reservedTokens: 0 });
await expect(control.command(owner, runId, { action: 'resume' }, 'resume-canceled')).rejects.toMatchObject({
code: 'TERMINAL_RUN',
});
});
it('invalidates pending releases, approvals and code reviews when a run is canceled', async () => {
const run = await control.run(owner, runId);
const releaseId = 'release-pending-1';
const approvalId = 'approval-pending-1';
const reviewId = 'review-pending-1';
await db.transaction('alpha', async tx => {
await tx.put('release', {
...record('alpha'),
id: releaseId,
runId,
projectId: run.projectId,
environmentId: 'env-1',
revision: run.revision,
status: 'awaiting_approval',
sourceCommit: 'a'.repeat(40),
imageDigest: `sha256:${'b'.repeat(64)}`,
architecture: 'linux/amd64',
configVersion: '1',
secretVersionRefs: [],
evidenceIds: [],
bindingDigest: 'd'.repeat(64),
policyVersion: run.policyVersion,
approvalId,
riskReasons: ['environment_policy'],
} satisfies Release);
await tx.put('approval', {
...record('alpha'),
id: approvalId,
releaseId,
bindingDigest: 'd'.repeat(64),
status: 'pending',
expiresAt: new Date(Date.now() + 60_000).toISOString(),
} satisfies Approval);
await tx.put('code_review', {
...record('alpha'),
id: reviewId,
runId,
projectId: run.projectId,
revision: run.revision,
sourceCommit: 'a'.repeat(40),
baseCommit: 'c'.repeat(40),
paths: ['internal/server.go'],
status: 'pending',
} satisfies CodeReview);
});
await control.command(owner, runId, { action: 'cancel' }, 'cancel-with-pending-release');
await db.transaction('alpha', async tx => {
expect(await tx.get<Release>('release', releaseId)).toMatchObject({
status: 'failed',
failureReason: 'The run was canceled',
});
expect(await tx.get<Approval>('approval', approvalId)).toMatchObject({ status: 'invalidated' });
expect(await tx.get<CodeReview>('code_review', reviewId)).toMatchObject({ status: 'invalidated' });
});
});
});
describe('pausing a run that is executing', () => {
const executor = (): Actor => ({ ...owner, userId: 'executor', kind: 'worker', runId });
const signals = () => db.state.get('alpha')!.outbox.filter(item => item.type === 'run.signal');
const edit = (change: (run: Run) => void) =>
db.transaction('alpha', async tx => {
const run = (await tx.get<Run>('run', runId))!;
change(run);
await tx.put('run', run);
});
/** Leases the run as its executor and asks for a pause; returns the executor's lease token. */
async function pausing(key: string) {
if ((await control.run(owner, runId)).status === 'paused')
await control.command(owner, runId, { action: 'resume' }, `resume-${key}`);
const leased = await control.lease(executor(), runId, 'executor');
await control.command(owner, runId, { action: 'pause' }, `pause-${key}`);
return leased;
}
it('lets the executor save the work before the run pauses', async () => {
const leased = await pausing('saves');
const token = leased.lease!.token;
expect(await control.run(owner, runId)).toMatchObject({
status: 'running',
pauseRequestedAt: expect.any(String),
lease: { token },
});
// The workflow is stopped only once the pause is complete; stopping it now would abort the save.
expect(signals()).toEqual([]);
// The agent being stopped gets no model credential, but the executor still uploads its work.
expect(
await new Connections(db, { NODE_ENV: 'test', WF_VAULT_KEY: 'a'.repeat(64) }).forRun(executor(), leased),
).toBe(undefined);
const bundle = Buffer.from('fixture bundle');
const saved = await control.checkpoint(
executor(),
runId,
{
gitBundleBase64: bundle.toString('base64'),
gitBundleSha256: createHash('sha256').update(bundle).digest('hex'),
},
token,
);
await expect(control.command(owner, runId, { action: 'resume' }, 'resume-while-pausing')).rejects.toMatchObject({
code: 'RUN_PAUSING',
status: 409,
});
expect((await control.command(owner, runId, { action: 'pause' }, 'pause-again')).status).toBe('running');
const paused = await control.completePause(executor(), runId, { leaseToken: token, saved: true });
expect(paused).toMatchObject({ status: 'paused', checkpointId: saved.id, lease: { token: token + 1 } });
expect(paused.pauseRequestedAt).toBeUndefined();
expect(signals()).toEqual([expect.objectContaining({ payload: { runId, action: 'pause' } })]);
expect(db.state.get('alpha')!.events.at(-1)).toMatchObject({
type: 'run.paused',
message: expect.stringContaining('saved as a checkpoint'),
});
await expect(control.completePause(executor(), runId, { leaseToken: token, saved: true })).rejects.toMatchObject({
code: 'STALE_LEASE',
});
expect((await control.command(owner, runId, { action: 'resume' }, 'resume-after-pause')).status).toBe('queued');
});
it('pauses anyway when the executor is late or gone', async () => {
const coordinator: Actor = { ...owner, userId: 'coordinator', kind: 'worker' };
const overdue = (run: Run) => {
run.pauseRequestedAt = new Date(Date.now() - PAUSE_GRACE_MS - 1000).toISOString();
};
const expired = (run: Run) => {
run.lease!.expiresAt = new Date(Date.now() - 1000).toISOString();
};
// A late executor learns from its heartbeat.
let leased = await pausing('late');
await edit(overdue);
expect(await control.heartbeat(executor(), runId, leased.lease!.token)).toMatchObject({ status: 'paused' });
// A stopped executor: the next lease attempt pauses the run instead.
leased = await pausing('gone');
await edit(expired);
await expect(control.lease(executor(), runId, 'another-executor')).rejects.toMatchObject({
code: 'RUN_NOT_RUNNABLE',
});
expect((await control.run(owner, runId)).status).toBe('paused');
// The workflow records why execution stopped: the run pauses instead of waiting.
await pausing('stopped');
await edit(expired);
expect(
await control.waitRun(coordinator, runId, { reason: 'infrastructure', detail: 'Executor lost' }),
).toMatchObject({
status: 'paused',
});
// Nobody asks at all: the background sweep completes it.
await pausing('forgotten');
await edit(overdue);
await control.finishOverduePauses('alpha');
expect((await control.run(owner, runId)).status).toBe('paused');
// Each completed pause stops the workflow once; the continues in between signal it too.
expect(signals().filter(item => item.payload.action === 'pause')).toHaveLength(4);
});
it('pauses at once when no executor holds the run, and cancelling overrides a pause in progress', async () => {
expect((await control.command(owner, runId, { action: 'pause' }, 'pause-queued')).status).toBe('paused');
await control.command(owner, runId, { action: 'resume' }, 'resume-queued');
const verifier: Actor = { ...owner, userId: 'verifier', kind: 'verifier', runId };
await control.lease(verifier, runId, 'verifier');
expect((await control.command(owner, runId, { action: 'pause' }, 'pause-verifier')).status).toBe('paused');
await pausing('then-cancel');
const canceled = await control.command(owner, runId, { action: 'cancel' }, 'cancel-while-pausing');
expect(canceled.status).toBe('canceled');
expect(canceled.pauseRequestedAt).toBeUndefined();
});
});