Canceling a run at the release stage left its awaiting_approval release and pending approval counted as waiting on the dashboard. Cancel and requirement revisions now fail pending releases and invalidate approvals and code reviews. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
267 lines
12 KiB
TypeScript
267 lines
12 KiB
TypeScript
/**
|
|
* Every way a run can stop has a documented way out (requirements FR-C5, FR-C6): each
|
|
* waiting reason is explained in the console with an action, continuing re-queues the run
|
|
* under a new execution generation, and stopping a run frees what it held.
|
|
*/
|
|
import { beforeEach, describe, expect, it } from 'vitest';
|
|
import {
|
|
waitingReasons,
|
|
type Actor,
|
|
type Approval,
|
|
type CodeReview,
|
|
type Release,
|
|
type Run,
|
|
type WaitingReason,
|
|
} from '@wishforge/contracts';
|
|
import { createHash } from 'node:crypto';
|
|
import { ControlPlane, PAUSE_GRACE_MS, record, setWaiting } from '../apps/api/src/domain/control-plane.js';
|
|
import { Connections } from '../apps/api/src/domain/connections.js';
|
|
import { waitingGuides } from '../apps/web/src/lib/waiting.js';
|
|
import { MemoryStore } from './helpers/memory-store.js';
|
|
|
|
const owner: Actor = { tenantId: 'alpha', userId: 'owner', role: 'owner', kind: 'human' };
|
|
let db: MemoryStore, control: ControlPlane, runId: string;
|
|
beforeEach(async () => {
|
|
db = new MemoryStore();
|
|
db.seed('alpha');
|
|
const env = { NODE_ENV: 'test', WF_VAULT_KEY: 'a'.repeat(64) };
|
|
control = new ControlPlane(db, env);
|
|
await new Connections(db, env).create(owner, { provider: 'codex', label: 'test', apiKey: 'fixture-key-only' });
|
|
const project = await control.createProject(owner, { name: 'Lifecycle', repositoryUrl: 'https://example.org/l.git' });
|
|
runId = (
|
|
await control.submit(
|
|
owner,
|
|
project.id,
|
|
{ goal: 'Lifecycle fixture', acceptanceCriteria: ['It works'], budget: { maxTokens: 10_000 } },
|
|
'lifecycle-wish',
|
|
)
|
|
).run_id;
|
|
});
|
|
async function wait(reason: WaitingReason) {
|
|
await db.transaction('alpha', async tx => {
|
|
const run = (await tx.get<Run>('run', runId))!;
|
|
setWaiting(run, reason, `Fixture: ${reason}`);
|
|
await tx.put('run', run);
|
|
});
|
|
}
|
|
|
|
describe('waiting reasons and their ways out', () => {
|
|
it('explains every waiting reason in Chinese with something the person can do', () => {
|
|
for (const reason of waitingReasons) {
|
|
const guide = waitingGuides[reason];
|
|
expect(guide, reason).toBeDefined();
|
|
expect(guide.title).toMatch(/[一-鿿]/);
|
|
expect(guide.action).toMatch(/[一-鿿]/);
|
|
// Either continuing helps, or the guide links to where the person acts, or they answer below.
|
|
expect(guide.resumable || guide.link !== undefined || reason === 'clarification', reason).toBe(true);
|
|
}
|
|
expect(Object.keys(waitingGuides).sort()).toEqual([...waitingReasons].sort());
|
|
});
|
|
|
|
it.each(waitingReasons.filter(reason => waitingGuides[reason].resumable))(
|
|
'continuing a run waiting for %s queues a new execution generation',
|
|
async reason => {
|
|
const before = await control.run(owner, runId);
|
|
await wait(reason);
|
|
const resumed = await control.command(owner, runId, { action: 'resume' }, `resume-${reason}`);
|
|
expect(resumed).toMatchObject({ status: 'queued' });
|
|
expect(resumed.waitingReason).toBeUndefined();
|
|
expect(resumed.pendingStartId).not.toBe(before.pendingStartId);
|
|
const starts = (await db.transaction('alpha', tx => tx.outbox())).filter(item => item.type === 'run.start');
|
|
expect(starts.map(item => item.id)).toContain(resumed.pendingStartId);
|
|
},
|
|
);
|
|
|
|
it('answers a clarification with a new requirement revision that starts from planning', async () => {
|
|
await wait('clarification');
|
|
const answered = await control.command(
|
|
owner,
|
|
runId,
|
|
{ action: 'answer', message: 'Use the existing login page' },
|
|
'answer-clarification',
|
|
);
|
|
expect(answered).toMatchObject({ status: 'queued', stage: 'plan', revision: 2, repairCount: 0 });
|
|
});
|
|
|
|
it('frees the reservation when paused and keeps nothing after cancelling', async () => {
|
|
expect((await control.usage(owner)).reservedTokens).toBe(10_000);
|
|
await control.command(owner, runId, { action: 'pause' }, 'pause-lifecycle');
|
|
expect((await control.usage(owner)).reservedTokens).toBe(0);
|
|
expect((await control.run(owner, runId)).waitingReason).toBeUndefined();
|
|
await control.command(owner, runId, { action: 'resume' }, 'resume-lifecycle');
|
|
expect((await control.usage(owner)).reservedTokens).toBe(10_000);
|
|
const canceled = await control.command(owner, runId, { action: 'cancel' }, 'cancel-lifecycle');
|
|
expect(canceled).toMatchObject({ status: 'canceled', reservedTokens: 0 });
|
|
await expect(control.command(owner, runId, { action: 'resume' }, 'resume-canceled')).rejects.toMatchObject({
|
|
code: 'TERMINAL_RUN',
|
|
});
|
|
});
|
|
|
|
it('invalidates pending releases, approvals and code reviews when a run is canceled', async () => {
|
|
const run = await control.run(owner, runId);
|
|
const releaseId = 'release-pending-1';
|
|
const approvalId = 'approval-pending-1';
|
|
const reviewId = 'review-pending-1';
|
|
await db.transaction('alpha', async tx => {
|
|
await tx.put('release', {
|
|
...record('alpha'),
|
|
id: releaseId,
|
|
runId,
|
|
projectId: run.projectId,
|
|
environmentId: 'env-1',
|
|
revision: run.revision,
|
|
status: 'awaiting_approval',
|
|
sourceCommit: 'a'.repeat(40),
|
|
imageDigest: `sha256:${'b'.repeat(64)}`,
|
|
architecture: 'linux/amd64',
|
|
configVersion: '1',
|
|
secretVersionRefs: [],
|
|
evidenceIds: [],
|
|
bindingDigest: 'd'.repeat(64),
|
|
policyVersion: run.policyVersion,
|
|
approvalId,
|
|
riskReasons: ['environment_policy'],
|
|
} satisfies Release);
|
|
await tx.put('approval', {
|
|
...record('alpha'),
|
|
id: approvalId,
|
|
releaseId,
|
|
bindingDigest: 'd'.repeat(64),
|
|
status: 'pending',
|
|
expiresAt: new Date(Date.now() + 60_000).toISOString(),
|
|
} satisfies Approval);
|
|
await tx.put('code_review', {
|
|
...record('alpha'),
|
|
id: reviewId,
|
|
runId,
|
|
projectId: run.projectId,
|
|
revision: run.revision,
|
|
sourceCommit: 'a'.repeat(40),
|
|
baseCommit: 'c'.repeat(40),
|
|
paths: ['internal/server.go'],
|
|
status: 'pending',
|
|
} satisfies CodeReview);
|
|
});
|
|
await control.command(owner, runId, { action: 'cancel' }, 'cancel-with-pending-release');
|
|
await db.transaction('alpha', async tx => {
|
|
expect(await tx.get<Release>('release', releaseId)).toMatchObject({
|
|
status: 'failed',
|
|
failureReason: 'The run was canceled',
|
|
});
|
|
expect(await tx.get<Approval>('approval', approvalId)).toMatchObject({ status: 'invalidated' });
|
|
expect(await tx.get<CodeReview>('code_review', reviewId)).toMatchObject({ status: 'invalidated' });
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('pausing a run that is executing', () => {
|
|
const executor = (): Actor => ({ ...owner, userId: 'executor', kind: 'worker', runId });
|
|
const signals = () => db.state.get('alpha')!.outbox.filter(item => item.type === 'run.signal');
|
|
const edit = (change: (run: Run) => void) =>
|
|
db.transaction('alpha', async tx => {
|
|
const run = (await tx.get<Run>('run', runId))!;
|
|
change(run);
|
|
await tx.put('run', run);
|
|
});
|
|
/** Leases the run as its executor and asks for a pause; returns the executor's lease token. */
|
|
async function pausing(key: string) {
|
|
if ((await control.run(owner, runId)).status === 'paused')
|
|
await control.command(owner, runId, { action: 'resume' }, `resume-${key}`);
|
|
const leased = await control.lease(executor(), runId, 'executor');
|
|
await control.command(owner, runId, { action: 'pause' }, `pause-${key}`);
|
|
return leased;
|
|
}
|
|
|
|
it('lets the executor save the work before the run pauses', async () => {
|
|
const leased = await pausing('saves');
|
|
const token = leased.lease!.token;
|
|
expect(await control.run(owner, runId)).toMatchObject({
|
|
status: 'running',
|
|
pauseRequestedAt: expect.any(String),
|
|
lease: { token },
|
|
});
|
|
// The workflow is stopped only once the pause is complete; stopping it now would abort the save.
|
|
expect(signals()).toEqual([]);
|
|
// The agent being stopped gets no model credential, but the executor still uploads its work.
|
|
expect(
|
|
await new Connections(db, { NODE_ENV: 'test', WF_VAULT_KEY: 'a'.repeat(64) }).forRun(executor(), leased),
|
|
).toBe(undefined);
|
|
const bundle = Buffer.from('fixture bundle');
|
|
const saved = await control.checkpoint(
|
|
executor(),
|
|
runId,
|
|
{
|
|
gitBundleBase64: bundle.toString('base64'),
|
|
gitBundleSha256: createHash('sha256').update(bundle).digest('hex'),
|
|
},
|
|
token,
|
|
);
|
|
await expect(control.command(owner, runId, { action: 'resume' }, 'resume-while-pausing')).rejects.toMatchObject({
|
|
code: 'RUN_PAUSING',
|
|
status: 409,
|
|
});
|
|
expect((await control.command(owner, runId, { action: 'pause' }, 'pause-again')).status).toBe('running');
|
|
|
|
const paused = await control.completePause(executor(), runId, { leaseToken: token, saved: true });
|
|
expect(paused).toMatchObject({ status: 'paused', checkpointId: saved.id, lease: { token: token + 1 } });
|
|
expect(paused.pauseRequestedAt).toBeUndefined();
|
|
expect(signals()).toEqual([expect.objectContaining({ payload: { runId, action: 'pause' } })]);
|
|
expect(db.state.get('alpha')!.events.at(-1)).toMatchObject({
|
|
type: 'run.paused',
|
|
message: expect.stringContaining('saved as a checkpoint'),
|
|
});
|
|
await expect(control.completePause(executor(), runId, { leaseToken: token, saved: true })).rejects.toMatchObject({
|
|
code: 'STALE_LEASE',
|
|
});
|
|
expect((await control.command(owner, runId, { action: 'resume' }, 'resume-after-pause')).status).toBe('queued');
|
|
});
|
|
|
|
it('pauses anyway when the executor is late or gone', async () => {
|
|
const coordinator: Actor = { ...owner, userId: 'coordinator', kind: 'worker' };
|
|
const overdue = (run: Run) => {
|
|
run.pauseRequestedAt = new Date(Date.now() - PAUSE_GRACE_MS - 1000).toISOString();
|
|
};
|
|
const expired = (run: Run) => {
|
|
run.lease!.expiresAt = new Date(Date.now() - 1000).toISOString();
|
|
};
|
|
// A late executor learns from its heartbeat.
|
|
let leased = await pausing('late');
|
|
await edit(overdue);
|
|
expect(await control.heartbeat(executor(), runId, leased.lease!.token)).toMatchObject({ status: 'paused' });
|
|
// A stopped executor: the next lease attempt pauses the run instead.
|
|
leased = await pausing('gone');
|
|
await edit(expired);
|
|
await expect(control.lease(executor(), runId, 'another-executor')).rejects.toMatchObject({
|
|
code: 'RUN_NOT_RUNNABLE',
|
|
});
|
|
expect((await control.run(owner, runId)).status).toBe('paused');
|
|
// The workflow records why execution stopped: the run pauses instead of waiting.
|
|
await pausing('stopped');
|
|
await edit(expired);
|
|
expect(
|
|
await control.waitRun(coordinator, runId, { reason: 'infrastructure', detail: 'Executor lost' }),
|
|
).toMatchObject({
|
|
status: 'paused',
|
|
});
|
|
// Nobody asks at all: the background sweep completes it.
|
|
await pausing('forgotten');
|
|
await edit(overdue);
|
|
await control.finishOverduePauses('alpha');
|
|
expect((await control.run(owner, runId)).status).toBe('paused');
|
|
// Each completed pause stops the workflow once; the continues in between signal it too.
|
|
expect(signals().filter(item => item.payload.action === 'pause')).toHaveLength(4);
|
|
});
|
|
|
|
it('pauses at once when no executor holds the run, and cancelling overrides a pause in progress', async () => {
|
|
expect((await control.command(owner, runId, { action: 'pause' }, 'pause-queued')).status).toBe('paused');
|
|
await control.command(owner, runId, { action: 'resume' }, 'resume-queued');
|
|
const verifier: Actor = { ...owner, userId: 'verifier', kind: 'verifier', runId };
|
|
await control.lease(verifier, runId, 'verifier');
|
|
expect((await control.command(owner, runId, { action: 'pause' }, 'pause-verifier')).status).toBe('paused');
|
|
await pausing('then-cancel');
|
|
const canceled = await control.command(owner, runId, { action: 'cancel' }, 'cancel-while-pausing');
|
|
expect(canceled.status).toBe('canceled');
|
|
expect(canceled.pauseRequestedAt).toBeUndefined();
|
|
});
|
|
});
|