A release's risk only counted what the target branch gained when the run integrated. Commits people pushed to the branch directly, and integrations that were never deployed, went out with the next release without being classified: an environment preauthorized for automatic release took a migration or a Dockerfile change pushed straight to the branch without approval. The POC chose what to rebuild from the same narrow diff, so worker and coordinator kept running old code; on the local POC, WishForge had in fact only ever rebuilt the api. The control plane now gives the verifier the commit of the release the environment runs, and the classification covers everything between it and the release, as well as what the integration brought. On the POC, each deployment records the commit and the image a release put there. The build plans from that snapshot, the one the release is checked against: worker and coordinator keep their images only while what they run differs from the source in web files alone. A component of unknown source (never deployed by a release, restored by a rollback, or set by hand, which leaves the annotations stale) is rebuilt, and the release is marked partial: it needs approval and does not roll back on its own. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
328 lines
16 KiB
TypeScript
328 lines
16 KiB
TypeScript
import { beforeEach, describe, expect, it } from 'vitest';
|
|
import type { Actor, CheckpointRecord, ProviderConnection, Run } from '@wishforge/contracts';
|
|
import { MemoryStore } from './helpers/memory-store.js';
|
|
import { ControlPlane } from '../apps/api/src/domain/control-plane.js';
|
|
import { Connections } from '../apps/api/src/domain/connections.js';
|
|
import { Releases } from '../apps/api/src/domain/releases.js';
|
|
import { AgentBundles } from '../apps/api/src/domain/agent-bundles.js';
|
|
import { digest, releaseRisk } from '../apps/api/src/domain/policy.js';
|
|
import { deliverable } from './helpers/delivery.js';
|
|
|
|
const owner: Actor = { tenantId: 'alpha', userId: 'owner', role: 'owner', kind: 'human' };
|
|
const worker: Actor = { ...owner, kind: 'worker', userId: 'worker' },
|
|
verifier: Actor = { ...owner, kind: 'verifier', userId: 'verifier' };
|
|
let db: MemoryStore, control: ControlPlane, releases: Releases, connections: Connections;
|
|
const goal = {
|
|
goal: 'Add search to the catalog',
|
|
acceptanceCriteria: ['Searching an existing item returns its row'],
|
|
provider: 'codex',
|
|
};
|
|
beforeEach(() => {
|
|
db = new MemoryStore();
|
|
db.seed('alpha');
|
|
db.seed('beta');
|
|
const env = { WF_VAULT_KEY: 'a'.repeat(64), NODE_ENV: 'test' };
|
|
control = new ControlPlane(db, env);
|
|
connections = new Connections(db, env);
|
|
releases = new Releases(db);
|
|
});
|
|
async function setup(connected = false) {
|
|
const p = await control.createProject(owner, {
|
|
name: 'Catalog',
|
|
repositoryUrl: 'https://example.org/catalog.git',
|
|
verificationCommands: [['node', '--test']],
|
|
});
|
|
if (connected)
|
|
await connections.create(owner, {
|
|
provider: 'codex',
|
|
label: 'Test envelope only',
|
|
apiKey: 'test-not-a-real-provider-key',
|
|
});
|
|
const result = await control.submit(owner, p.id, goal, 'submission-001');
|
|
return { p, runId: result.run_id };
|
|
}
|
|
|
|
describe('tenant-scoped durable task invariants', () => {
|
|
it('deduplicates concurrent submissions and reserves budget once', async () => {
|
|
const p = await control.createProject(owner, { name: 'Catalog', repositoryUrl: 'https://example.org/catalog.git' });
|
|
await connections.create(owner, { provider: 'codex', label: 'test', apiKey: 'test-fixture-value' });
|
|
const results = await Promise.all(
|
|
Array.from({ length: 10 }, () => control.submit(owner, p.id, goal, 'same-submission')),
|
|
);
|
|
expect(new Set(results.map(r => r.run_id)).size).toBe(1);
|
|
expect((await control.usage(owner)).reservedTokens).toBe(100000);
|
|
expect(await control.runs(owner)).toHaveLength(1);
|
|
});
|
|
it('frees the token reservation of paused and waiting runs (FR-J2)', async () => {
|
|
db.seed('gamma', 150_000);
|
|
const tenantOwner: Actor = { ...owner, tenantId: 'gamma' };
|
|
const p = await control.createProject(tenantOwner, { name: 'Quota', repositoryUrl: 'https://example.org/q.git' });
|
|
await connections.create(tenantOwner, { provider: 'codex', label: 'test', apiKey: 'test-fixture-value' });
|
|
const wish = { ...goal, budget: { maxTokens: 100_000 } };
|
|
const first = await control.submit(tenantOwner, p.id, wish, 'quota-first');
|
|
await expect(control.submit(tenantOwner, p.id, wish, 'quota-second')).rejects.toMatchObject({
|
|
code: 'QUOTA_EXCEEDED',
|
|
});
|
|
await control.command(tenantOwner, first.run_id, { action: 'pause' }, 'quota-pause');
|
|
expect((await control.usage(tenantOwner)).reservedTokens).toBe(0);
|
|
const second = await control.submit(tenantOwner, p.id, wish, 'quota-second');
|
|
expect((await control.usage(tenantOwner)).reservedTokens).toBe(100_000);
|
|
// Continuing the paused run would reserve its budget again, which no longer fits.
|
|
await expect(
|
|
control.command(tenantOwner, first.run_id, { action: 'resume' }, 'quota-resume'),
|
|
).rejects.toMatchObject({
|
|
code: 'QUOTA_EXCEEDED',
|
|
});
|
|
await control.command(tenantOwner, second.run_id, { action: 'cancel' }, 'quota-cancel');
|
|
expect((await control.command(tenantOwner, first.run_id, { action: 'resume' }, 'quota-resume-2')).status).toBe(
|
|
'queued',
|
|
);
|
|
});
|
|
it('rejects reused keys with different intent', async () => {
|
|
const { p } = await setup();
|
|
await expect(
|
|
control.submit(owner, p.id, { ...goal, goal: 'Delete the catalog' }, 'submission-001'),
|
|
).rejects.toMatchObject({ code: 'IDEMPOTENCY_CONFLICT' });
|
|
});
|
|
it('does not execute an unconfigured provider or report fabricated completion', async () => {
|
|
const { runId } = await setup();
|
|
expect(await control.run(owner, runId)).toMatchObject({ status: 'waiting', waitingReason: 'credentials' });
|
|
await expect(control.lease(worker, runId, 'executor')).rejects.toMatchObject({ code: 'RUN_NOT_RUNNABLE' });
|
|
});
|
|
it('denies access across tenants and to nonmember project users', async () => {
|
|
const { runId } = await setup();
|
|
await expect(control.run({ ...owner, tenantId: 'beta' }, runId)).rejects.toMatchObject({ code: 'NOT_FOUND' });
|
|
await expect(control.run({ ...owner, userId: 'viewer', role: 'viewer' }, runId)).rejects.toMatchObject({
|
|
code: 'NOT_FOUND',
|
|
});
|
|
});
|
|
it('checks capacity atomically under simultaneous claims', async () => {
|
|
const p = await control.createProject(owner, { name: 'Catalog', repositoryUrl: 'https://example.org/catalog.git' });
|
|
await connections.create(owner, { provider: 'codex', label: 'test', apiKey: 'test-fixture-value' });
|
|
const ids = await Promise.all([1, 2, 3].map(n => control.submit(owner, p.id, goal, `capacity-${n}`)));
|
|
const results = await Promise.allSettled(ids.map(({ run_id }) => control.lease(worker, run_id, run_id)));
|
|
expect(results.filter(r => r.status === 'fulfilled')).toHaveLength(2);
|
|
expect(results.filter(r => r.status === 'rejected')).toHaveLength(1);
|
|
});
|
|
it('revokes an old worker after pause and does not discard the project', async () => {
|
|
const { runId, p } = await setup(true);
|
|
const run = await control.lease(worker, runId, 'executor');
|
|
await control.command(owner, runId, { action: 'pause' }, 'pause-001');
|
|
// The executor saves its work first; once it completes the pause, its lease is gone.
|
|
await control.completePause({ ...worker, runId }, runId, { leaseToken: run.lease!.token, saved: false });
|
|
await expect(
|
|
control.transition(worker, runId, {
|
|
leaseToken: run.lease!.token,
|
|
stage: 'plan',
|
|
status: 'running',
|
|
message: 'late worker',
|
|
}),
|
|
).rejects.toMatchObject({ code: 'STALE_LEASE' });
|
|
expect((await control.projects(owner))[0].id).toBe(p.id);
|
|
});
|
|
it('only widens the trusted classification of a commit, so a later one cannot lower the release risk', async () => {
|
|
const { runId } = await setup(true);
|
|
const leased = await control.lease(verifier, runId, 'verifier');
|
|
const commit = 'a'.repeat(40);
|
|
await db.transaction('alpha', async tx => {
|
|
const run = (await tx.get<Run>('run', runId))!;
|
|
const checkpoint = {
|
|
id: 'checkpoint-1',
|
|
tenantId: 'alpha',
|
|
version: 1,
|
|
createdAt: run.createdAt,
|
|
updatedAt: run.createdAt,
|
|
runId,
|
|
revision: run.revision,
|
|
checkpoint: {},
|
|
};
|
|
await tx.put('checkpoint', checkpoint);
|
|
await tx.put('run', { ...run, checkpointId: checkpoint.id, verifiedCommit: commit });
|
|
});
|
|
const classify = (sourceCommit: string, changedPaths: string[], migration: boolean, partial = false) =>
|
|
control.classify(verifier, runId, {
|
|
leaseToken: leased.lease!.token,
|
|
sourceCommit,
|
|
changedPaths,
|
|
migration,
|
|
permissionsExpanded: false,
|
|
partial,
|
|
});
|
|
const stored = async () =>
|
|
(await db.transaction('alpha', tx => tx.get<CheckpointRecord>('checkpoint', 'checkpoint-1')))!
|
|
.trustedClassification;
|
|
await classify(commit, ['migrations/001.sql', 'src/app.ts'], true);
|
|
// A resumed build that diffed from a later base sees less of what the target branch gained.
|
|
await classify(commit, ['src/app.ts'], false);
|
|
expect(await stored()).toMatchObject({ migration: true, changedPaths: ['migrations/001.sql', 'src/app.ts'] });
|
|
// Once part of what the environment ran was of unknown source, the commit's changes stay incomplete.
|
|
await classify(commit, ['src/app.ts'], false, true);
|
|
await classify(commit, ['src/app.ts'], false);
|
|
expect(await stored()).toMatchObject({ migration: true, partial: true });
|
|
expect(releaseRisk({ ...(await stored()), evidenceComplete: true, preauthorized: true })).toEqual([
|
|
'The environment runs code of unknown source; not every change could be classified',
|
|
'Database migration',
|
|
'Protected change: migrations/001.sql',
|
|
]);
|
|
// Another commit is classified on its own.
|
|
await db.transaction('alpha', async tx => {
|
|
const run = (await tx.get<Run>('run', runId))!;
|
|
await tx.put('run', { ...run, verifiedCommit: 'b'.repeat(40) });
|
|
});
|
|
await classify('b'.repeat(40), ['src/app.ts'], false);
|
|
expect(await stored()).toMatchObject({ migration: false, changedPaths: ['src/app.ts'] });
|
|
});
|
|
it('requires verifier identity and protected commands before accepting evidence', async () => {
|
|
const { runId } = await setup(true);
|
|
const lease = await control.lease(verifier, runId, 'verifier');
|
|
const data = {
|
|
leaseToken: lease.lease!.token,
|
|
runId,
|
|
revision: 1,
|
|
sourceCommit: 'a'.repeat(40),
|
|
type: 'verification',
|
|
passed: true,
|
|
isolation: 'sandbox',
|
|
commands: [],
|
|
};
|
|
await expect(control.evidence(worker, data)).rejects.toMatchObject({ code: 'FORBIDDEN' });
|
|
await expect(control.evidence(verifier, data)).rejects.toMatchObject({ code: 'INCOMPLETE_VERIFICATION' });
|
|
});
|
|
it('cannot skip phases or let any worker declare success or move the run back', async () => {
|
|
const { runId } = await setup(true);
|
|
const run = await control.lease(worker, runId, 'executor');
|
|
const move = (actor: Actor, stage: string, status: string) =>
|
|
control.transition(actor, runId, { leaseToken: run.lease!.token, stage, status, message: 'fixture' });
|
|
await expect(move(worker, 'verify', 'running')).rejects.toMatchObject({ code: 'INVALID_TRANSITION' });
|
|
// Executors only reach implementation stages; nobody can mark a run succeeded or failed.
|
|
await expect(move(worker, 'build', 'running')).rejects.toMatchObject({ code: 'TRANSITION_NOT_PERMITTED' });
|
|
await expect(move(worker, 'accept', 'succeeded')).rejects.toMatchObject({ code: 'TRANSITION_NOT_PERMITTED' });
|
|
await expect(move(verifier, 'integrate', 'succeeded')).rejects.toMatchObject({ code: 'TRANSITION_NOT_PERMITTED' });
|
|
await expect(move(worker, 'plan', 'failed')).rejects.toMatchObject({ code: 'TRANSITION_NOT_PERMITTED' });
|
|
await move(worker, 'plan', 'running');
|
|
await expect(move(worker, 'clarify', 'running')).rejects.toMatchObject({ code: 'INVALID_TRANSITION' });
|
|
});
|
|
it('versions requirements and clears stale verification', async () => {
|
|
const { runId } = await setup(true);
|
|
const lease = await control.lease(verifier, runId, 'verifier');
|
|
await control.evidence(verifier, {
|
|
leaseToken: lease.lease!.token,
|
|
runId,
|
|
revision: 1,
|
|
sourceCommit: 'a'.repeat(40),
|
|
type: 'verification',
|
|
passed: true,
|
|
isolation: 'sandbox',
|
|
commands: [{ argv: ['node', '--test'], exitCode: 0, outputDigest: digest('output') }],
|
|
});
|
|
const next = await control.command(
|
|
owner,
|
|
runId,
|
|
{ action: 'answer', message: 'Search must ignore case' },
|
|
'answer-001',
|
|
);
|
|
expect(next.revision).toBe(2);
|
|
expect(next.verifiedCommit).toBeUndefined();
|
|
});
|
|
it('does not return secrets and rejects another tenant decrypting the envelope', async () => {
|
|
const c = await connections.create(owner, {
|
|
provider: 'codex',
|
|
label: 'tenant-owned',
|
|
apiKey: 'sensitive-fixture-key',
|
|
});
|
|
expect(JSON.stringify(c)).not.toContain('sensitive-fixture-key');
|
|
const raw = await db.transaction('alpha', tx => tx.get<ProviderConnection>('provider_connection', c.id));
|
|
await expect(connections.vault.open('beta', String(raw!.ciphertext))).rejects.toThrow();
|
|
});
|
|
it('lets only a person, not an agent client acting for an admin, add or revoke provider credentials', async () => {
|
|
const c = await connections.create(owner, { provider: 'codex', label: 'original', apiKey: 'original-fixture-key' });
|
|
const client: Actor = { ...owner, kind: 'connector' };
|
|
await expect(
|
|
connections.create(client, { provider: 'codex', label: 'replaced', apiKey: 'replacement-fixture-key' }),
|
|
).rejects.toMatchObject({ code: 'HUMAN_APPROVAL_REQUIRED', status: 403 });
|
|
await expect(connections.revoke(client, c.id)).rejects.toMatchObject({ code: 'HUMAN_APPROVAL_REQUIRED' });
|
|
expect((await connections.list(owner)).map(({ label, status }) => ({ label, status }))).toEqual([
|
|
{ label: 'original', status: 'active' },
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe('artifact-bound publication', () => {
|
|
async function prepared() {
|
|
const { p, runId } = await setup(true);
|
|
await db.transaction('alpha', async tx => {
|
|
const project = (await tx.get('project', p.id))!;
|
|
await tx.put('project', {
|
|
...project,
|
|
deliveryConfig: {
|
|
containerPort: 3000,
|
|
healthPath: '/health',
|
|
smokeChecks: [{ path: '/', expectedStatus: 200 }],
|
|
},
|
|
});
|
|
});
|
|
const commit = 'a'.repeat(40),
|
|
image = `registry.example/catalog@sha256:${'b'.repeat(64)}`;
|
|
const { verification: v, build: b } = await deliverable(control, 'alpha', runId, { commit, image });
|
|
const env = await releases.createEnvironment(owner, {
|
|
projectId: p.id,
|
|
name: 'production',
|
|
type: 'compose',
|
|
production: true,
|
|
});
|
|
const rel = await releases.request(
|
|
owner,
|
|
{ runId, environmentId: env.id, verificationEvidenceId: v.id, buildEvidenceId: b.id, changedPaths: [] },
|
|
'release-001',
|
|
);
|
|
return { rel, env, runId, commit, image };
|
|
}
|
|
it('does not trust caller risk classification and prohibits connector self-approval', async () => {
|
|
const { rel } = await prepared();
|
|
expect(rel.status).toBe('awaiting_approval');
|
|
expect(rel.riskReasons).toContain('Missing trusted change classification');
|
|
await expect(
|
|
releases.decide({ ...owner, kind: 'connector' }, rel.approvalId!, {
|
|
decision: 'approve',
|
|
bindingDigest: rel.bindingDigest,
|
|
}),
|
|
).rejects.toMatchObject({ code: 'HUMAN_APPROVAL_REQUIRED' });
|
|
});
|
|
it('invalidates release approval when requirements change', async () => {
|
|
const { rel, runId } = await prepared();
|
|
await control.command(owner, runId, { action: 'answer', message: 'Also add sorting' }, 'answer-release');
|
|
await expect(
|
|
releases.decide(owner, rel.approvalId!, { decision: 'approve', bindingDigest: rel.bindingDigest }),
|
|
).rejects.toMatchObject({ code: 'APPROVAL_EXPIRED' });
|
|
});
|
|
});
|
|
|
|
describe('agent configuration versions', () => {
|
|
const config = { schemaVersion: 1, prompts: { builder: 'Prefer small, reviewed commits.' } };
|
|
it('pins the active version at submission and keeps it after a later rollback', async () => {
|
|
const bundles = new AgentBundles(db);
|
|
const created = await bundles.create(owner, { name: 'Small commits', config });
|
|
expect(created.version).toMatch(/^sha256:[0-9a-f]{64}$/);
|
|
expect((await bundles.create(owner, { name: 'Duplicate', config })).created).toBe(false);
|
|
await bundles.activate(owner, created.version);
|
|
const { p } = await setup(true);
|
|
const { run_id } = await control.submit(owner, p.id, goal, 'bundle-pinned-run');
|
|
expect((await control.run(owner, run_id)).bundleVersion).toBe(created.version);
|
|
|
|
expect(await bundles.rollback(owner)).toEqual({ activeVersion: 'baseline-v1', previousVersion: created.version });
|
|
expect((await control.run(owner, run_id)).bundleVersion).toBe(created.version);
|
|
const second = await control.submit(owner, p.id, goal, 'bundle-baseline-run');
|
|
expect((await control.run(owner, second.run_id)).bundleVersion).toBe('baseline-v1');
|
|
});
|
|
it('restricts configuration changes to human owners and admins', async () => {
|
|
const bundles = new AgentBundles(db);
|
|
await expect(bundles.create({ ...owner, kind: 'connector' }, { name: 'x', config })).rejects.toMatchObject({
|
|
status: 403,
|
|
});
|
|
await expect(bundles.create({ ...owner, role: 'developer' }, { name: 'x', config })).rejects.toMatchObject({
|
|
status: 403,
|
|
});
|
|
await expect(bundles.activate(owner, digest('missing'))).rejects.toMatchObject({ code: 'BUNDLE_UNAVAILABLE' });
|
|
});
|
|
});
|