The dispatcher read the run, checked that the start it was delivering was still the pending one, and reserved the execution grant in a later transaction. A pause and a continue in between enqueued a newer start, and the older one went ahead anyway: it took the run's execution generation back and asked the provisioner for a sandbox for a generation that had already been replaced. Reserving the grant now checks the pending start again, in the transaction that takes the generation, and refuses a replaced one with START_SUPERSEDED. The dispatcher acknowledges such a start without marking the run as waiting; the newer start proceeds as usual. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
362 lines
16 KiB
TypeScript
362 lines
16 KiB
TypeScript
import { beforeEach, describe, expect, it } from 'vitest';
|
|
import type { Actor, Run, TenantEntitlementSnapshot } from '@wishforge/contracts';
|
|
import { MemoryStore } from './helpers/memory-store.js';
|
|
import { ControlPlane, record } from '../apps/api/src/domain/control-plane.js';
|
|
import { Connections } from '../apps/api/src/domain/connections.js';
|
|
import { Usage } from '../apps/api/src/domain/usage.js';
|
|
import {
|
|
Entitlements,
|
|
PILOT_PLAN,
|
|
requireCapacity,
|
|
reserveExecutionGrant,
|
|
recordAllocation,
|
|
} from '../apps/api/src/domain/entitlements.js';
|
|
import { purgeRetainedStorage, releaseStoppedAllocations } from '../apps/api/src/domain/resource-dispatch.js';
|
|
import type { Transaction } from '../apps/api/src/db/store.js';
|
|
const owner: Actor = { tenantId: 'alpha', userId: 'owner', kind: 'human', role: 'owner' },
|
|
operator: Actor = { ...owner, userId: 'operations-user', role: 'admin' };
|
|
const env = {
|
|
NODE_ENV: 'test',
|
|
WF_VAULT_KEY: 'a'.repeat(64),
|
|
WF_OPERATOR_SUBJECTS: operator.userId,
|
|
WF_RESOURCE_GRANT_SECRET: 'resource-fixture-signing-secret-only-12345',
|
|
WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'a'.repeat(64)}`,
|
|
WF_PROVISIONER_URL: 'http://127.0.0.1:4450',
|
|
WF_PROVISIONER_TOKEN: 'fixture-provisioner-token-only-long-enough',
|
|
WF_PROVISIONER_TENANT_ID: 'alpha',
|
|
};
|
|
let db: MemoryStore, control: ControlPlane, entitlements: Entitlements, usage: Usage;
|
|
/** Reserves as the dispatcher does: for the start the control plane last enqueued for the run. */
|
|
async function reserve(tx: Transaction, runId: string, generation: string, environment: NodeJS.ProcessEnv = env) {
|
|
const run = (await tx.get<Run>('run', runId))!;
|
|
if (run.pendingStartId !== generation) await tx.put('run', { ...run, pendingStartId: generation });
|
|
return reserveExecutionGrant(tx, runId, generation, environment);
|
|
}
|
|
beforeEach(() => {
|
|
db = new MemoryStore();
|
|
db.seed('alpha');
|
|
db.seed('beta');
|
|
const limits = structuredClone(PILOT_PLAN.limits);
|
|
limits.maxProjects = 1;
|
|
limits.maxWorkspaces = 1;
|
|
limits.maxMembers = 3;
|
|
limits.maxConcurrentRuns = 1;
|
|
const snapshot: TenantEntitlementSnapshot = {
|
|
schemaVersion: 1,
|
|
planId: 'pilot',
|
|
planVersion: 'pilot-v1',
|
|
limits,
|
|
assignedAt: '2019-01-01T00:00:00.000Z',
|
|
assignedBy: 'fixture-operator',
|
|
};
|
|
db.state.get('alpha')!.tenant.entitlements = snapshot;
|
|
control = new ControlPlane(db, env);
|
|
entitlements = new Entitlements(db, env);
|
|
usage = new Usage(db);
|
|
});
|
|
async function queuedRuns(count = 2) {
|
|
const project = await control.createProject(owner, {
|
|
name: 'Fixture',
|
|
repositoryUrl: 'https://example.org/fixture.git',
|
|
});
|
|
await new Connections(db, env).create(owner, {
|
|
provider: 'codex',
|
|
label: 'fixture',
|
|
apiKey: 'fixture-only-not-real',
|
|
});
|
|
return Promise.all(
|
|
Array.from({ length: count }, (_, i) =>
|
|
control.submit(
|
|
owner,
|
|
project.id,
|
|
{
|
|
goal: `Fixture request ${i}`,
|
|
acceptanceCriteria: ['Fixture is never executed'],
|
|
budget: { maxTokens: 1000 },
|
|
},
|
|
`billing-run-${i}`,
|
|
),
|
|
),
|
|
);
|
|
}
|
|
describe('fixed pilot entitlements and resource admission', () => {
|
|
it('rejects defaults below provisioner minimums before consuming an execution slot', async () => {
|
|
const [created] = await queuedRuns(1);
|
|
for (const defaults of [
|
|
{ WF_DEFAULT_EXECUTION_CPU_MILLI: '1' },
|
|
{ WF_DEFAULT_EXECUTION_MEMORY_MIB: '1' },
|
|
{ WF_DEFAULT_EXECUTION_STORAGE_MIB: '1' },
|
|
])
|
|
await expect(
|
|
db.transaction('alpha', tx => reserve(tx, created.run_id, 'invalid-default', { ...env, ...defaults })),
|
|
).rejects.toMatchObject({ code: 'RESOURCE_CONFIGURATION_INVALID' });
|
|
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(0);
|
|
});
|
|
it('atomically limits projects and workspaces without blocking idempotent existing data', async () => {
|
|
const results = await Promise.allSettled([
|
|
control.createProject(owner, { name: 'First' }),
|
|
control.createProject(owner, { name: 'Second' }),
|
|
]);
|
|
expect(results.filter(r => r.status === 'fulfilled')).toHaveLength(1);
|
|
const workspaces = await Promise.allSettled(
|
|
[1, 2].map(() =>
|
|
db.transaction('alpha', async tx => {
|
|
await requireCapacity(tx, 'workspace');
|
|
await tx.put('workspace', { ...record('alpha'), projectId: 'fixture' });
|
|
}),
|
|
),
|
|
);
|
|
expect(workspaces.filter(r => r.status === 'fulfilled')).toHaveLength(1);
|
|
expect((await entitlements.view(owner)).counts).toMatchObject({ projects: 1, workspaces: 1 });
|
|
});
|
|
it('requires a human operations admin and enforces member capacity in the same transaction', async () => {
|
|
await expect(
|
|
entitlements.inviteMember(owner, { userId: 'alice', role: 'developer' }, 'invite-owner'),
|
|
).rejects.toMatchObject({ code: 'PLATFORM_OPERATOR_REQUIRED' });
|
|
await expect(
|
|
entitlements.inviteMember(
|
|
{ ...operator, kind: 'connector' },
|
|
{ userId: 'alice', role: 'developer' },
|
|
'invite-agent',
|
|
),
|
|
).rejects.toMatchObject({ code: 'HUMAN_APPROVAL_REQUIRED' });
|
|
const result = await Promise.allSettled(
|
|
['alice', 'bob'].map(userId =>
|
|
entitlements.inviteMember(operator, { userId, role: 'developer' }, `invite-${userId}`),
|
|
),
|
|
);
|
|
expect(result.filter(r => r.status === 'fulfilled')).toHaveLength(1);
|
|
expect((await entitlements.view(owner)).counts.members).toBe(3);
|
|
expect(await db.transaction('alpha', tx => tx.list('entitlement_event'))).toHaveLength(1);
|
|
});
|
|
it('permits queued runs but reserves only one sandbox slot before provisioning and retains unknown allocations', async () => {
|
|
const runs = await queuedRuns(3);
|
|
expect(await control.runs(owner)).toHaveLength(3);
|
|
const results = await Promise.allSettled(
|
|
runs.map(run => db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env))),
|
|
);
|
|
expect(results.filter(r => r.status === 'fulfilled')).toHaveLength(1);
|
|
const first = runs[0].run_id;
|
|
const again = await db.transaction('alpha', tx => reserve(tx, first, 'generation-1', env));
|
|
expect(again.image).toBe(env.WF_RUN_WORKER_IMAGE);
|
|
expect(again.cpuMilli).toBe(2000);
|
|
await db.transaction('alpha', async tx => {
|
|
await recordAllocation(tx, first, 'generation-1', 'unknown');
|
|
const run = (await tx.get<Run>('run', first))!;
|
|
run.status = 'canceled';
|
|
await tx.put('run', run);
|
|
});
|
|
await expect(db.transaction('alpha', tx => reserve(tx, runs[1].run_id, 'generation-1', env))).rejects.toMatchObject(
|
|
{ code: 'EXECUTION_CAPACITY' },
|
|
);
|
|
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(1);
|
|
});
|
|
it('pins the original runtime digest across resumes and rejects legacy checkpoint image guesses', async () => {
|
|
const [run] = await queuedRuns(1);
|
|
const original = await db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env));
|
|
await db.transaction('alpha', tx => recordAllocation(tx, run.run_id, 'generation-1', 'released'));
|
|
const resumed = await db.transaction('alpha', tx =>
|
|
reserve(tx, run.run_id, 'generation-2', {
|
|
...env,
|
|
WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'b'.repeat(64)}`,
|
|
}),
|
|
);
|
|
expect(resumed.image).toBe(original.image);
|
|
await db.transaction('alpha', async tx => {
|
|
const value = (await tx.get<Run>('run', run.run_id))!;
|
|
value.runtimeImage = undefined;
|
|
value.checkpointId = 'legacy-checkpoint';
|
|
await tx.put('run', value);
|
|
});
|
|
await expect(db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-3', env))).rejects.toMatchObject({
|
|
code: 'RUNTIME_IMAGE_MIGRATION_REQUIRED',
|
|
});
|
|
});
|
|
it('does not replace an unresolved previous generation on resume', async () => {
|
|
const [created] = await queuedRuns(1);
|
|
await db.transaction('alpha', async tx => {
|
|
await reserve(tx, created.run_id, 'old-generation', env);
|
|
await recordAllocation(tx, created.run_id, 'old-generation', 'unknown');
|
|
});
|
|
await expect(
|
|
db.transaction('alpha', tx => reserve(tx, created.run_id, 'new-generation', env)),
|
|
).rejects.toMatchObject({ code: 'RESOURCE_PREVIOUS_GENERATION' });
|
|
expect((await control.run(owner, created.run_id)).resourceAllocation).toMatchObject({
|
|
generation: 'old-generation',
|
|
status: 'unknown',
|
|
});
|
|
});
|
|
it('refuses a start that a newer one replaced, so it cannot take the run back', async () => {
|
|
const [created] = await queuedRuns(1);
|
|
const first = (await db.transaction('alpha', tx => tx.get<Run>('run', created.run_id)))!.pendingStartId!;
|
|
// A pause and a continue enqueue a newer start before the dispatcher reserves the first.
|
|
await db.transaction('alpha', async tx => {
|
|
const run = (await tx.get<Run>('run', created.run_id))!;
|
|
await tx.put('run', { ...run, pendingStartId: 'newer-start' });
|
|
});
|
|
await expect(
|
|
db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, first, env)),
|
|
).rejects.toMatchObject({ code: 'START_SUPERSEDED' });
|
|
const untouched = (await db.transaction('alpha', tx => tx.get<Run>('run', created.run_id)))!;
|
|
expect(untouched.executionGeneration).toBeUndefined();
|
|
expect(untouched.resourceAllocation).toBeUndefined();
|
|
// The newer start reserves as usual.
|
|
await db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'newer-start', env));
|
|
expect((await control.run(owner, created.run_id)).executionGeneration).toBe('newer-start');
|
|
});
|
|
it('issues a grant accepted by the independent provisioner without allowing changed resources', async () => {
|
|
const { verifyResourceGrant } = await import('../apps/provisioner/resource-grants.mjs');
|
|
const [created] = await queuedRuns(1);
|
|
const grant = await db.transaction('alpha', tx => reserve(tx, created.run_id, 'generation-1', env));
|
|
const configuration = {
|
|
tenantId: 'alpha',
|
|
image: env.WF_RUN_WORKER_IMAGE,
|
|
resourceGrantSecret: env.WF_RESOURCE_GRANT_SECRET,
|
|
storageSize: '10Gi',
|
|
cpuMilliCap: 2000,
|
|
memoryMiBCap: 4096,
|
|
};
|
|
const request = { tenantId: 'alpha', runId: created.run_id, generation: 'generation-1', resourceGrant: grant };
|
|
expect(verifyResourceGrant(configuration, request)).toMatchObject({
|
|
runId: created.run_id,
|
|
image: env.WF_RUN_WORKER_IMAGE,
|
|
});
|
|
expect(() =>
|
|
verifyResourceGrant(configuration, { ...request, resourceGrant: { ...grant, cpuMilli: 2001 } }),
|
|
).toThrow('RESOURCE_GRANT_INVALID');
|
|
});
|
|
it('releases normal compute slots only after a matching provisioner receipt and requires operator reconciliation after unknown outcomes', async () => {
|
|
const [created] = await queuedRuns(1),
|
|
id = created.run_id;
|
|
await db.transaction('alpha', async tx => {
|
|
await reserve(tx, id, 'generation-1', env);
|
|
await recordAllocation(tx, id, 'generation-1', 'unknown');
|
|
const run = (await tx.get<Run>('run', id))!;
|
|
run.status = 'canceled';
|
|
await tx.put('run', run);
|
|
});
|
|
const receipt = {
|
|
tenantId: 'alpha',
|
|
runId: id,
|
|
generation: 'generation-1',
|
|
status: 'released',
|
|
computeReleased: true,
|
|
persistentStorageRetained: true,
|
|
};
|
|
await releaseStoppedAllocations(control, 'alpha', (async () => Response.json(receipt)) as typeof fetch);
|
|
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(1);
|
|
await entitlements.reconcileAllocation(
|
|
operator,
|
|
id,
|
|
{
|
|
confirmedStopped: true,
|
|
reference: 'fixture-inspection',
|
|
note: 'Operator inspected all prior fixture generations',
|
|
},
|
|
'reconcile-fixture',
|
|
);
|
|
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(0);
|
|
});
|
|
});
|
|
describe('retained run volumes', () => {
|
|
it('removes them once the retention period after release has passed, and only then', async () => {
|
|
const [old, recent] = (await queuedRuns(2)).map(created => created.run_id);
|
|
await db.transaction('alpha', async tx => {
|
|
for (const id of [old!, recent!]) {
|
|
await reserve(tx, id, `generation-${id}`, env);
|
|
await recordAllocation(tx, id, `generation-${id}`, 'released');
|
|
const run = (await tx.get<Run>('run', id))!;
|
|
run.status = 'succeeded';
|
|
if (id === old) run.resourceAllocation = { ...run.resourceAllocation!, updatedAt: '2020-01-01T00:00:00.000Z' };
|
|
await tx.put('run', run);
|
|
}
|
|
});
|
|
const requests: Record<string, unknown>[] = [];
|
|
const fetcher = (async (url: string, init: RequestInit) => {
|
|
const body = JSON.parse(String(init.body));
|
|
requests.push({ url, ...body });
|
|
return Response.json({
|
|
tenantId: 'alpha',
|
|
runId: body.runId,
|
|
generation: body.generation,
|
|
status: 'purged',
|
|
persistentStorageRetained: false,
|
|
});
|
|
}) as typeof fetch;
|
|
await purgeRetainedStorage(control, 'alpha', fetcher);
|
|
expect(requests).toEqual([
|
|
{
|
|
url: 'http://127.0.0.1:4450/runs/purge',
|
|
tenantId: 'alpha',
|
|
runId: old,
|
|
generation: `generation-${old}`,
|
|
resourceGrant: expect.any(Object),
|
|
},
|
|
]);
|
|
const runs = await db.transaction('alpha', tx => tx.list<Run>('run'));
|
|
expect(runs.find(run => run.id === old)?.resourceAllocation?.storagePurgedAt).toEqual(expect.any(String));
|
|
expect(runs.find(run => run.id === recent)?.resourceAllocation?.storagePurgedAt).toBeUndefined();
|
|
// Purged volumes are not requested again, and 0 keeps volumes forever.
|
|
await purgeRetainedStorage(control, 'alpha', fetcher);
|
|
expect(requests).toHaveLength(1);
|
|
await purgeRetainedStorage(new ControlPlane(db, { ...env, WF_RUN_STORAGE_RETENTION_HOURS: '0' }), 'alpha', fetcher);
|
|
expect(requests).toHaveLength(1);
|
|
});
|
|
});
|
|
describe('usage reporting', () => {
|
|
it('neutralizes spreadsheet formulas preceded by whitespace in exported ledger cells', async () => {
|
|
await db.transaction('alpha', tx =>
|
|
tx.meter({ id: '\t=1+2', tokens: 1, computeSeconds: 0, storageBytes: 0, at: '2020-01-02T00:00:00.000Z' }),
|
|
);
|
|
expect((await usage.exportCsv(owner, '2020-01')).content).toContain('"\'\t=1+2"');
|
|
});
|
|
it('reports an empty month as unknown model usage rather than zero', async () => {
|
|
const report = await usage.report(owner, '2020-01');
|
|
expect(report).toMatchObject({ lines: [], modelCoverage: 'unknown', totals: { modelTokens: null } });
|
|
const csv = await usage.exportCsv(owner, '2020-01');
|
|
expect(csv.filename).toBe('wishforge-usage-2020-01.csv');
|
|
expect(csv.content.trim().split('\r\n')).toHaveLength(1);
|
|
});
|
|
it('separates provider usage, executor reports and artifact bytes by UTC month and tenant', async () => {
|
|
await db.transaction('alpha', async tx => {
|
|
await tx.meter({
|
|
id: 'provider-proxy:fixture',
|
|
tokens: 12,
|
|
computeSeconds: 0,
|
|
storageBytes: 0,
|
|
at: '2020-01-05T00:00:00.000Z',
|
|
});
|
|
await tx.meter({
|
|
id: 'storage:artifact-fixture',
|
|
tokens: 0,
|
|
computeSeconds: 0,
|
|
storageBytes: 500,
|
|
at: '2020-01-06T00:00:00.000Z',
|
|
});
|
|
await tx.meter({
|
|
id: 'legacy-worker',
|
|
tokens: 7,
|
|
computeSeconds: 12,
|
|
storageBytes: 0,
|
|
at: '2020-01-07T00:00:00.000Z',
|
|
});
|
|
await tx.meter({
|
|
id: 'provider-proxy:next-month',
|
|
tokens: 99,
|
|
computeSeconds: 0,
|
|
storageBytes: 0,
|
|
at: '2020-02-01T00:00:00.000Z',
|
|
});
|
|
});
|
|
const report = await usage.report(owner, '2020-01');
|
|
expect(report.lines).toHaveLength(3);
|
|
expect(report.modelCoverage).toBe('partial');
|
|
expect(report.totals).toMatchObject({ modelTokens: 19, reportedComputeSeconds: 12, artifactBytesAdded: 500 });
|
|
expect((await usage.report({ ...owner, tenantId: 'beta' }, '2020-01')).totals.modelTokens).toBeNull();
|
|
});
|
|
it('restricts detailed usage to owners and admins', async () => {
|
|
await expect(usage.report({ ...owner, role: 'developer' }, '2020-01')).rejects.toMatchObject({ status: 403 });
|
|
await expect(usage.report(owner, '2020-13')).rejects.toMatchObject({ code: 'INVALID_PERIOD' });
|
|
});
|
|
});
|