wishforge-ide/tests/entitlements-usage.test.ts
eust-w df87c5dca6 🐛 Drop a run start that a newer one replaced before it takes the run back
The dispatcher read the run, checked that the start it was delivering
was still the pending one, and reserved the execution grant in a later
transaction. A pause and a continue in between enqueued a newer start,
and the older one went ahead anyway: it took the run's execution
generation back and asked the provisioner for a sandbox for a
generation that had already been replaced.

Reserving the grant now checks the pending start again, in the
transaction that takes the generation, and refuses a replaced one with
START_SUPERSEDED. The dispatcher acknowledges such a start without
marking the run as waiting; the newer start proceeds as usual.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 15:56:14 +08:00

362 lines
16 KiB
TypeScript

import { beforeEach, describe, expect, it } from 'vitest';
import type { Actor, Run, TenantEntitlementSnapshot } from '@wishforge/contracts';
import { MemoryStore } from './helpers/memory-store.js';
import { ControlPlane, record } from '../apps/api/src/domain/control-plane.js';
import { Connections } from '../apps/api/src/domain/connections.js';
import { Usage } from '../apps/api/src/domain/usage.js';
import {
Entitlements,
PILOT_PLAN,
requireCapacity,
reserveExecutionGrant,
recordAllocation,
} from '../apps/api/src/domain/entitlements.js';
import { purgeRetainedStorage, releaseStoppedAllocations } from '../apps/api/src/domain/resource-dispatch.js';
import type { Transaction } from '../apps/api/src/db/store.js';
const owner: Actor = { tenantId: 'alpha', userId: 'owner', kind: 'human', role: 'owner' },
operator: Actor = { ...owner, userId: 'operations-user', role: 'admin' };
const env = {
NODE_ENV: 'test',
WF_VAULT_KEY: 'a'.repeat(64),
WF_OPERATOR_SUBJECTS: operator.userId,
WF_RESOURCE_GRANT_SECRET: 'resource-fixture-signing-secret-only-12345',
WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'a'.repeat(64)}`,
WF_PROVISIONER_URL: 'http://127.0.0.1:4450',
WF_PROVISIONER_TOKEN: 'fixture-provisioner-token-only-long-enough',
WF_PROVISIONER_TENANT_ID: 'alpha',
};
let db: MemoryStore, control: ControlPlane, entitlements: Entitlements, usage: Usage;
/** Reserves as the dispatcher does: for the start the control plane last enqueued for the run. */
async function reserve(tx: Transaction, runId: string, generation: string, environment: NodeJS.ProcessEnv = env) {
const run = (await tx.get<Run>('run', runId))!;
if (run.pendingStartId !== generation) await tx.put('run', { ...run, pendingStartId: generation });
return reserveExecutionGrant(tx, runId, generation, environment);
}
beforeEach(() => {
db = new MemoryStore();
db.seed('alpha');
db.seed('beta');
const limits = structuredClone(PILOT_PLAN.limits);
limits.maxProjects = 1;
limits.maxWorkspaces = 1;
limits.maxMembers = 3;
limits.maxConcurrentRuns = 1;
const snapshot: TenantEntitlementSnapshot = {
schemaVersion: 1,
planId: 'pilot',
planVersion: 'pilot-v1',
limits,
assignedAt: '2019-01-01T00:00:00.000Z',
assignedBy: 'fixture-operator',
};
db.state.get('alpha')!.tenant.entitlements = snapshot;
control = new ControlPlane(db, env);
entitlements = new Entitlements(db, env);
usage = new Usage(db);
});
async function queuedRuns(count = 2) {
const project = await control.createProject(owner, {
name: 'Fixture',
repositoryUrl: 'https://example.org/fixture.git',
});
await new Connections(db, env).create(owner, {
provider: 'codex',
label: 'fixture',
apiKey: 'fixture-only-not-real',
});
return Promise.all(
Array.from({ length: count }, (_, i) =>
control.submit(
owner,
project.id,
{
goal: `Fixture request ${i}`,
acceptanceCriteria: ['Fixture is never executed'],
budget: { maxTokens: 1000 },
},
`billing-run-${i}`,
),
),
);
}
describe('fixed pilot entitlements and resource admission', () => {
it('rejects defaults below provisioner minimums before consuming an execution slot', async () => {
const [created] = await queuedRuns(1);
for (const defaults of [
{ WF_DEFAULT_EXECUTION_CPU_MILLI: '1' },
{ WF_DEFAULT_EXECUTION_MEMORY_MIB: '1' },
{ WF_DEFAULT_EXECUTION_STORAGE_MIB: '1' },
])
await expect(
db.transaction('alpha', tx => reserve(tx, created.run_id, 'invalid-default', { ...env, ...defaults })),
).rejects.toMatchObject({ code: 'RESOURCE_CONFIGURATION_INVALID' });
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(0);
});
it('atomically limits projects and workspaces without blocking idempotent existing data', async () => {
const results = await Promise.allSettled([
control.createProject(owner, { name: 'First' }),
control.createProject(owner, { name: 'Second' }),
]);
expect(results.filter(r => r.status === 'fulfilled')).toHaveLength(1);
const workspaces = await Promise.allSettled(
[1, 2].map(() =>
db.transaction('alpha', async tx => {
await requireCapacity(tx, 'workspace');
await tx.put('workspace', { ...record('alpha'), projectId: 'fixture' });
}),
),
);
expect(workspaces.filter(r => r.status === 'fulfilled')).toHaveLength(1);
expect((await entitlements.view(owner)).counts).toMatchObject({ projects: 1, workspaces: 1 });
});
it('requires a human operations admin and enforces member capacity in the same transaction', async () => {
await expect(
entitlements.inviteMember(owner, { userId: 'alice', role: 'developer' }, 'invite-owner'),
).rejects.toMatchObject({ code: 'PLATFORM_OPERATOR_REQUIRED' });
await expect(
entitlements.inviteMember(
{ ...operator, kind: 'connector' },
{ userId: 'alice', role: 'developer' },
'invite-agent',
),
).rejects.toMatchObject({ code: 'HUMAN_APPROVAL_REQUIRED' });
const result = await Promise.allSettled(
['alice', 'bob'].map(userId =>
entitlements.inviteMember(operator, { userId, role: 'developer' }, `invite-${userId}`),
),
);
expect(result.filter(r => r.status === 'fulfilled')).toHaveLength(1);
expect((await entitlements.view(owner)).counts.members).toBe(3);
expect(await db.transaction('alpha', tx => tx.list('entitlement_event'))).toHaveLength(1);
});
it('permits queued runs but reserves only one sandbox slot before provisioning and retains unknown allocations', async () => {
const runs = await queuedRuns(3);
expect(await control.runs(owner)).toHaveLength(3);
const results = await Promise.allSettled(
runs.map(run => db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env))),
);
expect(results.filter(r => r.status === 'fulfilled')).toHaveLength(1);
const first = runs[0].run_id;
const again = await db.transaction('alpha', tx => reserve(tx, first, 'generation-1', env));
expect(again.image).toBe(env.WF_RUN_WORKER_IMAGE);
expect(again.cpuMilli).toBe(2000);
await db.transaction('alpha', async tx => {
await recordAllocation(tx, first, 'generation-1', 'unknown');
const run = (await tx.get<Run>('run', first))!;
run.status = 'canceled';
await tx.put('run', run);
});
await expect(db.transaction('alpha', tx => reserve(tx, runs[1].run_id, 'generation-1', env))).rejects.toMatchObject(
{ code: 'EXECUTION_CAPACITY' },
);
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(1);
});
it('pins the original runtime digest across resumes and rejects legacy checkpoint image guesses', async () => {
const [run] = await queuedRuns(1);
const original = await db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env));
await db.transaction('alpha', tx => recordAllocation(tx, run.run_id, 'generation-1', 'released'));
const resumed = await db.transaction('alpha', tx =>
reserve(tx, run.run_id, 'generation-2', {
...env,
WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'b'.repeat(64)}`,
}),
);
expect(resumed.image).toBe(original.image);
await db.transaction('alpha', async tx => {
const value = (await tx.get<Run>('run', run.run_id))!;
value.runtimeImage = undefined;
value.checkpointId = 'legacy-checkpoint';
await tx.put('run', value);
});
await expect(db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-3', env))).rejects.toMatchObject({
code: 'RUNTIME_IMAGE_MIGRATION_REQUIRED',
});
});
it('does not replace an unresolved previous generation on resume', async () => {
const [created] = await queuedRuns(1);
await db.transaction('alpha', async tx => {
await reserve(tx, created.run_id, 'old-generation', env);
await recordAllocation(tx, created.run_id, 'old-generation', 'unknown');
});
await expect(
db.transaction('alpha', tx => reserve(tx, created.run_id, 'new-generation', env)),
).rejects.toMatchObject({ code: 'RESOURCE_PREVIOUS_GENERATION' });
expect((await control.run(owner, created.run_id)).resourceAllocation).toMatchObject({
generation: 'old-generation',
status: 'unknown',
});
});
it('refuses a start that a newer one replaced, so it cannot take the run back', async () => {
const [created] = await queuedRuns(1);
const first = (await db.transaction('alpha', tx => tx.get<Run>('run', created.run_id)))!.pendingStartId!;
// A pause and a continue enqueue a newer start before the dispatcher reserves the first.
await db.transaction('alpha', async tx => {
const run = (await tx.get<Run>('run', created.run_id))!;
await tx.put('run', { ...run, pendingStartId: 'newer-start' });
});
await expect(
db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, first, env)),
).rejects.toMatchObject({ code: 'START_SUPERSEDED' });
const untouched = (await db.transaction('alpha', tx => tx.get<Run>('run', created.run_id)))!;
expect(untouched.executionGeneration).toBeUndefined();
expect(untouched.resourceAllocation).toBeUndefined();
// The newer start reserves as usual.
await db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'newer-start', env));
expect((await control.run(owner, created.run_id)).executionGeneration).toBe('newer-start');
});
it('issues a grant accepted by the independent provisioner without allowing changed resources', async () => {
const { verifyResourceGrant } = await import('../apps/provisioner/resource-grants.mjs');
const [created] = await queuedRuns(1);
const grant = await db.transaction('alpha', tx => reserve(tx, created.run_id, 'generation-1', env));
const configuration = {
tenantId: 'alpha',
image: env.WF_RUN_WORKER_IMAGE,
resourceGrantSecret: env.WF_RESOURCE_GRANT_SECRET,
storageSize: '10Gi',
cpuMilliCap: 2000,
memoryMiBCap: 4096,
};
const request = { tenantId: 'alpha', runId: created.run_id, generation: 'generation-1', resourceGrant: grant };
expect(verifyResourceGrant(configuration, request)).toMatchObject({
runId: created.run_id,
image: env.WF_RUN_WORKER_IMAGE,
});
expect(() =>
verifyResourceGrant(configuration, { ...request, resourceGrant: { ...grant, cpuMilli: 2001 } }),
).toThrow('RESOURCE_GRANT_INVALID');
});
it('releases normal compute slots only after a matching provisioner receipt and requires operator reconciliation after unknown outcomes', async () => {
const [created] = await queuedRuns(1),
id = created.run_id;
await db.transaction('alpha', async tx => {
await reserve(tx, id, 'generation-1', env);
await recordAllocation(tx, id, 'generation-1', 'unknown');
const run = (await tx.get<Run>('run', id))!;
run.status = 'canceled';
await tx.put('run', run);
});
const receipt = {
tenantId: 'alpha',
runId: id,
generation: 'generation-1',
status: 'released',
computeReleased: true,
persistentStorageRetained: true,
};
await releaseStoppedAllocations(control, 'alpha', (async () => Response.json(receipt)) as typeof fetch);
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(1);
await entitlements.reconcileAllocation(
operator,
id,
{
confirmedStopped: true,
reference: 'fixture-inspection',
note: 'Operator inspected all prior fixture generations',
},
'reconcile-fixture',
);
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(0);
});
});
describe('retained run volumes', () => {
it('removes them once the retention period after release has passed, and only then', async () => {
const [old, recent] = (await queuedRuns(2)).map(created => created.run_id);
await db.transaction('alpha', async tx => {
for (const id of [old!, recent!]) {
await reserve(tx, id, `generation-${id}`, env);
await recordAllocation(tx, id, `generation-${id}`, 'released');
const run = (await tx.get<Run>('run', id))!;
run.status = 'succeeded';
if (id === old) run.resourceAllocation = { ...run.resourceAllocation!, updatedAt: '2020-01-01T00:00:00.000Z' };
await tx.put('run', run);
}
});
const requests: Record<string, unknown>[] = [];
const fetcher = (async (url: string, init: RequestInit) => {
const body = JSON.parse(String(init.body));
requests.push({ url, ...body });
return Response.json({
tenantId: 'alpha',
runId: body.runId,
generation: body.generation,
status: 'purged',
persistentStorageRetained: false,
});
}) as typeof fetch;
await purgeRetainedStorage(control, 'alpha', fetcher);
expect(requests).toEqual([
{
url: 'http://127.0.0.1:4450/runs/purge',
tenantId: 'alpha',
runId: old,
generation: `generation-${old}`,
resourceGrant: expect.any(Object),
},
]);
const runs = await db.transaction('alpha', tx => tx.list<Run>('run'));
expect(runs.find(run => run.id === old)?.resourceAllocation?.storagePurgedAt).toEqual(expect.any(String));
expect(runs.find(run => run.id === recent)?.resourceAllocation?.storagePurgedAt).toBeUndefined();
// Purged volumes are not requested again, and 0 keeps volumes forever.
await purgeRetainedStorage(control, 'alpha', fetcher);
expect(requests).toHaveLength(1);
await purgeRetainedStorage(new ControlPlane(db, { ...env, WF_RUN_STORAGE_RETENTION_HOURS: '0' }), 'alpha', fetcher);
expect(requests).toHaveLength(1);
});
});
describe('usage reporting', () => {
it('neutralizes spreadsheet formulas preceded by whitespace in exported ledger cells', async () => {
await db.transaction('alpha', tx =>
tx.meter({ id: '\t=1+2', tokens: 1, computeSeconds: 0, storageBytes: 0, at: '2020-01-02T00:00:00.000Z' }),
);
expect((await usage.exportCsv(owner, '2020-01')).content).toContain('"\'\t=1+2"');
});
it('reports an empty month as unknown model usage rather than zero', async () => {
const report = await usage.report(owner, '2020-01');
expect(report).toMatchObject({ lines: [], modelCoverage: 'unknown', totals: { modelTokens: null } });
const csv = await usage.exportCsv(owner, '2020-01');
expect(csv.filename).toBe('wishforge-usage-2020-01.csv');
expect(csv.content.trim().split('\r\n')).toHaveLength(1);
});
it('separates provider usage, executor reports and artifact bytes by UTC month and tenant', async () => {
await db.transaction('alpha', async tx => {
await tx.meter({
id: 'provider-proxy:fixture',
tokens: 12,
computeSeconds: 0,
storageBytes: 0,
at: '2020-01-05T00:00:00.000Z',
});
await tx.meter({
id: 'storage:artifact-fixture',
tokens: 0,
computeSeconds: 0,
storageBytes: 500,
at: '2020-01-06T00:00:00.000Z',
});
await tx.meter({
id: 'legacy-worker',
tokens: 7,
computeSeconds: 12,
storageBytes: 0,
at: '2020-01-07T00:00:00.000Z',
});
await tx.meter({
id: 'provider-proxy:next-month',
tokens: 99,
computeSeconds: 0,
storageBytes: 0,
at: '2020-02-01T00:00:00.000Z',
});
});
const report = await usage.report(owner, '2020-01');
expect(report.lines).toHaveLength(3);
expect(report.modelCoverage).toBe('partial');
expect(report.totals).toMatchObject({ modelTokens: 19, reportedComputeSeconds: 12, artifactBytesAdded: 500 });
expect((await usage.report({ ...owner, tenantId: 'beta' }, '2020-01')).totals.modelTokens).toBeNull();
});
it('restricts detailed usage to owners and admins', async () => {
await expect(usage.report({ ...owner, role: 'developer' }, '2020-01')).rejects.toMatchObject({ status: 403 });
await expect(usage.report(owner, '2020-13')).rejects.toMatchObject({ code: 'INVALID_PERIOD' });
});
});