import { beforeEach, describe, expect, it } from 'vitest'; import type { Actor, Run, TenantEntitlementSnapshot } from '@wishforge/contracts'; import { MemoryStore } from './helpers/memory-store.js'; import { ControlPlane } from '../apps/api/src/domain/control-plane.js'; import { Connections } from '../apps/api/src/domain/connections.js'; import { Usage } from '../apps/api/src/domain/usage.js'; import { Entitlements, PILOT_PLAN, reserveExecutionGrant, recordAllocation, } from '../apps/api/src/domain/entitlements.js'; import { purgeRetainedStorage, releaseStoppedAllocations } from '../apps/api/src/domain/resource-dispatch.js'; import type { Transaction } from '../apps/api/src/db/store.js'; const owner: Actor = { tenantId: 'alpha', userId: 'owner', kind: 'human', role: 'owner' }, operator: Actor = { ...owner, userId: 'operations-user', role: 'admin' }; const env = { NODE_ENV: 'test', WF_VAULT_KEY: 'a'.repeat(64), WF_OPERATOR_SUBJECTS: operator.userId, WF_RESOURCE_GRANT_SECRET: 'resource-fixture-signing-secret-only-12345', WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'a'.repeat(64)}`, WF_PROVISIONER_URL: 'http://127.0.0.1:4450', WF_PROVISIONER_TOKEN: 'fixture-provisioner-token-only-long-enough', WF_PROVISIONER_TENANT_ID: 'alpha', }; let db: MemoryStore, control: ControlPlane, entitlements: Entitlements, usage: Usage; /** Reserves as the dispatcher does: for the start the control plane last enqueued for the run. */ async function reserve(tx: Transaction, runId: string, generation: string, environment: NodeJS.ProcessEnv = env) { const run = (await tx.get('run', runId))!; if (run.pendingStartId !== generation) await tx.put('run', { ...run, pendingStartId: generation }); return reserveExecutionGrant(tx, runId, generation, environment); } beforeEach(() => { db = new MemoryStore(); db.seed('alpha'); db.seed('beta'); const limits = structuredClone(PILOT_PLAN.limits); limits.maxProjects = 1; limits.maxWorkspaces = 1; limits.maxMembers = 3; limits.maxConcurrentRuns = 1; const snapshot: TenantEntitlementSnapshot = { schemaVersion: 1, planId: 'pilot', planVersion: 'pilot-v1', limits, assignedAt: '2019-01-01T00:00:00.000Z', assignedBy: 'fixture-operator', }; db.state.get('alpha')!.tenant.entitlements = snapshot; control = new ControlPlane(db, env); entitlements = new Entitlements(db, env); usage = new Usage(db); }); async function queuedRuns(count = 2) { const project = await control.createProject(owner, { name: 'Fixture', repositoryUrl: 'https://example.org/fixture.git', }); await new Connections(db, env).create(owner, { provider: 'codex', label: 'fixture', apiKey: 'fixture-only-not-real', }); return Promise.all( Array.from({ length: count }, (_, i) => control.submit( owner, project.id, { goal: `Fixture request ${i}`, acceptanceCriteria: ['Fixture is never executed'], budget: { maxTokens: 1000 }, }, `billing-run-${i}`, ), ), ); } describe('fixed pilot entitlements and resource admission', () => { it('rejects defaults below provisioner minimums before consuming an execution slot', async () => { const [created] = await queuedRuns(1); for (const defaults of [ { WF_DEFAULT_EXECUTION_CPU_MILLI: '1' }, { WF_DEFAULT_EXECUTION_MEMORY_MIB: '1' }, { WF_DEFAULT_EXECUTION_STORAGE_MIB: '1' }, ]) await expect( db.transaction('alpha', tx => reserve(tx, created.run_id, 'invalid-default', { ...env, ...defaults })), ).rejects.toMatchObject({ code: 'RESOURCE_CONFIGURATION_INVALID' }); expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(0); }); it('atomically limits projects without blocking idempotent existing data', async () => { const results = await Promise.allSettled([ control.createProject(owner, { name: 'First' }), control.createProject(owner, { name: 'Second' }), ]); expect(results.filter(r => r.status === 'fulfilled')).toHaveLength(1); expect((await entitlements.view(owner)).counts).toMatchObject({ projects: 1 }); }); it('requires a human operations admin and enforces member capacity in the same transaction', async () => { await expect( entitlements.inviteMember(owner, { userId: 'alice', role: 'developer' }, 'invite-owner'), ).rejects.toMatchObject({ code: 'PLATFORM_OPERATOR_REQUIRED' }); await expect( entitlements.inviteMember( { ...operator, kind: 'connector' }, { userId: 'alice', role: 'developer' }, 'invite-agent', ), ).rejects.toMatchObject({ code: 'HUMAN_APPROVAL_REQUIRED' }); const result = await Promise.allSettled( ['alice', 'bob'].map(userId => entitlements.inviteMember(operator, { userId, role: 'developer' }, `invite-${userId}`), ), ); expect(result.filter(r => r.status === 'fulfilled')).toHaveLength(1); expect((await entitlements.view(owner)).counts.members).toBe(3); expect(await db.transaction('alpha', tx => tx.list('entitlement_event'))).toHaveLength(1); }); it('permits queued runs but reserves only one sandbox slot before provisioning and retains unknown allocations', async () => { const runs = await queuedRuns(3); expect(await control.runs(owner)).toHaveLength(3); const results = await Promise.allSettled( runs.map(run => db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env))), ); expect(results.filter(r => r.status === 'fulfilled')).toHaveLength(1); const first = runs[0].run_id; const again = await db.transaction('alpha', tx => reserve(tx, first, 'generation-1', env)); expect(again.image).toBe(env.WF_RUN_WORKER_IMAGE); expect(again.cpuMilli).toBe(2000); await db.transaction('alpha', async tx => { await recordAllocation(tx, first, 'generation-1', 'unknown'); const run = (await tx.get('run', first))!; run.status = 'canceled'; await tx.put('run', run); }); await expect(db.transaction('alpha', tx => reserve(tx, runs[1].run_id, 'generation-1', env))).rejects.toMatchObject( { code: 'EXECUTION_CAPACITY' }, ); expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(1); }); it('pins the original runtime digest across resumes and rejects legacy checkpoint image guesses', async () => { const [run] = await queuedRuns(1); const original = await db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env)); await db.transaction('alpha', tx => recordAllocation(tx, run.run_id, 'generation-1', 'released')); const resumed = await db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-2', { ...env, WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'b'.repeat(64)}`, }), ); expect(resumed.image).toBe(original.image); await db.transaction('alpha', async tx => { const value = (await tx.get('run', run.run_id))!; value.runtimeImage = undefined; value.checkpointId = 'legacy-checkpoint'; await tx.put('run', value); }); await expect(db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-3', env))).rejects.toMatchObject({ code: 'RUNTIME_IMAGE_MIGRATION_REQUIRED', }); }); it('does not replace an unresolved previous generation on resume', async () => { const [created] = await queuedRuns(1); await db.transaction('alpha', async tx => { await reserve(tx, created.run_id, 'old-generation', env); await recordAllocation(tx, created.run_id, 'old-generation', 'unknown'); }); await expect( db.transaction('alpha', tx => reserve(tx, created.run_id, 'new-generation', env)), ).rejects.toMatchObject({ code: 'RESOURCE_PREVIOUS_GENERATION' }); expect((await control.run(owner, created.run_id)).resourceAllocation).toMatchObject({ generation: 'old-generation', status: 'unknown', }); }); it('refuses a start that a newer one replaced, so it cannot take the run back', async () => { const [created] = await queuedRuns(1); const first = (await db.transaction('alpha', tx => tx.get('run', created.run_id)))!.pendingStartId!; // A pause and a continue enqueue a newer start before the dispatcher reserves the first. await db.transaction('alpha', async tx => { const run = (await tx.get('run', created.run_id))!; await tx.put('run', { ...run, pendingStartId: 'newer-start' }); }); await expect( db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, first, env)), ).rejects.toMatchObject({ code: 'START_SUPERSEDED' }); const untouched = (await db.transaction('alpha', tx => tx.get('run', created.run_id)))!; expect(untouched.executionGeneration).toBeUndefined(); expect(untouched.resourceAllocation).toBeUndefined(); // The newer start reserves as usual. await db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'newer-start', env)); expect((await control.run(owner, created.run_id)).executionGeneration).toBe('newer-start'); }); it('issues a grant accepted by the independent provisioner without allowing changed resources', async () => { const { verifyResourceGrant } = await import('../apps/provisioner/resource-grants.mjs'); const [created] = await queuedRuns(1); const grant = await db.transaction('alpha', tx => reserve(tx, created.run_id, 'generation-1', env)); const configuration = { tenantId: 'alpha', image: env.WF_RUN_WORKER_IMAGE, resourceGrantSecret: env.WF_RESOURCE_GRANT_SECRET, storageSize: '10Gi', cpuMilliCap: 2000, memoryMiBCap: 4096, }; const request = { tenantId: 'alpha', runId: created.run_id, generation: 'generation-1', resourceGrant: grant }; expect(verifyResourceGrant(configuration, request)).toMatchObject({ runId: created.run_id, image: env.WF_RUN_WORKER_IMAGE, }); expect(() => verifyResourceGrant(configuration, { ...request, resourceGrant: { ...grant, cpuMilli: 2001 } }), ).toThrow('RESOURCE_GRANT_INVALID'); }); it('releases normal compute slots only after a matching provisioner receipt and requires operator reconciliation after unknown outcomes', async () => { const [created] = await queuedRuns(1), id = created.run_id; await db.transaction('alpha', async tx => { await reserve(tx, id, 'generation-1', env); await recordAllocation(tx, id, 'generation-1', 'unknown'); const run = (await tx.get('run', id))!; run.status = 'canceled'; await tx.put('run', run); }); const receipt = { tenantId: 'alpha', runId: id, generation: 'generation-1', status: 'released', computeReleased: true, persistentStorageRetained: true, }; await releaseStoppedAllocations(control, 'alpha', (async () => Response.json(receipt)) as typeof fetch); expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(1); await entitlements.reconcileAllocation( operator, id, { confirmedStopped: true, reference: 'fixture-inspection', note: 'Operator inspected all prior fixture generations', }, 'reconcile-fixture', ); expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(0); }); }); describe('retained run volumes', () => { it('removes them once the retention period after release has passed, and only then', async () => { const [old, recent] = (await queuedRuns(2)).map(created => created.run_id); await db.transaction('alpha', async tx => { for (const id of [old!, recent!]) { await reserve(tx, id, `generation-${id}`, env); await recordAllocation(tx, id, `generation-${id}`, 'released'); const run = (await tx.get('run', id))!; run.status = 'succeeded'; if (id === old) run.resourceAllocation = { ...run.resourceAllocation!, updatedAt: '2020-01-01T00:00:00.000Z' }; await tx.put('run', run); } }); const requests: Record[] = []; const fetcher = (async (url: string, init: RequestInit) => { const body = JSON.parse(String(init.body)); requests.push({ url, ...body }); return Response.json({ tenantId: 'alpha', runId: body.runId, generation: body.generation, status: 'purged', persistentStorageRetained: false, }); }) as typeof fetch; await purgeRetainedStorage(control, 'alpha', fetcher); expect(requests).toEqual([ { url: 'http://127.0.0.1:4450/runs/purge', tenantId: 'alpha', runId: old, generation: `generation-${old}`, resourceGrant: expect.any(Object), }, ]); const runs = await db.transaction('alpha', tx => tx.list('run')); expect(runs.find(run => run.id === old)?.resourceAllocation?.storagePurgedAt).toEqual(expect.any(String)); expect(runs.find(run => run.id === recent)?.resourceAllocation?.storagePurgedAt).toBeUndefined(); // Purged volumes are not requested again, and 0 keeps volumes forever. await purgeRetainedStorage(control, 'alpha', fetcher); expect(requests).toHaveLength(1); await purgeRetainedStorage(new ControlPlane(db, { ...env, WF_RUN_STORAGE_RETENTION_HOURS: '0' }), 'alpha', fetcher); expect(requests).toHaveLength(1); }); }); describe('usage reporting', () => { it('neutralizes spreadsheet formulas preceded by whitespace in exported ledger cells', async () => { await db.transaction('alpha', tx => tx.meter({ id: '\t=1+2', tokens: 1, computeSeconds: 0, storageBytes: 0, at: '2020-01-02T00:00:00.000Z' }), ); expect((await usage.exportCsv(owner, '2020-01')).content).toContain('"\'\t=1+2"'); }); it('reports an empty month as unknown model usage rather than zero', async () => { const report = await usage.report(owner, '2020-01'); expect(report).toMatchObject({ lines: [], modelCoverage: 'unknown', totals: { modelTokens: null } }); const csv = await usage.exportCsv(owner, '2020-01'); expect(csv.filename).toBe('wishforge-usage-2020-01.csv'); expect(csv.content.trim().split('\r\n')).toHaveLength(1); }); it('separates provider usage, executor reports and artifact bytes by UTC month and tenant', async () => { await db.transaction('alpha', async tx => { await tx.meter({ id: 'provider-proxy:fixture', tokens: 12, computeSeconds: 0, storageBytes: 0, at: '2020-01-05T00:00:00.000Z', }); await tx.meter({ id: 'storage:artifact-fixture', tokens: 0, computeSeconds: 0, storageBytes: 500, at: '2020-01-06T00:00:00.000Z', }); await tx.meter({ id: 'legacy-worker', tokens: 7, computeSeconds: 12, storageBytes: 0, at: '2020-01-07T00:00:00.000Z', }); await tx.meter({ id: 'provider-proxy:next-month', tokens: 99, computeSeconds: 0, storageBytes: 0, at: '2020-02-01T00:00:00.000Z', }); }); const report = await usage.report(owner, '2020-01'); expect(report.lines).toHaveLength(3); expect(report.modelCoverage).toBe('partial'); expect(report.totals).toMatchObject({ modelTokens: 19, reportedComputeSeconds: 12, artifactBytesAdded: 500 }); expect((await usage.report({ ...owner, tenantId: 'beta' }, '2020-01')).totals.modelTokens).toBeNull(); }); it('restricts detailed usage to owners and admins', async () => { await expect(usage.report({ ...owner, role: 'developer' }, '2020-01')).rejects.toMatchObject({ status: 403 }); await expect(usage.report(owner, '2020-13')).rejects.toMatchObject({ code: 'INVALID_PERIOD' }); }); });