Imports the previously untracked working tree as-is so that the review changes that follow (requirements -> design -> implementation) can be reviewed as diffs and reverted if needed. No content changes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
61 lines
6.9 KiB
TypeScript
61 lines
6.9 KiB
TypeScript
import { generateKeyPairSync, randomUUID } from 'node:crypto';
|
|
import { expect, it } from 'vitest';
|
|
import type { Actor } from '@wishforge/contracts';
|
|
import { contentDigest, signRecord } from '../packages/evolution/src/index.js';
|
|
import { Evolution } from '../apps/api/src/domain/evolution.js';
|
|
import { MemoryStore } from './helpers/memory-store.js';
|
|
|
|
const owner: Actor = { tenantId: 'platform', userId: 'owner', role: 'owner', kind: 'human' };
|
|
const evaluator: Actor = { ...owner, userId: 'external-evaluator', kind: 'evaluator' };
|
|
async function fixture() {
|
|
const db = new MemoryStore(); db.seed('platform'); let clock = Date.now();
|
|
const evaluationKeys = generateKeyPairSync('ed25519'), releaseKeys = generateKeyPairSync('ed25519');
|
|
const pem = (keys: typeof evaluationKeys) => keys.privateKey.export({ type: 'pkcs8', format: 'pem' }).toString();
|
|
const pub = (keys: typeof evaluationKeys) => keys.publicKey.export({ type: 'spki', format: 'pem' }).toString();
|
|
const env = { WF_SYSTEM_TENANT: 'platform', WF_EVOLUTION_AUTO_PROMOTION: 'true', WF_EVALUATOR_PUBLIC_KEYS: JSON.stringify({ evaluation: pub(evaluationKeys) }), WF_PLATFORM_RELEASE_PUBLIC_KEYS: JSON.stringify({ release: pub(releaseKeys) }) };
|
|
const evolution = new Evolution(db, env, () => clock);
|
|
const candidate = await evolution.create(owner, { name: 'Source fixture', scope: 'platform', parentBundle: 'baseline-v1', candidateDigest: contentDigest('source-fixture'), targetMetric: 'cost', hypothesis: 'Fixture only', dataScope: 'platform' });
|
|
const time = () => new Date(clock).toISOString();
|
|
const base = () => ({ schemaVersion: 1, tenantId: 'platform', candidateId: candidate.id, candidateDigest: candidate.candidateDigest, issuedAt: time(), operationId: randomUUID() });
|
|
const release = { deploymentId: 'external-fixture-deployment', sourceCommit: 'a'.repeat(40), imageDigest: `registry.example/platform@sha256:${'b'.repeat(64)}`, previousImageDigest: `registry.example/platform@sha256:${'c'.repeat(64)}` };
|
|
// Signed measurements are explicit test fixtures, not production evaluation evidence.
|
|
await evolution.evaluate(evaluator, candidate.id, signRecord({ schemaVersion: 1, tenantId: 'platform', candidateId: candidate.id, candidateDigest: candidate.candidateDigest, issuedAt: time(), keyId: 'evaluation', nonce: randomUUID(), targetMetric: 'cost', baselineDigest: 'baseline-v1', datasetVersion: 'fixture', evaluatorVersion: 'fixture', samplesDigest: contentDigest('samples'), datasetDigest: contentDigest('dataset'), conditionsDigest: contentDigest('conditions'), distinctTasks: 50, repeats: 3, hardGateFailures: 0, baselineSuccess: 1, candidateSuccess: 1, successDeltaLower95: 0, successDeltaUpper95: 0, intervalMethod: 'paired-task-cluster-t95', baselineCost: 100, candidateCost: 80, baselineLatencyP95: 100, candidateLatencyP95: 100, costRatio: 0.8, latencyP95Ratio: 1, targetImproved: true }, pem(evaluationKeys)));
|
|
await evolution.observe(evaluator, candidate.id, signRecord({ ...base(), keyId: 'evaluation', kind: 'shadow', productionWrites: 0, isolated: true, completedTasks: 50, hardGateFailures: 0, samplesDigest: contentDigest('shadow') }, pem(evaluationKeys)));
|
|
await evolution.observe(evaluator, candidate.id, signRecord({ ...base(), keyId: 'release', kind: 'platform_release', releaseDigest: candidate.candidateDigest, ...release, policyGatePassed: true, target: 'independent_canary' }, pem(releaseKeys)));
|
|
const external = (payload: Record<string, unknown>) => signRecord({ ...base(), keyId: 'release', ...payload }, pem(releaseKeys));
|
|
return { db, env, evolution, candidate, release, external, time, later: () => { clock += 86400001; } };
|
|
}
|
|
it('requires maintainer approval before an automatic code rollout and retains every observation window', async () => {
|
|
const f = await fixture();
|
|
expect((await f.evolution.rolloutTick(evaluator)).outcomes[0]).toMatchObject({ blocked: 'EXTERNAL_RELEASE_REQUIRED' });
|
|
await f.evolution.command(owner, f.candidate.id, 'approve');
|
|
await f.evolution.rolloutTick(evaluator);
|
|
for (const expected of [['internal', 5], ['opt_in', 5], ['opt_in', 25], ['opt_in', 100]] as const) {
|
|
let c = (await f.evolution.list(owner))[0]!;
|
|
expect([c.canaryAudience, c.canaryPercent]).toEqual(expected); expect(c.externalAction?.status).toBe('requested');
|
|
const sample = { kind: 'platform_canary', deploymentId: f.release.deploymentId, imageDigest: f.release.imageDigest, sourceCommit: f.release.sourceCommit, stageStartedAt: c.canaryStartedAt, allocationPercent: c.canaryPercent, audience: c.canaryAudience, hardGateFailures: 0, samples: Array.from({ length: 30 }, (_, i) => ({ id: `${c.canaryStartedAt}-${i}`, startedAt: f.time(), completedAt: f.time(), passed: true })) };
|
|
await f.evolution.observe(evaluator, c.id, f.external(sample));
|
|
await f.evolution.observe(evaluator, c.id, f.external(sample));
|
|
c = (await f.evolution.list(owner))[0]!; expect(c.canaryCompletedRuns).toBe(30);
|
|
expect((await f.evolution.rolloutTick(evaluator)).outcomes).toHaveLength(0);
|
|
f.later(); await f.evolution.rolloutTick(evaluator);
|
|
}
|
|
expect((await f.evolution.list(owner))[0]?.status).toBe('promoted');
|
|
expect(await f.db.transaction('platform', tx => tx.get('policy', 'evolution-routing-v1'))).toBeUndefined();
|
|
});
|
|
it('freezes failed code canaries but reports rolled_back only after a matching signed runtime receipt', async () => {
|
|
const f = await fixture(); await f.evolution.command(owner, f.candidate.id, 'approve');
|
|
const c = await f.evolution.command(owner, f.candidate.id, 'start_canary');
|
|
const failed = await f.evolution.observe(evaluator, c.id, f.external({ kind: 'platform_canary', deploymentId: f.release.deploymentId, imageDigest: f.release.imageDigest, sourceCommit: f.release.sourceCommit, stageStartedAt: c.canaryStartedAt, allocationPercent: 5, audience: 'internal', samples: [], hardGateFailures: 1 }));
|
|
expect(failed.status).toBe('paused'); expect(failed.externalAction).toMatchObject({ action: 'rollback', status: 'requested', imageDigest: f.release.previousImageDigest });
|
|
const receipt = { kind: 'platform_rollback', deploymentId: f.release.deploymentId, actionId: failed.externalAction!.id, restoredImageDigest: f.release.previousImageDigest, healthPassed: true, businessPassed: true };
|
|
await expect(f.evolution.observe(evaluator, c.id, f.external({ ...receipt, restoredImageDigest: f.release.imageDigest }))).rejects.toMatchObject({ code: 'EVIDENCE_BINDING_MISMATCH' });
|
|
expect((await f.evolution.observe(evaluator, c.id, f.external(receipt))).status).toBe('rolled_back');
|
|
});
|
|
it('does not activate automatic promotion without operator policy or from a customer/executor identity', async () => {
|
|
const f = await fixture(); f.env.WF_EVOLUTION_AUTO_PROMOTION = 'false';
|
|
expect((await f.evolution.rolloutTick(evaluator)).status).toBe('unconfigured');
|
|
await expect(f.evolution.rolloutTick({ ...evaluator, tenantId: 'customer' })).rejects.toMatchObject({ code: 'FORBIDDEN' });
|
|
await expect(f.evolution.rolloutTick({ ...evaluator, kind: 'worker' })).rejects.toMatchObject({ code: 'FORBIDDEN' });
|
|
});
|