wishforge-ide/tests/evolution.test.ts
eust-w b47a07e8d2 chore: baseline snapshot of the Codex-delivered code before review
Imports the previously untracked working tree as-is so that the review
changes that follow (requirements -> design -> implementation) can be
reviewed as diffs and reverted if needed. No content changes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 13:18:04 +08:00

66 lines
14 KiB
TypeScript

import { generateKeyPairSync, randomUUID } from 'node:crypto';
import { describe, expect, it } from 'vitest';
import type { Actor, Run } from '@wishforge/contracts';
import { BASELINE_BUNDLE, bundleDigest, bundlePrompt, canaryBucket, contentDigest, evaluateOffline, runShadow, signRecord, verifyRecord, type AgentBundleConfig, type Benchmark, type EvaluationRunner, type SignedEvaluationReport } from '../packages/evolution/src/index.js';
import { Evolution, resolveBundle, selectBundle } from '../apps/api/src/domain/evolution.js';
import { MemoryStore } from './helpers/memory-store.js';
const keys=generateKeyPairSync('ed25519');
const privateKeyPem=keys.privateKey.export({type:'pkcs8',format:'pem'}).toString();
const publicKeyPem=keys.publicKey.export({type:'spki',format:'pem'}).toString();
const config:AgentBundleConfig={schemaVersion:1,prompts:{builder:'Use bounded context and preserve the acceptance criteria.'}};
const candidateDigest=bundleDigest(config);
const owner:Actor={tenantId:'tenant',userId:'owner',role:'owner',kind:'human'};
const evaluator:Actor={...owner,userId:'independent-evaluator',kind:'evaluator'};
// These measurements are test fixtures only; the production CLI never generates benchmark tasks.
function benchmark():Benchmark{return {schemaVersion:1,datasetVersion:'test-fixture-v1',conditions:{environmentDigest:contentDigest('test-vm'),budgetTokens:1000,timeoutSeconds:5,modelParameters:{temperature:0}},tasks:Array.from({length:50},(_,i)=>({id:`task-${i}`,input:{requirement:`fixture-${i}`},provenance:'synthetic' as const}))};}
function runner(onRequest?:(r:Parameters<EvaluationRunner['run']>[0])=>void):EvaluationRunner{return {capabilities:()=>({available:true,isolated:true,productionWrites:false,evaluatorVersion:'fixture-runner-v1'}),run:async request=>{onRequest?.(request);return {accepted:true,cost:request.bundleDigest===candidateDigest?0.8:1,durationMs:100,productionWrites:0,hardGates:{tenantIsolation:true,authorization:true,evidenceIntegrity:true,budgetEnforcement:true},evidenceDigest:contentDigest(request)};}};}
async function report(candidateId='candidate'):Promise<SignedEvaluationReport>{const result=await evaluateOffline({tenantId:'tenant',candidateId,candidateDigest,baselineDigest:'baseline-v1',baseline:BASELINE_BUNDLE,candidate:config,targetMetric:'cost',benchmark:benchmark(),runner:runner(),privateKeyPem,keyId:'eval'});if(result.status!=='evaluated')throw new Error('fixture evaluation missing');return result.report;}
async function setup(){const db=new MemoryStore();db.seed('tenant');let clock=Date.now();const evolution=new Evolution(db,{WF_EVALUATOR_PUBLIC_KEYS:JSON.stringify({eval:publicKeyPem})},()=>clock);const candidate=await evolution.create(owner,{name:'Context compression',scope:'prompt',parentBundle:'baseline-v1',candidateDigest,bundleConfig:config,hypothesis:'Reduce cost with unchanged acceptance',targetMetric:'cost'});return {db,evolution,candidate,advanceClock:()=>{clock+=86400001;},time:()=>new Date(clock).toISOString()};}
async function shadow(s:Awaited<ReturnType<typeof setup>>){await s.evolution.evaluate(evaluator,s.candidate.id,await report(s.candidate.id));const result=await runShadow({tenantId:'tenant',candidateId:s.candidate.id,candidate:config,benchmark:benchmark(),runner:runner(),privateKeyPem,keyId:'eval'});if(result.status!=='observed')throw new Error('fixture shadow missing');await s.evolution.observe(evaluator,s.candidate.id,result.observation);return s.evolution.command(owner,s.candidate.id,'start_canary');}
function observation(s:Awaited<ReturnType<typeof setup>>,stageStartedAt:string,runIds:string[],extra:Record<string,unknown>={}) {return signRecord({schemaVersion:1,kind:'canary',tenantId:'tenant',candidateId:s.candidate.id,candidateDigest,keyId:'eval',issuedAt:s.time(),operationId:randomUUID(),stageStartedAt,runIds,hardGateFailures:0,...extra},privateKeyPem);}
describe('trusted offline evaluation',()=>{
it('reports unconfigured without external inputs and never substitutes synthetic successes',async()=>{const result=await evaluateOffline({tenantId:'tenant',candidateId:'x',candidateDigest,baselineDigest:'baseline-v1',baseline:BASELINE_BUNDLE,candidate:config,targetMetric:'cost'});expect(result.status).toBe('unconfigured');});
it('pairs 50 distinct tasks x 3 repeats under identical seeds, budgets and environments and signs all report fields',async()=>{
const requests:Parameters<EvaluationRunner['run']>[0][]=[];const result=await evaluateOffline({tenantId:'tenant',candidateId:'x',candidateDigest,baselineDigest:'baseline-v1',baseline:BASELINE_BUNDLE,candidate:config,targetMetric:'cost',benchmark:benchmark(),runner:runner(r=>requests.push(r)),privateKeyPem,keyId:'eval'});
expect(requests).toHaveLength(300);for(let i=0;i<requests.length;i+=2){expect(requests[i]!.seed).toBe(requests[i+1]!.seed);expect(requests[i]!.conditions).toEqual(requests[i+1]!.conditions);expect(requests[i]!.productionWritesAllowed).toBe(false);}
expect(result.status).toBe('evaluated');if(result.status!=='evaluated')return;expect(result.report.distinctTasks).toBe(50);expect(result.report.repeats).toBe(3);expect(result.report.costRatio).toBeCloseTo(.8);expect(result.report.successDeltaLower95).toBe(0);expect(result.report.targetImproved).toBe(true);expect(verifyRecord(result.report,publicKeyPem)).toBe(true);expect(verifyRecord({...result.report,hardGateFailures:2},publicKeyPem)).toBe(false);
});
it('rejects relabeled duplicate task inputs and too few repeats',async()=>{const input={tenantId:'tenant',candidateId:'x',candidateDigest,baselineDigest:'baseline-v1',baseline:BASELINE_BUNDLE,candidate:config,targetMetric:'cost' as const,benchmark:benchmark(),runner:runner(),privateKeyPem,keyId:'eval'};await expect(evaluateOffline({...input,repeats:2})).rejects.toThrow('repeats');input.benchmark.tasks.forEach(t=>t.input={same:true});await expect(evaluateOffline(input)).rejects.toThrow('distinct task inputs');});
it('records hard gate failures and negative paired quality confidence despite lower cost',async()=>{const r=runner();const input={tenantId:'tenant',candidateId:'x',candidateDigest,baselineDigest:'baseline-v1',baseline:BASELINE_BUNDLE,candidate:config,targetMetric:'cost' as const,benchmark:benchmark(),runner:{...r,run:async (q:Parameters<EvaluationRunner['run']>[0])=>({...await r.run(q) as object,accepted:q.bundleDigest!==candidateDigest,hardGates:{tenantIsolation:false,authorization:true,evidenceIntegrity:true,budgetEnforcement:true}})},privateKeyPem,keyId:'eval'};const result=await evaluateOffline(input);if(result.status!=='evaluated')throw new Error('missing');expect(result.report.successDeltaLower95).toBe(-1);expect(result.report.hardGateFailures).toBe(300);});
it('refuses unsafe shadow capabilities and any reported production side effect',async()=>{const unsafe:EvaluationRunner={capabilities:()=>({available:true,isolated:false,productionWrites:false,evaluatorVersion:'x'}),run:async()=>{throw new Error('must not run');}};expect((await runShadow({tenantId:'tenant',candidateId:'x',candidate:config,benchmark:benchmark(),runner:unsafe,privateKeyPem,keyId:'eval'})).status).toBe('unconfigured');const r=runner();await expect(runShadow({tenantId:'tenant',candidateId:'x',candidate:config,benchmark:benchmark(),runner:{...r,run:async q=>({...await r.run(q) as object,productionWrites:1})},privateKeyPem,keyId:'eval'})).rejects.toThrow();});
});
describe('immutable bundles and evidence-driven rollout',()=>{
it('rejects forged signatures, wrong candidate bindings and incorrect declared targets',async()=>{const s=await setup();const r=await report(s.candidate.id);await expect(s.evolution.evaluate(evaluator,s.candidate.id,{...r,costRatio:0})).rejects.toMatchObject({code:'INVALID_EVALUATION_SIGNATURE'});await expect(s.evolution.evaluate(evaluator,s.candidate.id,await report('different-candidate'))).rejects.toMatchObject({code:'EVIDENCE_BINDING_MISMATCH'});const {signature,...payload}=r;await expect(s.evolution.evaluate(evaluator,s.candidate.id,signRecord({...payload,targetMetric:'success'},privateKeyPem))).rejects.toMatchObject({code:'EVIDENCE_BINDING_MISMATCH'});});
it('rejects unknown evaluator keys and candidate-provided permission settings',async()=>{const s=await setup();await expect(new Evolution(s.db,{}).evaluate(evaluator,s.candidate.id,await report(s.candidate.id))).rejects.toMatchObject({code:'EVALUATOR_UNCONFIGURED'});expect(()=>s.evolution.create(owner,{name:'unsafe',scope:'prompt',parentBundle:'baseline-v1',candidateDigest,hypothesis:'test',targetMetric:'cost',bundleConfig:{...config,permissions:['production']}})).toThrow();});
it('routes deterministic internal canaries only to new eligible runs and preserves historical bundle resolution after rollback',async()=>{const s=await setup();await shadow(s);let runId='';for(let i=0;i<1000;i++)if(canaryBucket('tenant','project',`run-${i}`,candidateDigest)<5){runId=`run-${i}`;break;}expect(runId).not.toBe('');const choose=(internalProject=false)=>s.db.transaction('tenant',tx=>selectBundle(tx,{tenantId:'tenant',projectId:'project',runId,internalProject}));expect((await choose()).version).toBe('baseline-v1');expect((await choose(true)).version).toBe(candidateDigest);expect((await choose(true)).version).toBe(candidateDigest);await s.evolution.command(owner,s.candidate.id,'rollback');expect((await choose(true)).version).toBe('baseline-v1');expect(await s.db.transaction('tenant',tx=>resolveBundle(tx,candidateDigest))).toEqual(config);});
it('cannot advance or reset a canary to skip a 24-hour stage',async()=>{const s=await setup();await shadow(s);await expect(s.evolution.command(owner,s.candidate.id,'advance')).rejects.toMatchObject({code:'CANARY_EVIDENCE_REQUIRED'});await expect(s.evolution.command(owner,s.candidate.id,'start_shadow')).rejects.toMatchObject({code:'EVALUATION_REQUIRED'});await expect(s.evolution.observe(evaluator,s.candidate.id,{shadowVerified:true,completedRuns:100,operationId:'forged-count'})).rejects.toThrow();});
it('requires observed runs from every stage including 100%, deduplicates observations and activates/rolls back only future runs',async()=>{
const s=await setup();let candidate=await shadow(s);
for(let stage=0;stage<4;stage++){
const ids=Array.from({length:30},(_,i)=>`stage-${stage}-run-${i}`);
await s.db.transaction('tenant',async tx=>{for(const id of ids)await tx.put('run',{id,tenantId:'tenant',version:1,createdAt:s.time(),updatedAt:s.time(),bundleVersion:candidateDigest,evolutionCandidateId:s.candidate.id,status:'succeeded'} as Run);});
const o=observation(s,candidate.canaryStartedAt!,ids);await s.evolution.observe(evaluator,s.candidate.id,o);expect((await s.evolution.observe(evaluator,s.candidate.id,o)).canaryCompletedRuns).toBe(30);
await expect(s.evolution.command(owner,s.candidate.id,'advance')).rejects.toMatchObject({code:'CANARY_EVIDENCE_REQUIRED'});s.advanceClock();candidate=await s.evolution.command(owner,s.candidate.id,'advance');
if(stage<3)expect(candidate.status).toBe('canary');
}
expect(candidate.status).toBe('promoted');const choose=()=>s.db.transaction('tenant',tx=>selectBundle(tx,{tenantId:'tenant',projectId:'project',runId:'future'}));expect((await choose()).version).toBe(candidateDigest);await s.evolution.command(owner,s.candidate.id,'rollback');expect((await choose()).version).toBe('baseline-v1');expect(await s.db.transaction('tenant',tx=>resolveBundle(tx,candidateDigest))).toEqual(config);
});
it('rolls back allocation immediately on an independent hard gate failure',async()=>{const s=await setup();const c=await shadow(s);expect((await s.evolution.observe(evaluator,c.id,observation(s,c.canaryStartedAt!,[],{hardGateFailures:1}))).status).toBe('rolled_back');});
it('refuses platform-code rollout without independent release evidence and refuses trust-root approval',async()=>{
const s=await setup();const codeDigest=contentDigest('platform-code');
const c=await s.evolution.create(owner,{name:'Platform code',scope:'platform',parentBundle:'baseline-v1',candidateDigest:codeDigest,hypothesis:'Improve cost',targetMetric:'cost'});
const {signature,...payload}=await report(c.id);await s.evolution.evaluate(evaluator,c.id,signRecord({...payload,candidateDigest:codeDigest},privateKeyPem));
await s.evolution.observe(evaluator,c.id,signRecord({schemaVersion:1,kind:'shadow',tenantId:'tenant',candidateId:c.id,candidateDigest:codeDigest,keyId:'eval',issuedAt:s.time(),operationId:randomUUID(),productionWrites:0,isolated:true,completedTasks:50,hardGateFailures:0,samplesDigest:contentDigest('external-shadow')},privateKeyPem));
await s.evolution.command(owner,c.id,'approve');await expect(s.evolution.command(owner,c.id,'start_canary')).rejects.toMatchObject({code:'EXTERNAL_RELEASE_REQUIRED'});
const trust=await s.evolution.create(owner,{name:'Trust root',scope:'trust_root',parentBundle:'baseline-v1',candidateDigest:contentDigest('root'),hypothesis:'Protected',targetMetric:'cost'});
await expect(s.evolution.command(owner,trust.id,'approve')).rejects.toMatchObject({code:'EXTERNAL_TRUST_ROOT_RELEASE_REQUIRED'});
});
it('rejects hidden replacement of the built-in baseline while keeping its alias',async()=>{
await expect(evaluateOffline({tenantId:'tenant',candidateId:'x',candidateDigest,baselineDigest:'baseline-v1',baseline:{schemaVersion:1,prompts:{builder:'Different baseline'}},candidate:config,targetMetric:'cost',benchmark:benchmark(),runner:runner(),privateKeyPem,keyId:'eval'})).rejects.toThrow('versions');
});
it('applies bundle instructions while retaining independent acceptance and deployment boundaries',()=>{const prompt=bundlePrompt(config,{goal:'Build app',acceptanceCriteria:['Works'],repairCount:0});expect(prompt).toContain(config.prompts.builder);expect(prompt).toContain('separate verifier');expect(prompt).toContain('deployment policies');});
});