wishforge-ide/tests/control-hardening.test.ts
eust-w 13ceca5c8f
Some checks failed
CI / Format, types and tests (pull_request) Has been cancelled
CI / PostgreSQL migrations and integration tests (pull_request) Has been cancelled
🔀 Merge review/overhaul into feat/wiki-copilot-diagnostics
Brings the branch up to the current base (executor settings, project
skills, Chinese run messages, cloud IDE removal, wish attachments) and
resolves the ten conflicted files, keeping both sides:

- wish-refiner: the tenant's executor settings pick the models first,
  then the Copilot model setting; the tenant credential still goes
  before the operator's login
- worker providers: executor settings apply, and a run with its own
  credential still gets an empty home
- main.ts: Copilot screenshot uploads and wish attachments each keep
  their own body limit
- contracts, http, web types: default budget and skill auto-activation,
  Copilot and attachment entity kinds, ./text and ./attachments exports
- globals.css: Copilot styles and attachment styles, one after the other
- tests: the canceled-release reason is Chinese; the refiner's default
  models have no [1m] suffix, as this branch names them

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-12 00:39:05 +08:00

505 lines
24 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { generateKeyPairSync } from 'node:crypto';
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import type {
Actor,
DeliveryConfig,
EntityKind,
Entity,
Evidence,
Project,
Run,
RunnerTask,
Environment,
Release,
Approval,
} from '@wishforge/contracts';
import { ControlPlane } from '../apps/api/src/domain/control-plane.js';
import { Connections } from '../apps/api/src/domain/connections.js';
import { Releases } from '../apps/api/src/domain/releases.js';
import { RunnerBroker } from '../apps/api/src/domain/runner-broker.js';
import { digest } from '../apps/api/src/domain/policy.js';
import { verifyRunnerTask, type SignedRunnerTask } from '@wishforge/runtime';
import { MemoryStore } from './helpers/memory-store.js';
import { deliverable } from './helpers/delivery.js';
const owner: Actor = { tenantId: 'alpha', userId: 'owner', role: 'owner', kind: 'human' };
const worker: Actor = { ...owner, userId: 'worker', kind: 'worker' };
const verifier: Actor = { ...owner, userId: 'verifier', kind: 'verifier' };
const deployer: Actor = { ...owner, userId: 'deployer', kind: 'deployer' };
const outsider: Actor = { ...owner, userId: 'outsider', role: 'developer' };
let db: MemoryStore,
control: ControlPlane,
connections: Connections,
releases: Releases,
broker: RunnerBroker,
temp: string;
beforeAll(async () => {
temp = await mkdtemp(path.join(os.tmpdir(), 'wishforge-broker-'));
const key = generateKeyPairSync('ed25519').privateKey.export({ type: 'pkcs8', format: 'pem' });
await writeFile(path.join(temp, 'signer.pem'), key, { mode: 0o600 });
});
afterAll(async () => {
await rm(temp, { recursive: true, force: true });
});
beforeEach(async () => {
db = new MemoryStore();
db.seed('alpha');
db.seed('beta');
const env = {
NODE_ENV: 'test',
WF_VAULT_KEY: 'c'.repeat(64),
WF_RUNNER_SIGNING_KEY_FILE: path.join(temp, 'signer.pem'),
};
control = new ControlPlane(db, env);
connections = new Connections(db, env);
releases = new Releases(db);
broker = new RunnerBroker(db, releases, env);
await connections.create(owner, { provider: 'codex', label: 'test', apiKey: 'fixture-key-only' });
});
async function edit<T extends Entity>(kind: EntityKind, id: string, change: (value: T) => void) {
return db.transaction('alpha', async tx => {
const v = (await tx.get<T>(kind, id))!;
change(v);
await tx.put(kind, v);
});
}
async function task(name = 'Catalog') {
const project = await control.createProject(owner, {
name,
repositoryUrl: 'https://example.org/catalog.git',
verificationCommands: [['node', '--test']],
});
await edit<Project>('project', project.id, p => {
p.deliveryConfig = {
containerPort: 3000,
healthPath: '/health',
smokeChecks: [{ path: '/catalog', expectedStatus: 200 }],
} as DeliveryConfig;
});
const submitted = await control.submit(
owner,
project.id,
{
goal: 'Implement catalog search',
acceptanceCriteria: ['Search returns matching items'],
budget: { maxTokens: 1000, maxMinutes: 1 },
},
`submit-${project.id}`,
);
return { project, runId: submitted.run_id };
}
async function evidenceFor(runId: string, letter = 'a') {
const { verification: v, build: b } = await deliverable(control, 'alpha', runId, { commit: letter.repeat(40) });
return { v, b };
}
async function prepared(name = 'Catalog') {
const { project, runId } = await task(name);
const { v, b } = await evidenceFor(runId);
const environment = await releases.createEnvironment(owner, {
projectId: project.id,
name: 'Production',
type: 'compose',
production: true,
});
const release = await releases.request(
owner,
{ runId, environmentId: environment.id, verificationEvidenceId: v.id, buildEvidenceId: b.id },
`release-${runId}`,
);
await releases.decide(owner, release.approvalId!, { decision: 'approve', bindingDigest: release.bindingDigest });
const registration = await broker.register(owner, environment.id);
const runner: Actor = { ...deployer, userId: `runner:${environment.id}`, clientId: registration.id };
return { project, runId, release, environment, runner, registration };
}
function verified(envelope: any) {
return {
result: {
releaseId: envelope.payload.release.id,
status: 'verified',
target: envelope.payload.target,
evidence: { identityMatched: true, trafficSwitched: true, checks: ['health', 'business-smoke'] },
},
};
}
describe('execution identity and quota enforcement', () => {
it('binds leases to authenticated principal, kind and run scope', async () => {
const { runId } = await task();
const leased = await control.lease(worker, runId, 'same-label');
const token = leased.lease!.token;
await expect(control.lease(verifier, runId, 'same-label')).rejects.toMatchObject({ code: 'LEASE_HELD' });
await expect(control.heartbeat(verifier, runId, token)).rejects.toMatchObject({ code: 'LEASE_IDENTITY_MISMATCH' });
await expect(control.checkpoint(owner, runId, {}, token)).rejects.toMatchObject({ code: 'FORBIDDEN' });
await expect(control.heartbeat({ ...worker, runId: 'different-run' }, runId, token)).rejects.toMatchObject({
code: 'FORBIDDEN',
});
await expect(connections.forRun({ ...worker, userId: 'another-worker' }, leased)).resolves.toBeUndefined();
});
it('gives a development run the operator’s gateway and the run’s model with the tenant key', async () => {
const { runId } = await task();
await edit<Run>('run', runId, run => {
run.providerModel = 'gateway-model';
});
const leased = await control.lease(worker, runId, 'executor');
const viaGateway = new Connections(db, {
NODE_ENV: 'test',
WF_VAULT_KEY: 'c'.repeat(64),
WF_OPENAI_BASE_URL: 'https://gateway.example/v1',
});
expect(await viaGateway.forRun(worker, leased)).toMatchObject({
apiKey: 'fixture-key-only',
baseUrl: 'https://gateway.example/v1',
model: 'gateway-model',
});
expect(await connections.forRun(worker, leased)).not.toHaveProperty('baseUrl');
});
it('does not release a credential from an expired or replaced run snapshot', async () => {
const { runId } = await task();
const leased = await control.lease(worker, runId, 'executor');
expect(await connections.forRun(worker, leased)).toMatchObject({ apiKey: 'fixture-key-only' });
await control.command(owner, runId, { action: 'pause' }, 'pause-credential');
expect(await connections.forRun(worker, leased)).toBeUndefined();
});
it('requires live verifier fencing for evidence and rejects stale same-revision results', async () => {
const { runId } = await task();
const leased = await control.lease(verifier, runId, 'executor');
const input = {
runId,
leaseToken: leased.lease!.token,
revision: 1,
sourceCommit: 'a'.repeat(40),
type: 'verification',
passed: true,
isolation: 'sandbox',
commands: [{ argv: ['node', '--test'], exitCode: 0, outputDigest: digest('passed') }],
};
await edit<Run>('run', runId, run => {
run.lease!.expiresAt = new Date(Date.now() - 1).toISOString();
});
await expect(control.evidence(verifier, input)).rejects.toMatchObject({ code: 'STALE_LEASE' });
const next = await control.lease(verifier, runId, 'new-executor');
expect(next.lease!.token).toBeGreaterThan(leased.lease!.token);
await expect(control.evidence(verifier, input)).rejects.toMatchObject({ code: 'STALE_LEASE' });
});
it('does not reset a running lease or enqueue new work on repeated resume', async () => {
const { runId } = await task();
const leased = await control.lease(worker, runId, 'executor');
const before = (await db.transaction('alpha', tx => tx.outbox())).length;
for (const key of ['resume-1', 'resume-2'])
expect((await control.command(owner, runId, { action: 'resume' }, key)).lease).toEqual(leased.lease);
expect((await db.transaction('alpha', tx => tx.outbox())).length).toBe(before);
});
it('deduplicates final usage after lease expiry and cannot resume exhausted tokens', async () => {
const { runId } = await task();
const leased = await control.lease(worker, runId, 'executor');
const measurement = {
operationId: 'provider-sequence-1',
tokens: 1000,
computeSeconds: 2,
leaseToken: leased.lease!.token,
};
await control.meter(worker, runId, measurement);
await control.meter(worker, runId, measurement);
expect(await control.usage(owner)).toMatchObject({ tokens: 1000, reservedTokens: 0 });
await expect(control.command(owner, runId, { action: 'resume' }, 'resume-budget')).rejects.toMatchObject({
code: 'TOKEN_BUDGET_EXHAUSTED',
});
});
it('stops a late stage transition at the active-time limit', async () => {
const { runId } = await task();
const leased = await control.lease(worker, runId, 'executor');
await edit<Run>('run', runId, r => {
r.activeMilliseconds = 59_999;
r.lastActiveAt = new Date(Date.now() - 100).toISOString();
});
expect(
await control.transition(worker, runId, {
leaseToken: leased.lease!.token,
stage: 'plan',
status: 'running',
message: 'late',
}),
).toMatchObject({ status: 'waiting', waitingReason: 'budget', stage: 'clarify' });
});
it('rejects short raw idempotency keys even after key namespacing', async () => {
const { project } = await task();
await expect(
control.submit(owner, project.id, { goal: 'Search', acceptanceCriteria: ['Search succeeds'] }, 'x'),
).rejects.toMatchObject({ code: 'IDEMPOTENCY_REQUIRED' });
});
});
describe('exact release bindings and project ACLs', () => {
it('filters environment and release lists by project membership', async () => {
const { release, environment } = await prepared();
expect(await releases.list(outsider)).toEqual([]);
expect(await releases.environments(outsider)).toEqual([]);
expect((await releases.list(owner))[0].id).toBe(release.id);
expect((await releases.environments(owner))[0].id).toBe(environment.id);
});
it.each(['secret', 'architecture', 'revision', 'delivery-config', 'verification-policy'] as const)(
'rejects %s drift after human approval',
async change => {
const { environment, runId, project, runner, release } = await prepared();
if (change === 'secret')
await edit<Environment>('environment', environment.id, e => {
e.secretVersionRefs = ['vault:new-version'];
});
if (change === 'architecture')
await edit<Environment>('environment', environment.id, e => {
e.architecture = 'linux/arm64';
});
if (change === 'revision')
await edit<Run>('run', runId, r => {
r.revision++;
});
if (change === 'delivery-config')
await edit<Project>('project', project.id, p => {
p.deliveryConfig = { ...p.deliveryConfig!, containerPort: 9000 };
});
if (change === 'verification-policy')
await edit<Project>('project', project.id, p => {
p.verificationCommands = [['node', '--test', 'new-policy']];
});
// Nothing deploys; the stale release fails instead of holding the environment for every later one.
expect(await broker.poll(runner, environment.id)).toBeNull();
expect(await db.transaction('alpha', tx => tx.get<Release>('release', release.id))).toMatchObject({
status: 'failed',
failureReason: expect.stringContaining('请申请新的发布'),
});
},
);
it('invalidates an already approved release when requirements are revised', async () => {
const { release, runId } = await prepared();
await control.command(owner, runId, { action: 'answer', message: 'Search must ignore case' }, 'revise-approved');
expect(await db.transaction('alpha', tx => tx.get<Approval>('approval', release.approvalId!))).toMatchObject({
status: 'invalidated',
});
});
});
describe('signed runner receipts and real rollback state machine', () => {
it('signs exact immutable release, commits acceptance once, and rejects conflicting duplicate receipts', async () => {
const { release, environment, runner, registration, runId } = await prepared();
const envelope = (await broker.poll(runner, environment.id))!;
verifyRunnerTask(envelope as SignedRunnerTask, {
tenantId: 'alpha',
environmentId: environment.id,
publicKeys: registration.publicKeys,
});
expect(envelope.payload.release).toMatchObject({ id: release.id, bindingDigest: release.bindingDigest });
const receipt = verified(envelope);
const results = await Promise.all([
broker.result(runner, envelope.payload.id, receipt),
broker.result(runner, envelope.payload.id, receipt),
]);
expect(results.every(r => r.delivered)).toBe(true);
expect((await control.run(owner, runId)).status).toBe('succeeded');
expect(
(await db.transaction('alpha', tx => tx.list<Evidence>('evidence'))).filter(e => e.type === 'deployment'),
).toHaveLength(1);
await expect(
broker.result(runner, envelope.payload.id, { result: { ...receipt.result, status: 'failed' } }),
).rejects.toMatchObject({ code: 'RECEIPT_CONFLICT' });
});
it('rejects wrong target, rotated identity, expired task and superseded receipts', async () => {
const { environment, runner } = await prepared();
const envelope = (await broker.poll(runner, environment.id))!;
await expect(
broker.result(runner, envelope.payload.id, { result: { ...verified(envelope).result, target: 'kubernetes' } }),
).rejects.toMatchObject({ code: 'RESULT_MISMATCH' });
await edit<RunnerTask>('runner_task', envelope.payload.id, t => {
(t.envelope as any).payload.expiresAt = new Date(Date.now() - 1).toISOString();
});
await expect(broker.result(runner, envelope.payload.id, verified(envelope))).rejects.toMatchObject({
code: 'STALE_RUNNER_TASK',
});
const reconciliation = (await broker.poll(runner, environment.id))!;
expect(reconciliation.payload.action).toBe('reconcile');
await expect(broker.result(runner, envelope.payload.id, verified(envelope))).rejects.toMatchObject({
code: 'STALE_RUNNER_TASK',
});
await broker.register(owner, environment.id);
await expect(broker.result(runner, reconciliation.payload.id, verified(reconciliation))).rejects.toMatchObject({
code: 'RUNNER_REVOKED',
});
});
it('does not acknowledge a verified claim that lacks business or traffic-switch evidence as delivered', async () => {
const { environment, runner, runId } = await prepared();
const envelope = (await broker.poll(runner, environment.id))!;
const receipt = verified(envelope);
receipt.result.evidence.checks = ['health'];
expect(await broker.result(runner, envelope.payload.id, receipt)).toMatchObject({ delivered: false });
expect((await control.run(owner, runId)).status).not.toBe('succeeded');
expect((await broker.poll(runner, environment.id))!.payload.action).toBe('reconcile');
});
it('frees the environment after a definite failure, and lets an admin unlock an unknown one (FR-G4)', async () => {
const { environment, runner, release } = await prepared();
const envelope = (await broker.poll(runner, environment.id))!;
const failure = {
result: {
...verified(envelope).result,
status: 'failed',
evidence: { reason: 'Candidate startup failed; stable proxy and previous version were retained' },
},
};
expect(await broker.result(runner, envelope.payload.id, failure)).toMatchObject({ delivered: false });
const failed = await db.transaction('alpha', tx => tx.get<Release>('release', release.id));
expect(failed).toMatchObject({ status: 'failed', failureReason: expect.stringContaining('previous version') });
expect(
(await db.transaction('alpha', tx => tx.get<Environment>('environment', environment.id)))?.activeReleaseId,
).toBeUndefined();
// An outcome the runner cannot confirm keeps the lock until an admin checked the target.
const second = await prepared('Catalog second');
const pending = (await broker.poll(second.runner, second.environment.id))!;
const unknown = verified(pending);
unknown.result.status = 'applied';
await broker.result(second.runner, pending.payload.id, unknown);
await expect(
releases.unlock({ ...owner, kind: 'connector' }, second.environment.id, { releaseId: second.release.id }),
).rejects.toMatchObject({ status: 403 });
expect(await releases.unlock(owner, second.environment.id, { releaseId: second.release.id })).toMatchObject({
status: 'failed',
});
expect(
(await db.transaction('alpha', tx => tx.get<Environment>('environment', second.environment.id)))?.activeReleaseId,
).toBeUndefined();
await expect(releases.unlock(owner, second.environment.id, { releaseId: second.release.id })).rejects.toMatchObject(
{
code: 'ENVIRONMENT_NOT_LOCKED',
},
);
});
it('deploys past ready releases that went stale, failing them, and keeps a paused run’s release waiting', async () => {
const { project, environment, runner, release: oldest } = await prepared();
const another = async (goal: string, letter: string) => {
const submitted = await control.submit(
owner,
project.id,
{ goal, acceptanceCriteria: ['It works'] },
`submit-${letter}-${project.id}`,
);
const { v, b } = await evidenceFor(submitted.run_id, letter);
const release = await releases.request(
owner,
{ runId: submitted.run_id, environmentId: environment.id, verificationEvidenceId: v.id, buildEvidenceId: b.id },
`release-${letter}-${project.id}`,
);
await releases.decide(owner, release.approvalId!, { decision: 'approve', bindingDigest: release.bindingDigest });
return { runId: submitted.run_id, release };
};
const canceled = await another('Add sorting', 'b');
const paused = await another('Add paging', 'c');
await control.command(owner, canceled.runId, { action: 'cancel' }, 'cancel-stale-release');
await control.command(owner, paused.runId, { action: 'pause' }, 'pause-ready-release');
const envelope = (await broker.poll(runner, environment.id))!;
expect(envelope.payload.release.id).toBe(oldest.id);
const status = async (id: string) => (await db.transaction('alpha', tx => tx.get<Release>('release', id)))!;
// Canceled runs fail their pending release outright (see run-lifecycle's invalidation case),
// so the stale release no longer waits on the queue with a "request a new release" reason.
expect(await status(canceled.release.id)).toMatchObject({
status: 'failed',
failureReason: '运行已取消',
});
// Continuing the paused run may make its release current again.
expect((await status(paused.release.id)).status).toBe('ready');
});
it('frees the environment when a rollback definitely failed, and lets an admin unlock an unknown one', async () => {
const { project, environment, runner } = await prepared();
const first = (await broker.poll(runner, environment.id))!;
await broker.result(runner, first.payload.id, verified(first));
const secondRun = await control.submit(
owner,
project.id,
{ goal: 'Add sorting to the catalog', acceptanceCriteria: ['Rows sort by name'] },
'submit-rollback-failure',
);
const { v, b } = await evidenceFor(secondRun.run_id, 'b');
const second = await releases.request(
owner,
{ runId: secondRun.run_id, environmentId: environment.id, verificationEvidenceId: v.id, buildEvidenceId: b.id },
'release-rollback-failure',
);
await releases.decide(owner, second.approvalId!, { decision: 'approve', bindingDigest: second.bindingDigest });
const deployed = (await broker.poll(runner, environment.id))!;
await broker.result(runner, deployed.payload.id, verified(deployed));
const environmentNow = async () =>
(await db.transaction('alpha', tx => tx.get<Environment>('environment', environment.id)))!;
const operationNow = async (id: string) =>
(await db.transaction('alpha', tx => tx.get<RunnerTask>('runner_task', id)))!;
const failing = await releases.rollback(owner, second.id, 'rollback-attempt-1');
const attempt = (await broker.poll(runner, environment.id))!;
const failed = verified(attempt);
failed.result.status = 'failed';
(failed.result.evidence as Record<string, unknown>).reason = 'Previous image failed its health check; current kept';
await broker.result(runner, attempt.payload.id, failed);
expect((await operationNow(failing.id)).status).toBe('failed');
expect(await environmentNow()).toMatchObject({ currentReleaseId: second.id, activeReleaseId: undefined });
// The environment is free to try again; an outcome the runner cannot confirm keeps it until an admin checked.
const retried = await releases.rollback(owner, second.id, 'rollback-attempt-2');
const again = (await broker.poll(runner, environment.id))!;
const unknown = verified(again);
unknown.result.status = 'applied';
await broker.result(runner, again.payload.id, unknown);
expect((await operationNow(retried.id)).status).toBe('unknown');
await expect(releases.rollback(owner, second.id, 'rollback-attempt-3')).rejects.toMatchObject({
code: 'ENVIRONMENT_CONFLICT',
});
await releases.unlock(owner, environment.id, { releaseId: first.payload.release.id });
expect((await operationNow(retried.id)).status).toBe('canceled');
expect((await environmentNow()).activeReleaseId).toBeUndefined();
// The release the rollback targeted was live before and stays so in the record.
expect((await db.transaction('alpha', tx => tx.get<Release>('release', first.payload.release.id)))?.status).toBe(
'healthy',
);
});
it('executes rollback as a signed deployment and advances state only after target acceptance', async () => {
const { project, release: first, environment, runner } = await prepared();
const original = (await broker.poll(runner, environment.id))!;
await broker.result(runner, original.payload.id, verified(original));
const secondRun = await control.submit(
owner,
project.id,
{ goal: 'Add sorting to the catalog', acceptanceCriteria: ['Rows sort by name'] },
'submit-second-version',
);
const { v, b } = await evidenceFor(secondRun.run_id, 'b');
const second = await releases.request(
owner,
{ runId: secondRun.run_id, environmentId: environment.id, verificationEvidenceId: v.id, buildEvidenceId: b.id },
'release-second',
);
await releases.decide(owner, second.approvalId!, { decision: 'approve', bindingDigest: second.bindingDigest });
const deployed = (await broker.poll(runner, environment.id))!;
await broker.result(runner, deployed.payload.id, verified(deployed));
const operation = await releases.rollback(owner, second.id, 'rollback-second');
expect(
(await db.transaction('alpha', tx => tx.get<Environment>('environment', environment.id)))?.currentReleaseId,
).toBe(second.id);
const rollback = (await broker.poll(runner, environment.id))!;
expect(rollback.payload.action).toBe('deploy');
expect(rollback.payload.release).toMatchObject({
id: first.id,
expectedPreviousReleaseId: second.id,
image: first.imageDigest,
});
expect(await broker.result(runner, rollback.payload.id, verified(rollback))).toMatchObject({
rolledBack: true,
delivered: false,
});
expect(
(await db.transaction('alpha', tx => tx.get<Environment>('environment', environment.id)))?.currentReleaseId,
).toBe(first.id);
expect((await db.transaction('alpha', tx => tx.get<Release>('release', second.id)))?.status).toBe('rolled_back');
expect((await db.transaction('alpha', tx => tx.get<RunnerTask>('runner_task', operation.id)))?.status).toBe(
'completed',
);
});
});