wishforge-ide/tests/local-loop.test.ts
eust-w ad06e82c67 ⚡ Reuse the verification of an integrated commit when its build continues
A build that stopped on an infrastructure fault, like the drops of the
POC's link to its cluster, repeated the whole protected verification
when continued: installation and every check, about two and a half
minutes on the POC, for a commit whose verdict could not change.

When the integrated commit is the verified checkpoint itself, the
verifier now reuses that verification evidence and only restores the
workspace for the build; the timeline says so. The control plane
offers only test and build evidence recorded under the current
protected policy, as a release requires: after the operator changes
the checks, the run verifies again. That also keeps an executor that
continues at the release stage from requesting a release with
evidence the release would refuse.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 14:54:01 +08:00

916 lines
46 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* The main loop end to end with real Git: wish → executor → independent verification
* → repair with failure feedback → integration → success, plus lessons and merge
* conflicts, and a change pushed from the IDE workspace. The control plane runs in process
* behind its real internal routes; only the agent is scripted.
*/
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { execFile } from 'node:child_process';
import { existsSync } from 'node:fs';
import { createRequire } from 'node:module';
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { promisify } from 'node:util';
import type { Actor, CheckpointRecord, Evidence, Lesson, Project, Run } from '@wishforge/contracts';
import { executeCommand, type CommandExecutor, type HostCheckSandbox, type ProviderStart } from '@wishforge/runtime';
import { ManagedProvider } from '../packages/runtime/src/providers/base.js';
import { ControlPlane } from '../apps/api/src/domain/control-plane.js';
import { Services } from '../apps/api/src/http.js';
import { ServiceTokens } from '../apps/api/src/service-tokens.js';
import { recordAllocation, reserveExecutionGrant } from '../apps/api/src/domain/entitlements.js';
import { installHooks } from '../apps/git-service/src/service.js';
import { createActivities, type ActivityOutcome, type RunInput } from '../apps/worker/src/activities.js';
import { MemoryStore } from './helpers/memory-store.js';
import { internalFetch } from './helpers/internal-fetch.js';
const run = promisify(execFile);
/** The IDE workspace backend that clones the project and keeps its credential (apps/ide/policy). */
const { createGitWorkspace } = createRequire(import.meta.url)('../apps/ide/policy/git-workspace.cjs') as {
createGitWorkspace: (config: { root: string; credentialDir: string; remote: string }) => {
apply: (delivery: Record<string, unknown>) => Promise<{ directory: string; cloning?: Promise<void> }>;
};
};
const TENANT = 'demo';
const REMOTE_URL = 'http://127.0.0.1:65530/demo.git';
const owner: Actor = { tenantId: TENANT, userId: 'owner', role: 'owner', kind: 'human' };
const env = {
NODE_ENV: 'test',
WF_DEPLOYMENT_MODE: 'development',
WF_DISPATCH_TENANTS: TENANT,
WF_INTERNAL_TOKEN_KEY: 'internal-token-signing-key-for-tests-only',
WF_WORKER_TOKEN: 'coordinator-static-token-for-tests-only',
WF_DEPLOYER_TOKEN: 'deployer-static-token-for-tests-only',
WF_RESOURCE_GRANT_SECRET: 'resource-grant-secret-for-tests-only-0000',
WF_RUN_WORKER_IMAGE: `registry.local/wishforge-run@sha256:${'a'.repeat(64)}`,
WF_GIT_ALLOW_LOCAL_HTTP: 'true',
};
const git = (cwd: string, ...args: string[]) =>
run('git', ['-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', ...args], { cwd }).then(r =>
r.stdout.trim(),
);
/** `stopped` aborts when the executor cancels the attempt, as it does on a pause. */
type Agent = (input: ProviderStart, stopped?: AbortSignal) => Promise<string>;
/** Stands in for Codex: edits the workspace through a script and reports like an adapter. */
class ScriptedAgent extends ManagedProvider {
constructor(
private readonly agent: Agent,
private readonly usage: Record<string, unknown> = { input_tokens: 100, output_tokens: 20 },
/** What a stopped session reports, as the Claude Code adapter estimates it. */
private readonly stoppedUsage?: Record<string, unknown>,
) {
super();
}
capabilities() {
return {
provider: 'codex',
availability: 'available' as const,
coding: true,
streaming: true,
cancellation: true,
portableCheckpoint: true,
};
}
async start(input: ProviderStart) {
const attempt = this.create(input);
void (async () => {
try {
const summary = await this.agent(input, attempt.controller.signal);
if (attempt.controller.signal.aborted) {
if (this.stoppedUsage) this.emit(attempt, 'usage', this.stoppedUsage);
return this.finish(attempt, 'canceled');
}
this.emit(attempt, 'command', { command: 'node edit.mjs', exitCode: 0 });
this.emit(attempt, 'file_change', { changes: [{ path: 'greeting.txt', kind: 'update' }] });
this.emit(attempt, 'output', { text: summary });
this.emit(attempt, 'usage', this.usage);
this.finish(attempt, 'completed');
} catch (error) {
this.finish(attempt, 'failed', error instanceof Error ? error.message : String(error));
}
})();
return { ...attempt.public };
}
}
let root: string;
let remote: string;
let store: MemoryStore;
let services: Services;
let prompts: string[];
beforeEach(async () => {
root = await mkdtemp(path.join(os.tmpdir(), 'wishforge-loop-'));
remote = path.join(root, 'remote.git');
await git(root, 'init', '--bare', '--initial-branch=main', remote);
const seed = path.join(root, 'seed');
await git(root, 'clone', remote, seed);
await writeFile(path.join(seed, 'greeting.txt'), 'hello\n');
await writeFile(
path.join(seed, 'check.mjs'),
[
"import { readFileSync } from 'node:fs';",
"const text = readFileSync('greeting.txt', 'utf8').trim();",
'if (!text.startsWith(\'hello world\')) { console.error(`expected "hello world", got "${text}"`); process.exit(1); }',
"console.log('greeting ok');",
].join('\n'),
);
await git(seed, 'add', '.');
await git(seed, 'commit', '-m', 'Initial project');
await git(seed, 'push', 'origin', 'main');
store = new MemoryStore();
store.seed(TENANT);
services = new Services(new ControlPlane(store, env));
prompts = [];
});
afterEach(async () => {
await rm(root, { recursive: true, force: true });
});
/**
* git clones of the loopback URL are served from the bare repository in the temporary directory;
* image builds are answered like a registry push, so no Docker is needed.
*/
let failingBuilds = 0;
/** How often the project's protected check actually ran. */
let checkRuns = 0;
const execute: CommandExecutor = async command => {
if (command.args.includes('check.mjs')) checkRuns++;
if (command.executable === 'docker' && failingBuilds > 0) {
failingBuilds--;
return { exitCode: 1, stdout: '', stderr: 'The image builder is unavailable', durationMs: 1 };
}
if (command.executable === 'docker') {
const metadata = command.args[command.args.indexOf('--metadata-file') + 1]!;
await writeFile(metadata, JSON.stringify({ 'containerimage.digest': `sha256:${'c'.repeat(64)}` }));
return { exitCode: 0, stdout: '', stderr: '', durationMs: 1 };
}
return executeCommand(
command.executable === 'git'
? { ...command, args: command.args.map(arg => (arg === REMOTE_URL ? remote : arg)) }
: command,
);
};
async function createProject() {
const project = await services.control.createProject(owner, {
name: 'Greeting',
repositoryUrl: REMOTE_URL,
verificationCommands: [[process.execPath, 'check.mjs']],
});
await store.transaction(TENANT, tx =>
tx.put('provider_connection', {
id: 'codex-connection',
tenantId: TENANT,
version: 1,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
provider: 'codex',
status: 'active',
}),
);
return project;
}
async function submit(projectId: string, goal: string, key: string, extra: Record<string, unknown> = {}) {
const { run_id } = await services.control.submit(
owner,
projectId,
{ goal, acceptanceCriteria: ['check.mjs passes'], provider: 'codex', budget: { maxRepairRounds: 2 }, ...extra },
key,
);
return run_id;
}
/** What the dispatcher and the local provisioner do: reserve resources and issue per-generation identities. */
async function provision(
runId: string,
agent: Agent,
settings: {
installProxy?: string;
usage?: Record<string, unknown>;
stoppedUsage?: Record<string, unknown>;
fetcher?: typeof fetch;
heartbeatMs?: number;
execute?: CommandExecutor;
hostCheckSandbox?: HostCheckSandbox;
} = {},
) {
const generation = (await services.control.run(owner, runId)).pendingStartId!;
await store.transaction(TENANT, tx => reserveExecutionGrant(tx, runId, generation, env));
const tokens = new ServiceTokens(env.WF_INTERNAL_TOKEN_KEY);
const input: RunInput = { tenantId: TENANT, runId, generation };
const shared = {
controlPlaneUrl: 'http://127.0.0.1:4000',
workspaceRoot: path.join(root, 'workspaces', generation),
isolatedProviderHomeRoot: path.join(root, 'homes'),
allowLocalGitHttp: true,
fetcher: internalFetch(services),
execute,
...settings,
};
const sandbox = {
sandboxMode: 'host' as const,
sandboxRunId: runId,
sandboxTenantId: TENANT,
sandboxGeneration: generation,
};
const executor = createActivities({
...shared,
...sandbox,
workerId: `executor-${generation}`,
workerToken: await tokens.issue(TENANT, runId, 'worker', generation),
providerFactory: () =>
new ScriptedAgent(
async (start, stopped) => {
prompts.push(start.prompt);
return agent(start, stopped);
},
settings.usage,
settings.stoppedUsage,
),
});
const verifier = createActivities({
...shared,
...sandbox,
workerId: `verifier-${generation}`,
verifierToken: await tokens.issue(TENANT, runId, 'verifier', generation),
});
const coordinator = createActivities({ ...shared, workerId: 'coordinator', workerToken: env.WF_WORKER_TOKEN });
return { input, generation, executor, verifier, coordinator };
}
/** The steps of runWorkflow, without Temporal. */
async function drive(session: Awaited<ReturnType<typeof provision>>, until?: ActivityOutcome['status']) {
for (let step = 0; step < 10; step++) {
let outcome = await session.executor.executeRun(session.input);
if (outcome.status === 'verify') outcome = await session.verifier.verifyRun(session.input);
if (until && outcome.status === until) return outcome;
if (outcome.status === 'repair') outcome = await session.coordinator.prepareRepair(session.input);
if (outcome.status === 'repair') continue;
if (outcome.reason)
await session.coordinator.recordWaiting({ ...session.input, reason: outcome.reason, detail: outcome.detail });
return outcome;
}
throw new Error('The loop did not settle');
}
async function release(runId: string, generation: string) {
await store.transaction(TENANT, tx => recordAllocation(tx, runId, generation, 'released'));
}
const writeGreeting =
(text: string, summary: string): Agent =>
async start => {
await writeFile(path.join(start.workspace, 'greeting.txt'), `${text}\n`);
return summary;
};
async function evidenceOf(runId: string) {
return store.transaction(TENANT, async tx => (await tx.list<Evidence>('evidence')).filter(e => e.runId === runId));
}
describe('local main loop', () => {
it('persists a completed checkpoint before final usage exhausts budget and resumes without another model call', async () => {
const project = await createProject();
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-final-budget', {
budget: { maxTokens: 1000 },
});
let workspace = '';
const session = await provision(
runId,
async start => {
workspace = start.workspace;
return writeGreeting('hello world', 'Completed the greeting.')(start);
},
{ usage: { input_tokens: 100, output_tokens: 100, cache_read_input_tokens: 1300, final: true } },
);
expect(await session.executor.executeRun(session.input)).toEqual({ status: 'waiting' });
const waiting = await services.control.run(owner, runId);
expect(waiting).toMatchObject({ status: 'waiting', waitingReason: 'budget', usedTokens: 1500, stage: 'implement' });
expect(waiting.checkpointId).toBeTruthy();
const checkpoint = await store.transaction(TENANT, tx =>
tx.get<CheckpointRecord>('checkpoint', waiting.checkpointId!),
);
expect(checkpoint?.checkpoint).toMatchObject({
patch: expect.stringContaining('+hello world'),
providerCompletion: { status: 'completed', finalUsage: [{ tokens: 1500, operationId: expect.any(String) }] },
});
expect(existsSync(workspace)).toBe(false); // Removal is safe because the source bundle was confirmed durable.
await store.transaction(TENANT, async tx => {
const current = (await tx.get<Run>('run', runId))!;
current.budget.maxTokens = 5000;
await tx.put('run', current);
});
await services.control.command(owner, runId, { action: 'resume' }, 'resume-final-budget');
await release(runId, session.generation);
const resumed = await provision(runId, async () => {
throw new Error('Completed source must not call a model again');
});
expect(await resumed.executor.executeRun(resumed.input)).toEqual({ status: 'verify' });
expect((await services.control.run(owner, runId)).usedTokens).toBe(1500); // The original operation ID is idempotent.
expect(prompts).toHaveLength(1);
expect(await resumed.verifier.verifyRun(resumed.input)).toEqual({ status: 'complete' });
});
it('continues metering non-final usage immediately and retains uncheckpointed work when budget cancels the lease', async () => {
const project = await createProject();
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-stream-budget', {
budget: { maxTokens: 1000 },
});
let workspace = '';
const session = await provision(
runId,
async start => {
workspace = start.workspace;
return writeGreeting('hello world', 'Work before streaming budget exhaustion.')(start);
},
{ usage: { input_tokens: 1500, output_tokens: 0 } },
);
expect(await session.executor.executeRun(session.input)).toEqual({ status: 'waiting' });
const waiting = await services.control.run(owner, runId);
expect(waiting).toMatchObject({ waitingReason: 'budget', usedTokens: 1500 });
expect(waiting.checkpointId).toBeUndefined();
expect(await readFile(path.join(workspace, 'greeting.txt'), 'utf8')).toBe('hello world\n');
expect(JSON.parse(await readFile(`${workspace}.recovery.json`, 'utf8'))).toMatchObject({
runId,
workspace,
reason: expect.stringContaining('retained'),
});
});
it('retains completed source when checkpoint persistence fails', async () => {
const project = await createProject();
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-checkpoint-down');
const delegate = internalFetch(services);
let workspace = '';
const session = await provision(
runId,
async start => {
workspace = start.workspace;
return writeGreeting('hello world', 'Completed source retained.')(start);
},
{
usage: { input_tokens: 100, output_tokens: 20, final: true },
fetcher: async (url, init) =>
String(url).endsWith('/checkpoint')
? Response.json(
{ error: { code: 'ARTIFACT_STORAGE_UNAVAILABLE', message: 'Storage unavailable' } },
{ status: 503 },
)
: delegate(url, init),
},
);
await expect(session.executor.executeRun(session.input)).rejects.toMatchObject({
domainCode: 'ARTIFACT_STORAGE_UNAVAILABLE',
});
expect(await readFile(path.join(workspace, 'greeting.txt'), 'utf8')).toBe('hello world\n');
expect(JSON.parse(await readFile(`${workspace}.recovery.json`, 'utf8'))).toMatchObject({
checkpoint: { providerCompletion: { finalUsage: [{ tokens: 120, operationId: expect.any(String) }] } },
});
});
it('retains the workspace when cancellation revokes the lease before checkpoint upload', async () => {
const project = await createProject();
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-canceled-source');
let workspace = '';
const session = await provision(runId, async start => {
workspace = start.workspace;
await writeGreeting('hello world', 'Work before cancellation.')(start);
await services.control.command(owner, runId, { action: 'cancel' }, 'cancel-before-checkpoint');
return 'Work stopped.';
});
// Cancel also revokes the scoped credential, so the worker cannot read run state again.
await expect(session.executor.executeRun(session.input)).rejects.toMatchObject({ domainCode: 'UNAUTHENTICATED' });
expect((await services.control.run(owner, runId)).status).toBe('canceled');
expect(await readFile(path.join(workspace, 'greeting.txt'), 'utf8')).toBe('hello world\n');
expect(existsSync(`${workspace}.recovery.json`)).toBe(true);
});
it('repairs a failed check with the verifier output, integrates the change and learns from it', async () => {
const project = await createProject();
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-repair-0001');
let round = 0;
const session = await provision(runId, async start => {
round++;
if (round === 1) return writeGreeting('hello wrld', 'Updated the greeting.')(start);
return writeGreeting('hello world', 'Fixed the typo in greeting.txt so it reads "hello world".')(start);
});
expect(await drive(session)).toEqual({ status: 'complete' });
const done = await services.control.run(owner, runId);
expect(done).toMatchObject({ status: 'succeeded', stage: 'integrate', repairCount: 1 });
expect(done.integratedCommit).toBe(await git(remote, 'rev-parse', 'main'));
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
expect(done.reservedTokens).toBe(0);
// The repair prompt quoted the failing command and its output (FR-E4).
expect(prompts).toHaveLength(2);
expect(prompts[0]).not.toContain('Repair round');
expect(prompts[1]).toContain('Repair round 1');
expect(prompts[1]).toContain('check.mjs');
expect(prompts[1]).toContain('expected "hello world", got "hello wrld"');
// Evidence records each check with its output, and the integration (FR-E3, FR-F2).
const evidence = await evidenceOf(runId);
const failed = evidence.find(e => e.type === 'verification' && !e.passed)!;
expect(failed.isolation).toBe('host');
expect(failed.commands?.[0]).toMatchObject({ exitCode: 1, outputTail: expect.stringContaining('hello wrld') });
expect(evidence.filter(e => e.type === 'verification' && e.passed)).toHaveLength(1);
expect(evidence.find(e => e.type === 'integration')).toMatchObject({
passed: true,
sourceCommit: done.integratedCommit,
targetRef: 'refs/heads/main',
});
// Progress shows what the agent did (FR-D7).
const events = await services.control.events(owner, runId);
expect(events.map(e => e.type)).toEqual(
expect.arrayContaining(['provider.command', 'provider.file_change', 'run.repair']),
);
expect(events.find(e => e.type === 'provider.output')?.message).toContain('Updated the greeting');
expect(events.find(e => e.type === 'provider.command')?.message).toBe('$ node edit.mjs (exit 0)');
// The repair became a project lesson that the next run is told about (FR-I2).
const lessons = await services.lessons.list(owner, project.id);
expect(lessons).toHaveLength(1);
expect(lessons[0]).toMatchObject({ kind: 'repair', status: 'active', sourceRunId: runId });
expect(lessons[0]!.summary).toContain('check.mjs');
expect(lessons[0]!.summary).toContain('Fixed the typo');
await release(runId, session.generation);
const next = await submit(project.id, 'Make the greeting say hello world again', 'wish-lesson-0002');
const second = await provision(next, writeGreeting('hello world!', 'Added an exclamation mark.'));
expect(await drive(second)).toEqual({ status: 'complete' });
expect(prompts[2]).toContain('Lessons from earlier work on this project');
expect(prompts[2]).toContain(lessons[0]!.summary);
await release(next, second.generation);
// A disabled lesson is no longer injected.
await services.lessons.setStatus(owner, lessons[0]!.id, { status: 'disabled' });
const third = await submit(project.id, 'Keep the greeting', 'wish-lesson-0003');
const thirdSession = await provision(third, writeGreeting('hello world!!', 'More emphasis.'));
expect(await drive(thirdSession)).toEqual({ status: 'complete' });
expect(prompts[3]).not.toContain('Lessons from earlier work');
});
it('waits instead of repairing when the repair budget is used up, and continues on request', async () => {
const project = await createProject();
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-budget-0001');
let rounds = 0;
const session = await provision(runId, async start => {
rounds++;
return writeGreeting(rounds < 4 ? `hello wrld ${rounds}` : 'hello world', `Attempt ${rounds}.`)(start);
});
expect(await drive(session)).toEqual({ status: 'waiting' });
let waiting = await services.control.run(owner, runId);
expect(waiting).toMatchObject({ status: 'waiting', waitingReason: 'repair_exhausted', repairCount: 2 });
expect(waiting.waitingDetail).toContain('check.mjs');
expect(rounds).toBe(3);
// Continuing grants exactly one more round; a new generation runs it.
await services.control.command(owner, runId, { action: 'resume' }, 'resume-budget-0001');
waiting = await services.control.run(owner, runId);
expect(waiting).toMatchObject({ status: 'queued', stage: 'implement', repairCount: 3 });
await release(runId, session.generation);
const resumed = await provision(runId, async start => {
rounds++;
return writeGreeting('hello world', 'Finally fixed it.')(start);
});
expect(await drive(resumed)).toEqual({ status: 'complete' });
expect((await services.control.run(owner, runId)).status).toBe('succeeded');
});
it('turns a merge conflict with a concurrent run into a repair that merges the new target', async () => {
const project = await createProject();
const first = await submit(project.id, 'Greet the world', 'wish-conflict-0001');
const second = await submit(project.id, 'Greet everyone', 'wish-conflict-0002');
const a = await provision(first, writeGreeting('hello world from A', 'A changed the greeting.'));
let secondRounds = 0;
const b = await provision(second, async start => {
secondRounds++;
if (secondRounds === 1) return writeGreeting('hello world from B', 'B changed the greeting.')(start);
// The repair round sees Git's conflict markers and resolves them.
const conflicted = await readFile(path.join(start.workspace, 'greeting.txt'), 'utf8');
expect(conflicted).toContain('<<<<<<<');
return writeGreeting('hello world from A and B', 'Resolved the greeting conflict by keeping both.')(start);
});
// Both executors start from the same base before either integrates.
expect(await a.executor.executeRun(a.input)).toEqual({ status: 'verify' });
expect(await b.executor.executeRun(b.input)).toEqual({ status: 'verify' });
expect(await a.verifier.verifyRun(a.input)).toEqual({ status: 'complete' });
expect(await b.verifier.verifyRun(b.input)).toEqual({ status: 'repair' });
const conflict = (await evidenceOf(second)).find(e => e.type === 'integration')!;
expect(conflict).toMatchObject({ passed: false, conflictPaths: ['greeting.txt'] });
expect((await services.control.run(owner, second)).lastFailure).toMatchObject({ kind: 'integration_conflict' });
expect(await b.coordinator.prepareRepair(b.input)).toEqual({ status: 'repair' });
expect(await drive(b)).toEqual({ status: 'complete' });
expect(prompts.at(-1)).toContain('could not be merged into the latest target branch');
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world from A and B');
// The resolved change contains A's commit; nothing was overwritten (FR-F1).
const integratedA = (await services.control.run(owner, first)).integratedCommit!;
await git(remote, 'merge-base', '--is-ancestor', integratedA, 'main');
const lessons: Lesson[] = await services.lessons.list(owner, project.id);
expect(lessons[0]?.summary).toContain('Merging into the target branch conflicted in greeting.txt');
});
it('verifies and merges a branch pushed from the IDE through a wish that starts from it', async () => {
const project = await createProject();
// The Git service's push hook: an IDE credential may update ide/ branches only.
await git(remote, 'config', 'core.hooksPath', await installHooks(path.join(root, 'git-service')));
const remoteUrl = `file://${remote}`;
const ide = createGitWorkspace({
root: path.join(root, 'ide'),
credentialDir: path.join(root, 'ide-credentials'),
remote: remoteUrl,
});
const { directory, cloning } = await ide.apply({
projectId: project.id,
projectName: project.name,
repositoryUrl: remoteUrl,
branch: 'main',
token: 'payload.signature',
expiresAt: new Date(Date.now() + 900_000).toISOString(),
author: { name: 'Ada Lovelace', email: 'ada@example.com' },
});
await cloning;
const inIde = (...args: string[]) =>
run('git', args, { cwd: directory, env: { ...process.env, WISHFORGE_REF_PREFIX: 'refs/heads/ide/' } });
await writeFile(path.join(directory, 'notes.md'), 'written in the IDE\n');
await inIde('add', 'notes.md');
await inIde('commit', '-m', 'Notes from the IDE');
const target = await git(remote, 'rev-parse', 'main');
await expect(inIde('push', 'origin', 'HEAD:main')).rejects.toThrow(/ide\//);
await inIde('push');
const pushed = await git(remote, 'rev-parse', 'ide/work');
expect(await git(remote, 'rev-parse', 'main')).toBe(target);
expect(await git(remote, 'log', '-1', '--format=%an', 'ide/work')).toBe('Ada Lovelace');
await expect(submit(project.id, 'Merge', 'wish-ide-0000', { sourceBranch: 'main' })).rejects.toMatchObject({
code: 'INVALID_SOURCE_BRANCH',
});
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-ide-0001', {
sourceBranch: 'ide/work',
});
let sawNotes = false;
const session = await provision(runId, async start => {
sawNotes = existsSync(path.join(start.workspace, 'notes.md'));
return writeGreeting('hello world', 'Updated the greeting.')(start);
});
expect(await drive(session)).toEqual({ status: 'complete' });
expect(sawNotes).toBe(true);
expect(prompts.at(-1)).toContain('starts from branch ide/work');
// The run started from the branch and was verified, so it lands on the target branch whole.
expect(await git(remote, 'rev-parse', 'main^')).toBe(pushed);
expect(await git(remote, 'show', 'main:notes.md')).toBe('written in the IDE');
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
expect(await services.control.run(owner, runId)).toMatchObject({ status: 'succeeded', sourceBranch: 'ide/work' });
});
it('weighs the release risk of everything the branch brings, not only the agent’s change', async () => {
const project = await createProject();
await store.transaction(TENANT, async tx => {
const current = (await tx.get<Project>('project', project.id))!;
await tx.put('project', {
...current,
deliveryConfig: {
imageRepository: 'registry.local/greeting',
architecture: 'linux/amd64',
environmentId: 'environment-1',
dockerfile: 'Dockerfile',
containerPort: 3000,
healthPath: '/health',
smokeChecks: [],
},
});
});
// A person adds a migration in the IDE; the agent then only touches the greeting.
const seed = path.join(root, 'person');
await git(root, 'clone', remote, seed);
await git(seed, 'switch', '--create', 'ide/work');
await mkdir(path.join(seed, 'migrations'));
await writeFile(path.join(seed, 'migrations', '001-notes.sql'), 'create table notes (body text);\n');
await writeFile(path.join(seed, 'Dockerfile'), 'FROM scratch\n');
await git(seed, 'add', '.');
await git(seed, 'commit', '-m', 'Add a notes table');
await git(seed, 'push', 'origin', 'ide/work');
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-ide-0003', {
sourceBranch: 'ide/work',
});
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'));
expect(await drive(session)).toMatchObject({ status: 'release' });
const checkpoint = await store.transaction(TENANT, async tx =>
tx.get<CheckpointRecord>('checkpoint', (await tx.get<Run>('run', runId))!.checkpointId!),
);
expect(checkpoint?.trustedClassification).toMatchObject({ migration: true });
expect(checkpoint?.trustedClassification?.changedPaths).toEqual(
expect.arrayContaining(['Dockerfile', 'greeting.txt', 'migrations/001-notes.sql']),
);
// The same when the build only succeeds after continuing, and nothing was classified before:
// a new branch with its own migration, so the target gains it only through this run.
await git(seed, 'fetch', 'origin');
await git(seed, 'switch', '--create', 'ide/tags', 'origin/main');
await writeFile(path.join(seed, 'migrations', '002-tags.sql'), 'create table tags (name text);\n');
await git(seed, 'add', '.');
await git(seed, 'commit', '-m', 'Add a tags table');
await git(seed, 'push', 'origin', 'ide/tags');
const second = await submit(project.id, 'Make the greeting say hello world again', 'wish-ide-0004', {
sourceBranch: 'ide/tags',
});
failingBuilds = 1;
const failing = await provision(second, writeGreeting('hello world again', 'Updated the greeting.'));
expect(await drive(failing)).toMatchObject({ status: 'waiting', reason: 'infrastructure' });
const integrated = await services.control.run(owner, second);
expect(integrated).toMatchObject({ stage: 'build', integratedCommit: expect.any(String) });
await store.transaction(TENANT, async tx => {
const current = (await tx.get<CheckpointRecord>('checkpoint', integrated.checkpointId!))!;
delete current.trustedClassification;
await tx.put('checkpoint', current);
});
await services.control.command(owner, second, { action: 'resume' }, 'resume-build-0004');
await release(second, failing.generation);
const resumed = await provision(second, async () => {
throw new Error('An integrated run must not call the model again');
});
const checksBefore = checkRuns;
const [verification] = (await evidenceOf(second)).filter(evidence => evidence.type === 'verification');
expect(await drive(resumed)).toMatchObject({ status: 'release', verificationEvidenceId: verification!.id });
// The integrated commit had passed the checks under the same policy: they did not run again.
expect(checkRuns).toBe(checksBefore);
expect((await evidenceOf(second)).filter(evidence => evidence.type === 'verification')).toHaveLength(1);
const reclassified = await store.transaction(TENANT, tx =>
tx.get<CheckpointRecord>('checkpoint', integrated.checkpointId!),
);
expect(reclassified?.trustedClassification).toMatchObject({ migration: true });
expect(reclassified?.trustedClassification?.changedPaths).toContain('migrations/002-tags.sql');
});
it('repeats the checks when a build continues after the protected policy changed', async () => {
const project = await createProject();
await store.transaction(TENANT, async tx => {
const current = (await tx.get<Project>('project', project.id))!;
await tx.put('project', {
...current,
deliveryConfig: {
imageRepository: 'registry.local/greeting',
architecture: 'linux/amd64',
environmentId: 'environment-1',
dockerfile: 'Dockerfile',
containerPort: 3000,
healthPath: '/health',
smokeChecks: [],
},
});
});
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-policy-0001');
failingBuilds = 1;
const failing = await provision(runId, async start => {
await writeFile(path.join(start.workspace, 'Dockerfile'), 'FROM scratch\n');
return writeGreeting('hello world', 'Updated the greeting.')(start);
});
expect(await drive(failing)).toMatchObject({ status: 'waiting', reason: 'infrastructure' });
expect(await services.control.run(owner, runId)).toMatchObject({ stage: 'build' });
// The operator tightens the protected checks before continuing: the earlier verdict no longer counts.
await store.transaction(TENANT, async tx => {
const current = (await tx.get<Project>('project', project.id))!;
await tx.put('project', {
...current,
verificationCommands: [...current.verificationCommands, [process.execPath, 'check.mjs', 'again']],
});
});
await services.control.command(owner, runId, { action: 'resume' }, 'resume-policy-0001');
await release(runId, failing.generation);
const resumed = await provision(runId, async () => {
throw new Error('An integrated run must not call the model again');
});
const checksBefore = checkRuns;
expect(await drive(resumed)).toMatchObject({ status: 'release' });
expect(checkRuns).toBeGreaterThan(checksBefore);
expect((await evidenceOf(runId)).filter(evidence => evidence.type === 'verification')).toHaveLength(2);
});
it('waits for answers when the agent asks before implementing, and continues from planning (FR-C4)', async () => {
const project = await createProject();
const runId = await submit(project.id, 'Greet people', 'wish-question-0001');
const asked = await provision(
runId,
async () =>
'The requirement leaves the wording open.\n\nQUESTIONS:\n- Which greeting should it show?\n- Should it be lowercase?',
);
expect(await drive(asked)).toMatchObject({ status: 'waiting', reason: 'clarification' });
const waiting = await services.control.run(owner, runId);
expect(waiting).toMatchObject({ status: 'waiting', waitingReason: 'clarification', repairCount: 0 });
expect(waiting.waitingDetail).toContain('1. Which greeting should it show?\n2. Should it be lowercase?');
expect(await evidenceOf(runId)).toEqual([]);
await services.control.command(
owner,
runId,
{ action: 'answer', message: 'Show "hello world", in lowercase.' },
'answer-question-0001',
);
expect(await services.control.run(owner, runId)).toMatchObject({ status: 'queued', stage: 'plan', revision: 2 });
await release(runId, asked.generation);
const answered = await provision(runId, writeGreeting('hello world', 'Set the greeting as answered.'));
expect(await drive(answered)).toEqual({ status: 'complete' });
expect(prompts.at(-1)).toContain('Show "hello world", in lowercase.');
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
});
it('installs dependencies through the operator’s egress proxy in both sandboxes', async () => {
const project = await createProject();
const needsProxy = [
process.execPath,
'-e',
"process.exit(process.env.HTTPS_PROXY === 'http://proxy.test:3128' ? 0 : 1)",
];
await store.transaction(TENANT, async tx => {
const current = (await tx.get<Project>('project', project.id))!;
await tx.put('project', { ...current, installCommands: [needsProxy] });
});
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-proxy-0001');
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'), {
installProxy: 'http://proxy.test:3128',
});
expect(await drive(session)).toEqual({ status: 'complete' });
// Without the proxy the installation cannot reach a registry and the run waits.
const second = await submit(project.id, 'Greet again', 'wish-proxy-0002');
const unproxied = await provision(second, writeGreeting('hello world again', 'Updated the greeting.'));
expect(await drive(unproxied)).toMatchObject({ status: 'waiting' });
// The agent never ran, so its workspace held nothing worth keeping; it does not pile up on disk.
expect(await readdir(path.join(root, 'workspaces', unproxied.generation, TENANT))).toEqual([]);
});
it('repairs an installation the agent’s manifest change broke, but waits out a network failure', async () => {
const project = await createProject();
// Stands in for a package manager: a dependency the registry does not have fails the install.
const install = [
process.execPath,
'-e',
[
"const fs = require('node:fs');",
"const wanted = fs.existsSync('package.json') ? JSON.parse(fs.readFileSync('package.json', 'utf8')).dependencies ?? {} : {};",
"if (wanted['left-pad-typo']) { console.error('npm ERR! 404 Not Found - GET https://registry.npmjs.org/left-pad-typo'); process.exit(1); }",
"if (wanted['slow-registry']) { console.error('npm ERR! network request failed, reason: connect ETIMEDOUT'); process.exit(1); }",
].join(' '),
];
await store.transaction(TENANT, async tx => {
const current = (await tx.get<Project>('project', project.id))!;
await tx.put('project', { ...current, installCommands: [install] });
});
const manifest = (dependencies: Record<string, string>) =>
JSON.stringify({ name: 'greeting', private: true, dependencies });
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-install-0001');
const session = await provision(runId, async start => {
await writeGreeting('hello world', 'Updated the greeting.')(start);
// The first attempt adds a misspelled dependency; the repair round is told why it broke.
const repairing = start.prompt.includes("installing the project's dependencies failed");
await writeFile(
path.join(start.workspace, 'package.json'),
manifest(repairing ? {} : { 'left-pad-typo': '1.0.0' }),
);
return repairing ? 'Removed the misspelled dependency.' : 'Updated the greeting and added a dependency.';
});
expect(await drive(session)).toEqual({ status: 'complete' });
expect(prompts).toHaveLength(2);
expect(prompts[1]).toContain('404 Not Found - GET https://registry.npmjs.org/left-pad-typo');
expect(await services.control.run(owner, runId)).toMatchObject({ status: 'succeeded', repairCount: 1 });
expect((await evidenceOf(runId)).find(evidence => evidence.installFailure)?.changedManifests).toEqual([
'package.json',
]);
const lessons = await store.transaction(TENANT, tx => tx.list<Lesson>('lesson', { projectId: project.id }));
expect(lessons.map(lesson => lesson.summary)).toEqual([
expect.stringMatching(/^Installing dependencies with `.+` failed \(exit code 1\): .*left-pad-typo/),
]);
// The same manifest change failing on the network is the environment's: no repair round is spent.
const second = await submit(project.id, 'Greet again', 'wish-install-0002');
const offline = await provision(second, async start => {
await writeGreeting('hello world again', 'Updated the greeting.')(start);
await writeFile(path.join(start.workspace, 'package.json'), manifest({ 'slow-registry': '1.0.0' }));
return 'Added a dependency.';
});
expect(await drive(offline)).toMatchObject({ status: 'waiting', reason: 'infrastructure' });
expect(await services.control.run(owner, second)).toMatchObject({ repairCount: 0 });
});
it('saves the work of a run paused mid-change, and continues from it', async () => {
const project = await createProject();
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-pause-0001');
const session = await provision(
runId,
async (start, stopped) => {
await writeFile(path.join(start.workspace, 'greeting.txt'), 'hello wor\n');
// The person pauses while the agent is still at work; it keeps going until it is stopped.
await services.control.command(owner, runId, { action: 'pause' }, 'pause-mid-change');
await new Promise(resolve => stopped!.addEventListener('abort', resolve, { once: true }));
return 'Stopped mid-change.';
},
{
heartbeatMs: 25,
// The stopped session never reports a total; what it did use still counts.
stoppedUsage: {
input_tokens: 40,
output_tokens: 10,
cache_read_input_tokens: 950,
final: true,
estimated: true,
},
},
);
expect(await drive(session)).toEqual({ status: 'paused' });
const paused = await services.control.run(owner, runId);
expect(paused).toMatchObject({ status: 'paused', checkpointId: expect.any(String), usedTokens: 1000 });
const saved = await store.transaction(TENANT, tx => tx.get<CheckpointRecord>('checkpoint', paused.checkpointId!));
expect(saved?.checkpoint).toMatchObject({
patch: expect.stringContaining('+hello wor'),
providerCompletion: { status: 'canceled', finalUsage: [{ tokens: 1000, operationId: expect.any(String) }] },
});
expect((await services.control.events(owner, runId)).map(event => event.message)).toContain(
'Model usage (estimated: the session stopped before reporting its total): 40 input, 10 output, 0 cache creation and 950 cache read tokens',
);
await services.control.command(owner, runId, { action: 'resume' }, 'resume-after-pause');
await release(runId, session.generation);
let seen = '';
const resumed = await provision(runId, async start => {
seen = await readFile(path.join(start.workspace, 'greeting.txt'), 'utf8');
return writeGreeting('hello world', 'Finished the greeting.')(start);
});
expect(await drive(resumed)).toEqual({ status: 'complete' });
expect(seen).toBe('hello wor\n');
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
// The stopped session is counted once, though continuing settles its saved usage again.
expect((await services.control.run(owner, runId)).usedTokens).toBe(1120);
});
it('records the checks as sandboxed when the operator sandboxes host checks', async () => {
const platform = Object.getOwnPropertyDescriptor(process, 'platform')!;
Object.defineProperty(process, 'platform', { value: 'darwin' });
try {
const project = await createProject();
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-host-sandbox-0001');
const sandboxed: string[][] = [];
// Stands in for sandbox-exec: notes the command it was given and runs it as it is.
const seatbelt: CommandExecutor = async command => {
if (command.executable !== '/usr/bin/sandbox-exec') return execute(command);
expect(command.args[0]).toBe('-p');
sandboxed.push(command.args.slice(2));
return execute({ ...command, executable: command.args[2]!, args: command.args.slice(3) });
};
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'), {
execute: seatbelt,
hostCheckSandbox: { readable: [], writable: [] },
});
expect(await drive(session)).toEqual({ status: 'complete' });
expect(sandboxed).toContainEqual([process.execPath, 'check.mjs']);
const verification = (await evidenceOf(runId)).find(evidence => evidence.type === 'verification');
expect(verification).toMatchObject({ passed: true, isolation: 'host', checkSandbox: 'seatbelt' });
} finally {
Object.defineProperty(process, 'platform', platform);
}
});
it('waits for a source branch that was not pushed, and starts once it exists', async () => {
const project = await createProject();
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-ide-0002', {
sourceBranch: 'ide/missing',
});
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'));
expect(await drive(session)).toMatchObject({ status: 'waiting', reason: 'git_storage' });
expect((await services.control.run(owner, runId)).waitingDetail).toContain('SOURCE_BRANCH_UNAVAILABLE');
// Once the branch is pushed, continuing the run starts from it.
const seed = path.join(root, 'person');
await git(root, 'clone', remote, seed);
await git(seed, 'push', 'origin', 'HEAD:refs/heads/ide/missing');
await services.control.command(owner, runId, { action: 'resume' }, 'resume-branch-0001');
await release(runId, session.generation);
const resumed = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'));
expect(await drive(resumed)).toEqual({ status: 'complete' });
});
it('keeps executors inside their own run and lets nothing but integration complete a run', async () => {
const project = await createProject();
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-scope-0001');
const other = await submit(project.id, 'Another wish', 'wish-scope-0002');
const session = await provision(runId, writeGreeting('hello world', 'Done.'));
const fetcher = internalFetch(services);
const tokens = new ServiceTokens(env.WF_INTERNAL_TOKEN_KEY);
const token = await tokens.issue(TENANT, runId, 'worker', session.generation);
const call = (route: string, body?: unknown) =>
fetcher(`http://127.0.0.1:4000/internal${route}`, {
method: body === undefined ? 'GET' : 'POST',
headers: { authorization: `Bearer ${token}`, 'x-wishforge-tenant': TENANT, 'content-type': 'application/json' },
body: body === undefined ? undefined : JSON.stringify(body),
});
expect((await call(`/runs/${other}/context`)).status).toBe(403);
const leased = (await (
await call(`/runs/${runId}/lease`, { workerId: 'w', generation: session.generation })
).json()) as {
data: Run;
};
const leaseToken = leased.data.lease!.token;
// An executor can neither declare success nor jump to integration.
for (const [stage, status] of [
['clarify', 'succeeded'],
['integrate', 'running'],
])
expect((await call(`/runs/${runId}/transition`, { leaseToken, stage, status, message: 'x' })).status).toBe(403);
// Waiting requires a reason from the enumeration.
expect(
(await call(`/runs/${runId}/transition`, { leaseToken, stage: 'clarify', status: 'waiting', message: 'x' }))
.status,
).toBe(400);
// Repairs are reserved for the coordinator.
expect((await call(`/runs/${runId}/repair`, { generation: session.generation })).status).toBe(403);
});
});