1660 lines
86 KiB
TypeScript
1660 lines
86 KiB
TypeScript
/**
|
||
* The main loop end to end with real Git: wish → executor → independent verification
|
||
* → repair with failure feedback → integration → success, plus lessons and merge
|
||
* conflicts, and a change pushed from the IDE workspace. The control plane runs in process
|
||
* behind its real internal routes; only the agent is scripted.
|
||
*/
|
||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||
import { execFile } from 'node:child_process';
|
||
import { existsSync } from 'node:fs';
|
||
import { createRequire } from 'node:module';
|
||
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
||
import os from 'node:os';
|
||
import path from 'node:path';
|
||
import { promisify } from 'node:util';
|
||
import {
|
||
accountTypes,
|
||
roleFor,
|
||
type AccountType,
|
||
type Actor,
|
||
type Permissions,
|
||
type CheckpointRecord,
|
||
type Environment,
|
||
type Evidence,
|
||
type Lesson,
|
||
type Project,
|
||
type Release,
|
||
type Role,
|
||
type Run,
|
||
} from '@wishforge/contracts';
|
||
import { executeCommand, type CommandExecutor, type HostCheckSandbox, type ProviderStart } from '@wishforge/runtime';
|
||
import { ManagedProvider } from '../packages/runtime/src/providers/base.js';
|
||
import { ControlPlane } from '../apps/api/src/domain/control-plane.js';
|
||
import { Skills } from '../apps/api/src/domain/skills.js';
|
||
import { WishReports } from '../apps/api/src/domain/wish-report.js';
|
||
import { Services } from '../apps/api/src/http.js';
|
||
import { ServiceTokens } from '../apps/api/src/service-tokens.js';
|
||
import { recordAllocation, reserveExecutionGrant } from '../apps/api/src/domain/entitlements.js';
|
||
import { installHooks } from '../apps/git-service/src/service.js';
|
||
import { createActivities, type ActivityOutcome, type RunInput } from '../apps/worker/src/activities.js';
|
||
import { MemoryStore } from './helpers/memory-store.js';
|
||
import { internalFetch } from './helpers/internal-fetch.js';
|
||
|
||
const run = promisify(execFile);
|
||
/** The IDE workspace backend that clones the project and keeps its credential (apps/ide/policy). */
|
||
const { createGitWorkspace } = createRequire(import.meta.url)('../apps/ide/policy/git-workspace.cjs') as {
|
||
createGitWorkspace: (config: { root: string; credentialDir: string; remote: string }) => {
|
||
apply: (delivery: Record<string, unknown>) => Promise<{ directory: string; cloning?: Promise<void> }>;
|
||
};
|
||
};
|
||
const TENANT = 'demo';
|
||
const REMOTE_URL = 'http://127.0.0.1:65530/demo.git';
|
||
const owner: Actor = { tenantId: TENANT, userId: 'owner', role: 'owner', kind: 'human' };
|
||
const env = {
|
||
NODE_ENV: 'test',
|
||
WF_DEPLOYMENT_MODE: 'development',
|
||
WF_DISPATCH_TENANTS: TENANT,
|
||
WF_INTERNAL_TOKEN_KEY: 'internal-token-signing-key-for-tests-only',
|
||
WF_WORKER_TOKEN: 'coordinator-static-token-for-tests-only',
|
||
WF_DEPLOYER_TOKEN: 'deployer-static-token-for-tests-only',
|
||
WF_RESOURCE_GRANT_SECRET: 'resource-grant-secret-for-tests-only-0000',
|
||
WF_RUN_WORKER_IMAGE: `registry.local/wishforge-run@sha256:${'a'.repeat(64)}`,
|
||
WF_GIT_ALLOW_LOCAL_HTTP: 'true',
|
||
};
|
||
const git = (cwd: string, ...args: string[]) =>
|
||
run('git', ['-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', ...args], { cwd }).then(r =>
|
||
r.stdout.trim(),
|
||
);
|
||
|
||
/** `stopped` aborts when the executor cancels the attempt, as it does on a pause. */
|
||
type Agent = (input: ProviderStart, stopped?: AbortSignal) => Promise<string>;
|
||
/** Stands in for Codex: edits the workspace through a script and reports like an adapter. */
|
||
class ScriptedAgent extends ManagedProvider {
|
||
constructor(
|
||
private readonly agent: Agent,
|
||
private readonly usage: Record<string, unknown> = { input_tokens: 100, output_tokens: 20 },
|
||
/** What a stopped session reports, as the Claude Code adapter estimates it. */
|
||
private readonly stoppedUsage?: Record<string, unknown>,
|
||
) {
|
||
super();
|
||
}
|
||
capabilities() {
|
||
return {
|
||
provider: 'codex',
|
||
availability: 'available' as const,
|
||
coding: true,
|
||
streaming: true,
|
||
cancellation: true,
|
||
portableCheckpoint: true,
|
||
};
|
||
}
|
||
async start(input: ProviderStart) {
|
||
const attempt = this.create(input);
|
||
void (async () => {
|
||
try {
|
||
const summary = await this.agent(input, attempt.controller.signal);
|
||
if (attempt.controller.signal.aborted) {
|
||
if (this.stoppedUsage) this.emit(attempt, 'usage', this.stoppedUsage);
|
||
return this.finish(attempt, 'canceled');
|
||
}
|
||
this.emit(attempt, 'command', { command: 'node edit.mjs', exitCode: 0 });
|
||
this.emit(attempt, 'file_change', { changes: [{ path: 'greeting.txt', kind: 'update' }] });
|
||
this.emit(attempt, 'output', { text: summary });
|
||
this.emit(attempt, 'usage', this.usage);
|
||
this.finish(attempt, 'completed');
|
||
} catch (error) {
|
||
this.finish(attempt, 'failed', error instanceof Error ? error.message : String(error));
|
||
}
|
||
})();
|
||
return { ...attempt.public };
|
||
}
|
||
}
|
||
|
||
let root: string;
|
||
let remote: string;
|
||
let store: MemoryStore;
|
||
let services: Services;
|
||
let prompts: string[];
|
||
beforeEach(async () => {
|
||
root = await mkdtemp(path.join(os.tmpdir(), 'wishforge-loop-'));
|
||
remote = path.join(root, 'remote.git');
|
||
await git(root, 'init', '--bare', '--initial-branch=main', remote);
|
||
const seed = path.join(root, 'seed');
|
||
await git(root, 'clone', remote, seed);
|
||
await writeFile(path.join(seed, 'greeting.txt'), 'hello\n');
|
||
await writeFile(
|
||
path.join(seed, 'check.mjs'),
|
||
[
|
||
"import { readFileSync } from 'node:fs';",
|
||
"const text = readFileSync('greeting.txt', 'utf8').trim();",
|
||
'if (!text.startsWith(\'hello world\')) { console.error(`expected "hello world", got "${text}"`); process.exit(1); }',
|
||
"console.log('greeting ok');",
|
||
].join('\n'),
|
||
);
|
||
await git(seed, 'add', '.');
|
||
await git(seed, 'commit', '-m', 'Initial project');
|
||
await git(seed, 'push', 'origin', 'main');
|
||
store = new MemoryStore();
|
||
store.seed(TENANT);
|
||
services = new Services(new ControlPlane(store, env));
|
||
prompts = [];
|
||
});
|
||
afterEach(async () => {
|
||
await rm(root, { recursive: true, force: true });
|
||
});
|
||
|
||
/**
|
||
* git clones of the loopback URL are served from the bare repository in the temporary directory;
|
||
* image builds are answered like a registry push, so no Docker is needed.
|
||
*/
|
||
let failingBuilds = 0;
|
||
/** How often the project's protected check actually ran. */
|
||
let checkRuns = 0;
|
||
const execute: CommandExecutor = async command => {
|
||
if (command.args.includes('check.mjs')) checkRuns++;
|
||
if (command.executable === 'docker' && failingBuilds > 0) {
|
||
failingBuilds--;
|
||
return { exitCode: 1, stdout: '', stderr: 'The image builder is unavailable', durationMs: 1 };
|
||
}
|
||
if (command.executable === 'docker') {
|
||
const metadata = command.args[command.args.indexOf('--metadata-file') + 1]!;
|
||
await writeFile(metadata, JSON.stringify({ 'containerimage.digest': `sha256:${'c'.repeat(64)}` }));
|
||
return { exitCode: 0, stdout: '', stderr: '', durationMs: 1 };
|
||
}
|
||
return executeCommand(
|
||
command.executable === 'git'
|
||
? { ...command, args: command.args.map(arg => (arg === REMOTE_URL ? remote : arg)) }
|
||
: command,
|
||
);
|
||
};
|
||
|
||
async function createProject() {
|
||
const project = await services.control.createProject(owner, {
|
||
name: 'Greeting',
|
||
repositoryUrl: REMOTE_URL,
|
||
verificationCommands: [[process.execPath, 'check.mjs']],
|
||
});
|
||
await store.transaction(TENANT, tx =>
|
||
tx.put('provider_connection', {
|
||
id: 'codex-connection',
|
||
tenantId: TENANT,
|
||
version: 1,
|
||
createdAt: new Date().toISOString(),
|
||
updatedAt: new Date().toISOString(),
|
||
provider: 'codex',
|
||
status: 'active',
|
||
}),
|
||
);
|
||
return project;
|
||
}
|
||
async function submit(
|
||
projectId: string,
|
||
goal: string,
|
||
key: string,
|
||
extra: Record<string, unknown> = {},
|
||
by: Actor = owner,
|
||
) {
|
||
const { run_id } = await services.control.submit(
|
||
by,
|
||
projectId,
|
||
{ goal, acceptanceCriteria: ['check.mjs passes'], provider: 'codex', budget: { maxRepairRounds: 2 }, ...extra },
|
||
key,
|
||
);
|
||
return run_id;
|
||
}
|
||
/** What the dispatcher and the local provisioner do: reserve resources and issue per-generation identities. */
|
||
async function provision(
|
||
runId: string,
|
||
agent: Agent,
|
||
settings: {
|
||
installProxy?: string;
|
||
usage?: Record<string, unknown>;
|
||
stoppedUsage?: Record<string, unknown>;
|
||
fetcher?: typeof fetch;
|
||
heartbeatMs?: number;
|
||
execute?: CommandExecutor;
|
||
hostCheckSandbox?: HostCheckSandbox;
|
||
} = {},
|
||
) {
|
||
const generation = (await services.control.run(owner, runId)).pendingStartId!;
|
||
await store.transaction(TENANT, tx => reserveExecutionGrant(tx, runId, generation, env));
|
||
const tokens = new ServiceTokens(env.WF_INTERNAL_TOKEN_KEY);
|
||
const input: RunInput = { tenantId: TENANT, runId, generation };
|
||
const shared = {
|
||
controlPlaneUrl: 'http://127.0.0.1:4000',
|
||
workspaceRoot: path.join(root, 'workspaces', generation),
|
||
isolatedProviderHomeRoot: path.join(root, 'homes'),
|
||
allowLocalGitHttp: true,
|
||
fetcher: internalFetch(services),
|
||
execute,
|
||
...settings,
|
||
};
|
||
const sandbox = {
|
||
sandboxMode: 'host' as const,
|
||
sandboxRunId: runId,
|
||
sandboxTenantId: TENANT,
|
||
sandboxGeneration: generation,
|
||
};
|
||
const executor = createActivities({
|
||
...shared,
|
||
...sandbox,
|
||
workerId: `executor-${generation}`,
|
||
workerToken: await tokens.issue(TENANT, runId, 'worker', generation),
|
||
providerFactory: () =>
|
||
new ScriptedAgent(
|
||
async (start, stopped) => {
|
||
prompts.push(start.prompt);
|
||
return agent(start, stopped);
|
||
},
|
||
settings.usage,
|
||
settings.stoppedUsage,
|
||
),
|
||
});
|
||
const verifier = createActivities({
|
||
...shared,
|
||
...sandbox,
|
||
workerId: `verifier-${generation}`,
|
||
verifierToken: await tokens.issue(TENANT, runId, 'verifier', generation),
|
||
});
|
||
const coordinator = createActivities({ ...shared, workerId: 'coordinator', workerToken: env.WF_WORKER_TOKEN });
|
||
return { input, generation, executor, verifier, coordinator };
|
||
}
|
||
/** The steps of runWorkflow, without Temporal. */
|
||
async function drive(session: Awaited<ReturnType<typeof provision>>, until?: ActivityOutcome['status']) {
|
||
for (let step = 0; step < 10; step++) {
|
||
let outcome = await session.executor.executeRun(session.input);
|
||
if (outcome.status === 'verify') outcome = await session.verifier.verifyRun(session.input);
|
||
if (until && outcome.status === until) return outcome;
|
||
if (outcome.status === 'repair') outcome = await session.coordinator.prepareRepair(session.input);
|
||
if (outcome.status === 'repair') continue;
|
||
if (outcome.reason)
|
||
await session.coordinator.recordWaiting({ ...session.input, reason: outcome.reason, detail: outcome.detail });
|
||
return outcome;
|
||
}
|
||
throw new Error('The loop did not settle');
|
||
}
|
||
async function release(runId: string, generation: string) {
|
||
await store.transaction(TENANT, tx => recordAllocation(tx, runId, generation, 'released'));
|
||
}
|
||
const writeGreeting =
|
||
(text: string, summary: string): Agent =>
|
||
async start => {
|
||
await writeFile(path.join(start.workspace, 'greeting.txt'), `${text}\n`);
|
||
return summary;
|
||
};
|
||
/** A person commits straight to the target branch, as anyone with access to the repository can. */
|
||
async function pushToTarget(files: Record<string, string>, message: string) {
|
||
const person = path.join(root, 'pusher');
|
||
if (existsSync(person)) await git(person, 'pull', '--ff-only', 'origin', 'main');
|
||
else await git(root, 'clone', remote, person);
|
||
for (const [file, content] of Object.entries(files)) await writeFile(path.join(person, file), content);
|
||
await git(person, 'add', '.');
|
||
await git(person, 'commit', '-m', message);
|
||
await git(person, 'push', 'origin', 'HEAD:main');
|
||
return git(person, 'rev-parse', 'HEAD');
|
||
}
|
||
/** The project's check with one more rule a person added to the branch. */
|
||
const checkWith = (rule: string) =>
|
||
[
|
||
"import { existsSync, readFileSync } from 'node:fs';",
|
||
"const text = readFileSync('greeting.txt', 'utf8').trim();",
|
||
'if (!text.startsWith(\'hello world\')) { console.error(`expected "hello world", got "${text}"`); process.exit(1); }',
|
||
rule,
|
||
"console.log('greeting ok');",
|
||
].join('\n');
|
||
/** Someone the role alone decides for, in the project, as a sign-in from an identity provider is. */
|
||
async function join(projectId: string, userId: string, role: Role): Promise<Actor> {
|
||
await store.transaction(TENANT, async tx => {
|
||
await tx.putMembership({ userId, role, active: true });
|
||
const project = (await tx.get<Project>('project', projectId))!;
|
||
await tx.put('project', { ...project, memberIds: [...project.memberIds, userId] });
|
||
});
|
||
return { tenantId: TENANT, userId, role, kind: 'human' };
|
||
}
|
||
/** Someone with an account of their own and its permissions, as the local console signs them in. */
|
||
async function person(projectId: string, name: string, type: AccountType | Permissions) {
|
||
const permissions = { ...(typeof type === 'string' ? accountTypes[type] : type) };
|
||
const actor = await join(projectId, `local:${name}`, roleFor(permissions));
|
||
const time = new Date().toISOString();
|
||
await store.transaction(TENANT, tx =>
|
||
tx.put('local_account', {
|
||
id: actor.userId,
|
||
tenantId: TENANT,
|
||
version: 1,
|
||
createdAt: time,
|
||
updatedAt: time,
|
||
username: name,
|
||
displayName: name,
|
||
permissions,
|
||
}),
|
||
);
|
||
return { ...actor, permissions };
|
||
}
|
||
/** The project's policy: changes to these paths wait for a person's review before integration. */
|
||
async function holdForReview(projectId: string, paths: string[]) {
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', projectId))!;
|
||
await tx.put('project', { ...current, codeReview: { paths, exceptPaths: [] } });
|
||
});
|
||
}
|
||
async function evidenceOf(runId: string) {
|
||
return store.transaction(TENANT, async tx => (await tx.list<Evidence>('evidence')).filter(e => e.runId === runId));
|
||
}
|
||
|
||
describe('local main loop', () => {
|
||
it('persists a completed checkpoint before final usage exhausts budget and resumes without another model call', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-final-budget', {
|
||
budget: { maxTokens: 1000 },
|
||
});
|
||
let workspace = '';
|
||
const session = await provision(
|
||
runId,
|
||
async start => {
|
||
workspace = start.workspace;
|
||
return writeGreeting('hello world', 'Completed the greeting.')(start);
|
||
},
|
||
// Cache reads count a tenth: 100 + 100 + 13000 / 10.
|
||
{ usage: { input_tokens: 100, output_tokens: 100, cache_read_input_tokens: 13000, final: true } },
|
||
);
|
||
expect(await session.executor.executeRun(session.input)).toEqual({ status: 'waiting' });
|
||
const waiting = await services.control.run(owner, runId);
|
||
expect(waiting).toMatchObject({ status: 'waiting', waitingReason: 'budget', usedTokens: 1500, stage: 'implement' });
|
||
expect(waiting.checkpointId).toBeTruthy();
|
||
const checkpoint = await store.transaction(TENANT, tx =>
|
||
tx.get<CheckpointRecord>('checkpoint', waiting.checkpointId!),
|
||
);
|
||
expect(checkpoint?.checkpoint).toMatchObject({
|
||
patch: expect.stringContaining('+hello world'),
|
||
providerCompletion: { status: 'completed', finalUsage: [{ tokens: 1500, operationId: expect.any(String) }] },
|
||
});
|
||
expect(existsSync(workspace)).toBe(false); // Removal is safe because the source bundle was confirmed durable.
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Run>('run', runId))!;
|
||
current.budget.maxTokens = 5000;
|
||
await tx.put('run', current);
|
||
});
|
||
await services.control.command(owner, runId, { action: 'resume' }, 'resume-final-budget');
|
||
await release(runId, session.generation);
|
||
const resumed = await provision(runId, async () => {
|
||
throw new Error('Completed source must not call a model again');
|
||
});
|
||
expect(await resumed.executor.executeRun(resumed.input)).toEqual({ status: 'verify' });
|
||
expect((await services.control.run(owner, runId)).usedTokens).toBe(1500); // The original operation ID is idempotent.
|
||
expect(prompts).toHaveLength(1);
|
||
expect(await resumed.verifier.verifyRun(resumed.input)).toEqual({ status: 'complete' });
|
||
});
|
||
|
||
it('continues metering non-final usage immediately and retains uncheckpointed work when budget cancels the lease', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-stream-budget', {
|
||
budget: { maxTokens: 1000 },
|
||
});
|
||
let workspace = '';
|
||
const session = await provision(
|
||
runId,
|
||
async start => {
|
||
workspace = start.workspace;
|
||
return writeGreeting('hello world', 'Work before streaming budget exhaustion.')(start);
|
||
},
|
||
{ usage: { input_tokens: 1500, output_tokens: 0 } },
|
||
);
|
||
expect(await session.executor.executeRun(session.input)).toEqual({ status: 'waiting' });
|
||
const waiting = await services.control.run(owner, runId);
|
||
expect(waiting).toMatchObject({ waitingReason: 'budget', usedTokens: 1500 });
|
||
expect(waiting.checkpointId).toBeUndefined();
|
||
expect(await readFile(path.join(workspace, 'greeting.txt'), 'utf8')).toBe('hello world\n');
|
||
expect(JSON.parse(await readFile(`${workspace}.recovery.json`, 'utf8'))).toMatchObject({
|
||
runId,
|
||
workspace,
|
||
reason: expect.stringContaining('retained'),
|
||
});
|
||
});
|
||
|
||
it('retains completed source when checkpoint persistence fails', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-checkpoint-down');
|
||
const delegate = internalFetch(services);
|
||
let workspace = '';
|
||
const session = await provision(
|
||
runId,
|
||
async start => {
|
||
workspace = start.workspace;
|
||
return writeGreeting('hello world', 'Completed source retained.')(start);
|
||
},
|
||
{
|
||
usage: { input_tokens: 100, output_tokens: 20, final: true },
|
||
fetcher: async (url, init) =>
|
||
String(url).endsWith('/checkpoint')
|
||
? Response.json(
|
||
{ error: { code: 'ARTIFACT_STORAGE_UNAVAILABLE', message: 'Storage unavailable' } },
|
||
{ status: 503 },
|
||
)
|
||
: delegate(url, init),
|
||
},
|
||
);
|
||
await expect(session.executor.executeRun(session.input)).rejects.toMatchObject({
|
||
domainCode: 'ARTIFACT_STORAGE_UNAVAILABLE',
|
||
});
|
||
expect(await readFile(path.join(workspace, 'greeting.txt'), 'utf8')).toBe('hello world\n');
|
||
expect(JSON.parse(await readFile(`${workspace}.recovery.json`, 'utf8'))).toMatchObject({
|
||
checkpoint: { providerCompletion: { finalUsage: [{ tokens: 120, operationId: expect.any(String) }] } },
|
||
});
|
||
});
|
||
|
||
it('retains the workspace when cancellation revokes the lease before checkpoint upload', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-canceled-source');
|
||
let workspace = '';
|
||
const session = await provision(runId, async start => {
|
||
workspace = start.workspace;
|
||
await writeGreeting('hello world', 'Work before cancellation.')(start);
|
||
await services.control.command(owner, runId, { action: 'cancel' }, 'cancel-before-checkpoint');
|
||
return 'Work stopped.';
|
||
});
|
||
// Cancel also revokes the scoped credential, so the worker cannot read run state again.
|
||
await expect(session.executor.executeRun(session.input)).rejects.toMatchObject({ domainCode: 'UNAUTHENTICATED' });
|
||
expect((await services.control.run(owner, runId)).status).toBe('canceled');
|
||
expect(await readFile(path.join(workspace, 'greeting.txt'), 'utf8')).toBe('hello world\n');
|
||
expect(existsSync(`${workspace}.recovery.json`)).toBe(true);
|
||
});
|
||
|
||
it('learns project skills from a succeeded run, uses the confirmed ones and reports the wish', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-skills-0001');
|
||
let round = 0;
|
||
const session = await provision(runId, async start => {
|
||
round++;
|
||
if (round === 1) return writeGreeting('hello wrld', 'Updated the greeting.')(start);
|
||
return writeGreeting('hello world', 'Fixed the typo in greeting.txt so it reads "hello world".')(start);
|
||
});
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
await release(runId, session.generation);
|
||
|
||
// The background harvest looks at the run once; the model proposes two skills, one named
|
||
// like a skill the project has.
|
||
const asked: string[] = [];
|
||
const skills = new Skills(store, { ...env, WF_TENANT_ID: TENANT }, async input => {
|
||
asked.push(input.prompt);
|
||
return {
|
||
skills: [
|
||
{
|
||
name: 'greeting-check',
|
||
description: 'When changing greeting.txt',
|
||
content: '1. Edit greeting.txt.\n2. Run `node check.mjs`.',
|
||
},
|
||
{ name: 'Not A Name', description: 'x', content: 'y' },
|
||
],
|
||
};
|
||
});
|
||
await skills.create(owner, project.id, {
|
||
name: 'greeting-check',
|
||
description: 'An older skill',
|
||
content: 'Keep greetings lowercase.',
|
||
});
|
||
expect(await skills.harvestNext(TENANT)).toBe(runId);
|
||
expect(await skills.harvestNext(TENANT)).toBeUndefined();
|
||
expect(JSON.parse(asked[0]!)).toMatchObject({
|
||
goal: 'Make the greeting say hello world',
|
||
changedFiles: ['greeting.txt'],
|
||
existingSkills: [{ name: 'greeting-check' }],
|
||
});
|
||
const listed = await skills.list(owner, project.id);
|
||
const proposed = listed.find(skill => skill.status === 'proposed')!;
|
||
expect(listed).toHaveLength(2);
|
||
expect(proposed).toMatchObject({ name: 'greeting-check-2', createdBy: 'wishforge', sourceRunIds: [runId] });
|
||
expect((await services.control.events(owner, runId)).map(event => event.type)).toContain('skill.proposed');
|
||
|
||
// Someone who may not wish for the project may not shape its skills; anyone who may can.
|
||
const viewer = await person(project.id, 'li', { wish: 'none', manage: false, releases: false, reviewCode: false });
|
||
await expect(skills.update(viewer, proposed.id, { status: 'active' })).rejects.toMatchObject({ code: 'FORBIDDEN' });
|
||
const frontend = await person(project.id, 'wang', 'frontend');
|
||
await skills.update(frontend, proposed.id, {
|
||
status: 'active',
|
||
content: 'Edit greeting.txt, then run node check.mjs.',
|
||
});
|
||
await expect(skills.update(owner, proposed.id, { name: 'greeting-check' })).rejects.toMatchObject({
|
||
code: 'SKILL_NAME_TAKEN',
|
||
});
|
||
|
||
// The confirmed skills reach the next run's prompt.
|
||
const next = await submit(project.id, 'Make the greeting say hello world again', 'wish-skills-0002');
|
||
let prompt = '';
|
||
const second = await provision(next, async start => {
|
||
prompt = start.prompt;
|
||
return writeGreeting('hello world', 'Nothing to change.')(start);
|
||
});
|
||
await second.executor.executeRun(second.input);
|
||
expect(prompt).toContain('### greeting-check-2');
|
||
expect(prompt).toContain('Edit greeting.txt, then run node check.mjs.');
|
||
expect(prompt).toContain('### greeting-check');
|
||
|
||
// Exports: one SKILL.md, or a zip with a directory per active skill.
|
||
const one = await skills.exportOne(owner, proposed.id);
|
||
expect(one.content).toMatch(/^---\nname: greeting-check-2\ndescription: "When changing greeting.txt"\n---\n\nEdit/);
|
||
const zip = await skills.exportAll(owner, project.id);
|
||
expect(zip.content.subarray(0, 4).toString('hex')).toBe('504b0304');
|
||
const unzipped = path.join(root, 'skills.zip');
|
||
await writeFile(unzipped, zip.content);
|
||
const listing = await run('unzip', ['-Z1', unzipped]).then(result => result.stdout.trim().split('\n'));
|
||
expect(listing.sort()).toEqual(['greeting-check-2/SKILL.md', 'greeting-check/SKILL.md']);
|
||
expect((await run('unzip', ['-p', unzipped, 'greeting-check-2/SKILL.md'])).stdout).toBe(one.content);
|
||
|
||
// The wish's report carries every stage, the agent's work, the change and the checks.
|
||
const report = await new WishReports(store).markdown(owner, (await services.control.run(owner, runId)).wishId);
|
||
expect(report.filename).toMatch(/^wish-[0-9a-f]{8}-\d{8}\.md$/);
|
||
for (const part of [
|
||
'# 愿望报告:Make the greeting say hello world',
|
||
'### 验收标准',
|
||
'### 阶段时间线',
|
||
'#### Agent 的回复',
|
||
'Fixed the typo in greeting.txt',
|
||
'#### Agent 改动的文件',
|
||
'### 代码改动',
|
||
'+hello world',
|
||
'### 验证证据',
|
||
'验证 · 通过',
|
||
'### 失败与修复',
|
||
'### 完整事件日志',
|
||
])
|
||
expect(report.content, part).toContain(part);
|
||
});
|
||
|
||
it('repairs a failed check with the verifier output, integrates the change and learns from it', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-repair-0001');
|
||
let round = 0;
|
||
const session = await provision(runId, async start => {
|
||
round++;
|
||
if (round === 1) return writeGreeting('hello wrld', 'Updated the greeting.')(start);
|
||
return writeGreeting('hello world', 'Fixed the typo in greeting.txt so it reads "hello world".')(start);
|
||
});
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
|
||
const done = await services.control.run(owner, runId);
|
||
expect(done).toMatchObject({ status: 'succeeded', stage: 'integrate', repairCount: 1 });
|
||
expect(done.integratedCommit).toBe(await git(remote, 'rev-parse', 'main'));
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
|
||
expect(done.reservedTokens).toBe(0);
|
||
|
||
// The repair prompt quoted the failing command and its output (FR-E4).
|
||
expect(prompts).toHaveLength(2);
|
||
expect(prompts[0]).not.toContain('Repair round');
|
||
expect(prompts[1]).toContain('Repair round 1');
|
||
expect(prompts[1]).toContain('check.mjs');
|
||
expect(prompts[1]).toContain('expected "hello world", got "hello wrld"');
|
||
|
||
// Evidence records each check with its output, and the integration (FR-E3, FR-F2).
|
||
const evidence = await evidenceOf(runId);
|
||
const failed = evidence.find(e => e.type === 'verification' && !e.passed)!;
|
||
expect(failed.isolation).toBe('host');
|
||
expect(failed.commands?.[0]).toMatchObject({ exitCode: 1, outputTail: expect.stringContaining('hello wrld') });
|
||
expect(evidence.filter(e => e.type === 'verification' && e.passed)).toHaveLength(1);
|
||
expect(evidence.find(e => e.type === 'integration')).toMatchObject({
|
||
passed: true,
|
||
sourceCommit: done.integratedCommit,
|
||
targetRef: 'refs/heads/main',
|
||
});
|
||
|
||
// Progress shows what the agent did (FR-D7).
|
||
const events = await services.control.events(owner, runId);
|
||
expect(events.map(e => e.type)).toEqual(
|
||
expect.arrayContaining(['provider.command', 'provider.file_change', 'run.repair']),
|
||
);
|
||
expect(events.find(e => e.type === 'provider.output')?.message).toContain('Updated the greeting');
|
||
expect(events.find(e => e.type === 'provider.command')?.message).toBe('$ node edit.mjs (exit 0)');
|
||
|
||
// The repair became a project lesson that the next run is told about (FR-I2).
|
||
const lessons = await services.lessons.list(owner, project.id);
|
||
expect(lessons).toHaveLength(1);
|
||
expect(lessons[0]).toMatchObject({ kind: 'repair', status: 'active', sourceRunId: runId });
|
||
expect(lessons[0]!.summary).toContain('check.mjs');
|
||
expect(lessons[0]!.summary).toContain('Fixed the typo');
|
||
await release(runId, session.generation);
|
||
|
||
const next = await submit(project.id, 'Make the greeting say hello world again', 'wish-lesson-0002');
|
||
const second = await provision(next, writeGreeting('hello world!', 'Added an exclamation mark.'));
|
||
expect(await drive(second)).toEqual({ status: 'complete' });
|
||
expect(prompts[2]).toContain('Lessons from earlier work on this project');
|
||
expect(prompts[2]).toContain(lessons[0]!.summary);
|
||
await release(next, second.generation);
|
||
|
||
// A disabled lesson is no longer injected.
|
||
await services.lessons.setStatus(owner, lessons[0]!.id, { status: 'disabled' });
|
||
const third = await submit(project.id, 'Keep the greeting', 'wish-lesson-0003');
|
||
const thirdSession = await provision(third, writeGreeting('hello world!!', 'More emphasis.'));
|
||
expect(await drive(thirdSession)).toEqual({ status: 'complete' });
|
||
expect(prompts[3]).not.toContain('Lessons from earlier work');
|
||
});
|
||
|
||
it('waits instead of repairing when the repair budget is used up, and continues on request', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-budget-0001');
|
||
let rounds = 0;
|
||
const session = await provision(runId, async start => {
|
||
rounds++;
|
||
return writeGreeting(rounds < 4 ? `hello wrld ${rounds}` : 'hello world', `Attempt ${rounds}.`)(start);
|
||
});
|
||
expect(await drive(session)).toEqual({ status: 'waiting' });
|
||
let waiting = await services.control.run(owner, runId);
|
||
expect(waiting).toMatchObject({ status: 'waiting', waitingReason: 'repair_exhausted', repairCount: 2 });
|
||
expect(waiting.waitingDetail).toContain('check.mjs');
|
||
expect(rounds).toBe(3);
|
||
|
||
// Continuing grants exactly one more round; a new generation runs it.
|
||
await services.control.command(owner, runId, { action: 'resume' }, 'resume-budget-0001');
|
||
waiting = await services.control.run(owner, runId);
|
||
expect(waiting).toMatchObject({ status: 'queued', stage: 'implement', repairCount: 3 });
|
||
await release(runId, session.generation);
|
||
const resumed = await provision(runId, async start => {
|
||
rounds++;
|
||
return writeGreeting('hello world', 'Finally fixed it.')(start);
|
||
});
|
||
expect(await drive(resumed)).toEqual({ status: 'complete' });
|
||
expect((await services.control.run(owner, runId)).status).toBe('succeeded');
|
||
});
|
||
|
||
it('turns a merge conflict with a concurrent run into a repair that merges the new target', async () => {
|
||
const project = await createProject();
|
||
const first = await submit(project.id, 'Greet the world', 'wish-conflict-0001');
|
||
const second = await submit(project.id, 'Greet everyone', 'wish-conflict-0002');
|
||
const a = await provision(first, writeGreeting('hello world from A', 'A changed the greeting.'));
|
||
let secondRounds = 0;
|
||
const b = await provision(second, async start => {
|
||
secondRounds++;
|
||
if (secondRounds === 1) return writeGreeting('hello world from B', 'B changed the greeting.')(start);
|
||
// The repair round sees Git's conflict markers and resolves them.
|
||
const conflicted = await readFile(path.join(start.workspace, 'greeting.txt'), 'utf8');
|
||
expect(conflicted).toContain('<<<<<<<');
|
||
return writeGreeting('hello world from A and B', 'Resolved the greeting conflict by keeping both.')(start);
|
||
});
|
||
// Both executors start from the same base before either integrates.
|
||
expect(await a.executor.executeRun(a.input)).toEqual({ status: 'verify' });
|
||
expect(await b.executor.executeRun(b.input)).toEqual({ status: 'verify' });
|
||
expect(await a.verifier.verifyRun(a.input)).toEqual({ status: 'complete' });
|
||
expect(await b.verifier.verifyRun(b.input)).toEqual({ status: 'repair' });
|
||
|
||
const conflict = (await evidenceOf(second)).find(e => e.type === 'integration')!;
|
||
expect(conflict).toMatchObject({ passed: false, conflictPaths: ['greeting.txt'] });
|
||
expect((await services.control.run(owner, second)).lastFailure).toMatchObject({ kind: 'integration_conflict' });
|
||
|
||
expect(await b.coordinator.prepareRepair(b.input)).toEqual({ status: 'repair' });
|
||
expect(await drive(b)).toEqual({ status: 'complete' });
|
||
expect(prompts.at(-1)).toContain('could not be merged into the latest target branch');
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world from A and B');
|
||
// The resolved change contains A's commit; nothing was overwritten (FR-F1).
|
||
const integratedA = (await services.control.run(owner, first)).integratedCommit!;
|
||
await git(remote, 'merge-base', '--is-ancestor', integratedA, 'main');
|
||
const lessons: Lesson[] = await services.lessons.list(owner, project.id);
|
||
expect(lessons[0]?.summary).toContain('Merging into the target branch conflicted in greeting.txt');
|
||
});
|
||
|
||
it('merges a change onto a target that moved meanwhile, and checks the merge', async () => {
|
||
const project = await createProject();
|
||
await pushToTarget({ 'greeting.txt': 'hello world\n' }, 'Greet the world');
|
||
const runId = await submit(project.id, 'Add notes', 'wish-moved-0001');
|
||
const session = await provision(runId, async start => {
|
||
await writeFile(path.join(start.workspace, 'notes.md'), 'Notes\n');
|
||
return 'Added the notes.';
|
||
});
|
||
expect(await session.executor.executeRun(session.input)).toEqual({ status: 'verify' });
|
||
const moved = await pushToTarget({ 'changelog.md': 'Changes\n' }, 'Start a changelog');
|
||
const checksBefore = checkRuns;
|
||
expect(await session.verifier.verifyRun(session.input)).toEqual({ status: 'complete' });
|
||
// The change on its own, then its merge with what the branch gained.
|
||
expect(checkRuns).toBe(checksBefore + 2);
|
||
expect(await git(remote, 'show', 'main:notes.md')).toBe('Notes');
|
||
await git(remote, 'merge-base', '--is-ancestor', moved, 'main');
|
||
});
|
||
it('waits for people when commits pushed to the target broke it, instead of having the agent repair them', async () => {
|
||
const project = await createProject();
|
||
// The branch passes its check when the run starts.
|
||
await pushToTarget({ 'greeting.txt': 'hello world\n' }, 'Greet the world');
|
||
const runId = await submit(project.id, 'Add notes', 'wish-broken-0001');
|
||
const session = await provision(runId, async start => {
|
||
await writeFile(path.join(start.workspace, 'notes.md'), 'Notes\n');
|
||
return 'Added the notes.';
|
||
});
|
||
expect(await session.executor.executeRun(session.input)).toEqual({ status: 'verify' });
|
||
// Meanwhile a person pushes a commit that breaks the check.
|
||
const broken = await pushToTarget({ 'greeting.txt': 'goodbye\n' }, 'Say goodbye');
|
||
const checksBefore = checkRuns;
|
||
const outcome = await session.verifier.verifyRun(session.input);
|
||
expect(outcome).toMatchObject({ status: 'waiting', reason: 'target_broken' });
|
||
await session.coordinator.recordWaiting({ ...session.input, reason: outcome.reason!, detail: outcome.detail });
|
||
// The change on its own, the merge, the branch without the change, and where the run started.
|
||
expect(checkRuns).toBe(checksBefore + 4);
|
||
const waiting = await services.control.run(owner, runId);
|
||
expect(waiting).toMatchObject({ status: 'waiting', waitingReason: 'target_broken', repairCount: 0 });
|
||
expect(waiting.waitingDetail).toContain(`at ${broken.slice(0, 12)} without this run's change`);
|
||
expect(waiting.waitingDetail).toContain('expected "hello world", got "goodbye"');
|
||
expect(prompts).toHaveLength(1);
|
||
// The run's own verified change is its checkpoint again, not the failed merge.
|
||
const checkpoint = await store.transaction(TENANT, async tx =>
|
||
tx.get<CheckpointRecord>('checkpoint', (await tx.get<Run>('run', runId))!.checkpointId!),
|
||
);
|
||
expect((checkpoint?.checkpoint as { sourceCommit?: string }).sourceCommit).toBe(waiting.verifiedCommit);
|
||
|
||
// People repair the branch; continuing merges the verified change onto it without asking the agent.
|
||
await pushToTarget({ 'greeting.txt': 'hello world again\n' }, 'Greet the world again');
|
||
await services.control.command(owner, runId, { action: 'resume' }, 'resume-broken-0001');
|
||
await release(runId, session.generation);
|
||
const resumed = await provision(runId, async () => {
|
||
throw new Error('A verified change must not call the model again');
|
||
});
|
||
const checksBeforeResume = checkRuns;
|
||
expect(await drive(resumed)).toEqual({ status: 'complete' });
|
||
// Only the merge with the repaired branch is checked; the change's own verification still holds.
|
||
expect(checkRuns).toBe(checksBeforeResume + 1);
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world again');
|
||
expect(await git(remote, 'show', 'main:notes.md')).toBe('Notes');
|
||
});
|
||
it('still repairs a change that fails only together with what the target gained', async () => {
|
||
const project = await createProject();
|
||
await pushToTarget({ 'greeting.txt': 'hello world\n' }, 'Greet the world');
|
||
const runId = await submit(project.id, 'Add notes', 'wish-broken-0002');
|
||
let rounds = 0;
|
||
const session = await provision(runId, async start => {
|
||
if (++rounds === 1) {
|
||
await writeFile(path.join(start.workspace, 'notes.md'), 'Notes\n');
|
||
return 'Added the notes.';
|
||
}
|
||
// The repair round follows the rule the branch gained.
|
||
await rm(path.join(start.workspace, 'notes.md'));
|
||
await mkdir(path.join(start.workspace, 'docs'));
|
||
await writeFile(path.join(start.workspace, 'docs', 'notes.md'), 'Notes\n');
|
||
return 'Moved the notes into docs/.';
|
||
});
|
||
expect(await session.executor.executeRun(session.input)).toEqual({ status: 'verify' });
|
||
// A person adds a rule the branch passes, but the run's change does not.
|
||
await pushToTarget(
|
||
{
|
||
'check.mjs': checkWith(
|
||
"if (existsSync('notes.md')) { console.error('notes belong in docs/'); process.exit(1); }",
|
||
),
|
||
},
|
||
'Keep notes in docs/',
|
||
);
|
||
const checksBefore = checkRuns;
|
||
expect(await session.verifier.verifyRun(session.input)).toEqual({ status: 'repair' });
|
||
// The branch passed without the change, so where the run started was not checked.
|
||
expect(checkRuns).toBe(checksBefore + 3);
|
||
expect(await session.coordinator.prepareRepair(session.input)).toEqual({ status: 'repair' });
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
expect(prompts.at(-1)).toContain('notes belong in docs/');
|
||
expect(await git(remote, 'show', 'main:docs/notes.md')).toBe('Notes');
|
||
});
|
||
it('still repairs when the branch already failed the check where the run started, which the run may fix', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-broken-0003');
|
||
let rounds = 0;
|
||
const session = await provision(runId, async start => {
|
||
if (++rounds === 1) return writeGreeting('hello world', 'Updated the greeting.')(start);
|
||
await writeFile(path.join(start.workspace, 'ready.txt'), 'ready\n');
|
||
return 'Added ready.txt.';
|
||
});
|
||
expect(await session.executor.executeRun(session.input)).toEqual({ status: 'verify' });
|
||
// A person pushes a rule without the file it requires; the branch fails the check anyway.
|
||
await pushToTarget(
|
||
{
|
||
'check.mjs': checkWith(
|
||
"if (!existsSync('ready.txt')) { console.error('ready.txt is missing'); process.exit(1); }",
|
||
),
|
||
},
|
||
'Require ready.txt',
|
||
);
|
||
const checksBefore = checkRuns;
|
||
expect(await session.verifier.verifyRun(session.input)).toEqual({ status: 'repair' });
|
||
expect(checkRuns).toBe(checksBefore + 4);
|
||
expect(await session.coordinator.prepareRepair(session.input)).toEqual({ status: 'repair' });
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
|
||
});
|
||
it('verifies and merges a branch pushed from the IDE through a wish that starts from it', async () => {
|
||
const project = await createProject();
|
||
// The Git service's push hook: an IDE credential may update ide/ branches only.
|
||
await git(remote, 'config', 'core.hooksPath', await installHooks(path.join(root, 'git-service')));
|
||
const remoteUrl = `file://${remote}`;
|
||
const ide = createGitWorkspace({
|
||
root: path.join(root, 'ide'),
|
||
credentialDir: path.join(root, 'ide-credentials'),
|
||
remote: remoteUrl,
|
||
});
|
||
const { directory, cloning } = await ide.apply({
|
||
projectId: project.id,
|
||
projectName: project.name,
|
||
repositoryUrl: remoteUrl,
|
||
branch: 'main',
|
||
token: 'payload.signature',
|
||
expiresAt: new Date(Date.now() + 900_000).toISOString(),
|
||
author: { name: 'Ada Lovelace', email: 'ada@example.com' },
|
||
});
|
||
await cloning;
|
||
const inIde = (...args: string[]) =>
|
||
run('git', args, { cwd: directory, env: { ...process.env, WISHFORGE_REF_PREFIX: 'refs/heads/ide/' } });
|
||
await writeFile(path.join(directory, 'notes.md'), 'written in the IDE\n');
|
||
await inIde('add', 'notes.md');
|
||
await inIde('commit', '-m', 'Notes from the IDE');
|
||
const target = await git(remote, 'rev-parse', 'main');
|
||
await expect(inIde('push', 'origin', 'HEAD:main')).rejects.toThrow(/ide\//);
|
||
await inIde('push');
|
||
const pushed = await git(remote, 'rev-parse', 'ide/work');
|
||
expect(await git(remote, 'rev-parse', 'main')).toBe(target);
|
||
expect(await git(remote, 'log', '-1', '--format=%an', 'ide/work')).toBe('Ada Lovelace');
|
||
|
||
await expect(submit(project.id, 'Merge', 'wish-ide-0000', { sourceBranch: 'main' })).rejects.toMatchObject({
|
||
code: 'INVALID_SOURCE_BRANCH',
|
||
});
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-ide-0001', {
|
||
sourceBranch: 'ide/work',
|
||
});
|
||
let sawNotes = false;
|
||
const session = await provision(runId, async start => {
|
||
sawNotes = existsSync(path.join(start.workspace, 'notes.md'));
|
||
return writeGreeting('hello world', 'Updated the greeting.')(start);
|
||
});
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
expect(sawNotes).toBe(true);
|
||
expect(prompts.at(-1)).toContain('starts from branch ide/work');
|
||
// The run started from the branch and was verified, so it lands on the target branch whole.
|
||
expect(await git(remote, 'rev-parse', 'main^')).toBe(pushed);
|
||
expect(await git(remote, 'show', 'main:notes.md')).toBe('written in the IDE');
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
|
||
expect(await services.control.run(owner, runId)).toMatchObject({ status: 'succeeded', sourceBranch: 'ide/work' });
|
||
});
|
||
it('weighs the release risk of everything the branch brings, not only the agent’s change', async () => {
|
||
const project = await createProject();
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', {
|
||
...current,
|
||
deliveryConfig: {
|
||
imageRepository: 'registry.local/greeting',
|
||
architecture: 'linux/amd64',
|
||
environmentId: 'environment-1',
|
||
dockerfile: 'Dockerfile',
|
||
containerPort: 3000,
|
||
healthPath: '/health',
|
||
smokeChecks: [],
|
||
},
|
||
});
|
||
});
|
||
// A person adds a migration in the IDE; the agent then only touches the greeting.
|
||
const seed = path.join(root, 'person');
|
||
await git(root, 'clone', remote, seed);
|
||
await git(seed, 'switch', '--create', 'ide/work');
|
||
await mkdir(path.join(seed, 'migrations'));
|
||
await writeFile(path.join(seed, 'migrations', '001-notes.sql'), 'create table notes (body text);\n');
|
||
await writeFile(path.join(seed, 'Dockerfile'), 'FROM scratch\n');
|
||
await git(seed, 'add', '.');
|
||
await git(seed, 'commit', '-m', 'Add a notes table');
|
||
await git(seed, 'push', 'origin', 'ide/work');
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-ide-0003', {
|
||
sourceBranch: 'ide/work',
|
||
});
|
||
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'));
|
||
expect(await drive(session)).toMatchObject({ status: 'release' });
|
||
const checkpoint = await store.transaction(TENANT, async tx =>
|
||
tx.get<CheckpointRecord>('checkpoint', (await tx.get<Run>('run', runId))!.checkpointId!),
|
||
);
|
||
expect(checkpoint?.trustedClassification).toMatchObject({ migration: true });
|
||
expect(checkpoint?.trustedClassification?.changedPaths).toEqual(
|
||
expect.arrayContaining(['Dockerfile', 'greeting.txt', 'migrations/001-notes.sql']),
|
||
);
|
||
|
||
// The same when the build only succeeds after continuing, and nothing was classified before:
|
||
// a new branch with its own migration, so the target gains it only through this run.
|
||
await git(seed, 'fetch', 'origin');
|
||
await git(seed, 'switch', '--create', 'ide/tags', 'origin/main');
|
||
await writeFile(path.join(seed, 'migrations', '002-tags.sql'), 'create table tags (name text);\n');
|
||
await git(seed, 'add', '.');
|
||
await git(seed, 'commit', '-m', 'Add a tags table');
|
||
await git(seed, 'push', 'origin', 'ide/tags');
|
||
const second = await submit(project.id, 'Make the greeting say hello world again', 'wish-ide-0004', {
|
||
sourceBranch: 'ide/tags',
|
||
});
|
||
failingBuilds = 1;
|
||
const failing = await provision(second, writeGreeting('hello world again', 'Updated the greeting.'));
|
||
expect(await drive(failing)).toMatchObject({ status: 'waiting', reason: 'infrastructure' });
|
||
const integrated = await services.control.run(owner, second);
|
||
expect(integrated).toMatchObject({ stage: 'build', integratedCommit: expect.any(String) });
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<CheckpointRecord>('checkpoint', integrated.checkpointId!))!;
|
||
delete current.trustedClassification;
|
||
await tx.put('checkpoint', current);
|
||
});
|
||
await services.control.command(owner, second, { action: 'resume' }, 'resume-build-0004');
|
||
await release(second, failing.generation);
|
||
const resumed = await provision(second, async () => {
|
||
throw new Error('An integrated run must not call the model again');
|
||
});
|
||
const checksBefore = checkRuns;
|
||
const [verification] = (await evidenceOf(second)).filter(evidence => evidence.type === 'verification');
|
||
expect(await drive(resumed)).toMatchObject({ status: 'release', verificationEvidenceId: verification!.id });
|
||
// The integrated commit had passed the checks under the same policy: they did not run again.
|
||
expect(checkRuns).toBe(checksBefore);
|
||
expect((await evidenceOf(second)).filter(evidence => evidence.type === 'verification')).toHaveLength(1);
|
||
const reclassified = await store.transaction(TENANT, tx =>
|
||
tx.get<CheckpointRecord>('checkpoint', integrated.checkpointId!),
|
||
);
|
||
expect(reclassified?.trustedClassification).toMatchObject({ migration: true });
|
||
expect(reclassified?.trustedClassification?.changedPaths).toContain('migrations/002-tags.sql');
|
||
});
|
||
it('weighs the release risk of everything the environment has not run yet, including direct pushes', async () => {
|
||
const project = await createProject();
|
||
const deployed = await git(remote, 'rev-parse', 'main');
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', {
|
||
...current,
|
||
deliveryConfig: {
|
||
imageRepository: 'registry.local/greeting',
|
||
architecture: 'linux/amd64',
|
||
environmentId: 'environment-1',
|
||
dockerfile: 'Dockerfile',
|
||
containerPort: 3000,
|
||
healthPath: '/health',
|
||
smokeChecks: [],
|
||
},
|
||
});
|
||
// The environment runs the project as it was before the person's push below.
|
||
const stamp = new Date().toISOString();
|
||
const entity = { tenantId: TENANT, version: 1, createdAt: stamp, updatedAt: stamp, projectId: project.id };
|
||
await tx.put('release', {
|
||
...entity,
|
||
id: 'release-live',
|
||
environmentId: 'environment-1',
|
||
status: 'healthy',
|
||
sourceCommit: deployed,
|
||
} as Release);
|
||
await tx.put('environment', { ...entity, id: 'environment-1', currentReleaseId: 'release-live' } as Environment);
|
||
});
|
||
// A person pushes a migration straight to the target branch; the agent then only touches the greeting.
|
||
const person = path.join(root, 'person');
|
||
await git(root, 'clone', remote, person);
|
||
await mkdir(path.join(person, 'migrations'));
|
||
await writeFile(path.join(person, 'migrations', '001-notes.sql'), 'create table notes (body text);\n');
|
||
await writeFile(path.join(person, 'Dockerfile'), 'FROM scratch\n');
|
||
await git(person, 'add', '.');
|
||
await git(person, 'commit', '-m', 'Add a notes table');
|
||
await git(person, 'push', 'origin', 'main');
|
||
const classification = (runId: string) =>
|
||
store.transaction(TENANT, async tx => {
|
||
const run = (await tx.get<Run>('run', runId))!;
|
||
return (await tx.get<CheckpointRecord>('checkpoint', run.checkpointId!))?.trustedClassification;
|
||
});
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-direct-0001');
|
||
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'));
|
||
expect(await drive(session)).toMatchObject({ status: 'release' });
|
||
expect(await classification(runId)).toMatchObject({
|
||
migration: true,
|
||
partial: false,
|
||
changedPaths: ['Dockerfile', 'greeting.txt', 'migrations/001-notes.sql'],
|
||
});
|
||
|
||
// The environment runs a commit the repository does not have, as after a rewrite of the branch.
|
||
await store.transaction(TENANT, async tx => {
|
||
const live = (await tx.get<Release>('release', 'release-live'))!;
|
||
await tx.put('release', { ...live, sourceCommit: 'f'.repeat(40) });
|
||
});
|
||
const second = await submit(project.id, 'Make the greeting say hello world again', 'wish-direct-0002');
|
||
const next = await provision(second, writeGreeting('hello world again', 'Updated the greeting.'));
|
||
expect(await drive(next)).toMatchObject({ status: 'release' });
|
||
expect(await classification(second)).toMatchObject({ partial: true, changedPaths: ['greeting.txt'] });
|
||
});
|
||
it('repeats the checks when a build continues after the protected policy changed', async () => {
|
||
const project = await createProject();
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', {
|
||
...current,
|
||
deliveryConfig: {
|
||
imageRepository: 'registry.local/greeting',
|
||
architecture: 'linux/amd64',
|
||
environmentId: 'environment-1',
|
||
dockerfile: 'Dockerfile',
|
||
containerPort: 3000,
|
||
healthPath: '/health',
|
||
smokeChecks: [],
|
||
},
|
||
});
|
||
});
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-policy-0001');
|
||
failingBuilds = 1;
|
||
const failing = await provision(runId, async start => {
|
||
await writeFile(path.join(start.workspace, 'Dockerfile'), 'FROM scratch\n');
|
||
return writeGreeting('hello world', 'Updated the greeting.')(start);
|
||
});
|
||
expect(await drive(failing)).toMatchObject({ status: 'waiting', reason: 'infrastructure' });
|
||
expect(await services.control.run(owner, runId)).toMatchObject({ stage: 'build' });
|
||
// The operator tightens the protected checks before continuing: the earlier verdict no longer counts.
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', {
|
||
...current,
|
||
verificationCommands: [...current.verificationCommands, [process.execPath, 'check.mjs', 'again']],
|
||
});
|
||
});
|
||
await services.control.command(owner, runId, { action: 'resume' }, 'resume-policy-0001');
|
||
await release(runId, failing.generation);
|
||
const resumed = await provision(runId, async () => {
|
||
throw new Error('An integrated run must not call the model again');
|
||
});
|
||
const checksBefore = checkRuns;
|
||
expect(await drive(resumed)).toMatchObject({ status: 'release' });
|
||
expect(checkRuns).toBeGreaterThan(checksBefore);
|
||
expect((await evidenceOf(runId)).filter(evidence => evidence.type === 'verification')).toHaveLength(2);
|
||
});
|
||
it('waits for answers when the agent asks before implementing, and continues from planning (FR-C4)', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Greet people', 'wish-question-0001');
|
||
const asked = await provision(
|
||
runId,
|
||
async () =>
|
||
'The requirement leaves the wording open.\n\nQUESTIONS:\n- Which greeting should it show?\n- Should it be lowercase?',
|
||
);
|
||
expect(await drive(asked)).toMatchObject({ status: 'waiting', reason: 'clarification' });
|
||
const waiting = await services.control.run(owner, runId);
|
||
expect(waiting).toMatchObject({ status: 'waiting', waitingReason: 'clarification', repairCount: 0 });
|
||
expect(waiting.waitingDetail).toContain('1. Which greeting should it show?\n2. Should it be lowercase?');
|
||
expect(await evidenceOf(runId)).toEqual([]);
|
||
|
||
await services.control.command(
|
||
owner,
|
||
runId,
|
||
{ action: 'answer', message: 'Show "hello world", in lowercase.' },
|
||
'answer-question-0001',
|
||
);
|
||
expect(await services.control.run(owner, runId)).toMatchObject({ status: 'queued', stage: 'plan', revision: 2 });
|
||
await release(runId, asked.generation);
|
||
const answered = await provision(runId, writeGreeting('hello world', 'Set the greeting as answered.'));
|
||
expect(await drive(answered)).toEqual({ status: 'complete' });
|
||
expect(prompts.at(-1)).toContain('Show "hello world", in lowercase.');
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
|
||
});
|
||
it('installs dependencies through the operator’s egress proxy in both sandboxes', async () => {
|
||
const project = await createProject();
|
||
const needsProxy = [
|
||
process.execPath,
|
||
'-e',
|
||
"process.exit(process.env.HTTPS_PROXY === 'http://proxy.test:3128' ? 0 : 1)",
|
||
];
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', { ...current, installCommands: [needsProxy] });
|
||
});
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-proxy-0001');
|
||
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'), {
|
||
installProxy: 'http://proxy.test:3128',
|
||
});
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
// Without the proxy the installation cannot reach a registry and the run waits.
|
||
const second = await submit(project.id, 'Greet again', 'wish-proxy-0002');
|
||
const unproxied = await provision(second, writeGreeting('hello world again', 'Updated the greeting.'));
|
||
expect(await drive(unproxied)).toMatchObject({ status: 'waiting' });
|
||
// The agent never ran, so its workspace held nothing worth keeping; it does not pile up on disk.
|
||
expect(await readdir(path.join(root, 'workspaces', unproxied.generation, TENANT))).toEqual([]);
|
||
});
|
||
it('repairs an installation the agent’s manifest change broke, but waits out a network failure', async () => {
|
||
const project = await createProject();
|
||
// Stands in for a package manager: a dependency the registry does not have fails the install.
|
||
const install = [
|
||
process.execPath,
|
||
'-e',
|
||
[
|
||
"const fs = require('node:fs');",
|
||
"const wanted = fs.existsSync('package.json') ? JSON.parse(fs.readFileSync('package.json', 'utf8')).dependencies ?? {} : {};",
|
||
"if (wanted['left-pad-typo']) { console.error('npm ERR! 404 Not Found - GET https://registry.npmjs.org/left-pad-typo'); process.exit(1); }",
|
||
"if (wanted['slow-registry']) { console.error('npm ERR! network request failed, reason: connect ETIMEDOUT'); process.exit(1); }",
|
||
].join(' '),
|
||
];
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', { ...current, installCommands: [install] });
|
||
});
|
||
const manifest = (dependencies: Record<string, string>) =>
|
||
JSON.stringify({ name: 'greeting', private: true, dependencies });
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-install-0001');
|
||
const session = await provision(runId, async start => {
|
||
await writeGreeting('hello world', 'Updated the greeting.')(start);
|
||
// The first attempt adds a misspelled dependency; the repair round is told why it broke.
|
||
const repairing = start.prompt.includes("installing the project's dependencies failed");
|
||
await writeFile(
|
||
path.join(start.workspace, 'package.json'),
|
||
manifest(repairing ? {} : { 'left-pad-typo': '1.0.0' }),
|
||
);
|
||
return repairing ? 'Removed the misspelled dependency.' : 'Updated the greeting and added a dependency.';
|
||
});
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
expect(prompts).toHaveLength(2);
|
||
expect(prompts[1]).toContain('404 Not Found - GET https://registry.npmjs.org/left-pad-typo');
|
||
expect(await services.control.run(owner, runId)).toMatchObject({ status: 'succeeded', repairCount: 1 });
|
||
expect((await evidenceOf(runId)).find(evidence => evidence.installFailure)?.changedManifests).toEqual([
|
||
'package.json',
|
||
]);
|
||
const lessons = await store.transaction(TENANT, tx => tx.list<Lesson>('lesson', { projectId: project.id }));
|
||
expect(lessons.map(lesson => lesson.summary)).toEqual([
|
||
expect.stringMatching(/^Installing dependencies with `.+` failed \(exit code 1\): .*left-pad-typo/),
|
||
]);
|
||
|
||
// The same manifest change failing on the network is the environment's: no repair round is spent.
|
||
const second = await submit(project.id, 'Greet again', 'wish-install-0002');
|
||
const offline = await provision(second, async start => {
|
||
await writeGreeting('hello world again', 'Updated the greeting.')(start);
|
||
await writeFile(path.join(start.workspace, 'package.json'), manifest({ 'slow-registry': '1.0.0' }));
|
||
return 'Added a dependency.';
|
||
});
|
||
expect(await drive(offline)).toMatchObject({ status: 'waiting', reason: 'infrastructure' });
|
||
expect(await services.control.run(owner, second)).toMatchObject({ repairCount: 0 });
|
||
});
|
||
it('saves the work of a run paused mid-change, and continues from it', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-pause-0001');
|
||
const session = await provision(
|
||
runId,
|
||
async (start, stopped) => {
|
||
await writeFile(path.join(start.workspace, 'greeting.txt'), 'hello wor\n');
|
||
// The person pauses while the agent is still at work; it keeps going until it is stopped.
|
||
await services.control.command(owner, runId, { action: 'pause' }, 'pause-mid-change');
|
||
await new Promise(resolve => stopped!.addEventListener('abort', resolve, { once: true }));
|
||
return 'Stopped mid-change.';
|
||
},
|
||
{
|
||
heartbeatMs: 25,
|
||
// The stopped session never reports a total; what it did use still counts.
|
||
stoppedUsage: {
|
||
input_tokens: 40,
|
||
output_tokens: 10,
|
||
cache_read_input_tokens: 9500,
|
||
final: true,
|
||
estimated: true,
|
||
},
|
||
},
|
||
);
|
||
expect(await drive(session)).toEqual({ status: 'paused' });
|
||
const paused = await services.control.run(owner, runId);
|
||
expect(paused).toMatchObject({ status: 'paused', checkpointId: expect.any(String), usedTokens: 1000 });
|
||
const saved = await store.transaction(TENANT, tx => tx.get<CheckpointRecord>('checkpoint', paused.checkpointId!));
|
||
expect(saved?.checkpoint).toMatchObject({
|
||
patch: expect.stringContaining('+hello wor'),
|
||
providerCompletion: { status: 'canceled', finalUsage: [{ tokens: 1000, operationId: expect.any(String) }] },
|
||
});
|
||
expect((await services.control.events(owner, runId)).map(event => event.message)).toContain(
|
||
'Model usage (estimated: the session stopped before reporting its total): 40 input, 10 output, 0 cache creation and 9500 cache read tokens; 1000 count toward the budget (cache reads a tenth)',
|
||
);
|
||
|
||
await services.control.command(owner, runId, { action: 'resume' }, 'resume-after-pause');
|
||
await release(runId, session.generation);
|
||
let seen = '';
|
||
const resumed = await provision(runId, async start => {
|
||
seen = await readFile(path.join(start.workspace, 'greeting.txt'), 'utf8');
|
||
return writeGreeting('hello world', 'Finished the greeting.')(start);
|
||
});
|
||
expect(await drive(resumed)).toEqual({ status: 'complete' });
|
||
expect(seen).toBe('hello wor\n');
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
|
||
// The stopped session is counted once, though continuing settles its saved usage again.
|
||
expect((await services.control.run(owner, runId)).usedTokens).toBe(1120);
|
||
});
|
||
it('records the checks as sandboxed when the operator sandboxes host checks', async () => {
|
||
const platform = Object.getOwnPropertyDescriptor(process, 'platform')!;
|
||
Object.defineProperty(process, 'platform', { value: 'darwin' });
|
||
try {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-host-sandbox-0001');
|
||
const sandboxed: string[][] = [];
|
||
// Stands in for sandbox-exec: notes the command it was given and runs it as it is.
|
||
const seatbelt: CommandExecutor = async command => {
|
||
if (command.executable !== '/usr/bin/sandbox-exec') return execute(command);
|
||
expect(command.args[0]).toBe('-p');
|
||
sandboxed.push(command.args.slice(2));
|
||
return execute({ ...command, executable: command.args[2]!, args: command.args.slice(3) });
|
||
};
|
||
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'), {
|
||
execute: seatbelt,
|
||
hostCheckSandbox: { readable: [], writable: [] },
|
||
});
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
expect(sandboxed).toContainEqual([process.execPath, 'check.mjs']);
|
||
const verification = (await evidenceOf(runId)).find(evidence => evidence.type === 'verification');
|
||
expect(verification).toMatchObject({ passed: true, isolation: 'host', checkSandbox: 'seatbelt' });
|
||
} finally {
|
||
Object.defineProperty(process, 'platform', platform);
|
||
}
|
||
});
|
||
it('waits for a source branch that was not pushed, and starts once it exists', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-ide-0002', {
|
||
sourceBranch: 'ide/missing',
|
||
});
|
||
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'));
|
||
expect(await drive(session)).toMatchObject({ status: 'waiting', reason: 'git_storage' });
|
||
expect((await services.control.run(owner, runId)).waitingDetail).toContain('SOURCE_BRANCH_UNAVAILABLE');
|
||
// Once the branch is pushed, continuing the run starts from it.
|
||
const seed = path.join(root, 'person');
|
||
await git(root, 'clone', remote, seed);
|
||
await git(seed, 'push', 'origin', 'HEAD:refs/heads/ide/missing');
|
||
await services.control.command(owner, runId, { action: 'resume' }, 'resume-branch-0001');
|
||
await release(runId, session.generation);
|
||
const resumed = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'));
|
||
expect(await drive(resumed)).toEqual({ status: 'complete' });
|
||
});
|
||
it('holds a change to reviewed paths until a person approves it, then integrates it without repeating the checks', async () => {
|
||
const project = await createProject();
|
||
await holdForReview(project.id, ['greeting.txt']);
|
||
const developer = await person(project.id, 'li', 'reviewed');
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-review-0001', {}, developer);
|
||
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'));
|
||
expect(await drive(session)).toMatchObject({ status: 'waiting', reason: 'code_review' });
|
||
const held = await services.control.run(owner, runId);
|
||
expect(held).toMatchObject({ status: 'waiting', waitingReason: 'code_review', stage: 'integrate' });
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello');
|
||
const review = (await services.control.codeReview(owner, runId))!;
|
||
expect(review).toMatchObject({ status: 'pending', paths: ['greeting.txt'], sourceCommit: held.verifiedCommit });
|
||
const checks = checkRuns;
|
||
// Continuing does not get around the review: the verifier asks again and the run waits again.
|
||
await services.control.command(owner, runId, { action: 'resume' }, 'resume-review-0001');
|
||
await release(runId, session.generation);
|
||
const resumed = await provision(runId, async () => {
|
||
throw new Error('The agent does not run again');
|
||
});
|
||
expect(await drive(resumed)).toMatchObject({ status: 'waiting', reason: 'code_review' });
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello');
|
||
// A developer cannot approve.
|
||
await expect(
|
||
services.control.decideCodeReview(
|
||
{ ...owner, userId: 'developer', role: 'developer' },
|
||
runId,
|
||
{ decision: 'approve', sourceCommit: review.sourceCommit },
|
||
'approve-developer',
|
||
),
|
||
).rejects.toMatchObject({ code: 'FORBIDDEN' });
|
||
await services.control.decideCodeReview(
|
||
owner,
|
||
runId,
|
||
{ decision: 'approve', sourceCommit: review.sourceCommit },
|
||
'approve-0001',
|
||
);
|
||
expect(await services.control.run(owner, runId)).toMatchObject({ status: 'queued' });
|
||
await release(runId, resumed.generation);
|
||
const approved = await provision(runId, async () => {
|
||
throw new Error('The agent does not run again');
|
||
});
|
||
expect(await drive(approved)).toEqual({ status: 'complete' });
|
||
// The approved commit integrated; the checks it passed were not repeated.
|
||
expect(checkRuns).toBe(checks);
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
|
||
expect(await services.control.codeReview(owner, runId)).toMatchObject({ status: 'approved', decidedBy: 'owner' });
|
||
});
|
||
|
||
it('lets a developer the project trusts merge backend changes without a review', async () => {
|
||
const project = await createProject();
|
||
const developer = await join(project.id, 'developer', 'developer');
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', {
|
||
...current,
|
||
codeReview: { paths: ['**'], exceptPaths: ['docs/'], trustedMembers: [developer.userId] },
|
||
});
|
||
});
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-review-trusted', {}, developer);
|
||
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'));
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
|
||
expect(await services.control.codeReview(owner, runId)).toBeNull();
|
||
const events = await services.control.events(owner, runId);
|
||
expect(events.find(event => event.type === 'code_review.skipped')).toMatchObject({
|
||
data: { requestedBy: developer.userId, paths: ['greeting.txt'] },
|
||
});
|
||
});
|
||
|
||
it('keeps a frontend-only wish out of backend code: such a change is put back and the agent tries again', async () => {
|
||
const project = await createProject();
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', { ...current, codeReview: { paths: ['**'], exceptPaths: ['greeting.txt'] } });
|
||
});
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-frontend-0001', {
|
||
frontendOnly: true,
|
||
});
|
||
const checks = checkRuns;
|
||
let round = 0;
|
||
const session = await provision(runId, async start => {
|
||
round++;
|
||
if (round > 1) return 'Kept to the greeting.';
|
||
await writeFile(path.join(start.workspace, 'check.mjs'), "console.log('weakened');\n");
|
||
await writeFile(path.join(start.workspace, 'server.txt'), 'backend\n');
|
||
return writeGreeting('hello world', 'Updated the greeting, the check and the server.')(start);
|
||
});
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
expect(await services.control.run(owner, runId)).toMatchObject({
|
||
status: 'succeeded',
|
||
repairCount: 1,
|
||
frontendOnly: true,
|
||
});
|
||
// The first change went back before any check ran, naming the backend files it touched.
|
||
const scope = (await evidenceOf(runId)).find(e => e.type === 'verification' && !e.passed)!;
|
||
expect(scope).toMatchObject({ backendPaths: ['check.mjs', 'server.txt'], commands: [] });
|
||
expect(checkRuns).toBe(checks + 1);
|
||
expect(prompts[0]).toContain('This wish may change only frontend code');
|
||
expect(prompts[0]).toContain('except those matching greeting.txt');
|
||
expect(prompts[1]).toContain('Repair round 1: this wish may change only frontend code');
|
||
expect(prompts[1]).toContain('server.txt');
|
||
// The platform put the backend files back; only the frontend change reached main, without a review.
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
|
||
expect(await git(remote, 'show', 'main:check.mjs')).toContain('greeting ok');
|
||
expect(await git(remote, 'ls-tree', '--name-only', 'main')).not.toContain('server.txt');
|
||
expect(await services.control.codeReview(owner, runId)).toBeNull();
|
||
});
|
||
|
||
it('lets an answer widen a frontend-only wish to backend code once the agent asks for it', async () => {
|
||
const project = await createProject();
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', {
|
||
...current,
|
||
codeReview: { paths: ['**'], exceptPaths: ['greeting.txt'], trustedMembers: [owner.userId] },
|
||
});
|
||
});
|
||
const runId = await submit(project.id, 'Greet and log it on the server', 'wish-frontend-0002', {
|
||
frontendOnly: true,
|
||
});
|
||
const asked = await provision(
|
||
runId,
|
||
async () => 'Logging needs server.txt, which is backend code.\n\nQUESTIONS:\n- May this wish change server.txt?',
|
||
);
|
||
expect(await drive(asked)).toMatchObject({ status: 'waiting', reason: 'clarification' });
|
||
expect(prompts.at(-1)).toContain('change no files and ask instead');
|
||
// Only an answer to a frontend-only wish widens it.
|
||
await expect(
|
||
services.control.command(owner, runId, { action: 'resume', allowBackend: true }, 'scope-resume-0001'),
|
||
).rejects.toMatchObject({ code: 'SCOPE_UNCHANGED' });
|
||
await services.control.command(owner, runId, { action: 'answer', allowBackend: true }, 'scope-answer-0001');
|
||
const widened = await services.control.run(owner, runId);
|
||
expect(widened).toMatchObject({ frontendOnly: false, revision: 2, status: 'queued', stage: 'plan' });
|
||
expect(widened.goal).toContain('代码范围改为前端和后端:这个愿望可以修改后端代码。');
|
||
const scoped = (await services.control.events(owner, runId)).find(event => event.type === 'run.scope');
|
||
expect(scoped).toMatchObject({ data: { frontendOnly: false, changedBy: owner.userId } });
|
||
await expect(
|
||
services.control.command(owner, runId, { action: 'answer', allowBackend: true }, 'scope-answer-0002'),
|
||
).rejects.toMatchObject({ code: 'SCOPE_UNCHANGED' });
|
||
await release(runId, asked.generation);
|
||
const answered = await provision(runId, async start => {
|
||
await writeFile(path.join(start.workspace, 'server.txt'), 'log greetings\n');
|
||
return writeGreeting('hello world', 'Greeted and logged on the server.')(start);
|
||
});
|
||
expect(await drive(answered)).toEqual({ status: 'complete' });
|
||
expect(prompts.at(-1)).not.toContain('This wish may change only frontend code');
|
||
expect(prompts.at(-1)).toContain('代码范围改为前端和后端');
|
||
// The submitter may change backend code, so it merged without a review.
|
||
expect(await git(remote, 'show', 'main:server.txt')).toBe('log greetings');
|
||
expect(await services.control.codeReview(owner, runId)).toBeNull();
|
||
});
|
||
|
||
it('runs a wish with a model its project offers, flash when offered unless another is chosen', async () => {
|
||
const project = await createProject();
|
||
// A project without models leaves the model to the executor: there is nothing to choose.
|
||
await expect(
|
||
submit(project.id, 'Make the greeting say hello world', 'wish-model-0001', {
|
||
model: 'deepseek-flash@latest',
|
||
}),
|
||
).rejects.toMatchObject({ code: 'MODEL_NOT_ALLOWED' });
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', { ...current, models: ['deepseek-pro@latest', 'deepseek-flash@latest'] });
|
||
});
|
||
const byDefault = await submit(project.id, 'Make the greeting say hello world', 'wish-model-0002');
|
||
expect(await services.control.run(owner, byDefault)).toMatchObject({ providerModel: 'deepseek-flash@latest' });
|
||
await expect(
|
||
submit(project.id, 'Make the greeting say hello world', 'wish-model-0003', { model: 'gpt-5.4' }),
|
||
).rejects.toMatchObject({ code: 'MODEL_NOT_ALLOWED' });
|
||
const chosen = await submit(project.id, 'Make the greeting say hello world', 'wish-model-0004', {
|
||
model: 'deepseek-flash@latest',
|
||
});
|
||
const run = await services.control.run(owner, chosen);
|
||
expect(run.providerModel).toBe('deepseek-flash@latest');
|
||
expect(run).not.toHaveProperty('model');
|
||
// The executor starts the agent with the model the run pinned.
|
||
let started: string | undefined;
|
||
const session = await provision(chosen, async start => {
|
||
started = start.model;
|
||
return writeGreeting('hello world', 'Completed the greeting.')(start);
|
||
});
|
||
await session.executor.executeRun(session.input);
|
||
expect(started).toBe('deepseek-flash@latest');
|
||
});
|
||
it('keeps an account that may only wish for frontend changes to frontend code', async () => {
|
||
const project = await createProject();
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', { ...current, codeReview: { paths: ['**'], exceptPaths: ['greeting.txt'] } });
|
||
});
|
||
const frontend = await person(project.id, 'wang', 'frontend');
|
||
// Its wishes are frontend-only whether it says so or not, and it cannot ask for more.
|
||
const own = await submit(project.id, 'Make the greeting say hello world', 'wish-scope-0001', {}, frontend);
|
||
expect(await services.control.run(owner, own)).toMatchObject({ frontendOnly: true, requestedBy: 'local:wang' });
|
||
await expect(
|
||
submit(project.id, 'Log greetings on the server', 'wish-scope-0002', { frontendOnly: false }, frontend),
|
||
).rejects.toMatchObject({ code: 'FRONTEND_ONLY_ACCOUNT' });
|
||
await expect(
|
||
services.control.command(frontend, own, { action: 'answer', allowBackend: true }, 'scope-widen-0001'),
|
||
).rejects.toMatchObject({ code: 'FRONTEND_ONLY_ACCOUNT' });
|
||
// Accounts that may not wish at all cannot submit.
|
||
const releaser = await person(project.id, 'zhao', 'releaser');
|
||
await expect(submit(project.id, 'Anything at all', 'wish-scope-0003', {}, releaser)).rejects.toMatchObject({
|
||
code: 'FORBIDDEN',
|
||
});
|
||
const done = await provision(own, writeGreeting('hello world', 'Updated the greeting.'));
|
||
expect(await drive(done)).toEqual({ status: 'complete' });
|
||
await release(own, done.generation);
|
||
// It can take back a frontend-only change, as a frontend-only run.
|
||
const { run_id: undo } = await services.control.revert(frontend, own, 'revert-scope-0001');
|
||
expect(await services.control.run(owner, undo)).toMatchObject({ frontendOnly: true, requestedBy: 'local:wang' });
|
||
await services.control.command(owner, undo, { action: 'cancel' }, 'cancel-scope-0001');
|
||
// Neither adding to, nor taking back, what may change backend code.
|
||
const wide = await submit(project.id, 'Greet louder', 'wish-scope-0004', { frontendOnly: false });
|
||
await expect(
|
||
services.control.command(frontend, wide, { action: 'answer', message: 'Also log it' }, 'scope-answer-0001'),
|
||
).rejects.toMatchObject({ code: 'FRONTEND_ONLY_ACCOUNT' });
|
||
const louder = await provision(wide, writeGreeting('hello world!', 'Greeted louder.'));
|
||
expect(await drive(louder)).toEqual({ status: 'complete' });
|
||
await release(wide, louder.generation);
|
||
await expect(services.control.revert(frontend, wide, 'revert-scope-0002')).rejects.toMatchObject({
|
||
code: 'FRONTEND_ONLY_ACCOUNT',
|
||
message: expect.stringContaining('to take it back'),
|
||
});
|
||
// Where the project does not say which code is backend, everything is frontend code.
|
||
await store.transaction(TENANT, async tx => {
|
||
const current = (await tx.get<Project>('project', project.id))!;
|
||
await tx.put('project', { ...current, codeReview: undefined });
|
||
});
|
||
const open = await submit(project.id, 'Greet in French', 'wish-scope-0005', { frontendOnly: false }, frontend);
|
||
expect((await services.control.run(owner, open)).frontendOnly).toBe(false);
|
||
});
|
||
|
||
it('merges a backend change directly only when everyone who asked for it may', async () => {
|
||
const project = await createProject();
|
||
await holdForReview(project.id, ['greeting.txt']);
|
||
const fullstack = await person(project.id, 'chen', 'fullstack');
|
||
const reviewed = await person(project.id, 'li', 'reviewed');
|
||
const own = await submit(project.id, 'Make the greeting say hello world', 'wish-trust-0001', {}, fullstack);
|
||
const first = await provision(own, writeGreeting('hello world', 'Updated the greeting.'));
|
||
expect(await drive(first)).toEqual({ status: 'complete' });
|
||
await release(own, first.generation);
|
||
// Someone whose backend changes are reviewed adds to the requirement: the change waits for a review.
|
||
const asked = await submit(project.id, 'Greet more warmly', 'wish-trust-0002', {}, fullstack);
|
||
const question = await provision(
|
||
asked,
|
||
async () => 'The wording is open.\n\nQUESTIONS:\n- Which greeting should it show?',
|
||
);
|
||
expect(await drive(question)).toMatchObject({ reason: 'clarification' });
|
||
await services.control.command(
|
||
reviewed,
|
||
asked,
|
||
{ action: 'answer', message: 'Show "hello world!"' },
|
||
'trust-answer-0001',
|
||
);
|
||
expect(await services.control.run(owner, asked)).toMatchObject({ contributors: ['local:li'] });
|
||
await release(asked, question.generation);
|
||
const answered = await provision(asked, writeGreeting('hello world!', 'Greeted as answered.'));
|
||
expect(await drive(answered)).toMatchObject({ status: 'waiting', reason: 'code_review' });
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
|
||
});
|
||
|
||
it('lets whoever may review code decide, but not on a change they could not merge themselves', async () => {
|
||
const project = await createProject();
|
||
await holdForReview(project.id, ['greeting.txt']);
|
||
const lead = await person(project.id, 'zhou', { ...accountTypes.reviewed, reviewCode: true });
|
||
const fullstack = await person(project.id, 'chen', 'fullstack');
|
||
const reviewer = await person(project.id, 'wu', { ...accountTypes.frontend, reviewCode: true });
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-reviewer-0001', {}, lead);
|
||
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'));
|
||
expect(await drive(session)).toMatchObject({ status: 'waiting', reason: 'code_review' });
|
||
const { sourceCommit } = (await services.control.codeReview(owner, runId))!;
|
||
const approve = (by: Actor, key: string) =>
|
||
services.control.decideCodeReview(by, runId, { decision: 'approve', sourceCommit }, key);
|
||
await expect(approve(fullstack, 'approve-reviewer-0001')).rejects.toMatchObject({ code: 'FORBIDDEN' });
|
||
await expect(approve(lead, 'approve-reviewer-0002')).rejects.toMatchObject({ code: 'SELF_REVIEW' });
|
||
await approve(reviewer, 'approve-reviewer-0003');
|
||
await release(runId, session.generation);
|
||
const approved = await provision(runId, async () => {
|
||
throw new Error('The agent does not run again');
|
||
});
|
||
expect(await drive(approved)).toEqual({ status: 'complete' });
|
||
expect(await services.control.codeReview(owner, runId)).toMatchObject({
|
||
status: 'approved',
|
||
decidedBy: 'local:wu',
|
||
});
|
||
});
|
||
|
||
it('hands a rejected change back to the agent with the reviewer’s comment', async () => {
|
||
const project = await createProject();
|
||
await holdForReview(project.id, ['*.txt']);
|
||
const developer = await person(project.id, 'li', 'reviewed');
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-review-0002', {}, developer);
|
||
const session = await provision(runId, writeGreeting('hello world', 'Updated the greeting.'));
|
||
expect(await drive(session)).toMatchObject({ reason: 'code_review' });
|
||
const first = (await services.control.codeReview(owner, runId))!;
|
||
await expect(
|
||
services.control.decideCodeReview(owner, runId, { decision: 'reject', sourceCommit: first.sourceCommit }, 'r-0'),
|
||
).rejects.toThrow();
|
||
await services.control.decideCodeReview(
|
||
owner,
|
||
runId,
|
||
{ decision: 'reject', sourceCommit: first.sourceCommit, comment: '问候语末尾要加感叹号' },
|
||
'reject-0002',
|
||
);
|
||
const rejected = await services.control.run(owner, runId);
|
||
expect(rejected).toMatchObject({ status: 'queued', revision: 2, stage: 'plan' });
|
||
expect(rejected.goal).toContain('问候语末尾要加感叹号');
|
||
await release(runId, session.generation);
|
||
const again = await provision(runId, writeGreeting('hello world!', 'Added the exclamation mark.'));
|
||
expect(await drive(again)).toMatchObject({ reason: 'code_review' });
|
||
expect(prompts.at(-1)).toContain('问候语末尾要加感叹号');
|
||
const second = (await services.control.codeReview(owner, runId))!;
|
||
expect(second).toMatchObject({ status: 'pending', revision: 2 });
|
||
expect(second.sourceCommit).not.toBe(first.sourceCommit);
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello');
|
||
});
|
||
|
||
it('takes back what a run integrated, keeping the commits that came after it', async () => {
|
||
const project = await createProject();
|
||
const first = await submit(project.id, 'Make the greeting say hello world', 'wish-revert-0001');
|
||
const firstSession = await provision(first, writeGreeting('hello world', 'Fixed the greeting.'));
|
||
expect(await drive(firstSession)).toEqual({ status: 'complete' });
|
||
await release(first, firstSession.generation);
|
||
const second = await submit(project.id, 'Add a farewell', 'wish-revert-0002');
|
||
const secondSession = await provision(second, async start => {
|
||
await writeFile(path.join(start.workspace, 'farewell.txt'), 'bye\n');
|
||
await writeFile(path.join(start.workspace, 'greeting.txt'), 'hello world, and bye\n');
|
||
return 'Added a farewell.';
|
||
});
|
||
expect(await drive(secondSession)).toEqual({ status: 'complete' });
|
||
await release(second, secondSession.generation);
|
||
await pushToTarget({ 'notes.md': 'Notes\n' }, 'Add notes');
|
||
const asked = prompts.length;
|
||
const { run_id: revertId } = await services.control.revert(owner, second, 'revert-0002');
|
||
// Asking again while it runs returns the same revert.
|
||
expect((await services.control.revert(owner, second, 'revert-0003')).run_id).toBe(revertId);
|
||
const integrated = (await services.control.run(owner, second)).integratedCommit;
|
||
expect((await services.control.run(owner, revertId)).revertOf).toMatchObject({
|
||
runId: second,
|
||
integratedCommit: integrated,
|
||
});
|
||
const session = await provision(revertId, async () => {
|
||
throw new Error('A clean revert needs no agent');
|
||
});
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
expect(prompts).toHaveLength(asked);
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
|
||
await expect(git(remote, 'show', 'main:farewell.txt')).rejects.toThrow();
|
||
expect(await git(remote, 'show', 'main:notes.md')).toBe('Notes');
|
||
// A run that integrated nothing has nothing to take back.
|
||
const empty = await submit(project.id, 'Nothing yet', 'wish-revert-0004');
|
||
await expect(services.control.revert(owner, empty, 'revert-0004')).rejects.toMatchObject({
|
||
code: 'REVERT_UNAVAILABLE',
|
||
});
|
||
});
|
||
|
||
it('lets the agent resolve a revert that later commits conflict with', async () => {
|
||
const project = await createProject();
|
||
const first = await submit(project.id, 'Make the greeting say hello world', 'wish-revert-0011');
|
||
const firstSession = await provision(first, writeGreeting('hello world', 'Fixed the greeting.'));
|
||
expect(await drive(firstSession)).toEqual({ status: 'complete' });
|
||
await release(first, firstSession.generation);
|
||
const second = await submit(project.id, 'Say it twice', 'wish-revert-0012');
|
||
const secondSession = await provision(second, writeGreeting('hello world, twice', 'Twice.'));
|
||
expect(await drive(secondSession)).toEqual({ status: 'complete' });
|
||
await release(second, secondSession.generation);
|
||
await pushToTarget({ 'greeting.txt': 'hello world, twice and thrice\n' }, 'And thrice');
|
||
const { run_id: revertId } = await services.control.revert(owner, second, 'revert-0012');
|
||
let conflicted = '';
|
||
const session = await provision(revertId, async start => {
|
||
conflicted = await readFile(path.join(start.workspace, 'greeting.txt'), 'utf8');
|
||
await writeFile(path.join(start.workspace, 'greeting.txt'), 'hello world, thrice\n');
|
||
return 'Kept the later change and removed "twice".';
|
||
});
|
||
expect(await drive(session)).toEqual({ status: 'complete' });
|
||
expect(conflicted).toContain('<<<<<<<');
|
||
expect(prompts.at(-1)).toContain('takes back the change commit');
|
||
expect(prompts.at(-1)).toContain('- greeting.txt');
|
||
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world, thrice');
|
||
});
|
||
|
||
it('keeps executors inside their own run and lets nothing but integration complete a run', async () => {
|
||
const project = await createProject();
|
||
const runId = await submit(project.id, 'Make the greeting say hello world', 'wish-scope-0001');
|
||
const other = await submit(project.id, 'Another wish', 'wish-scope-0002');
|
||
const session = await provision(runId, writeGreeting('hello world', 'Done.'));
|
||
const fetcher = internalFetch(services);
|
||
const tokens = new ServiceTokens(env.WF_INTERNAL_TOKEN_KEY);
|
||
const token = await tokens.issue(TENANT, runId, 'worker', session.generation);
|
||
const call = (route: string, body?: unknown) =>
|
||
fetcher(`http://127.0.0.1:4000/internal${route}`, {
|
||
method: body === undefined ? 'GET' : 'POST',
|
||
headers: { authorization: `Bearer ${token}`, 'x-wishforge-tenant': TENANT, 'content-type': 'application/json' },
|
||
body: body === undefined ? undefined : JSON.stringify(body),
|
||
});
|
||
expect((await call(`/runs/${other}/context`)).status).toBe(403);
|
||
const leased = (await (
|
||
await call(`/runs/${runId}/lease`, { workerId: 'w', generation: session.generation })
|
||
).json()) as {
|
||
data: Run;
|
||
};
|
||
const leaseToken = leased.data.lease!.token;
|
||
// An executor can neither declare success nor jump to integration.
|
||
for (const [stage, status] of [
|
||
['clarify', 'succeeded'],
|
||
['integrate', 'running'],
|
||
])
|
||
expect((await call(`/runs/${runId}/transition`, { leaseToken, stage, status, message: 'x' })).status).toBe(403);
|
||
// Waiting requires a reason from the enumeration.
|
||
expect(
|
||
(await call(`/runs/${runId}/transition`, { leaseToken, stage: 'clarify', status: 'waiting', message: 'x' }))
|
||
.status,
|
||
).toBe(400);
|
||
// Repairs are reserved for the coordinator.
|
||
expect((await call(`/runs/${runId}/repair`, { generation: session.generation })).status).toBe(403);
|
||
});
|
||
});
|