wishforge-ide/tests/executor-settings.test.ts
wishforge dbf9fbedab
Some checks failed
CI / Format, types and tests (pull_request) Has been cancelled
CI / PostgreSQL migrations and integration tests (pull_request) Has been cancelled
🐛 Align ExecutorSettings across WishRefiner, Copilot, CodeWiki and Run resume
2026-11-11 17:34:05 -08:00

358 lines
16 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* A tenant admin's executor settings: gateway, models and time limit per executor. Unset fields
* keep the operator's configuration; the tenant's gateway goes only with the tenant's own key.
*/
import { beforeEach, describe, expect, it } from 'vitest';
import type { Actor, Run } from '@wishforge/contracts';
import { CodeWiki } from '../apps/api/src/domain/code-wiki.js';
import { ControlPlane } from '../apps/api/src/domain/control-plane.js';
import { Connections, CredentialVault } from '../apps/api/src/domain/connections.js';
import { Copilot } from '../apps/api/src/domain/copilot.js';
import { ExecutorSettingsService } from '../apps/api/src/domain/executor-settings.js';
import { WishRefiner, askModels } from '../apps/api/src/domain/wish-refiner.js';
import { MemoryStore } from './helpers/memory-store.js';
const owner: Actor = { tenantId: 'alpha', userId: 'owner', role: 'owner', kind: 'human' };
const admin: Actor = { ...owner, userId: 'admin', role: 'admin' };
const developer: Actor = { ...owner, userId: 'developer', role: 'developer' };
const connector: Actor = { ...owner, kind: 'connector' };
const worker: Actor = { ...owner, userId: 'worker', kind: 'worker' };
const env = {
NODE_ENV: 'test',
WF_VAULT_KEY: 'd'.repeat(64),
WF_CLAUDE_BIN: '/usr/local/bin/claude',
WF_ANTHROPIC_BASE_URL: 'https://operator-gateway.example',
};
/** The deployment's own Claude account runs this tenant's work. */
const nativeEnv = {
...env,
WF_DEPLOYMENT_MODE: 'development',
WF_PROVISIONER_BACKEND: 'local',
WF_TENANT_ID: 'alpha',
WF_CLAUDE_LOCAL_AUTH_HOME: '/Users/operator',
};
let db: MemoryStore;
beforeEach(() => {
db = new MemoryStore();
db.seed('alpha');
});
async function leasedRun(
environment: Record<string, string>,
provider: Run['provider'] = 'claude-code',
model?: string,
) {
const control = new ControlPlane(db, environment);
const project = await control.createProject(owner, {
name: 'Catalog',
repositoryUrl: 'https://example.org/catalog.git',
verificationCommands: [['node', '--test']],
});
if (model) await db.transaction('alpha', tx => tx.put('project', { ...project, models: [model] }));
if (!(await db.transaction('alpha', tx => tx.list('provider_connection'))).length)
await new Connections(db, environment).create(owner, { provider, label: 'key', apiKey: 'tenant-key-1' });
const submitted = await control.submit(
owner,
project.id,
{
goal: 'Implement catalog search',
acceptanceCriteria: ['Search returns matching items'],
provider,
budget: { maxTokens: 1000, maxMinutes: 1 },
},
`submit-${provider}-${model ?? ''}`,
);
return { control, run: await control.lease(worker, submitted.run_id, 'executor') };
}
describe('executor settings', () => {
it('lets owners and admins read and replace them, idempotently', async () => {
const settings = new ExecutorSettingsService(db, env);
expect(await settings.get(owner, 'claude-code')).toMatchObject({
provider: 'claude-code',
baseUrlAllowed: true,
supported: expect.arrayContaining(['smallFastModel', 'askModel']),
});
const body = {
defaultModel: 'deepseek-pro@latest[1m]',
smallFastModel: 'deepseek-flash@latest[1m]',
fallbackModel: 'glm-5:cloud',
timeoutMinutes: 90,
askModel: 'deepseek-flash@latest[1m]',
askFallbackModel: '',
};
const saved = await settings.put(owner, 'claude-code', body);
const { askFallbackModel: _, ...set } = body;
expect(saved).toMatchObject({ ...set, version: 1 });
expect(saved).not.toHaveProperty('askFallbackModel');
// The same body again changes nothing.
expect(await settings.put(admin, 'claude-code', body)).toMatchObject({ version: 1 });
// A replacement clears what it leaves out.
const replaced = await settings.put(admin, 'claude-code', { timeoutMinutes: 45 });
expect(replaced).toMatchObject({ timeoutMinutes: 45, version: 2 });
expect(replaced).not.toHaveProperty('defaultModel');
expect((await settings.list(owner)).map(entry => entry.provider)).toEqual(['codex', 'codebuddy', 'claude-code']);
await expect(settings.get(developer, 'claude-code')).rejects.toMatchObject({ code: 'FORBIDDEN' });
await expect(settings.put(developer, 'claude-code', {})).rejects.toMatchObject({ code: 'FORBIDDEN' });
await expect(settings.put(connector, 'claude-code', {})).rejects.toMatchObject({
code: 'HUMAN_APPROVAL_REQUIRED',
});
});
it('validates each field and what the executor takes', async () => {
const settings = new ExecutorSettingsService(db, env);
for (const body of [
{ defaultModel: 'has space' },
{ fallbackModel: '-leading-dash' },
{ timeoutMinutes: 4 },
{ timeoutMinutes: 151 },
{ timeoutMinutes: 30.5 },
{ unknown: 'field' },
])
await expect(settings.put(owner, 'claude-code', body)).rejects.toThrow();
await expect(settings.put(owner, 'codex', { smallFastModel: 'gpt-5.4-mini' })).rejects.toMatchObject({
code: 'SETTING_UNSUPPORTED',
});
await expect(settings.put(owner, 'codebuddy', { baseUrl: 'https://gateway.example' })).rejects.toMatchObject({
code: 'SETTING_UNSUPPORTED',
});
await expect(
settings.put(owner, 'claude-code', { defaultModel: 'same-model', fallbackModel: 'same-model' }),
).rejects.toMatchObject({ code: 'INVALID_FALLBACK_MODEL' });
await expect(settings.put(owner, 'unknown-provider', {})).rejects.toThrow();
});
it('accepts a tenant gateway only over HTTPS, and only where the tenant key goes to it', async () => {
const settings = new ExecutorSettingsService(db, env);
expect(await settings.put(owner, 'claude-code', { baseUrl: 'https://gateway.example/v1/' })).toMatchObject({
baseUrl: 'https://gateway.example/v1',
});
expect(await settings.put(owner, 'claude-code', { baseUrl: 'http://127.0.0.1:8080' })).toMatchObject({
baseUrl: 'http://127.0.0.1:8080',
});
for (const baseUrl of [
'http://gateway.example',
'https://user:pass@gateway.example',
'https://gateway.example/?key=1',
'not a url',
])
await expect(settings.put(owner, 'claude-code', { baseUrl })).rejects.toMatchObject({
code: 'INVALID_BASE_URL',
});
const production = new ExecutorSettingsService(db, { ...env, WF_DEPLOYMENT_MODE: 'production' });
expect(await production.get(owner, 'codex')).toMatchObject({
baseUrlAllowed: false,
baseUrlBlocked: 'operator_managed',
});
await expect(production.put(owner, 'codex', { baseUrl: 'https://gateway.example' })).rejects.toMatchObject({
code: 'BASE_URL_OPERATOR_MANAGED',
});
const native = new ExecutorSettingsService(db, nativeEnv);
expect(await native.get(owner, 'claude-code')).toMatchObject({ baseUrlBlocked: 'native_account' });
await expect(native.put(owner, 'claude-code', { baseUrl: 'https://gateway.example' })).rejects.toMatchObject({
code: 'BASE_URL_OPERATOR_MANAGED',
});
// Models and the time limit still apply to the native account.
expect(await native.put(owner, 'claude-code', { timeoutMinutes: 60 })).toMatchObject({ timeoutMinutes: 60 });
});
});
describe('executor settings in runs', () => {
it('sends the tenant key to the tenant’s gateway instead of the operator’s', async () => {
const { run } = await leasedRun(env);
const connections = new Connections(db, env);
expect(await connections.forRun(worker, run)).toMatchObject({ baseUrl: 'https://operator-gateway.example' });
await new ExecutorSettingsService(db, env).put(owner, 'claude-code', { baseUrl: 'https://tenant-gateway.example' });
expect(await connections.forRun(worker, run)).toEqual({
apiKey: 'tenant-key-1',
baseUrl: 'https://tenant-gateway.example',
});
// A deployment that later moves to its native account keeps that account’s gateway.
const stored = await db.transaction('alpha', tx => tx.get('executor_settings', 'claude-code'));
expect(stored).toMatchObject({ baseUrl: 'https://tenant-gateway.example' });
expect(await new Connections(db, nativeEnv).forRun(worker, run)).toMatchObject({
baseUrl: 'https://operator-gateway.example',
});
});
it('pins the tenant’s default model below the wish and the agent configuration', async () => {
await new ExecutorSettingsService(db, env).put(owner, 'claude-code', {
defaultModel: 'tenant-default',
smallFastModel: 'tenant-small',
fallbackModel: 'tenant-fallback',
timeoutMinutes: 75,
askModel: 'tenant-ask',
});
const plain = await leasedRun(env);
expect(plain.run.providerModel).toBe('tenant-default');
// Only what the executor applies at run time travels in the run context.
expect((await plain.control.context(worker, plain.run.id)).executorSettings).toEqual({
smallFastModel: 'tenant-small',
fallbackModel: 'tenant-fallback',
timeoutMinutes: 75,
});
const chosen = await leasedRun(env, 'claude-code', 'deepseek-pro@latest[1m]');
expect(chosen.run.providerModel).toBe('deepseek-pro@latest[1m]');
});
it('leaves runs exactly as before without settings', async () => {
const { control, run } = await leasedRun(env);
expect(run.providerModel).toBeUndefined();
expect((await control.context(worker, run.id)).executorSettings).toEqual({});
});
it('gives the wish refiner the tenant’s models first', async () => {
expect(askModels({})).toEqual(['deepseek-flash@latest', 'deepseek-pro@latest']);
expect(askModels({ WF_CLAUDE_ASK_MODEL: 'op-ask', WF_CLAUDE_ASK_FALLBACK_MODEL: '' })).toEqual(['op-ask']);
expect(askModels({ WF_CLAUDE_ASK_MODEL: 'op-ask' }, { askModel: 'tenant-ask' })).toEqual([
'tenant-ask',
'deepseek-pro@latest',
]);
expect(askModels({}, { askFallbackModel: 'tenant-fallback' })).toEqual([
'deepseek-flash@latest',
'tenant-fallback',
]);
await new ExecutorSettingsService(db, nativeEnv).put(owner, 'claude-code', { askModel: 'tenant-ask' });
const project = await new ControlPlane(db, nativeEnv).createProject(owner, {
name: 'Pilot',
repositoryUrl: 'https://example.org/pilot.git',
});
const models: (string | undefined)[] = [];
const refiner = new WishRefiner(db, nativeEnv, async input => {
models.push(input.model);
return { goal: '下载文件名带上数据集类型。', acceptanceCriteria: ['现有测试、类型检查和构建全部通过'] };
});
await refiner.refine(owner, project.id, { goal: '下载文件名要带上类型', acceptanceCriteria: [] });
expect(models).toEqual(['tenant-ask']);
});
it('sends the wish refiner, Copilot and code wiki to the tenant’s gateway and ask models', async () => {
const byokEnv = {
NODE_ENV: 'test',
WF_VAULT_KEY: 'd'.repeat(64),
WF_CLAUDE_BIN: '/usr/local/bin/claude',
};
const control = new ControlPlane(db, byokEnv);
const project = await control.createProject(owner, {
name: 'Pilot',
repositoryUrl: 'https://example.org/pilot.git',
});
await new Connections(db, byokEnv).create(owner, {
provider: 'claude-code',
label: 'key',
apiKey: 'tenant-key-1',
});
await new ExecutorSettingsService(db, byokEnv).put(owner, 'claude-code', {
baseUrl: 'https://tenant-gateway.example/v1/',
askModel: 'tenant-ask',
askFallbackModel: 'tenant-ask-fallback',
});
const refinerCalls: Array<{ model?: string; credentialEnv?: Record<string, string> }> = [];
const refiner = new WishRefiner(db, byokEnv, async input => {
refinerCalls.push({ model: input.model, credentialEnv: input.credentialEnv });
if (input.model === 'tenant-ask') throw new Error('primary overloaded');
return { goal: '下载文件名带上数据集类型。', acceptanceCriteria: ['现有测试全部通过'] };
});
await refiner.refine(owner, project.id, { goal: '下载文件名要带上类型', acceptanceCriteria: [] });
expect(refinerCalls).toEqual([
{
model: 'tenant-ask',
credentialEnv: {
ANTHROPIC_API_KEY: 'tenant-key-1',
ANTHROPIC_BASE_URL: 'https://tenant-gateway.example/v1',
ANTHROPIC_MODEL: 'tenant-ask',
ANTHROPIC_SMALL_FAST_MODEL: 'tenant-ask',
},
},
{
model: 'tenant-ask-fallback',
credentialEnv: {
ANTHROPIC_API_KEY: 'tenant-key-1',
ANTHROPIC_BASE_URL: 'https://tenant-gateway.example/v1',
ANTHROPIC_MODEL: 'tenant-ask-fallback',
ANTHROPIC_SMALL_FAST_MODEL: 'tenant-ask-fallback',
},
},
]);
const wiki = {
snapshot: async () => ({
project,
version: 'a'.repeat(40),
files: [{ path: 'wiki/features/wishes.md', content: '# 提交 Wish\n填写目标和验收标准。' }],
}),
write: async () => ({ version: 'b'.repeat(40) }),
};
let copilotInput: { model?: string; credentialEnv?: Record<string, string> } | undefined;
const copilot = new Copilot(
control,
byokEnv,
async input => {
copilotInput = { model: input.model, credentialEnv: input.credentialEnv };
return {
kind: 'faq',
answer: '填写目标和验收标准。',
citations: [{ path: 'wiki/features/wishes.md', quote: '填写目标和验收标准。' }],
};
},
wiki,
);
const session = await copilot.createSession(owner, project.id);
await copilot.ask(owner, project.id, session.id, {
content: '如何提交 Wish?',
clientId: 'client-settings-ask',
});
expect(copilotInput).toEqual({
model: 'tenant-ask',
credentialEnv: {
ANTHROPIC_API_KEY: 'tenant-key-1',
ANTHROPIC_BASE_URL: 'https://tenant-gateway.example/v1',
ANTHROPIC_MODEL: 'tenant-ask',
ANTHROPIC_SMALL_FAST_MODEL: 'tenant-ask',
},
});
const codeWikiRequests: Array<{ url: string; model: string }> = [];
const codeWiki = new CodeWiki({
store: db,
env: byokEnv,
wiki: wiki as never,
vault: new CredentialVault(byokEnv),
runs: async () => [],
code: {
forProject: () => ({
tree: async () => ({ entries: [{ path: 'README.md', type: 'blob' }] }),
blob: async () => ({ content: '# Pilot\n项目说明。' }),
}),
} as never,
fetcher: (async (url, init) => {
const body = JSON.parse(String(init?.body ?? '{}')) as { model: string };
codeWikiRequests.push({ url: String(url), model: body.model });
return new Response(JSON.stringify({ content: [{ type: 'text', text: '# 项目概览\n这是项目概览。' }] }), {
status: 200,
});
}) as typeof fetch,
});
await codeWiki.refresh(owner, project.id, ['overview']);
expect(codeWikiRequests).toEqual([{ url: 'https://tenant-gateway.example/v1/v1/messages', model: 'tenant-ask' }]);
});
it('fills in a waiting run’s missing model from newly configured executor settings on resume', async () => {
const { control, run } = await leasedRun(env);
expect(run.providerModel).toBeUndefined();
await control.releaseLease(worker, run.id, run.lease!.token);
await control.waitRun(worker, run.id, {
reason: 'provider_error',
detail: "403 The requested model 'claude-opus-5-5' is not available",
});
await new ExecutorSettingsService(db, env).put(owner, 'claude-code', {
defaultModel: 'deepseek-pro@latest',
});
const resumed = await control.command(owner, run.id, { action: 'resume' }, 'resume-waiting-run-1');
expect(resumed).toMatchObject({
status: 'queued',
providerModel: 'deepseek-pro@latest',
});
});
});