358 lines
16 KiB
TypeScript
358 lines
16 KiB
TypeScript
/**
|
||
* A tenant admin's executor settings: gateway, models and time limit per executor. Unset fields
|
||
* keep the operator's configuration; the tenant's gateway goes only with the tenant's own key.
|
||
*/
|
||
import { beforeEach, describe, expect, it } from 'vitest';
|
||
import type { Actor, Run } from '@wishforge/contracts';
|
||
import { CodeWiki } from '../apps/api/src/domain/code-wiki.js';
|
||
import { ControlPlane } from '../apps/api/src/domain/control-plane.js';
|
||
import { Connections, CredentialVault } from '../apps/api/src/domain/connections.js';
|
||
import { Copilot } from '../apps/api/src/domain/copilot.js';
|
||
import { ExecutorSettingsService } from '../apps/api/src/domain/executor-settings.js';
|
||
import { WishRefiner, askModels } from '../apps/api/src/domain/wish-refiner.js';
|
||
import { MemoryStore } from './helpers/memory-store.js';
|
||
|
||
const owner: Actor = { tenantId: 'alpha', userId: 'owner', role: 'owner', kind: 'human' };
|
||
const admin: Actor = { ...owner, userId: 'admin', role: 'admin' };
|
||
const developer: Actor = { ...owner, userId: 'developer', role: 'developer' };
|
||
const connector: Actor = { ...owner, kind: 'connector' };
|
||
const worker: Actor = { ...owner, userId: 'worker', kind: 'worker' };
|
||
const env = {
|
||
NODE_ENV: 'test',
|
||
WF_VAULT_KEY: 'd'.repeat(64),
|
||
WF_CLAUDE_BIN: '/usr/local/bin/claude',
|
||
WF_ANTHROPIC_BASE_URL: 'https://operator-gateway.example',
|
||
};
|
||
/** The deployment's own Claude account runs this tenant's work. */
|
||
const nativeEnv = {
|
||
...env,
|
||
WF_DEPLOYMENT_MODE: 'development',
|
||
WF_PROVISIONER_BACKEND: 'local',
|
||
WF_TENANT_ID: 'alpha',
|
||
WF_CLAUDE_LOCAL_AUTH_HOME: '/Users/operator',
|
||
};
|
||
let db: MemoryStore;
|
||
beforeEach(() => {
|
||
db = new MemoryStore();
|
||
db.seed('alpha');
|
||
});
|
||
async function leasedRun(
|
||
environment: Record<string, string>,
|
||
provider: Run['provider'] = 'claude-code',
|
||
model?: string,
|
||
) {
|
||
const control = new ControlPlane(db, environment);
|
||
const project = await control.createProject(owner, {
|
||
name: 'Catalog',
|
||
repositoryUrl: 'https://example.org/catalog.git',
|
||
verificationCommands: [['node', '--test']],
|
||
});
|
||
if (model) await db.transaction('alpha', tx => tx.put('project', { ...project, models: [model] }));
|
||
if (!(await db.transaction('alpha', tx => tx.list('provider_connection'))).length)
|
||
await new Connections(db, environment).create(owner, { provider, label: 'key', apiKey: 'tenant-key-1' });
|
||
const submitted = await control.submit(
|
||
owner,
|
||
project.id,
|
||
{
|
||
goal: 'Implement catalog search',
|
||
acceptanceCriteria: ['Search returns matching items'],
|
||
provider,
|
||
budget: { maxTokens: 1000, maxMinutes: 1 },
|
||
},
|
||
`submit-${provider}-${model ?? ''}`,
|
||
);
|
||
return { control, run: await control.lease(worker, submitted.run_id, 'executor') };
|
||
}
|
||
|
||
describe('executor settings', () => {
|
||
it('lets owners and admins read and replace them, idempotently', async () => {
|
||
const settings = new ExecutorSettingsService(db, env);
|
||
expect(await settings.get(owner, 'claude-code')).toMatchObject({
|
||
provider: 'claude-code',
|
||
baseUrlAllowed: true,
|
||
supported: expect.arrayContaining(['smallFastModel', 'askModel']),
|
||
});
|
||
const body = {
|
||
defaultModel: 'deepseek-pro@latest[1m]',
|
||
smallFastModel: 'deepseek-flash@latest[1m]',
|
||
fallbackModel: 'glm-5:cloud',
|
||
timeoutMinutes: 90,
|
||
askModel: 'deepseek-flash@latest[1m]',
|
||
askFallbackModel: '',
|
||
};
|
||
const saved = await settings.put(owner, 'claude-code', body);
|
||
const { askFallbackModel: _, ...set } = body;
|
||
expect(saved).toMatchObject({ ...set, version: 1 });
|
||
expect(saved).not.toHaveProperty('askFallbackModel');
|
||
// The same body again changes nothing.
|
||
expect(await settings.put(admin, 'claude-code', body)).toMatchObject({ version: 1 });
|
||
// A replacement clears what it leaves out.
|
||
const replaced = await settings.put(admin, 'claude-code', { timeoutMinutes: 45 });
|
||
expect(replaced).toMatchObject({ timeoutMinutes: 45, version: 2 });
|
||
expect(replaced).not.toHaveProperty('defaultModel');
|
||
expect((await settings.list(owner)).map(entry => entry.provider)).toEqual(['codex', 'codebuddy', 'claude-code']);
|
||
await expect(settings.get(developer, 'claude-code')).rejects.toMatchObject({ code: 'FORBIDDEN' });
|
||
await expect(settings.put(developer, 'claude-code', {})).rejects.toMatchObject({ code: 'FORBIDDEN' });
|
||
await expect(settings.put(connector, 'claude-code', {})).rejects.toMatchObject({
|
||
code: 'HUMAN_APPROVAL_REQUIRED',
|
||
});
|
||
});
|
||
|
||
it('validates each field and what the executor takes', async () => {
|
||
const settings = new ExecutorSettingsService(db, env);
|
||
for (const body of [
|
||
{ defaultModel: 'has space' },
|
||
{ fallbackModel: '-leading-dash' },
|
||
{ timeoutMinutes: 4 },
|
||
{ timeoutMinutes: 151 },
|
||
{ timeoutMinutes: 30.5 },
|
||
{ unknown: 'field' },
|
||
])
|
||
await expect(settings.put(owner, 'claude-code', body)).rejects.toThrow();
|
||
await expect(settings.put(owner, 'codex', { smallFastModel: 'gpt-5.4-mini' })).rejects.toMatchObject({
|
||
code: 'SETTING_UNSUPPORTED',
|
||
});
|
||
await expect(settings.put(owner, 'codebuddy', { baseUrl: 'https://gateway.example' })).rejects.toMatchObject({
|
||
code: 'SETTING_UNSUPPORTED',
|
||
});
|
||
await expect(
|
||
settings.put(owner, 'claude-code', { defaultModel: 'same-model', fallbackModel: 'same-model' }),
|
||
).rejects.toMatchObject({ code: 'INVALID_FALLBACK_MODEL' });
|
||
await expect(settings.put(owner, 'unknown-provider', {})).rejects.toThrow();
|
||
});
|
||
|
||
it('accepts a tenant gateway only over HTTPS, and only where the tenant key goes to it', async () => {
|
||
const settings = new ExecutorSettingsService(db, env);
|
||
expect(await settings.put(owner, 'claude-code', { baseUrl: 'https://gateway.example/v1/' })).toMatchObject({
|
||
baseUrl: 'https://gateway.example/v1',
|
||
});
|
||
expect(await settings.put(owner, 'claude-code', { baseUrl: 'http://127.0.0.1:8080' })).toMatchObject({
|
||
baseUrl: 'http://127.0.0.1:8080',
|
||
});
|
||
for (const baseUrl of [
|
||
'http://gateway.example',
|
||
'https://user:pass@gateway.example',
|
||
'https://gateway.example/?key=1',
|
||
'not a url',
|
||
])
|
||
await expect(settings.put(owner, 'claude-code', { baseUrl })).rejects.toMatchObject({
|
||
code: 'INVALID_BASE_URL',
|
||
});
|
||
const production = new ExecutorSettingsService(db, { ...env, WF_DEPLOYMENT_MODE: 'production' });
|
||
expect(await production.get(owner, 'codex')).toMatchObject({
|
||
baseUrlAllowed: false,
|
||
baseUrlBlocked: 'operator_managed',
|
||
});
|
||
await expect(production.put(owner, 'codex', { baseUrl: 'https://gateway.example' })).rejects.toMatchObject({
|
||
code: 'BASE_URL_OPERATOR_MANAGED',
|
||
});
|
||
const native = new ExecutorSettingsService(db, nativeEnv);
|
||
expect(await native.get(owner, 'claude-code')).toMatchObject({ baseUrlBlocked: 'native_account' });
|
||
await expect(native.put(owner, 'claude-code', { baseUrl: 'https://gateway.example' })).rejects.toMatchObject({
|
||
code: 'BASE_URL_OPERATOR_MANAGED',
|
||
});
|
||
// Models and the time limit still apply to the native account.
|
||
expect(await native.put(owner, 'claude-code', { timeoutMinutes: 60 })).toMatchObject({ timeoutMinutes: 60 });
|
||
});
|
||
});
|
||
|
||
describe('executor settings in runs', () => {
|
||
it('sends the tenant key to the tenant’s gateway instead of the operator’s', async () => {
|
||
const { run } = await leasedRun(env);
|
||
const connections = new Connections(db, env);
|
||
expect(await connections.forRun(worker, run)).toMatchObject({ baseUrl: 'https://operator-gateway.example' });
|
||
await new ExecutorSettingsService(db, env).put(owner, 'claude-code', { baseUrl: 'https://tenant-gateway.example' });
|
||
expect(await connections.forRun(worker, run)).toEqual({
|
||
apiKey: 'tenant-key-1',
|
||
baseUrl: 'https://tenant-gateway.example',
|
||
});
|
||
// A deployment that later moves to its native account keeps that account’s gateway.
|
||
const stored = await db.transaction('alpha', tx => tx.get('executor_settings', 'claude-code'));
|
||
expect(stored).toMatchObject({ baseUrl: 'https://tenant-gateway.example' });
|
||
expect(await new Connections(db, nativeEnv).forRun(worker, run)).toMatchObject({
|
||
baseUrl: 'https://operator-gateway.example',
|
||
});
|
||
});
|
||
|
||
it('pins the tenant’s default model below the wish and the agent configuration', async () => {
|
||
await new ExecutorSettingsService(db, env).put(owner, 'claude-code', {
|
||
defaultModel: 'tenant-default',
|
||
smallFastModel: 'tenant-small',
|
||
fallbackModel: 'tenant-fallback',
|
||
timeoutMinutes: 75,
|
||
askModel: 'tenant-ask',
|
||
});
|
||
const plain = await leasedRun(env);
|
||
expect(plain.run.providerModel).toBe('tenant-default');
|
||
// Only what the executor applies at run time travels in the run context.
|
||
expect((await plain.control.context(worker, plain.run.id)).executorSettings).toEqual({
|
||
smallFastModel: 'tenant-small',
|
||
fallbackModel: 'tenant-fallback',
|
||
timeoutMinutes: 75,
|
||
});
|
||
const chosen = await leasedRun(env, 'claude-code', 'deepseek-pro@latest[1m]');
|
||
expect(chosen.run.providerModel).toBe('deepseek-pro@latest[1m]');
|
||
});
|
||
|
||
it('leaves runs exactly as before without settings', async () => {
|
||
const { control, run } = await leasedRun(env);
|
||
expect(run.providerModel).toBeUndefined();
|
||
expect((await control.context(worker, run.id)).executorSettings).toEqual({});
|
||
});
|
||
|
||
it('gives the wish refiner the tenant’s models first', async () => {
|
||
expect(askModels({})).toEqual(['deepseek-flash@latest', 'deepseek-pro@latest']);
|
||
expect(askModels({ WF_CLAUDE_ASK_MODEL: 'op-ask', WF_CLAUDE_ASK_FALLBACK_MODEL: '' })).toEqual(['op-ask']);
|
||
expect(askModels({ WF_CLAUDE_ASK_MODEL: 'op-ask' }, { askModel: 'tenant-ask' })).toEqual([
|
||
'tenant-ask',
|
||
'deepseek-pro@latest',
|
||
]);
|
||
expect(askModels({}, { askFallbackModel: 'tenant-fallback' })).toEqual([
|
||
'deepseek-flash@latest',
|
||
'tenant-fallback',
|
||
]);
|
||
await new ExecutorSettingsService(db, nativeEnv).put(owner, 'claude-code', { askModel: 'tenant-ask' });
|
||
const project = await new ControlPlane(db, nativeEnv).createProject(owner, {
|
||
name: 'Pilot',
|
||
repositoryUrl: 'https://example.org/pilot.git',
|
||
});
|
||
const models: (string | undefined)[] = [];
|
||
const refiner = new WishRefiner(db, nativeEnv, async input => {
|
||
models.push(input.model);
|
||
return { goal: '下载文件名带上数据集类型。', acceptanceCriteria: ['现有测试、类型检查和构建全部通过'] };
|
||
});
|
||
await refiner.refine(owner, project.id, { goal: '下载文件名要带上类型', acceptanceCriteria: [] });
|
||
expect(models).toEqual(['tenant-ask']);
|
||
});
|
||
|
||
it('sends the wish refiner, Copilot and code wiki to the tenant’s gateway and ask models', async () => {
|
||
const byokEnv = {
|
||
NODE_ENV: 'test',
|
||
WF_VAULT_KEY: 'd'.repeat(64),
|
||
WF_CLAUDE_BIN: '/usr/local/bin/claude',
|
||
};
|
||
const control = new ControlPlane(db, byokEnv);
|
||
const project = await control.createProject(owner, {
|
||
name: 'Pilot',
|
||
repositoryUrl: 'https://example.org/pilot.git',
|
||
});
|
||
await new Connections(db, byokEnv).create(owner, {
|
||
provider: 'claude-code',
|
||
label: 'key',
|
||
apiKey: 'tenant-key-1',
|
||
});
|
||
await new ExecutorSettingsService(db, byokEnv).put(owner, 'claude-code', {
|
||
baseUrl: 'https://tenant-gateway.example/v1/',
|
||
askModel: 'tenant-ask',
|
||
askFallbackModel: 'tenant-ask-fallback',
|
||
});
|
||
|
||
const refinerCalls: Array<{ model?: string; credentialEnv?: Record<string, string> }> = [];
|
||
const refiner = new WishRefiner(db, byokEnv, async input => {
|
||
refinerCalls.push({ model: input.model, credentialEnv: input.credentialEnv });
|
||
if (input.model === 'tenant-ask') throw new Error('primary overloaded');
|
||
return { goal: '下载文件名带上数据集类型。', acceptanceCriteria: ['现有测试全部通过'] };
|
||
});
|
||
await refiner.refine(owner, project.id, { goal: '下载文件名要带上类型', acceptanceCriteria: [] });
|
||
expect(refinerCalls).toEqual([
|
||
{
|
||
model: 'tenant-ask',
|
||
credentialEnv: {
|
||
ANTHROPIC_API_KEY: 'tenant-key-1',
|
||
ANTHROPIC_BASE_URL: 'https://tenant-gateway.example/v1',
|
||
ANTHROPIC_MODEL: 'tenant-ask',
|
||
ANTHROPIC_SMALL_FAST_MODEL: 'tenant-ask',
|
||
},
|
||
},
|
||
{
|
||
model: 'tenant-ask-fallback',
|
||
credentialEnv: {
|
||
ANTHROPIC_API_KEY: 'tenant-key-1',
|
||
ANTHROPIC_BASE_URL: 'https://tenant-gateway.example/v1',
|
||
ANTHROPIC_MODEL: 'tenant-ask-fallback',
|
||
ANTHROPIC_SMALL_FAST_MODEL: 'tenant-ask-fallback',
|
||
},
|
||
},
|
||
]);
|
||
|
||
const wiki = {
|
||
snapshot: async () => ({
|
||
project,
|
||
version: 'a'.repeat(40),
|
||
files: [{ path: 'wiki/features/wishes.md', content: '# 提交 Wish\n填写目标和验收标准。' }],
|
||
}),
|
||
write: async () => ({ version: 'b'.repeat(40) }),
|
||
};
|
||
let copilotInput: { model?: string; credentialEnv?: Record<string, string> } | undefined;
|
||
const copilot = new Copilot(
|
||
control,
|
||
byokEnv,
|
||
async input => {
|
||
copilotInput = { model: input.model, credentialEnv: input.credentialEnv };
|
||
return {
|
||
kind: 'faq',
|
||
answer: '填写目标和验收标准。',
|
||
citations: [{ path: 'wiki/features/wishes.md', quote: '填写目标和验收标准。' }],
|
||
};
|
||
},
|
||
wiki,
|
||
);
|
||
const session = await copilot.createSession(owner, project.id);
|
||
await copilot.ask(owner, project.id, session.id, {
|
||
content: '如何提交 Wish?',
|
||
clientId: 'client-settings-ask',
|
||
});
|
||
expect(copilotInput).toEqual({
|
||
model: 'tenant-ask',
|
||
credentialEnv: {
|
||
ANTHROPIC_API_KEY: 'tenant-key-1',
|
||
ANTHROPIC_BASE_URL: 'https://tenant-gateway.example/v1',
|
||
ANTHROPIC_MODEL: 'tenant-ask',
|
||
ANTHROPIC_SMALL_FAST_MODEL: 'tenant-ask',
|
||
},
|
||
});
|
||
|
||
const codeWikiRequests: Array<{ url: string; model: string }> = [];
|
||
const codeWiki = new CodeWiki({
|
||
store: db,
|
||
env: byokEnv,
|
||
wiki: wiki as never,
|
||
vault: new CredentialVault(byokEnv),
|
||
runs: async () => [],
|
||
code: {
|
||
forProject: () => ({
|
||
tree: async () => ({ entries: [{ path: 'README.md', type: 'blob' }] }),
|
||
blob: async () => ({ content: '# Pilot\n项目说明。' }),
|
||
}),
|
||
} as never,
|
||
fetcher: (async (url, init) => {
|
||
const body = JSON.parse(String(init?.body ?? '{}')) as { model: string };
|
||
codeWikiRequests.push({ url: String(url), model: body.model });
|
||
return new Response(JSON.stringify({ content: [{ type: 'text', text: '# 项目概览\n这是项目概览。' }] }), {
|
||
status: 200,
|
||
});
|
||
}) as typeof fetch,
|
||
});
|
||
await codeWiki.refresh(owner, project.id, ['overview']);
|
||
expect(codeWikiRequests).toEqual([{ url: 'https://tenant-gateway.example/v1/v1/messages', model: 'tenant-ask' }]);
|
||
});
|
||
|
||
it('fills in a waiting run’s missing model from newly configured executor settings on resume', async () => {
|
||
const { control, run } = await leasedRun(env);
|
||
expect(run.providerModel).toBeUndefined();
|
||
await control.releaseLease(worker, run.id, run.lease!.token);
|
||
await control.waitRun(worker, run.id, {
|
||
reason: 'provider_error',
|
||
detail: "403 The requested model 'claude-opus-5-5' is not available",
|
||
});
|
||
await new ExecutorSettingsService(db, env).put(owner, 'claude-code', {
|
||
defaultModel: 'deepseek-pro@latest',
|
||
});
|
||
const resumed = await control.command(owner, run.id, { action: 'resume' }, 'resume-waiting-run-1');
|
||
expect(resumed).toMatchObject({
|
||
status: 'queued',
|
||
providerModel: 'deepseek-pro@latest',
|
||
});
|
||
});
|
||
});
|