wishforge-ide/tests/wish-refiner.test.ts

229 lines
11 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* Optimizing a draft wish rewrites only text, with the local Claude account and no tools; the
* project's own rules join the built-in ones, and nothing is stored or submitted.
*/
import { beforeEach, describe, expect, it } from 'vitest';
import { wishDetails, wishTitle, withAnswers, type Actor } from '@wishforge/contracts';
import { ControlPlane } from '../apps/api/src/domain/control-plane.js';
import { Connections } from '../apps/api/src/domain/connections.js';
import { WISH_RULES, WishRefiner } from '../apps/api/src/domain/wish-refiner.js';
import { Releases } from '../apps/api/src/domain/releases.js';
import { MemoryStore } from './helpers/memory-store.js';
const owner: Actor = { tenantId: 'demo', userId: 'owner', role: 'owner', kind: 'human' };
const env = {
NODE_ENV: 'test',
WF_DEPLOYMENT_MODE: 'development',
WF_PROVISIONER_BACKEND: 'local',
WF_TENANT_ID: 'demo',
WF_CLAUDE_BIN: '/usr/local/bin/claude',
WF_CLAUDE_LOCAL_AUTH_HOME: '/Users/operator',
};
let store: MemoryStore, projectId: string;
beforeEach(async () => {
store = new MemoryStore();
store.seed('demo');
const project = await new ControlPlane(store, env).createProject(owner, {
name: 'Pilot',
repositoryUrl: 'https://example.org/pilot.git',
});
projectId = project.id;
await store.transaction('demo', async tx => {
await tx.put('project', { ...project, wishRules: '验收标准必须包含新增的单元测试' });
});
});
describe('wish optimization', () => {
it('rewrites the draft with the built-in and the project’s rules', async () => {
const asked: Parameters<ConstructorParameters<typeof WishRefiner>[2] & object>[0][] = [];
const refiner = new WishRefiner(store, env, async input => {
asked.push(input);
return {
goal: '下载文件名带上数据集类型。\n范围:例如 xxx-episode.zip。\n不改:接口返回格式。',
acceptanceCriteria: ['文件名包含数据集类型', '新增单元测试覆盖文件名生成', '现有测试、类型检查和构建全部通过'],
questions: [
{
question: '数据集类型为空时怎么办?',
options: [
{ label: '不加类型', wish: '数据集类型为空时文件名不带类型。' },
{ label: '写 unknown', wish: '数据集类型为空时文件名写 unknown。' },
{ label: '', wish: 'An option without a label is dropped.' },
],
},
{ question: 'A question with one option is dropped', options: [{ label: 'only', wish: 'Only.' }] },
'not a question',
],
notes: [],
extra: 'dropped',
};
});
const result = await refiner.refine(owner, projectId, { goal: '下载文件名要带上类型', acceptanceCriteria: [] });
expect(result).toEqual({
goal: expect.stringContaining('xxx-episode.zip'),
acceptanceCriteria: expect.arrayContaining(['新增单元测试覆盖文件名生成']),
// A malformed question or option is left out rather than failing the whole answer.
questions: [
{
question: '数据集类型为空时怎么办?',
options: [
{ label: '不加类型', wish: '数据集类型为空时文件名不带类型。' },
{ label: '写 unknown', wish: '数据集类型为空时文件名写 unknown。' },
],
},
],
notes: [],
});
expect(asked[0]!.system).toContain(WISH_RULES);
// A short wish, the details in the criteria, and questions only the person can decide.
expect(WISH_RULES).toContain('愿望最多三行');
expect(WISH_RULES).toContain('细节写在验收标准里');
expect(WISH_RULES).toContain('编码 Agent 读代码就能确定的事');
expect(asked[0]!.system).toContain('同时优化「愿望」和「验收标准」');
// Optimizing only the criteria leaves the wish alone, so nothing is asked about it.
const criteria = await refiner.refine(owner, projectId, { goal: '下载文件名要带上类型', focus: 'criteria' });
expect(asked[1]!.system).toContain('goal 原样返回用户写的愿望');
expect(criteria.questions).toEqual([]);
expect(asked[0]!.system).toContain('验收标准必须包含新增的单元测试');
expect(JSON.parse(asked[0]!.prompt)).toMatchObject({ project: 'Pilot', goal: '下载文件名要带上类型' });
expect(asked[0]).toMatchObject({
binary: '/usr/local/bin/claude',
home: '/Users/operator',
model: 'deepseek-flash@latest',
timeoutMs: 180_000,
});
});
it('lets the operator name a different model for optimization', async () => {
const asked: { model?: string }[] = [];
const refiner = new WishRefiner(store, { ...env, WF_CLAUDE_ASK_MODEL: 'deepseek-pro@latest' }, async input => {
asked.push(input);
return {
goal: '下载文件名带上数据集类型。\n范围:例如 xxx-episode.zip。',
acceptanceCriteria: ['现有测试、类型检查和构建全部通过'],
questions: [],
};
});
await refiner.refine(owner, projectId, { goal: '下载文件名要带上类型' });
expect(asked[0]!.model).toBe('deepseek-pro@latest');
expect(asked).toHaveLength(1);
});
it('retries with the fallback model when the first times out or returns unusable JSON', async () => {
const asked: { model?: string }[] = [];
const usable = {
goal: '下载文件名带上数据集类型。\n范围:例如 xxx-episode.zip。',
acceptanceCriteria: ['现有测试、类型检查和构建全部通过'],
questions: [],
};
const refiner = new WishRefiner(store, env, async input => {
asked.push(input);
if (asked.length === 1) throw new Error('Command exceeded its time limit');
return usable;
});
expect(await refiner.refine(owner, projectId, { goal: '下载文件名要带上类型' })).toMatchObject({
goal: expect.stringContaining('xxx-episode.zip'),
});
expect(asked.map(call => call.model)).toEqual(['deepseek-flash@latest', 'deepseek-pro@latest']);
const unusable = new WishRefiner(store, env, async input => {
asked.push(input);
return asked.length === 4 ? usable : { goal: '', acceptanceCriteria: 'none' };
});
await unusable.refine(owner, projectId, { goal: '下载文件名要带上类型' });
expect(asked.slice(2).map(call => call.model)).toEqual(['deepseek-flash@latest', 'deepseek-pro@latest']);
});
it('optimizes in production from the platform model connection, with no local login', async () => {
// Sealed where connections are made; the deployed call only ever sees its id.
await new Connections(store, { ...env, WF_VAULT_KEY: 'a'.repeat(64) }).create(owner, {
provider: 'claude-code',
label: '平台方模型连接',
apiKey: 'tenant-fixture-key',
});
const production = {
...env,
NODE_ENV: 'production',
WF_DEPLOYMENT_MODE: 'production',
WF_VAULT_KEY: 'a'.repeat(64),
WF_ANTHROPIC_BASE_URL: 'https://model.example',
};
const asked: { home?: string; model?: string; credentialEnv?: Record<string, string> }[] = [];
const refiner = new WishRefiner(
store,
production,
async input => {
asked.push(input);
return {
goal: '下载文件名带上数据集类型。\n范围:例如 xxx-episode.zip。',
acceptanceCriteria: ['现有测试、类型检查和构建全部通过'],
questions: [],
};
},
{ open: async () => 'tenant-fixture-key' },
);
expect(await refiner.available('demo')).toBe(true);
await refiner.refine(owner, projectId, { goal: '下载文件名要带上类型' });
expect(asked[0]).not.toHaveProperty('home');
expect(asked[0]!.model).toBe('deepseek-flash@latest');
expect(asked[0]!.credentialEnv).toStrictEqual({
ANTHROPIC_API_KEY: 'tenant-fixture-key',
ANTHROPIC_MODEL: 'deepseek-flash@latest',
ANTHROPIC_SMALL_FAST_MODEL: 'deepseek-flash@latest',
ANTHROPIC_BASE_URL: 'https://model.example',
});
});
it('says so when no local Claude account serves this tenant, and when the answer is unusable', async () => {
const unavailable = new WishRefiner(store, { ...env, WF_DEPLOYMENT_MODE: 'production' }, async () => ({}));
await expect(unavailable.refine(owner, projectId, { goal: 'Make it better please' })).rejects.toMatchObject({
code: 'REFINER_UNAVAILABLE',
});
const garbled = new WishRefiner(store, env, async () => ({ goal: '', acceptanceCriteria: 'none' }));
await expect(garbled.refine(owner, projectId, { goal: 'Make it better please' })).rejects.toMatchObject({
code: 'REFINER_FAILED',
});
const viewer = { ...owner, userId: 'viewer', role: 'viewer' as const };
await expect(
new WishRefiner(store, env).refine(viewer, projectId, { goal: 'Make it better' }),
).rejects.toMatchObject({ code: 'FORBIDDEN' });
});
});
describe('wish format', () => {
it('names the goal on the first line and keeps the requirements below it', () => {
const goal = '\n为每次执行保存记录。\r\n1. 写明开始和结束时间\n2. 不修改现有执行流程\n';
expect(wishTitle(goal)).toBe('为每次执行保存记录。');
expect(wishDetails(goal)).toBe('1. 写明开始和结束时间\n2. 不修改现有执行流程');
expect(wishTitle('只改一处文案。')).toBe('只改一处文案。');
expect(wishDetails('只改一处文案。')).toBe('');
});
it('adds the chosen answers before the line saying what must not change', () => {
const wish = '下载文件名带上数据集类型。\n范围:下载按钮。\n不改:接口返回格式。';
expect(withAnswers(wish, ['类型为空时不带类型。', '', ' 只改\n前端。 '])).toBe(
'下载文件名带上数据集类型。\n范围:下载按钮。\n类型为空时不带类型。\n只改 前端。\n不改:接口返回格式。',
);
expect(withAnswers('下载文件名带上数据集类型。', ['类型为空时不带类型。'])).toBe(
'下载文件名带上数据集类型。\n类型为空时不带类型。',
);
// A first line that happens to start with 不改 is the goal, not the limits.
expect(withAnswers('不改:接口;只改文件名。', ['A'])).toBe('不改:接口;只改文件名。\nA');
expect(withAnswers(wish, [])).toBe(wish);
});
});
describe('release confirmation', () => {
it('lets owners make every release of an environment wait for a person', async () => {
const releases = new Releases(store, env);
const environment = await releases.createEnvironment(owner, {
projectId,
name: 'POC',
type: 'kubernetes',
autoDeploy: true,
});
expect(environment.autoDeploy).toBe(true);
expect(await releases.updateReleasePolicy(owner, environment.id, { autoDeploy: false })).toMatchObject({
autoDeploy: false,
});
const developer = { ...owner, userId: 'developer', role: 'developer' as const };
await expect(releases.updateReleasePolicy(developer, environment.id, { autoDeploy: true })).rejects.toMatchObject({
code: 'FORBIDDEN',
});
await expect(releases.updateReleasePolicy(owner, environment.id, { autoDeploy: 'yes' })).rejects.toThrow();
});
});