Add the read-only Fanyu diagnostic client (cluster_overview, task_diagnosis, request_trace, egress) with private file-based key loading, mutual CA verification, a 40 req/min token bucket, short TTL caches, and strict separation between unavailable telemetry and healthy state. Feed the last ten conversation turns into streaming answers, verify every reported fact against the server-side evidence ledger before persisting a diagnosis, withhold model thinking frames from the client stream, classify stream error codes, and cover secret isolation and diagnosis downgrades with dedicated unit tests.
384 lines
15 KiB
TypeScript
384 lines
15 KiB
TypeScript
import { describe, expect, it } from 'vitest';
|
||
import type { CopilotMessage } from '@wishforge/contracts';
|
||
import { sessionHistory } from '../apps/api/src/domain/copilot.js';
|
||
import { verifiedDiagnosis, type Diagnostics, type Evidence } from '../apps/api/src/domain/copilot-diagnostics.js';
|
||
import { DomainError } from '@wishforge/contracts';
|
||
import {
|
||
answerStreaming,
|
||
historyMessages,
|
||
streamErrorCode,
|
||
type StreamEvent,
|
||
} from '../apps/api/src/domain/copilot-stream.js';
|
||
import { deniedToolPath, runTool, type ToolContext } from '../apps/api/src/domain/copilot-tools.js';
|
||
|
||
/**
|
||
* Locating a reported problem: the Copilot remembers the conversation, reads production evidence
|
||
* through read-only tools, and the platform holds its conclusion to what that evidence says.
|
||
*/
|
||
const missingEvidence = (tool: Evidence['tool'], subject: string): Evidence => ({
|
||
tool,
|
||
available: false,
|
||
subject,
|
||
facts: [],
|
||
limitations: ['这项证据暂时拿不到,不能当成正常。'],
|
||
error: 'unavailable',
|
||
});
|
||
const diagnostics: Diagnostics = {
|
||
clusterOverview: async () => ({
|
||
tool: 'cluster_overview',
|
||
available: true,
|
||
observedAt: '2026-11-10T17:31:33Z',
|
||
subject: '全部节点',
|
||
facts: ['其余 17 个运行中的实例全部就绪、没有重启。'],
|
||
limitations: [],
|
||
}),
|
||
task: async taskId => ({
|
||
tool: 'task_diagnosis',
|
||
available: true,
|
||
observedAt: '2026-11-10T17:40:00Z',
|
||
subject: taskId,
|
||
facts: ['任务:status=validation_failed,attempt=5,errorCategory=imu_invalid'],
|
||
limitations: [],
|
||
}),
|
||
trace: async requestId => missingEvidence('request_trace', requestId),
|
||
egress: async () => missingEvidence('egress', '公网出口'),
|
||
};
|
||
function toolStub(withDiagnostics = true): ToolContext {
|
||
const code: ToolContext['code'] = {
|
||
tree: async () => ({}),
|
||
blob: async () => ({}),
|
||
search: async () => ({}),
|
||
log: async () => ({}),
|
||
show: async () => ({}),
|
||
forProject: () => code,
|
||
};
|
||
return {
|
||
actor: { userId: 'u1', tenantId: 'demo', role: 'owner', kind: 'human' },
|
||
ref: 'a'.repeat(40),
|
||
searchWiki: async () => [],
|
||
code,
|
||
run: async () => {
|
||
throw new Error('not used');
|
||
},
|
||
namespaces: [],
|
||
...(withDiagnostics ? { diagnostics } : {}),
|
||
};
|
||
}
|
||
const sse = (frames: unknown[]) =>
|
||
new Response(frames.map(frame => `data: ${JSON.stringify(frame)}\n\n`).join(''), {
|
||
status: 200,
|
||
headers: { 'content-type': 'text/event-stream' },
|
||
});
|
||
const toolCall = (id: string, name: string, input: unknown) =>
|
||
sse([
|
||
{ type: 'content_block_start', index: 0, content_block: { type: 'tool_use', id, name } },
|
||
{ type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: JSON.stringify(input) } },
|
||
{ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { input_tokens: 1, output_tokens: 1 } },
|
||
]);
|
||
|
||
describe('Copilot problem diagnosis', () => {
|
||
it('sends the earlier turns, offers the production tools and numbers the evidence it read', async () => {
|
||
const bodies: Record<string, unknown>[] = [];
|
||
let round = 0;
|
||
const fetcher = (async (_url: string, init: RequestInit) => {
|
||
bodies.push(JSON.parse(String(init.body)));
|
||
round += 1;
|
||
if (round === 1) return toolCall('t1', 'task_diagnosis', { taskId: 'annotation-task-abc' });
|
||
return toolCall('t2', 'submit_answer', {
|
||
kind: 'bug',
|
||
answer: '已确认这个任务校验失败了 5 次。',
|
||
goal: '送检后任务不再回退',
|
||
acceptanceCriteria: ['送检成功后任务停在验收'],
|
||
diagnosis: {
|
||
verdict: 'problem',
|
||
facts: [
|
||
{ text: '任务校验失败 5 次', evidenceId: 'ev-1' },
|
||
{ text: '数据库写锁超时', evidenceId: 'ev-9' },
|
||
],
|
||
hypotheses: [{ text: '送检时写入冲突', confidence: 'medium' }],
|
||
},
|
||
});
|
||
}) as unknown as typeof fetch;
|
||
const tools = toolStub();
|
||
const result = await answerStreaming({
|
||
apiKey: 'k',
|
||
baseUrl: 'https://gateway.invalid',
|
||
model: 'm',
|
||
system: 's',
|
||
question: '编号是 annotation-task-abc',
|
||
history: [
|
||
{ role: 'user', content: '送检后任务回退了' },
|
||
{ role: 'assistant', content: '方便给一下任务编号吗?' },
|
||
],
|
||
wiki: [],
|
||
tools,
|
||
fetcher,
|
||
emit: () => undefined,
|
||
});
|
||
const first = bodies[0] as { messages: { role: string; content: unknown }[]; tools: { name: string }[] };
|
||
expect(first.messages.map(message => message.role)).toEqual(['user', 'assistant', 'user']);
|
||
expect(first.messages[0]!.content).toBe('送检后任务回退了');
|
||
expect(first.tools.map(tool => tool.name)).toEqual(
|
||
expect.arrayContaining(['task_diagnosis', 'request_trace', 'cluster_overview', 'egress']),
|
||
);
|
||
const toolResult = JSON.stringify((bodies[1] as { messages: unknown[] }).messages.at(-1));
|
||
expect(toolResult).toContain('ev-1');
|
||
expect(toolResult).toContain('validation_failed');
|
||
expect(result.evidence.map(item => item.id)).toEqual(['ev-1']);
|
||
|
||
const checked = verifiedDiagnosis(result.answer.diagnosis, result.evidence, { cited: false })!;
|
||
expect(checked.verdict).toBe('problem');
|
||
expect(checked.facts).toEqual([{ text: '任务校验失败 5 次', evidenceId: 'ev-1' }]);
|
||
// A "fact" that points at evidence nobody read is only a guess.
|
||
expect(checked.hypotheses).toContainEqual({ text: '数据库写锁超时', confidence: 'low' });
|
||
expect(checked.evidence[0]).toMatchObject({
|
||
source: 'task',
|
||
label: '线上任务记录',
|
||
observedAt: '2026-11-10T17:40:00Z',
|
||
});
|
||
});
|
||
|
||
it('keeps the production tools away from projects that do not have them', async () => {
|
||
let names: string[] = [];
|
||
const fetcher = (async (_url: string, init: RequestInit) => {
|
||
names = (JSON.parse(String(init.body)) as { tools: { name: string }[] }).tools.map(tool => tool.name);
|
||
return toolCall('t1', 'submit_answer', { kind: 'unknown', answer: '好的' });
|
||
}) as unknown as typeof fetch;
|
||
await answerStreaming({
|
||
apiKey: 'k',
|
||
baseUrl: 'https://gateway.invalid',
|
||
model: 'm',
|
||
system: 's',
|
||
question: '很慢',
|
||
wiki: [],
|
||
tools: toolStub(false),
|
||
fetcher,
|
||
emit: () => undefined,
|
||
});
|
||
expect(names).not.toContain('cluster_overview');
|
||
// No log reader configured: the tool that could only refuse is not offered either.
|
||
expect(names).not.toContain('read_logs');
|
||
await expect(runTool(toolStub(false), 'cluster_overview', {})).rejects.toThrow('does not exist');
|
||
});
|
||
|
||
it('lists what could not be read as missing, and never proves a problem from it', () => {
|
||
const ledger = [{ ...missingEvidence('request_trace', 'f'.repeat(32)), id: 'ev-1' }];
|
||
const checked = verifiedDiagnosis(
|
||
{ verdict: 'problem', facts: [{ text: '请求超时', evidenceId: 'ev-1' }], hypotheses: [], missing: [] },
|
||
ledger,
|
||
{ cited: false },
|
||
)!;
|
||
expect(checked.verdict).toBe('insufficient');
|
||
expect(checked.facts).toEqual([]);
|
||
expect(checked.missing[0]).toContain('线上请求记录');
|
||
expect(checked.missing[0]).toContain('不能当成正常');
|
||
});
|
||
|
||
it('keeps the caveats of a snapshot out of what was observed', () => {
|
||
const snapshot: Evidence & { id: string } = {
|
||
tool: 'cluster_overview',
|
||
available: true,
|
||
observedAt: '2026-11-10T18:16:34Z',
|
||
facts: ['其余 17 个运行中的实例全部就绪、没有重启。'],
|
||
limitations: ['资源快照约每 30 秒刷新一次,是近似的实时值,不是历史曲线;CPU 低不代表没有被限流。'],
|
||
id: 'ev-1',
|
||
};
|
||
const checked = verifiedDiagnosis(
|
||
{
|
||
verdict: 'insufficient',
|
||
facts: [
|
||
{ text: '查询时全部 17 个运行中的实例都就绪、没有重启', evidenceId: 'ev-1' },
|
||
{
|
||
text: '资源快照约每 30 秒刷新一次,是近似实时值,不是历史曲线;CPU 低不代表没有被限流',
|
||
evidenceId: 'ev-1',
|
||
},
|
||
],
|
||
hypotheses: [],
|
||
missing: [],
|
||
},
|
||
[snapshot],
|
||
{ cited: false },
|
||
)!;
|
||
expect(checked.facts.map(fact => fact.text)).toEqual(['查询时全部 17 个运行中的实例都就绪、没有重启']);
|
||
expect(checked.hypotheses).toEqual([]);
|
||
});
|
||
|
||
it('calls "working as designed" unproven unless a rule was quoted', () => {
|
||
const evidence: Evidence & { id: string } = {
|
||
tool: 'cluster_overview',
|
||
available: true,
|
||
facts: ['都正常'],
|
||
limitations: [],
|
||
id: 'ev-1',
|
||
};
|
||
const claimed = {
|
||
verdict: 'expected' as const,
|
||
facts: [{ text: '实例都就绪', evidenceId: 'ev-1' }],
|
||
hypotheses: [],
|
||
missing: [],
|
||
};
|
||
expect(verifiedDiagnosis(claimed, [evidence], { cited: false })!.verdict).toBe('insufficient');
|
||
expect(verifiedDiagnosis(claimed, [evidence], { cited: true })!.verdict).toBe('expected');
|
||
expect(verifiedDiagnosis(claimed, [], { cited: true })).toBeUndefined();
|
||
});
|
||
|
||
it('refuses a fourth lookup of the same kind in one answer', async () => {
|
||
const tools = toolStub();
|
||
for (let i = 0; i < 3; i += 1) await runTool(tools, 'task_diagnosis', { taskId: `annotation-task-${i}` });
|
||
const fourth = await runTool(tools, 'task_diagnosis', { taskId: 'annotation-task-9' });
|
||
expect(fourth.content).toContain('足够多');
|
||
expect(tools.evidence).toHaveLength(3);
|
||
});
|
||
|
||
it('keeps the diagnostic kit out of the code tools', () => {
|
||
expect(deniedToolPath('.wishforge/diagnostics/fanyu-agent-readonly-kit/agent.key')).toBe(true);
|
||
expect(deniedToolPath('fanyu-agent-readonly-kit/ca.crt')).toBe(true);
|
||
expect(deniedToolPath('agent.key')).toBe(true);
|
||
expect(deniedToolPath('internal/diagnostics/handler.go')).toBe(false);
|
||
});
|
||
|
||
it('joins turns into alternating messages that start with the customer and end before the new question', () => {
|
||
expect(
|
||
historyMessages([
|
||
{ role: 'assistant', content: '你好' },
|
||
{ role: 'user', content: '很慢' },
|
||
{ role: 'user', content: '一直转圈' },
|
||
{ role: 'assistant', content: '哪个页面?' },
|
||
{ role: 'user', content: '没答完的问题' },
|
||
]),
|
||
).toEqual([
|
||
{ role: 'user', content: '很慢\n\n一直转圈' },
|
||
{ role: 'assistant', content: '哪个页面?' },
|
||
]);
|
||
});
|
||
|
||
it('carries an earlier diagnosis forward as an outline, not as evidence', () => {
|
||
const messages: CopilotMessage[] = [
|
||
{ id: '1:user', role: 'user', content: '送检后回退了', status: 'complete', createdAt: 't' },
|
||
{
|
||
id: '1:assistant',
|
||
role: 'assistant',
|
||
content: '证据还不够。',
|
||
status: 'complete',
|
||
createdAt: 't',
|
||
diagnosis: {
|
||
verdict: 'insufficient',
|
||
facts: [],
|
||
hypotheses: [],
|
||
missing: ['线上请求记录:只保留 15 分钟'],
|
||
evidence: [{ id: 'ev-1', source: 'trace', label: '线上请求记录', subject: 'abc', available: false }],
|
||
},
|
||
},
|
||
{ id: '2:user', role: 'user', content: '坏掉的', status: 'error', createdAt: 't' },
|
||
];
|
||
const history = sessionHistory(messages);
|
||
expect(history).toHaveLength(2);
|
||
expect(history[1]!.content).toContain('上次诊断:证据不足');
|
||
expect(history[1]!.content).toContain('线上请求记录 abc');
|
||
});
|
||
|
||
it('sends each look as a start and an end, and ends a lookup that found nothing as missing', async () => {
|
||
let round = 0;
|
||
const fetcher = (async () => {
|
||
round += 1;
|
||
if (round === 1) return toolCall('t1', 'request_trace', { requestId: 'f'.repeat(32) });
|
||
if (round === 2) return toolCall('t2', 'task_diagnosis', { taskId: 'annotation-task-abc' });
|
||
return toolCall('t3', 'submit_answer', { kind: 'unknown', answer: '还缺请求记录。' });
|
||
}) as unknown as typeof fetch;
|
||
const events: StreamEvent[] = [];
|
||
await answerStreaming({
|
||
apiKey: 'k',
|
||
baseUrl: 'https://gateway.invalid',
|
||
model: 'm',
|
||
system: 's',
|
||
question: '报错了',
|
||
wiki: [],
|
||
tools: toolStub(),
|
||
fetcher,
|
||
emit: event => events.push(event),
|
||
});
|
||
const steps = events.filter(event => event.type === 'tool_use');
|
||
expect(steps).toEqual([
|
||
{
|
||
type: 'tool_use',
|
||
stepId: 'step-1',
|
||
phase: 'start',
|
||
name: 'request_trace',
|
||
progress: '正在查找这次请求的记录…',
|
||
},
|
||
{
|
||
type: 'tool_use',
|
||
stepId: 'step-1',
|
||
phase: 'end',
|
||
name: 'request_trace',
|
||
progress: '正在查找这次请求的记录…',
|
||
outcome: 'missing',
|
||
evidenceId: 'ev-1',
|
||
},
|
||
{ type: 'tool_use', stepId: 'step-2', phase: 'start', name: 'task_diagnosis', progress: '正在查看任务状态…' },
|
||
{
|
||
type: 'tool_use',
|
||
stepId: 'step-2',
|
||
phase: 'end',
|
||
name: 'task_diagnosis',
|
||
progress: '正在查看任务状态…',
|
||
outcome: 'ok',
|
||
evidenceId: 'ev-2',
|
||
observedAt: '2026-11-10T17:40:00Z',
|
||
},
|
||
]);
|
||
});
|
||
|
||
it('keeps the model reasoning on the server: it goes back to the model, never to the customer', async () => {
|
||
const bodies: string[] = [];
|
||
let round = 0;
|
||
const fetcher = (async (_url: string, init: RequestInit) => {
|
||
bodies.push(String(init.body));
|
||
round += 1;
|
||
if (round === 1)
|
||
return sse([
|
||
{ type: 'content_block_start', index: 0, content_block: { type: 'thinking', thinking: '' } },
|
||
{
|
||
type: 'content_block_delta',
|
||
index: 0,
|
||
delta: { type: 'thinking_delta', thinking: 'Per rules I must not guess' },
|
||
},
|
||
{
|
||
type: 'content_block_start',
|
||
index: 1,
|
||
content_block: { type: 'tool_use', id: 't1', name: 'cluster_overview' },
|
||
},
|
||
{ type: 'content_block_delta', index: 1, delta: { type: 'input_json_delta', partial_json: '{}' } },
|
||
{ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { input_tokens: 1, output_tokens: 1 } },
|
||
]);
|
||
return toolCall('t2', 'submit_answer', { kind: 'unknown', answer: '现在查看时实例都就绪。' });
|
||
}) as unknown as typeof fetch;
|
||
const events: StreamEvent[] = [];
|
||
await answerStreaming({
|
||
apiKey: 'k',
|
||
baseUrl: 'https://gateway.invalid',
|
||
model: 'm',
|
||
system: 's',
|
||
question: '报错了',
|
||
wiki: [],
|
||
tools: toolStub(),
|
||
fetcher,
|
||
emit: event => events.push(event),
|
||
});
|
||
expect(events.map(event => event.type)).not.toContain('thinking');
|
||
expect(JSON.stringify(events)).not.toContain('Per rules');
|
||
expect(bodies[1]).toContain('Per rules I must not guess');
|
||
});
|
||
|
||
it('names why an answer stopped, so a gateway 503 is never shown as an answer', () => {
|
||
const upstream = (status: number) => new DomainError('COPILOT_UPSTREAM', `refused (${status})`, 502, { status });
|
||
expect(streamErrorCode(upstream(503))).toBe('upstream_unavailable');
|
||
expect(streamErrorCode(upstream(429))).toBe('rate_limited');
|
||
expect(streamErrorCode(upstream(401))).toBe('upstream_refused');
|
||
expect(streamErrorCode(new DomainError('COPILOT_FAILED', 'x', 502))).toBe('no_answer');
|
||
expect(streamErrorCode(new DomainError('COPILOT_NO_PROVIDER', 'x', 409))).toBe('not_configured');
|
||
expect(streamErrorCode(Object.assign(new Error('aborted'), { name: 'AbortError' }))).toBe('timeout');
|
||
expect(streamErrorCode(new Error('boom'))).toBe('internal');
|
||
});
|
||
});
|