wishforge-ide/tests/copilot-diagnostics.test.ts
wishforge 9dde59e8c6 ✨ Diagnose reported problems from read-only production evidence
Add the read-only Fanyu diagnostic client (cluster_overview, task_diagnosis, request_trace, egress) with private file-based key loading, mutual CA verification, a 40 req/min token bucket, short TTL caches, and strict separation between unavailable telemetry and healthy state.

Feed the last ten conversation turns into streaming answers, verify every reported fact against the server-side evidence ledger before persisting a diagnosis, withhold model thinking frames from the client stream, classify stream error codes, and cover secret isolation and diagnosis downgrades with dedicated unit tests.
2026-11-11 07:59:03 -08:00

211 lines
9.2 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { mkdtempSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, expect, it, vi } from 'vitest';
import {
cpuCores,
fanyuDiagnostics,
gatewayDiagnostics,
memoryGi,
summarizeCluster,
type Transport,
} from '../apps/api/src/domain/copilot-diagnostics.js';
const FAKE_KEY = 'fake-key-0123456789abcdef0123456789abcdef';
/** A clock the tests move by hand: retries wait 15 s and the rate window is a minute. */
const clock = () => {
let now = 1_000_000;
return { now: () => now, sleep: async (ms: number) => void (now += ms), advance: (ms: number) => void (now += ms) };
};
const snapshot = (data: unknown[]) => ({
status: 200,
body: { available: true, observedAt: '2026-11-10T17:27:33Z', data },
});
const nodes = [
{
name: 'n1',
status: {
allocatable: { cpu: '4000m', memory: '8Gi' },
conditions: [
{ type: 'Ready', status: 'True' },
{ type: 'MemoryPressure', status: 'False' },
],
},
},
];
const pods = [
{
name: 'api-1',
node: 'n1',
containers: [{ image: 'reg/droboticflow-api:e36617c0' }],
status: { phase: 'Running', containerStatuses: [{ name: 'api', ready: true, restartCount: 3 }] },
},
{
name: 'web-1',
node: 'n1',
containers: [{ image: 'web' }],
status: { phase: 'Running', containerStatuses: [{ name: 'web', ready: true, restartCount: 0 }] },
},
{ name: 'job', node: 'n1', status: { phase: 'Succeeded', containerStatuses: [{ name: 'q', ready: false }] } },
];
const routes = (overrides: Record<string, { status: number; body: unknown }> = {}): Transport =>
vi.fn(async (path: string) => {
if (overrides[path]) return overrides[path];
if (path === 'nodes') return snapshot(nodes);
if (path === 'pods') return snapshot(pods);
if (path === 'node-usage') return snapshot([{ name: 'n1', usage: { cpu: '2000000000n', memory: '4Gi' } }]);
if (path === 'pod-usage')
return snapshot([{ name: 'api-1', containerUsage: [{ usage: { cpu: '500m', memory: '1Gi' } }] }]);
return { status: 404, body: {} };
});
describe('Copilot production diagnostics', () => {
it('reads Kubernetes quantities in their own units', () => {
expect(cpuCores('54600m')).toBeCloseTo(54.6);
expect(cpuCores('357200u')).toBeCloseTo(0.3572);
expect(cpuCores('66750n')).toBeCloseTo(0.00006675);
expect(cpuCores('56')).toBe(56);
expect(memoryGi('2Gi')).toBe(2);
expect(memoryGi('1048576Ki')).toBe(1);
});
it('turns the four snapshots into sentences: usage per node, unhealthy instances, finished jobs skipped', () => {
const facts = summarizeCluster(
snapshot(nodes).body,
snapshot(pods).body,
snapshot([{ name: 'n1', usage: { cpu: '2', memory: '4Gi' } }]).body,
{},
);
expect(facts[0]).toContain('CPU 使用 50%');
expect(facts[0]).toContain('内存使用 50%');
expect(facts[0]).toContain('状态正常');
expect(facts[0]).toMatch(/^节点 1:/);
expect(facts.join('\n')).not.toContain('n1');
expect(facts.join('\n')).toContain('api-1');
expect(facts.join('\n')).toContain('累计重启 3 次');
expect(facts.join('\n')).not.toContain('job');
expect(facts.join('\n')).toContain('其余 1 个运行中的实例');
});
it('answers a cluster overview with the gateway time and its limits', async () => {
const diagnostics = gatewayDiagnostics(routes(), clock());
const evidence = await diagnostics.clusterOverview();
expect(evidence).toMatchObject({ tool: 'cluster_overview', available: true, observedAt: '2026-11-10T17:27:33Z' });
expect(evidence.limitations.join()).toContain('不是历史曲线');
});
it('never reports a 503 as healthy: it retries once after 15 s, then says the evidence is missing', async () => {
const time = clock();
const transport = routes({ egress: { status: 503, body: { error: 'bcm_not_configured' } } });
const evidence = await gatewayDiagnostics(transport, time).egress();
expect(transport).toHaveBeenCalledTimes(2);
expect(evidence.available).toBe(false);
expect(evidence.error).toBe('unavailable');
expect(evidence.facts).toEqual([]);
expect(evidence.limitations.join()).toContain('不能当成正常');
expect(evidence.limitations.join()).toContain('不代表流量为零');
});
it('treats a body that says available:false as missing, whatever the status', async () => {
const transport = routes({ pods: { status: 200, body: { available: false, data: [] } } });
const evidence = await gatewayDiagnostics(transport, clock()).clusterOverview();
expect(evidence.available).toBe(false);
expect(evidence.facts).toEqual([]);
});
it('retries a 429 once and recovers', async () => {
let calls = 0;
const transport: Transport = async () =>
++calls === 1
? { status: 429, body: {} }
: { status: 200, body: { available: true, data: { status: 'validation_failed', attempt: 5 } } };
const evidence = await gatewayDiagnostics(transport, clock()).task('annotation-task-abc');
expect(calls).toBe(2);
expect(evidence.available).toBe(true);
expect(evidence.facts[0]).toContain('status=validation_failed');
});
it('stops for ten minutes after the credential is refused, without retrying', async () => {
const time = clock();
const transport = vi.fn(async () => ({ status: 401, body: {} }));
const diagnostics = gatewayDiagnostics(transport, time);
expect((await diagnostics.task('annotation-task-a')).error).toBe('auth_failed');
expect((await diagnostics.task('annotation-task-b')).error).toBe('auth_failed');
expect(transport).toHaveBeenCalledTimes(1);
time.advance(10 * 60_000);
await diagnostics.task('annotation-task-c');
expect(transport).toHaveBeenCalledTimes(2);
});
it('refuses malformed ids before calling the gateway and says how to get a real one', async () => {
const transport = vi.fn(routes());
const diagnostics = gatewayDiagnostics(transport, clock());
const task = await diagnostics.task('FYLX-0001/part3');
const trace = await diagnostics.trace('../../nodes');
expect(transport).not.toHaveBeenCalled();
expect(task.error).toBe('invalid_id');
expect(task.limitations.join()).toContain('annotation-task-');
expect(trace.limitations.join()).toContain('X-Request-ID');
});
it('reports an empty trace as missing with the way to collect one, never as an empty success', async () => {
const transport = routes({ [`logs/api/${'a'.repeat(32)}`]: { status: 200, body: { available: true, data: [] } } });
const evidence = await gatewayDiagnostics(transport, clock()).trace('a'.repeat(32));
expect(evidence.available).toBe(false);
expect(evidence.error).toBe('not_found');
expect(evidence.limitations.join()).toContain('最近 15 分钟');
expect(evidence.limitations.join()).toContain('复现');
});
it('caches snapshots for 30 s and items for 15 s, failures included', async () => {
const time = clock();
const transport = vi.fn(routes());
const diagnostics = gatewayDiagnostics(transport, time);
await diagnostics.clusterOverview();
await diagnostics.clusterOverview();
expect(transport).toHaveBeenCalledTimes(4);
time.advance(30_000);
await diagnostics.clusterOverview();
expect(transport).toHaveBeenCalledTimes(8);
await diagnostics.task('annotation-task-missing');
await diagnostics.task('annotation-task-missing');
expect(transport).toHaveBeenCalledTimes(9);
});
it('stays under 40 calls a minute and refuses the rest at once instead of queueing', async () => {
const time = clock();
const transport = vi.fn(async () => ({ status: 200, body: { available: true, data: { status: 'ok' } } }));
const diagnostics = gatewayDiagnostics(transport, time);
const results = [];
for (let i = 0; i < 45; i += 1) results.push(await diagnostics.task(`annotation-task-${i}`));
expect(transport).toHaveBeenCalledTimes(40);
expect(results.at(-1)?.error).toBe('rate_limited');
time.advance(60_000);
expect((await diagnostics.task('annotation-task-later')).available).toBe(true);
});
it('keeps the key out of every result, error and log line', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
const echo: Transport = async path => {
if (path === 'egress') throw new Error(`connect failed with Bearer ${FAKE_KEY}`);
return { status: 200, body: { available: true, observedAt: 'x', data: { note: FAKE_KEY } } };
};
const diagnostics = gatewayDiagnostics(echo, clock());
const all = [await diagnostics.egress(), await diagnostics.clusterOverview()];
const logged = warn.mock.calls.flat().join('\n');
warn.mockRestore();
expect(JSON.stringify(all)).not.toContain(FAKE_KEY);
expect(logged).not.toContain(FAKE_KEY);
});
it('is off unless the kit is configured, and never copies the key into the environment', () => {
expect(fanyuDiagnostics({})).toBeUndefined();
const dir = mkdtempSync(join(tmpdir(), 'kit-'));
writeFileSync(join(dir, 'agent.key'), FAKE_KEY);
writeFileSync(join(dir, 'ca.crt'), 'ca');
const env: NodeJS.ProcessEnv = { WF_DIAGNOSTIC_KIT_DIR: dir };
expect(fanyuDiagnostics(env)).toBeDefined();
expect(JSON.stringify(env)).not.toContain(FAKE_KEY);
});
});