import { mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; import { cpuCores, fanyuDiagnostics, gatewayDiagnostics, memoryGi, summarizeCluster, type Transport, } from '../apps/api/src/domain/copilot-diagnostics.js'; const FAKE_KEY = 'fake-key-0123456789abcdef0123456789abcdef'; /** A clock the tests move by hand: retries wait 15 s and the rate window is a minute. */ const clock = () => { let now = 1_000_000; return { now: () => now, sleep: async (ms: number) => void (now += ms), advance: (ms: number) => void (now += ms) }; }; const snapshot = (data: unknown[]) => ({ status: 200, body: { available: true, observedAt: '2026-11-10T17:27:33Z', data }, }); const nodes = [ { name: 'n1', status: { allocatable: { cpu: '4000m', memory: '8Gi' }, conditions: [ { type: 'Ready', status: 'True' }, { type: 'MemoryPressure', status: 'False' }, ], }, }, ]; const pods = [ { name: 'api-1', node: 'n1', containers: [{ image: 'reg/droboticflow-api:e36617c0' }], status: { phase: 'Running', containerStatuses: [{ name: 'api', ready: true, restartCount: 3 }] }, }, { name: 'web-1', node: 'n1', containers: [{ image: 'web' }], status: { phase: 'Running', containerStatuses: [{ name: 'web', ready: true, restartCount: 0 }] }, }, { name: 'job', node: 'n1', status: { phase: 'Succeeded', containerStatuses: [{ name: 'q', ready: false }] } }, ]; const routes = (overrides: Record = {}): Transport => vi.fn(async (path: string) => { if (overrides[path]) return overrides[path]; if (path === 'nodes') return snapshot(nodes); if (path === 'pods') return snapshot(pods); if (path === 'node-usage') return snapshot([{ name: 'n1', usage: { cpu: '2000000000n', memory: '4Gi' } }]); if (path === 'pod-usage') return snapshot([{ name: 'api-1', containerUsage: [{ usage: { cpu: '500m', memory: '1Gi' } }] }]); return { status: 404, body: {} }; }); describe('Copilot production diagnostics', () => { it('reads Kubernetes quantities in their own units', () => { expect(cpuCores('54600m')).toBeCloseTo(54.6); expect(cpuCores('357200u')).toBeCloseTo(0.3572); expect(cpuCores('66750n')).toBeCloseTo(0.00006675); expect(cpuCores('56')).toBe(56); expect(memoryGi('2Gi')).toBe(2); expect(memoryGi('1048576Ki')).toBe(1); }); it('turns the four snapshots into sentences: usage per node, unhealthy instances, finished jobs skipped', () => { const facts = summarizeCluster( snapshot(nodes).body, snapshot(pods).body, snapshot([{ name: 'n1', usage: { cpu: '2', memory: '4Gi' } }]).body, {}, ); expect(facts[0]).toContain('CPU 使用 50%'); expect(facts[0]).toContain('内存使用 50%'); expect(facts[0]).toContain('状态正常'); expect(facts[0]).toMatch(/^节点 1:/); expect(facts.join('\n')).not.toContain('n1'); expect(facts.join('\n')).toContain('api-1'); expect(facts.join('\n')).toContain('累计重启 3 次'); expect(facts.join('\n')).not.toContain('job'); expect(facts.join('\n')).toContain('其余 1 个运行中的实例'); }); it('answers a cluster overview with the gateway time and its limits', async () => { const diagnostics = gatewayDiagnostics(routes(), clock()); const evidence = await diagnostics.clusterOverview(); expect(evidence).toMatchObject({ tool: 'cluster_overview', available: true, observedAt: '2026-11-10T17:27:33Z' }); expect(evidence.limitations.join()).toContain('不是历史曲线'); }); it('never reports a 503 as healthy: it retries once after 15 s, then says the evidence is missing', async () => { const time = clock(); const transport = routes({ egress: { status: 503, body: { error: 'bcm_not_configured' } } }); const evidence = await gatewayDiagnostics(transport, time).egress(); expect(transport).toHaveBeenCalledTimes(2); expect(evidence.available).toBe(false); expect(evidence.error).toBe('unavailable'); expect(evidence.facts).toEqual([]); expect(evidence.limitations.join()).toContain('不能当成正常'); expect(evidence.limitations.join()).toContain('不代表流量为零'); }); it('treats a body that says available:false as missing, whatever the status', async () => { const transport = routes({ pods: { status: 200, body: { available: false, data: [] } } }); const evidence = await gatewayDiagnostics(transport, clock()).clusterOverview(); expect(evidence.available).toBe(false); expect(evidence.facts).toEqual([]); }); it('retries a 429 once and recovers', async () => { let calls = 0; const transport: Transport = async () => ++calls === 1 ? { status: 429, body: {} } : { status: 200, body: { available: true, data: { status: 'validation_failed', attempt: 5 } } }; const evidence = await gatewayDiagnostics(transport, clock()).task('annotation-task-abc'); expect(calls).toBe(2); expect(evidence.available).toBe(true); expect(evidence.facts[0]).toContain('status=validation_failed'); }); it('stops for ten minutes after the credential is refused, without retrying', async () => { const time = clock(); const transport = vi.fn(async () => ({ status: 401, body: {} })); const diagnostics = gatewayDiagnostics(transport, time); expect((await diagnostics.task('annotation-task-a')).error).toBe('auth_failed'); expect((await diagnostics.task('annotation-task-b')).error).toBe('auth_failed'); expect(transport).toHaveBeenCalledTimes(1); time.advance(10 * 60_000); await diagnostics.task('annotation-task-c'); expect(transport).toHaveBeenCalledTimes(2); }); it('refuses malformed ids before calling the gateway and says how to get a real one', async () => { const transport = vi.fn(routes()); const diagnostics = gatewayDiagnostics(transport, clock()); const task = await diagnostics.task('FYLX-0001/part3'); const trace = await diagnostics.trace('../../nodes'); expect(transport).not.toHaveBeenCalled(); expect(task.error).toBe('invalid_id'); expect(task.limitations.join()).toContain('annotation-task-'); expect(trace.limitations.join()).toContain('X-Request-ID'); }); it('reports an empty trace as missing with the way to collect one, never as an empty success', async () => { const transport = routes({ [`logs/api/${'a'.repeat(32)}`]: { status: 200, body: { available: true, data: [] } } }); const evidence = await gatewayDiagnostics(transport, clock()).trace('a'.repeat(32)); expect(evidence.available).toBe(false); expect(evidence.error).toBe('not_found'); expect(evidence.limitations.join()).toContain('最近 15 分钟'); expect(evidence.limitations.join()).toContain('复现'); }); it('caches snapshots for 30 s and items for 15 s, failures included', async () => { const time = clock(); const transport = vi.fn(routes()); const diagnostics = gatewayDiagnostics(transport, time); await diagnostics.clusterOverview(); await diagnostics.clusterOverview(); expect(transport).toHaveBeenCalledTimes(4); time.advance(30_000); await diagnostics.clusterOverview(); expect(transport).toHaveBeenCalledTimes(8); await diagnostics.task('annotation-task-missing'); await diagnostics.task('annotation-task-missing'); expect(transport).toHaveBeenCalledTimes(9); }); it('stays under 40 calls a minute and refuses the rest at once instead of queueing', async () => { const time = clock(); const transport = vi.fn(async () => ({ status: 200, body: { available: true, data: { status: 'ok' } } })); const diagnostics = gatewayDiagnostics(transport, time); const results = []; for (let i = 0; i < 45; i += 1) results.push(await diagnostics.task(`annotation-task-${i}`)); expect(transport).toHaveBeenCalledTimes(40); expect(results.at(-1)?.error).toBe('rate_limited'); time.advance(60_000); expect((await diagnostics.task('annotation-task-later')).available).toBe(true); }); it('keeps the key out of every result, error and log line', async () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); const echo: Transport = async path => { if (path === 'egress') throw new Error(`connect failed with Bearer ${FAKE_KEY}`); return { status: 200, body: { available: true, observedAt: 'x', data: { note: FAKE_KEY } } }; }; const diagnostics = gatewayDiagnostics(echo, clock()); const all = [await diagnostics.egress(), await diagnostics.clusterOverview()]; const logged = warn.mock.calls.flat().join('\n'); warn.mockRestore(); expect(JSON.stringify(all)).not.toContain(FAKE_KEY); expect(logged).not.toContain(FAKE_KEY); }); it('is off unless the kit is configured, and never copies the key into the environment', () => { expect(fanyuDiagnostics({})).toBeUndefined(); const dir = mkdtempSync(join(tmpdir(), 'kit-')); writeFileSync(join(dir, 'agent.key'), FAKE_KEY); writeFileSync(join(dir, 'ca.crt'), 'ca'); const env: NodeJS.ProcessEnv = { WF_DIAGNOSTIC_KIT_DIR: dir }; expect(fanyuDiagnostics(env)).toBeDefined(); expect(JSON.stringify(env)).not.toContain(FAKE_KEY); }); });