import { existsSync, readdirSync, readFileSync } from 'node:fs'; import path from 'node:path'; import { describe, expect, it } from 'vitest'; import { redactSecrets } from '../apps/api/src/domain/copilot-tools.js'; /** * The image path, end to end, against the real gateway. It is skipped unless an operator puts * screenshots in `apps/api/src/__tests__/fixtures` and sets WF_VISION_ACCEPTANCE=1: a fake image * here would prove nothing about a model that has to read a Chinese error dialog. * * The gateway is called directly, with the same credential shape the Copilot uses, because this * test is about what the model sees in a screenshot, not about the console around it. */ const DIRECTORY = path.join(import.meta.dirname, '..', 'apps', 'api', 'src', '__tests__', 'fixtures'); const images = existsSync(DIRECTORY) ? readdirSync(DIRECTORY) .filter(file => /\.(?:png|jpe?g)$/i.test(file)) .slice(0, 5) : []; const baseUrl = process.env.WF_ANTHROPIC_BASE_URL; const apiKey = process.env.WF_ANTHROPIC_ACCEPTANCE_KEY; const enabled = process.env.WF_VISION_ACCEPTANCE === '1' && images.length > 0 && Boolean(baseUrl && apiKey); const MEDIA = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg' } as const; describe.skipIf(!enabled)('screenshot acceptance', () => { it('reads what went wrong from each screenshot, without leaking anything', async () => { for (const file of images) { const bytes = readFileSync(path.join(DIRECTORY, file)); expect(bytes.byteLength).toBeLessThan(4 * 1024 * 1024); const extension = file.split('.').pop()!.toLowerCase() as keyof typeof MEDIA; const response = await fetch(`${baseUrl!.replace(/\/$/, '')}/v1/messages`, { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey!, authorization: `Bearer ${apiKey}`, 'anthropic-version': '2023-06-01', }, body: JSON.stringify({ model: process.env.WF_COPILOT_VISION_MODEL ?? process.env.ANTHROPIC_MODEL ?? 'deepseek-flash@latest', max_tokens: 1024, messages: [ { role: 'user', content: [ { type: 'image', source: { type: 'base64', media_type: MEDIA[extension], data: bytes.toString('base64'), }, }, { type: 'text', text: '这张截图里用户遇到了什么问题?只描述界面上能看到的内容。' }, ], }, ], }), signal: AbortSignal.timeout(120_000), }); expect(response.status, `${file}: gateway rejected the image`).toBe(200); const payload = (await response.json()) as { content?: { type?: string; text?: string }[] }; const answer = redactSecrets( (payload.content ?? []) .filter(block => block.type === 'text') .map(block => block.text ?? '') .join(''), ); expect(answer.trim(), `${file}: no description`).not.toBe(''); expect(answer, `${file}: leaked a credential`).not.toMatch(/sk-[A-Za-z0-9_-]{8,}|Bearer\s+\S{8,}/); // Console output for the operator: which screenshot produced what. console.log(`\n${file}\n${answer.slice(0, 400)}\n`); } }); });