wishforge-ide/tests/copilot-vision.test.ts
wishforge 3f879c37d3 ✅ Red-team the Copilot, prepare screenshot acceptance, and tidy two names
Ten ways a customer, a page or a log line may try to make the Copilot hand over
what it must not: a key, the connection string, the system prompt, a kubeconfig.
Where a mock model leaks, the platform still refuses; where it obeys an
injected instruction, the prompt already says a document is data, not a command.
The tool layer is tested separately: an environment file is refused even when
the model names it, and logs outside the project's namespaces are refused.

Screenshot acceptance is prepared, not faked: the fixtures directory explains
what to put there, and the test runs only when an operator does, against the
real gateway.

Model names stored with a context suffix are rewritten by a dry-run-by-default
script (the local database currently has no projects), and the console calls
/providers what it is: 模型连接.
2026-11-09 10:09:59 -08:00

76 lines
3.3 KiB
TypeScript

import { existsSync, readdirSync, readFileSync } from 'node:fs';
import path from 'node:path';
import { describe, expect, it } from 'vitest';
import { redactSecrets } from '../apps/api/src/domain/copilot-tools.js';
/**
* The image path, end to end, against the real gateway. It is skipped unless an operator puts
* screenshots in `apps/api/src/__tests__/fixtures` and sets WF_VISION_ACCEPTANCE=1: a fake image
* here would prove nothing about a model that has to read a Chinese error dialog.
*
* The gateway is called directly, with the same credential shape the Copilot uses, because this
* test is about what the model sees in a screenshot, not about the console around it.
*/
const DIRECTORY = path.join(import.meta.dirname, '..', 'apps', 'api', 'src', '__tests__', 'fixtures');
const images = existsSync(DIRECTORY)
? readdirSync(DIRECTORY)
.filter(file => /\.(?:png|jpe?g)$/i.test(file))
.slice(0, 5)
: [];
const baseUrl = process.env.WF_ANTHROPIC_BASE_URL;
const apiKey = process.env.WF_ANTHROPIC_ACCEPTANCE_KEY;
const enabled = process.env.WF_VISION_ACCEPTANCE === '1' && images.length > 0 && Boolean(baseUrl && apiKey);
const MEDIA = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg' } as const;
describe.skipIf(!enabled)('screenshot acceptance', () => {
it('reads what went wrong from each screenshot, without leaking anything', async () => {
for (const file of images) {
const bytes = readFileSync(path.join(DIRECTORY, file));
expect(bytes.byteLength).toBeLessThan(4 * 1024 * 1024);
const extension = file.split('.').pop()!.toLowerCase() as keyof typeof MEDIA;
const response = await fetch(`${baseUrl!.replace(/\/$/, '')}/v1/messages`, {
method: 'POST',
headers: {
'content-type': 'application/json',
'x-api-key': apiKey!,
authorization: `Bearer ${apiKey}`,
'anthropic-version': '2023-06-01',
},
body: JSON.stringify({
model: process.env.WF_COPILOT_VISION_MODEL ?? process.env.ANTHROPIC_MODEL ?? 'deepseek-flash@latest',
max_tokens: 1024,
messages: [
{
role: 'user',
content: [
{
type: 'image',
source: {
type: 'base64',
media_type: MEDIA[extension],
data: bytes.toString('base64'),
},
},
{ type: 'text', text: '这张截图里用户遇到了什么问题?只描述界面上能看到的内容。' },
],
},
],
}),
signal: AbortSignal.timeout(120_000),
});
expect(response.status, `${file}: gateway rejected the image`).toBe(200);
const payload = (await response.json()) as { content?: { type?: string; text?: string }[] };
const answer = redactSecrets(
(payload.content ?? [])
.filter(block => block.type === 'text')
.map(block => block.text ?? '')
.join(''),
);
expect(answer.trim(), `${file}: no description`).not.toBe('');
expect(answer, `${file}: leaked a credential`).not.toMatch(/sk-[A-Za-z0-9_-]{8,}|Bearer\s+\S{8,}/);
// Console output for the operator: which screenshot produced what.
console.log(`\n${file}\n${answer.slice(0, 400)}\n`);
}
});
});