Ten ways a customer, a page or a log line may try to make the Copilot hand over what it must not: a key, the connection string, the system prompt, a kubeconfig. Where a mock model leaks, the platform still refuses; where it obeys an injected instruction, the prompt already says a document is data, not a command. The tool layer is tested separately: an environment file is refused even when the model names it, and logs outside the project's namespaces are refused. Screenshot acceptance is prepared, not faked: the fixtures directory explains what to put there, and the test runs only when an operator does, against the real gateway. Model names stored with a context suffix are rewritten by a dry-run-by-default script (the local database currently has no projects), and the console calls /providers what it is: 模型连接.
76 lines
3.3 KiB
TypeScript
76 lines
3.3 KiB
TypeScript
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
|
import path from 'node:path';
|
|
import { describe, expect, it } from 'vitest';
|
|
import { redactSecrets } from '../apps/api/src/domain/copilot-tools.js';
|
|
|
|
/**
|
|
* The image path, end to end, against the real gateway. It is skipped unless an operator puts
|
|
* screenshots in `apps/api/src/__tests__/fixtures` and sets WF_VISION_ACCEPTANCE=1: a fake image
|
|
* here would prove nothing about a model that has to read a Chinese error dialog.
|
|
*
|
|
* The gateway is called directly, with the same credential shape the Copilot uses, because this
|
|
* test is about what the model sees in a screenshot, not about the console around it.
|
|
*/
|
|
const DIRECTORY = path.join(import.meta.dirname, '..', 'apps', 'api', 'src', '__tests__', 'fixtures');
|
|
const images = existsSync(DIRECTORY)
|
|
? readdirSync(DIRECTORY)
|
|
.filter(file => /\.(?:png|jpe?g)$/i.test(file))
|
|
.slice(0, 5)
|
|
: [];
|
|
const baseUrl = process.env.WF_ANTHROPIC_BASE_URL;
|
|
const apiKey = process.env.WF_ANTHROPIC_ACCEPTANCE_KEY;
|
|
const enabled = process.env.WF_VISION_ACCEPTANCE === '1' && images.length > 0 && Boolean(baseUrl && apiKey);
|
|
const MEDIA = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg' } as const;
|
|
|
|
describe.skipIf(!enabled)('screenshot acceptance', () => {
|
|
it('reads what went wrong from each screenshot, without leaking anything', async () => {
|
|
for (const file of images) {
|
|
const bytes = readFileSync(path.join(DIRECTORY, file));
|
|
expect(bytes.byteLength).toBeLessThan(4 * 1024 * 1024);
|
|
const extension = file.split('.').pop()!.toLowerCase() as keyof typeof MEDIA;
|
|
const response = await fetch(`${baseUrl!.replace(/\/$/, '')}/v1/messages`, {
|
|
method: 'POST',
|
|
headers: {
|
|
'content-type': 'application/json',
|
|
'x-api-key': apiKey!,
|
|
authorization: `Bearer ${apiKey}`,
|
|
'anthropic-version': '2023-06-01',
|
|
},
|
|
body: JSON.stringify({
|
|
model: process.env.WF_COPILOT_VISION_MODEL ?? process.env.ANTHROPIC_MODEL ?? 'deepseek-flash@latest',
|
|
max_tokens: 1024,
|
|
messages: [
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'image',
|
|
source: {
|
|
type: 'base64',
|
|
media_type: MEDIA[extension],
|
|
data: bytes.toString('base64'),
|
|
},
|
|
},
|
|
{ type: 'text', text: '这张截图里用户遇到了什么问题?只描述界面上能看到的内容。' },
|
|
],
|
|
},
|
|
],
|
|
}),
|
|
signal: AbortSignal.timeout(120_000),
|
|
});
|
|
expect(response.status, `${file}: gateway rejected the image`).toBe(200);
|
|
const payload = (await response.json()) as { content?: { type?: string; text?: string }[] };
|
|
const answer = redactSecrets(
|
|
(payload.content ?? [])
|
|
.filter(block => block.type === 'text')
|
|
.map(block => block.text ?? '')
|
|
.join(''),
|
|
);
|
|
expect(answer.trim(), `${file}: no description`).not.toBe('');
|
|
expect(answer, `${file}: leaked a credential`).not.toMatch(/sk-[A-Za-z0-9_-]{8,}|Bearer\s+\S{8,}/);
|
|
// Console output for the operator: which screenshot produced what.
|
|
console.log(`\n${file}\n${answer.slice(0, 400)}\n`);
|
|
}
|
|
});
|
|
});
|