wishforge-ide/scripts/copilot-smoke.mjs

146 lines
5.8 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env node
/**
* Copilot smoke test over HTTP only: it needs no browser and no local model, so it can run from an
* operator's machine against a pre-production or production address and exercise exactly the answer
* path customers use.
*
* WF_API_URL=https://wishforge.example WF_API_TOKEN=… WF_PROJECT_ID=… node scripts/copilot-smoke.mjs
*
* The token belongs to an account that may read the project and create Wishes; it is never printed.
* The run asks a FAQ question, asks a Bug question and turns that answer into a Wish, which starts a
* real run. Nothing is pushed, opened as a PR or deployed.
*
* Every check is an assertion: an answer must be grounded in `wiki/index.md`, `wiki/features/**` or
* `wiki/glossary/**`, and no answer may mention a source path. The report is one JSON object.
*/
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
const base = (process.env.WF_API_URL ?? '').replace(/\/$/, '');
const token = process.env.WF_API_TOKEN;
const projectId = process.env.WF_PROJECT_ID;
const missing = [
['WF_API_URL', base],
['WF_API_TOKEN', token],
['WF_PROJECT_ID', projectId],
]
.filter(([, value]) => !value)
.map(([name]) => name);
if (missing.length) {
console.error(`Missing ${missing.join(', ')}; see the header of this script.`);
process.exit(2);
}
/** The pages a customer may see, and the only ones an answer may cite. */
const CUSTOMER_PAGE = /^wiki\/(?:index\.md|features\/[A-Za-z0-9._/-]+\.md|glossary\/[A-Za-z0-9._/-]+\.md)$/;
/** A repository path, with or without a line number: customers read answers, not the codebase. */
const CODE_PATH = /(?:^|[\s((])(?:apps|packages|scripts|tests|infra)\/[\w./-]+(?::\d+)?/;
const FILE_WITH_LINE = /\b[\w-]+\.(?:ts|tsx|js|mjs|json):\d+\b/;
async function call(route, body, timeoutMs = 360_000) {
const response = await fetch(`${base}${route}`, {
method: body === undefined ? 'GET' : 'POST',
headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(timeoutMs),
});
const text = await response.text();
let payload;
try {
payload = JSON.parse(text);
} catch {
throw new Error(`${route} returned ${response.status} without JSON: ${text.slice(0, 300)}`);
}
if (!response.ok) {
const error = payload?.error ?? payload;
throw new Error(`${route} returned ${response.status}: ${JSON.stringify(error).slice(0, 500)}`);
}
return payload.data;
}
function grounded(message, label) {
assert.equal(message.status, 'complete', `${label}: the answer did not complete (${message.status})`);
for (const source of message.sources ?? [])
assert.match(source.path, CUSTOMER_PAGE, `${label}: the answer cites a page customers cannot see`);
assert.doesNotMatch(message.content, CODE_PATH, `${label}: the answer mentions a source path`);
assert.doesNotMatch(message.content, FILE_WITH_LINE, `${label}: the answer mentions a file and line number`);
assert.doesNotMatch(JSON.stringify(message.goal ?? ''), CODE_PATH, `${label}: the draft mentions a source path`);
}
async function ask(sessionId, content) {
const answered = await call(`/v1/projects/${projectId}/copilot/sessions/${sessionId}/messages`, {
content,
clientId: randomUUID(),
context: { pathname: '/projects' },
});
const message = answered.messages.at(-1);
assert.equal(message?.role, 'assistant', 'The last message of a session is the answer');
return { answered, message };
}
const started = Date.now();
const passed = [];
try {
const capabilities = await call('/v1/capabilities');
assert.equal(
capabilities.copilot,
true,
'Copilot is not configured for this deployment: record the platform model connection, or install the Claude CLI',
);
passed.push('copilot-configured');
const session = await call(`/v1/projects/${projectId}/copilot/sessions`, {});
passed.push('session-created');
const faq = await ask(session.id, '如何提交 Wish?请根据当前项目 Wiki 回答。');
grounded(faq.message, 'FAQ');
assert.equal(faq.message.kind, 'faq', `FAQ: expected a FAQ answer, got ${faq.message.kind}`);
assert.ok((faq.message.sources ?? []).length > 0, 'FAQ: a grounded answer brings at least one Wiki source');
passed.push('faq-grounded');
const bug = await ask(session.id, '我点击提交 Wish 后接口返回 500,页面没有创建记录。请记录复现信息。');
grounded(bug.message, 'Bug');
assert.equal(bug.message.kind, 'bug', `Bug: expected a Bug answer, got ${bug.message.kind}`);
assert.ok(bug.message.goal?.trim(), 'Bug: a draft goal is required before creating a Wish');
assert.ok(bug.message.acceptanceCriteria?.length, 'Bug: acceptance criteria are required before creating a Wish');
passed.push('bug-draft');
const wish = await call(`/v1/projects/${projectId}/copilot/sessions/${session.id}/wish`, {
messageId: bug.message.id,
goal: bug.message.goal,
acceptanceCriteria: bug.message.acceptanceCriteria,
answers: [],
});
assert.ok(wish.wishId && wish.runId, 'Wish: the API returns the Wish and its run');
assert.equal(wish.url, `/runs/${wish.runId}`, 'Wish: the console link points at the run');
passed.push('wish-created');
console.log(
JSON.stringify(
{
passed,
projectId,
faq: { kind: faq.message.kind, sources: (faq.message.sources ?? []).map(source => source.path) },
bug: { kind: bug.message.kind, sources: (bug.message.sources ?? []).map(source => source.path) },
wish,
durationMs: Date.now() - started,
},
null,
2,
),
);
} catch (error) {
console.error(
JSON.stringify(
{
passed,
failure: error instanceof Error ? error.message : String(error),
durationMs: Date.now() - started,
},
null,
2,
),
);
process.exit(1);
}