146 lines
5.8 KiB
JavaScript
146 lines
5.8 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* Copilot smoke test over HTTP only: it needs no browser and no local model, so it can run from an
|
||
* operator's machine against a pre-production or production address and exercise exactly the answer
|
||
* path customers use.
|
||
*
|
||
* WF_API_URL=https://wishforge.example WF_API_TOKEN=… WF_PROJECT_ID=… node scripts/copilot-smoke.mjs
|
||
*
|
||
* The token belongs to an account that may read the project and create Wishes; it is never printed.
|
||
* The run asks a FAQ question, asks a Bug question and turns that answer into a Wish, which starts a
|
||
* real run. Nothing is pushed, opened as a PR or deployed.
|
||
*
|
||
* Every check is an assertion: an answer must be grounded in `wiki/index.md`, `wiki/features/**` or
|
||
* `wiki/glossary/**`, and no answer may mention a source path. The report is one JSON object.
|
||
*/
|
||
import assert from 'node:assert/strict';
|
||
import { randomUUID } from 'node:crypto';
|
||
|
||
const base = (process.env.WF_API_URL ?? '').replace(/\/$/, '');
|
||
const token = process.env.WF_API_TOKEN;
|
||
const projectId = process.env.WF_PROJECT_ID;
|
||
const missing = [
|
||
['WF_API_URL', base],
|
||
['WF_API_TOKEN', token],
|
||
['WF_PROJECT_ID', projectId],
|
||
]
|
||
.filter(([, value]) => !value)
|
||
.map(([name]) => name);
|
||
if (missing.length) {
|
||
console.error(`Missing ${missing.join(', ')}; see the header of this script.`);
|
||
process.exit(2);
|
||
}
|
||
|
||
/** The pages a customer may see, and the only ones an answer may cite. */
|
||
const CUSTOMER_PAGE = /^wiki\/(?:index\.md|features\/[A-Za-z0-9._/-]+\.md|glossary\/[A-Za-z0-9._/-]+\.md)$/;
|
||
/** A repository path, with or without a line number: customers read answers, not the codebase. */
|
||
const CODE_PATH = /(?:^|[\s((])(?:apps|packages|scripts|tests|infra)\/[\w./-]+(?::\d+)?/;
|
||
const FILE_WITH_LINE = /\b[\w-]+\.(?:ts|tsx|js|mjs|json):\d+\b/;
|
||
|
||
async function call(route, body, timeoutMs = 360_000) {
|
||
const response = await fetch(`${base}${route}`, {
|
||
method: body === undefined ? 'GET' : 'POST',
|
||
headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
|
||
body: body === undefined ? undefined : JSON.stringify(body),
|
||
signal: AbortSignal.timeout(timeoutMs),
|
||
});
|
||
const text = await response.text();
|
||
let payload;
|
||
try {
|
||
payload = JSON.parse(text);
|
||
} catch {
|
||
throw new Error(`${route} returned ${response.status} without JSON: ${text.slice(0, 300)}`);
|
||
}
|
||
if (!response.ok) {
|
||
const error = payload?.error ?? payload;
|
||
throw new Error(`${route} returned ${response.status}: ${JSON.stringify(error).slice(0, 500)}`);
|
||
}
|
||
return payload.data;
|
||
}
|
||
|
||
function grounded(message, label) {
|
||
assert.equal(message.status, 'complete', `${label}: the answer did not complete (${message.status})`);
|
||
for (const source of message.sources ?? [])
|
||
assert.match(source.path, CUSTOMER_PAGE, `${label}: the answer cites a page customers cannot see`);
|
||
assert.doesNotMatch(message.content, CODE_PATH, `${label}: the answer mentions a source path`);
|
||
assert.doesNotMatch(message.content, FILE_WITH_LINE, `${label}: the answer mentions a file and line number`);
|
||
assert.doesNotMatch(JSON.stringify(message.goal ?? ''), CODE_PATH, `${label}: the draft mentions a source path`);
|
||
}
|
||
|
||
async function ask(sessionId, content) {
|
||
const answered = await call(`/v1/projects/${projectId}/copilot/sessions/${sessionId}/messages`, {
|
||
content,
|
||
clientId: randomUUID(),
|
||
context: { pathname: '/projects' },
|
||
});
|
||
const message = answered.messages.at(-1);
|
||
assert.equal(message?.role, 'assistant', 'The last message of a session is the answer');
|
||
return { answered, message };
|
||
}
|
||
|
||
const started = Date.now();
|
||
const passed = [];
|
||
try {
|
||
const capabilities = await call('/v1/capabilities');
|
||
assert.equal(
|
||
capabilities.copilot,
|
||
true,
|
||
'Copilot is not configured for this deployment: record the platform model connection, or install the Claude CLI',
|
||
);
|
||
passed.push('copilot-configured');
|
||
|
||
const session = await call(`/v1/projects/${projectId}/copilot/sessions`, {});
|
||
passed.push('session-created');
|
||
|
||
const faq = await ask(session.id, '如何提交 Wish?请根据当前项目 Wiki 回答。');
|
||
grounded(faq.message, 'FAQ');
|
||
assert.equal(faq.message.kind, 'faq', `FAQ: expected a FAQ answer, got ${faq.message.kind}`);
|
||
assert.ok((faq.message.sources ?? []).length > 0, 'FAQ: a grounded answer brings at least one Wiki source');
|
||
passed.push('faq-grounded');
|
||
|
||
const bug = await ask(session.id, '我点击提交 Wish 后接口返回 500,页面没有创建记录。请记录复现信息。');
|
||
grounded(bug.message, 'Bug');
|
||
assert.equal(bug.message.kind, 'bug', `Bug: expected a Bug answer, got ${bug.message.kind}`);
|
||
assert.ok(bug.message.goal?.trim(), 'Bug: a draft goal is required before creating a Wish');
|
||
assert.ok(bug.message.acceptanceCriteria?.length, 'Bug: acceptance criteria are required before creating a Wish');
|
||
passed.push('bug-draft');
|
||
|
||
const wish = await call(`/v1/projects/${projectId}/copilot/sessions/${session.id}/wish`, {
|
||
messageId: bug.message.id,
|
||
goal: bug.message.goal,
|
||
acceptanceCriteria: bug.message.acceptanceCriteria,
|
||
answers: [],
|
||
});
|
||
assert.ok(wish.wishId && wish.runId, 'Wish: the API returns the Wish and its run');
|
||
assert.equal(wish.url, `/runs/${wish.runId}`, 'Wish: the console link points at the run');
|
||
passed.push('wish-created');
|
||
|
||
console.log(
|
||
JSON.stringify(
|
||
{
|
||
passed,
|
||
projectId,
|
||
faq: { kind: faq.message.kind, sources: (faq.message.sources ?? []).map(source => source.path) },
|
||
bug: { kind: bug.message.kind, sources: (bug.message.sources ?? []).map(source => source.path) },
|
||
wish,
|
||
durationMs: Date.now() - started,
|
||
},
|
||
null,
|
||
2,
|
||
),
|
||
);
|
||
} catch (error) {
|
||
console.error(
|
||
JSON.stringify(
|
||
{
|
||
passed,
|
||
failure: error instanceof Error ? error.message : String(error),
|
||
durationMs: Date.now() - started,
|
||
},
|
||
null,
|
||
2,
|
||
),
|
||
);
|
||
process.exit(1);
|
||
}
|