#!/usr/bin/env node /** * Copilot smoke test over HTTP only: it needs no browser and no local model, so it can run from an * operator's machine against a pre-production or production address and exercise exactly the answer * path customers use. * * WF_API_URL=https://wishforge.example WF_API_TOKEN=… WF_PROJECT_ID=… node scripts/copilot-smoke.mjs * * The token belongs to an account that may read the project and create Wishes; it is never printed. * The run asks a FAQ question, asks a Bug question and turns that answer into a Wish, which starts a * real run. Nothing is pushed, opened as a PR or deployed. * * Every check is an assertion: an answer must be grounded in `wiki/index.md`, `wiki/features/**` or * `wiki/glossary/**`, and no answer may mention a source path. The report is one JSON object. */ import assert from 'node:assert/strict'; import { randomUUID } from 'node:crypto'; const base = (process.env.WF_API_URL ?? '').replace(/\/$/, ''); const token = process.env.WF_API_TOKEN; const projectId = process.env.WF_PROJECT_ID; const missing = [ ['WF_API_URL', base], ['WF_API_TOKEN', token], ['WF_PROJECT_ID', projectId], ] .filter(([, value]) => !value) .map(([name]) => name); if (missing.length) { console.error(`Missing ${missing.join(', ')}; see the header of this script.`); process.exit(2); } /** The pages a customer may see, and the only ones an answer may cite. */ const CUSTOMER_PAGE = /^wiki\/(?:index\.md|features\/[A-Za-z0-9._/-]+\.md|glossary\/[A-Za-z0-9._/-]+\.md)$/; /** A repository path, with or without a line number: customers read answers, not the codebase. */ const CODE_PATH = /(?:^|[\s((])(?:apps|packages|scripts|tests|infra)\/[\w./-]+(?::\d+)?/; const FILE_WITH_LINE = /\b[\w-]+\.(?:ts|tsx|js|mjs|json):\d+\b/; async function call(route, body, timeoutMs = 360_000) { const response = await fetch(`${base}${route}`, { method: body === undefined ? 'GET' : 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(timeoutMs), }); const text = await response.text(); let payload; try { payload = JSON.parse(text); } catch { throw new Error(`${route} returned ${response.status} without JSON: ${text.slice(0, 300)}`); } if (!response.ok) { const error = payload?.error ?? payload; throw new Error(`${route} returned ${response.status}: ${JSON.stringify(error).slice(0, 500)}`); } return payload.data; } function grounded(message, label) { assert.equal(message.status, 'complete', `${label}: the answer did not complete (${message.status})`); for (const source of message.sources ?? []) assert.match(source.path, CUSTOMER_PAGE, `${label}: the answer cites a page customers cannot see`); assert.doesNotMatch(message.content, CODE_PATH, `${label}: the answer mentions a source path`); assert.doesNotMatch(message.content, FILE_WITH_LINE, `${label}: the answer mentions a file and line number`); assert.doesNotMatch(JSON.stringify(message.goal ?? ''), CODE_PATH, `${label}: the draft mentions a source path`); } async function ask(sessionId, content) { const answered = await call(`/v1/projects/${projectId}/copilot/sessions/${sessionId}/messages`, { content, clientId: randomUUID(), context: { pathname: '/projects' }, }); const message = answered.messages.at(-1); assert.equal(message?.role, 'assistant', 'The last message of a session is the answer'); return { answered, message }; } const started = Date.now(); const passed = []; try { const capabilities = await call('/v1/capabilities'); assert.equal( capabilities.copilot, true, 'Copilot is not configured for this deployment: record the platform model connection, or install the Claude CLI', ); passed.push('copilot-configured'); const session = await call(`/v1/projects/${projectId}/copilot/sessions`, {}); passed.push('session-created'); const faq = await ask(session.id, '如何提交 Wish?请根据当前项目 Wiki 回答。'); grounded(faq.message, 'FAQ'); assert.equal(faq.message.kind, 'faq', `FAQ: expected a FAQ answer, got ${faq.message.kind}`); assert.ok((faq.message.sources ?? []).length > 0, 'FAQ: a grounded answer brings at least one Wiki source'); passed.push('faq-grounded'); const bug = await ask(session.id, '我点击提交 Wish 后接口返回 500,页面没有创建记录。请记录复现信息。'); grounded(bug.message, 'Bug'); assert.equal(bug.message.kind, 'bug', `Bug: expected a Bug answer, got ${bug.message.kind}`); assert.ok(bug.message.goal?.trim(), 'Bug: a draft goal is required before creating a Wish'); assert.ok(bug.message.acceptanceCriteria?.length, 'Bug: acceptance criteria are required before creating a Wish'); passed.push('bug-draft'); const wish = await call(`/v1/projects/${projectId}/copilot/sessions/${session.id}/wish`, { messageId: bug.message.id, goal: bug.message.goal, acceptanceCriteria: bug.message.acceptanceCriteria, answers: [], }); assert.ok(wish.wishId && wish.runId, 'Wish: the API returns the Wish and its run'); assert.equal(wish.url, `/runs/${wish.runId}`, 'Wish: the console link points at the run'); passed.push('wish-created'); console.log( JSON.stringify( { passed, projectId, faq: { kind: faq.message.kind, sources: (faq.message.sources ?? []).map(source => source.path) }, bug: { kind: bug.message.kind, sources: (bug.message.sources ?? []).map(source => source.path) }, wish, durationMs: Date.now() - started, }, null, 2, ), ); } catch (error) { console.error( JSON.stringify( { passed, failure: error instanceof Error ? error.message : String(error), durationMs: Date.now() - started, }, null, 2, ), ); process.exit(1); }