Six follow-ups from the round8-12 hotfix retrospective: - web-ci.yml: lint + Go/frontend tests on every push and PR (nothing ran app checks before; the round10 dev-mode regression shipped green) - tests/known-failures.json + check-test-baseline.mjs: gate by exact failing-test id instead of hand-counted failure totals (14 frontend, 70 Go upstream debts recorded from the dev tip) - devboot.test.mjs: boots the real Vite dev server and pins the movi loader contract that broke round10 (query-less DEV url, /vendor stays off the module pipeline) - scripts/local/smoke-workbench.mjs: pre-release smoke covering the seven recurring acceptance points (login/pool/claim/play-while-loading/ captions/4x/playhead), 3x green against the local replica - docs/CODING_STANDARDS.md + AGENTS pointer: eligibility flags are server-shipped, the UI renders them (rounds 9+12 lesson) - vendor README host-element quirks section, HANDOFF gate pointers and movi-path verification rule, perf-audit report pinned with the post-round12 production 500 observation - husky pre-commit: lint-staged (repo prettier scope) + tsc + gofmt - gofmt: restore lint:go green on the dev tip (drift shipped silently in the perf merge and round12 because nothing gated it)
127 lines
4.5 KiB
JavaScript
127 lines
4.5 KiB
JavaScript
#!/usr/bin/env node
|
|
// Known-failure baseline gate for the two suites with upstream test debt.
|
|
//
|
|
// node scripts/check-test-baseline.mjs frontend # npm run test:frontend:gate
|
|
// node scripts/check-test-baseline.mjs go # npm run test:go:gate
|
|
// node scripts/check-test-baseline.mjs <suite> --update
|
|
//
|
|
// Exit 0: no NEW failures vs tests/known-failures.json. Exit 1: new failures
|
|
// (printed by exact id) or a runner crash. Fixed baseline entries are reported
|
|
// as candidates to prune with --update but never fail the gate.
|
|
//
|
|
// Baseline ids: frontend "<file basename> :: <test name>" (spec-reporter
|
|
// failure detail pairs `test at <file>` with the following `✖ <name>` line);
|
|
// go "<package path> :: <TestName>" (top-level `--- FAIL:` lines assigned to
|
|
// the package whose `FAIL\t<pkg>` block marker closes them).
|
|
|
|
import { spawn } from "node:child_process";
|
|
import { readFileSync, writeFileSync } from "node:fs";
|
|
import { fileURLToPath } from "node:url";
|
|
import { dirname, join } from "node:path";
|
|
|
|
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
|
|
const baselinePath = join(repoRoot, "tests", "known-failures.json");
|
|
const suite = process.argv[2];
|
|
const update = process.argv.includes("--update");
|
|
|
|
if (suite !== "frontend" && suite !== "go") {
|
|
console.error(`usage: check-test-baseline.mjs <frontend|go> [--update]`);
|
|
process.exit(2);
|
|
}
|
|
|
|
function run(command, args) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn(command, args, { cwd: repoRoot, shell: false });
|
|
let out = "";
|
|
let err = "";
|
|
child.stdout.on("data", (chunk) => {
|
|
out += chunk;
|
|
});
|
|
child.stderr.on("data", (chunk) => {
|
|
err += chunk;
|
|
});
|
|
child.on("error", (error) => resolve({ code: null, out: "", err: String(error) }));
|
|
child.on("close", (code) => resolve({ code, out, err }));
|
|
});
|
|
}
|
|
|
|
function parseFrontendFailures(output) {
|
|
const failures = new Set();
|
|
const lines = output.split("\n");
|
|
for (let i = 0; i < lines.length; i++) {
|
|
const match = lines[i].match(/^test at (tests\/frontend\/(\S+\.test\.mjs)):\d+:\d+$/);
|
|
if (match && lines[i + 1]?.startsWith("✖")) {
|
|
const name = lines[i + 1].replace(/^✖\s*/, "").replace(/\s+\([\d.]+ms\)$/, "");
|
|
failures.add(`${match[2]} :: ${name}`);
|
|
}
|
|
}
|
|
return failures;
|
|
}
|
|
|
|
function parseGoFailures(output) {
|
|
const failures = new Set();
|
|
let pending = [];
|
|
for (const line of output.split("\n")) {
|
|
const fail = line.match(/^--- FAIL: (\w+)/);
|
|
if (fail) {
|
|
pending.push(fail[1]);
|
|
continue;
|
|
}
|
|
const block = line.match(/^FAIL\t(\S+)\t/);
|
|
if (block && pending.length > 0) {
|
|
const pkg = block[1].replace(/^droboticflow\//, "");
|
|
for (const name of pending) failures.add(`${pkg} :: ${name}`);
|
|
pending = [];
|
|
}
|
|
}
|
|
return failures;
|
|
}
|
|
|
|
async function main() {
|
|
const baselineFile = JSON.parse(readFileSync(baselinePath, "utf8"));
|
|
const known = new Set(baselineFile[suite] ?? []);
|
|
if (known.size === 0 && !update) {
|
|
console.error(`baseline file has no "${suite}" entries; run with --update first`);
|
|
process.exit(2);
|
|
}
|
|
|
|
const result =
|
|
suite === "frontend"
|
|
? await run("npm", ["run", "--silent", "test:frontend"])
|
|
: await run("npm", ["run", "--silent", "test"]);
|
|
|
|
const failures =
|
|
suite === "frontend" ? parseFrontendFailures(result.out + result.err) : parseGoFailures(result.out + result.err);
|
|
|
|
const runnerCrashed = result.code === null;
|
|
const noFailuresParsed = failures.size === 0 && result.code !== 0;
|
|
if (runnerCrashed || noFailuresParsed) {
|
|
console.error(`test runner did not produce parseable results (code=${result.code})`);
|
|
console.error(result.err.slice(0, 2000));
|
|
process.exit(1);
|
|
}
|
|
|
|
if (update) {
|
|
baselineFile[suite] = [...failures].sort();
|
|
writeFileSync(baselinePath, JSON.stringify(baselineFile, null, 2) + "\n");
|
|
console.log(`baseline updated: ${suite} = ${failures.size} known failures -> ${baselinePath}`);
|
|
return;
|
|
}
|
|
|
|
const fresh = [...failures].filter((id) => !known.has(id));
|
|
const fixed = [...known].filter((id) => !failures.has(id));
|
|
|
|
if (fresh.length > 0) {
|
|
console.error(`NEW ${suite} failures (${fresh.length}) not in tests/known-failures.json:`);
|
|
for (const id of fresh) console.error(` + ${id}`);
|
|
process.exit(1);
|
|
}
|
|
if (fixed.length > 0) {
|
|
console.log(`fixed vs baseline (${fixed.length}) — prune with --update when confirmed:`);
|
|
for (const id of fixed) console.log(` - ${id}`);
|
|
}
|
|
console.log(`${suite}: ${failures.size} failures, all in baseline (${known.size} known). OK`);
|
|
}
|
|
|
|
await main();
|