droboticflow/scripts/check-test-baseline.mjs
d-robotics 58e0e50570 🧪 Retro 2026-10-12: CI gate, known-failure baselines, devboot regression test, workbench smoke, review standards
Six follow-ups from the round8-12 hotfix retrospective:

- web-ci.yml: lint + Go/frontend tests on every push and PR (nothing ran
  app checks before; the round10 dev-mode regression shipped green)
- tests/known-failures.json + check-test-baseline.mjs: gate by exact
  failing-test id instead of hand-counted failure totals (14 frontend,
  70 Go upstream debts recorded from the dev tip)
- devboot.test.mjs: boots the real Vite dev server and pins the movi
  loader contract that broke round10 (query-less DEV url, /vendor stays
  off the module pipeline)
- scripts/local/smoke-workbench.mjs: pre-release smoke covering the
  seven recurring acceptance points (login/pool/claim/play-while-loading/
  captions/4x/playhead), 3x green against the local replica
- docs/CODING_STANDARDS.md + AGENTS pointer: eligibility flags are
  server-shipped, the UI renders them (rounds 9+12 lesson)
- vendor README host-element quirks section, HANDOFF gate pointers and
  movi-path verification rule, perf-audit report pinned with the
  post-round12 production 500 observation
- husky pre-commit: lint-staged (repo prettier scope) + tsc + gofmt
- gofmt: restore lint:go green on the dev tip (drift shipped silently
  in the perf merge and round12 because nothing gated it)
2026-10-12 18:07:02 +05:30

127 lines
4.5 KiB
JavaScript

#!/usr/bin/env node
// Known-failure baseline gate for the two suites with upstream test debt.
//
// node scripts/check-test-baseline.mjs frontend # npm run test:frontend:gate
// node scripts/check-test-baseline.mjs go # npm run test:go:gate
// node scripts/check-test-baseline.mjs <suite> --update
//
// Exit 0: no NEW failures vs tests/known-failures.json. Exit 1: new failures
// (printed by exact id) or a runner crash. Fixed baseline entries are reported
// as candidates to prune with --update but never fail the gate.
//
// Baseline ids: frontend "<file basename> :: <test name>" (spec-reporter
// failure detail pairs `test at <file>` with the following `✖ <name>` line);
// go "<package path> :: <TestName>" (top-level `--- FAIL:` lines assigned to
// the package whose `FAIL\t<pkg>` block marker closes them).
import { spawn } from "node:child_process";
import { readFileSync, writeFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
const baselinePath = join(repoRoot, "tests", "known-failures.json");
const suite = process.argv[2];
const update = process.argv.includes("--update");
if (suite !== "frontend" && suite !== "go") {
console.error(`usage: check-test-baseline.mjs <frontend|go> [--update]`);
process.exit(2);
}
function run(command, args) {
return new Promise((resolve) => {
const child = spawn(command, args, { cwd: repoRoot, shell: false });
let out = "";
let err = "";
child.stdout.on("data", (chunk) => {
out += chunk;
});
child.stderr.on("data", (chunk) => {
err += chunk;
});
child.on("error", (error) => resolve({ code: null, out: "", err: String(error) }));
child.on("close", (code) => resolve({ code, out, err }));
});
}
function parseFrontendFailures(output) {
const failures = new Set();
const lines = output.split("\n");
for (let i = 0; i < lines.length; i++) {
const match = lines[i].match(/^test at (tests\/frontend\/(\S+\.test\.mjs)):\d+:\d+$/);
if (match && lines[i + 1]?.startsWith("✖")) {
const name = lines[i + 1].replace(/^✖\s*/, "").replace(/\s+\([\d.]+ms\)$/, "");
failures.add(`${match[2]} :: ${name}`);
}
}
return failures;
}
function parseGoFailures(output) {
const failures = new Set();
let pending = [];
for (const line of output.split("\n")) {
const fail = line.match(/^--- FAIL: (\w+)/);
if (fail) {
pending.push(fail[1]);
continue;
}
const block = line.match(/^FAIL\t(\S+)\t/);
if (block && pending.length > 0) {
const pkg = block[1].replace(/^droboticflow\//, "");
for (const name of pending) failures.add(`${pkg} :: ${name}`);
pending = [];
}
}
return failures;
}
async function main() {
const baselineFile = JSON.parse(readFileSync(baselinePath, "utf8"));
const known = new Set(baselineFile[suite] ?? []);
if (known.size === 0 && !update) {
console.error(`baseline file has no "${suite}" entries; run with --update first`);
process.exit(2);
}
const result =
suite === "frontend"
? await run("npm", ["run", "--silent", "test:frontend"])
: await run("npm", ["run", "--silent", "test"]);
const failures =
suite === "frontend" ? parseFrontendFailures(result.out + result.err) : parseGoFailures(result.out + result.err);
const runnerCrashed = result.code === null;
const noFailuresParsed = failures.size === 0 && result.code !== 0;
if (runnerCrashed || noFailuresParsed) {
console.error(`test runner did not produce parseable results (code=${result.code})`);
console.error(result.err.slice(0, 2000));
process.exit(1);
}
if (update) {
baselineFile[suite] = [...failures].sort();
writeFileSync(baselinePath, JSON.stringify(baselineFile, null, 2) + "\n");
console.log(`baseline updated: ${suite} = ${failures.size} known failures -> ${baselinePath}`);
return;
}
const fresh = [...failures].filter((id) => !known.has(id));
const fixed = [...known].filter((id) => !failures.has(id));
if (fresh.length > 0) {
console.error(`NEW ${suite} failures (${fresh.length}) not in tests/known-failures.json:`);
for (const id of fresh) console.error(` + ${id}`);
process.exit(1);
}
if (fixed.length > 0) {
console.log(`fixed vs baseline (${fixed.length}) — prune with --update when confirmed:`);
for (const id of fixed) console.log(` - ${id}`);
}
console.log(`${suite}: ${failures.size} failures, all in baseline (${known.size} known). OK`);
}
await main();