droboticflow/scripts/check-test-baseline.mjs

166 lines
5.9 KiB
JavaScript

#!/usr/bin/env node
// Known-failure baseline gate for the two suites with upstream test debt.
//
// node scripts/check-test-baseline.mjs frontend # npm run test:frontend:gate
// node scripts/check-test-baseline.mjs go # npm run test:go:gate
// node scripts/check-test-baseline.mjs <suite> --update
//
// Exit 0: no NEW failures vs tests/known-failures.json. Exit 1: new failures
// (printed by exact id) or a runner crash. Fixed baseline entries are reported
// as candidates to prune with --update but never fail the gate.
//
// Baseline ids: frontend "<file basename> :: <test name>" (spec-reporter
// failure detail pairs `test at <file>` with the following `✖ <name>` line);
// go "<package path> :: <TestName>" (top-level `--- FAIL:` lines assigned to
// the package whose `FAIL\t<pkg>` block marker closes them).
import { spawn } from "node:child_process";
import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname, join, resolve } from "node:path";
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
const baselinePath = join(repoRoot, "tests", "known-failures.json");
const suite = process.argv[2];
const update = process.argv.includes("--update");
if (suite !== "frontend" && suite !== "go") {
console.error(`usage: check-test-baseline.mjs <frontend|go> [--update]`);
process.exit(2);
}
function run(command, args) {
return new Promise((resolve) => {
const child = spawn(command, args, { cwd: repoRoot, shell: false });
let out = "";
let err = "";
child.stdout.on("data", (chunk) => {
out += chunk;
});
child.stderr.on("data", (chunk) => {
err += chunk;
});
child.on("error", (error) => resolve({ code: null, out: "", err: String(error) }));
child.on("close", (code) => resolve({ code, out, err }));
});
}
function parseFrontendFailures(output) {
const failures = new Set();
const lines = output.split("\n");
for (let i = 0; i < lines.length; i++) {
const match = lines[i].match(/^test at (tests\/frontend\/(\S+\.test\.mjs)):\d+:\d+$/);
if (match && lines[i + 1]?.startsWith("✖")) {
const name = lines[i + 1].replace(/^✖\s*/, "").replace(/\s+\([\d.]+ms\)$/, "");
failures.add(`${match[2]} :: ${name}`);
}
}
return failures;
}
function parseGoFailures(output) {
const failures = new Set();
let pending = [];
for (const line of output.split("\n")) {
const fail = line.match(/^--- FAIL: (\w+)/);
if (fail) {
pending.push(fail[1]);
continue;
}
const block = line.match(/^FAIL\t(\S+)\t/);
if (block && pending.length > 0) {
const pkg = block[1].replace(/^droboticflow\//, "");
for (const name of pending) failures.add(`${pkg} :: ${name}`);
pending = [];
}
}
return failures;
}
async function main() {
const logOption = process.argv.indexOf("--log-dir");
if (logOption >= 0 && (!process.argv[logOption + 1] || process.argv[logOption + 1].startsWith("--"))) {
console.error("--log-dir requires a directory path");
process.exit(2);
}
const baselineFile = JSON.parse(readFileSync(baselinePath, "utf8"));
const known = new Set(baselineFile[suite] ?? []);
if (known.size === 0 && !update) {
console.error(`baseline file has no "${suite}" entries; run with --update first`);
process.exit(2);
}
const result =
suite === "frontend"
? await run("npm", ["run", "--silent", "test:frontend"])
: await run("npm", ["run", "--silent", "test"]);
// Preserve every invocation, including crashes and runs which pass only
// because failures are known. No automatic retries or baseline mutation.
const option = logOption;
const logDir = resolve(
option >= 0
? process.argv[option + 1]
: join(
repoRoot,
"local-data/test-results",
`${suite}-${new Date().toISOString().replaceAll(":", "-")}-${process.pid}`,
),
);
mkdirSync(logDir, { recursive: true, mode: 0o700 });
writeFileSync(join(logDir, "stdout.log"), result.out, { mode: 0o600 });
writeFileSync(join(logDir, "stderr.log"), result.err, { mode: 0o600 });
console.log(`Full test output: ${logDir}`);
const failures =
suite === "frontend" ? parseFrontendFailures(result.out + result.err) : parseGoFailures(result.out + result.err);
const fresh = [...failures].filter((id) => !known.has(id));
const fixed = [...known].filter((id) => !failures.has(id));
writeFileSync(
join(logDir, "summary.json"),
JSON.stringify(
{
suite,
at: new Date().toISOString(),
exitCode: result.code,
failures: [...failures],
newFailures: fresh,
fixed,
updateRequested: update,
},
null,
2,
) + "\n",
{ mode: 0o600 },
);
const runnerCrashed = result.code === null;
const noFailuresParsed = failures.size === 0 && result.code !== 0;
if (runnerCrashed || noFailuresParsed) {
console.error(`test runner did not produce parseable results (code=${result.code})`);
console.error(result.err.slice(0, 2000));
process.exit(1);
}
if (update) {
baselineFile[suite] = [...failures].sort();
writeFileSync(baselinePath, JSON.stringify(baselineFile, null, 2) + "\n");
console.log(`baseline updated: ${suite} = ${failures.size} known failures -> ${baselinePath}`);
return;
}
if (fresh.length > 0) {
console.error(`NEW ${suite} failures (${fresh.length}) not in tests/known-failures.json:`);
for (const id of fresh) console.error(` + ${id}`);
// Put a bounded diagnostic tail in CI output; full streams remain on disk.
console.error((result.out + result.err).slice(-4000));
process.exit(1);
}
if (fixed.length > 0) {
console.log(`fixed vs baseline (${fixed.length}) — prune with --update when confirmed:`);
for (const id of fixed) console.log(` - ${id}`);
}
console.log(`${suite}: ${failures.size} failures, all in baseline (${known.size} known). OK`);
}
await main();