166 lines
5.9 KiB
JavaScript
166 lines
5.9 KiB
JavaScript
#!/usr/bin/env node
|
|
// Known-failure baseline gate for the two suites with upstream test debt.
|
|
//
|
|
// node scripts/check-test-baseline.mjs frontend # npm run test:frontend:gate
|
|
// node scripts/check-test-baseline.mjs go # npm run test:go:gate
|
|
// node scripts/check-test-baseline.mjs <suite> --update
|
|
//
|
|
// Exit 0: no NEW failures vs tests/known-failures.json. Exit 1: new failures
|
|
// (printed by exact id) or a runner crash. Fixed baseline entries are reported
|
|
// as candidates to prune with --update but never fail the gate.
|
|
//
|
|
// Baseline ids: frontend "<file basename> :: <test name>" (spec-reporter
|
|
// failure detail pairs `test at <file>` with the following `✖ <name>` line);
|
|
// go "<package path> :: <TestName>" (top-level `--- FAIL:` lines assigned to
|
|
// the package whose `FAIL\t<pkg>` block marker closes them).
|
|
|
|
import { spawn } from "node:child_process";
|
|
import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
|
import { fileURLToPath } from "node:url";
|
|
import { dirname, join, resolve } from "node:path";
|
|
|
|
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
|
|
const baselinePath = join(repoRoot, "tests", "known-failures.json");
|
|
const suite = process.argv[2];
|
|
const update = process.argv.includes("--update");
|
|
|
|
if (suite !== "frontend" && suite !== "go") {
|
|
console.error(`usage: check-test-baseline.mjs <frontend|go> [--update]`);
|
|
process.exit(2);
|
|
}
|
|
|
|
function run(command, args) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn(command, args, { cwd: repoRoot, shell: false });
|
|
let out = "";
|
|
let err = "";
|
|
child.stdout.on("data", (chunk) => {
|
|
out += chunk;
|
|
});
|
|
child.stderr.on("data", (chunk) => {
|
|
err += chunk;
|
|
});
|
|
child.on("error", (error) => resolve({ code: null, out: "", err: String(error) }));
|
|
child.on("close", (code) => resolve({ code, out, err }));
|
|
});
|
|
}
|
|
|
|
function parseFrontendFailures(output) {
|
|
const failures = new Set();
|
|
const lines = output.split("\n");
|
|
for (let i = 0; i < lines.length; i++) {
|
|
const match = lines[i].match(/^test at (tests\/frontend\/(\S+\.test\.mjs)):\d+:\d+$/);
|
|
if (match && lines[i + 1]?.startsWith("✖")) {
|
|
const name = lines[i + 1].replace(/^✖\s*/, "").replace(/\s+\([\d.]+ms\)$/, "");
|
|
failures.add(`${match[2]} :: ${name}`);
|
|
}
|
|
}
|
|
return failures;
|
|
}
|
|
|
|
function parseGoFailures(output) {
|
|
const failures = new Set();
|
|
let pending = [];
|
|
for (const line of output.split("\n")) {
|
|
const fail = line.match(/^--- FAIL: (\w+)/);
|
|
if (fail) {
|
|
pending.push(fail[1]);
|
|
continue;
|
|
}
|
|
const block = line.match(/^FAIL\t(\S+)\t/);
|
|
if (block && pending.length > 0) {
|
|
const pkg = block[1].replace(/^droboticflow\//, "");
|
|
for (const name of pending) failures.add(`${pkg} :: ${name}`);
|
|
pending = [];
|
|
}
|
|
}
|
|
return failures;
|
|
}
|
|
|
|
async function main() {
|
|
const logOption = process.argv.indexOf("--log-dir");
|
|
if (logOption >= 0 && (!process.argv[logOption + 1] || process.argv[logOption + 1].startsWith("--"))) {
|
|
console.error("--log-dir requires a directory path");
|
|
process.exit(2);
|
|
}
|
|
const baselineFile = JSON.parse(readFileSync(baselinePath, "utf8"));
|
|
const known = new Set(baselineFile[suite] ?? []);
|
|
if (known.size === 0 && !update) {
|
|
console.error(`baseline file has no "${suite}" entries; run with --update first`);
|
|
process.exit(2);
|
|
}
|
|
|
|
const result =
|
|
suite === "frontend"
|
|
? await run("npm", ["run", "--silent", "test:frontend"])
|
|
: await run("npm", ["run", "--silent", "test"]);
|
|
|
|
// Preserve every invocation, including crashes and runs which pass only
|
|
// because failures are known. No automatic retries or baseline mutation.
|
|
const option = logOption;
|
|
const logDir = resolve(
|
|
option >= 0
|
|
? process.argv[option + 1]
|
|
: join(
|
|
repoRoot,
|
|
"local-data/test-results",
|
|
`${suite}-${new Date().toISOString().replaceAll(":", "-")}-${process.pid}`,
|
|
),
|
|
);
|
|
mkdirSync(logDir, { recursive: true, mode: 0o700 });
|
|
writeFileSync(join(logDir, "stdout.log"), result.out, { mode: 0o600 });
|
|
writeFileSync(join(logDir, "stderr.log"), result.err, { mode: 0o600 });
|
|
console.log(`Full test output: ${logDir}`);
|
|
const failures =
|
|
suite === "frontend" ? parseFrontendFailures(result.out + result.err) : parseGoFailures(result.out + result.err);
|
|
|
|
const fresh = [...failures].filter((id) => !known.has(id));
|
|
const fixed = [...known].filter((id) => !failures.has(id));
|
|
writeFileSync(
|
|
join(logDir, "summary.json"),
|
|
JSON.stringify(
|
|
{
|
|
suite,
|
|
at: new Date().toISOString(),
|
|
exitCode: result.code,
|
|
failures: [...failures],
|
|
newFailures: fresh,
|
|
fixed,
|
|
updateRequested: update,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
{ mode: 0o600 },
|
|
);
|
|
const runnerCrashed = result.code === null;
|
|
const noFailuresParsed = failures.size === 0 && result.code !== 0;
|
|
if (runnerCrashed || noFailuresParsed) {
|
|
console.error(`test runner did not produce parseable results (code=${result.code})`);
|
|
console.error(result.err.slice(0, 2000));
|
|
process.exit(1);
|
|
}
|
|
|
|
if (update) {
|
|
baselineFile[suite] = [...failures].sort();
|
|
writeFileSync(baselinePath, JSON.stringify(baselineFile, null, 2) + "\n");
|
|
console.log(`baseline updated: ${suite} = ${failures.size} known failures -> ${baselinePath}`);
|
|
return;
|
|
}
|
|
|
|
if (fresh.length > 0) {
|
|
console.error(`NEW ${suite} failures (${fresh.length}) not in tests/known-failures.json:`);
|
|
for (const id of fresh) console.error(` + ${id}`);
|
|
// Put a bounded diagnostic tail in CI output; full streams remain on disk.
|
|
console.error((result.out + result.err).slice(-4000));
|
|
process.exit(1);
|
|
}
|
|
if (fixed.length > 0) {
|
|
console.log(`fixed vs baseline (${fixed.length}) — prune with --update when confirmed:`);
|
|
for (const id of fixed) console.log(` - ${id}`);
|
|
}
|
|
console.log(`${suite}: ${failures.size} failures, all in baseline (${known.size} known). OK`);
|
|
}
|
|
|
|
await main();
|