#!/usr/bin/env node // Known-failure baseline gate for the two suites with upstream test debt. // // node scripts/check-test-baseline.mjs frontend # npm run test:frontend:gate // node scripts/check-test-baseline.mjs go # npm run test:go:gate // node scripts/check-test-baseline.mjs --update // // Exit 0: no NEW failures vs tests/known-failures.json. Exit 1: new failures // (printed by exact id) or a runner crash. Fixed baseline entries are reported // as candidates to prune with --update but never fail the gate. // // Baseline ids: frontend " :: " (spec-reporter // failure detail pairs `test at ` with the following `✖ ` line); // go " :: " (top-level `--- FAIL:` lines assigned to // the package whose `FAIL\t` block marker closes them). import { spawn } from "node:child_process"; import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { dirname, join, resolve } from "node:path"; const repoRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); const baselinePath = join(repoRoot, "tests", "known-failures.json"); const suite = process.argv[2]; const update = process.argv.includes("--update"); if (suite !== "frontend" && suite !== "go") { console.error(`usage: check-test-baseline.mjs [--update]`); process.exit(2); } function run(command, args) { return new Promise((resolve) => { const child = spawn(command, args, { cwd: repoRoot, shell: false }); let out = ""; let err = ""; child.stdout.on("data", (chunk) => { out += chunk; }); child.stderr.on("data", (chunk) => { err += chunk; }); child.on("error", (error) => resolve({ code: null, out: "", err: String(error) })); child.on("close", (code) => resolve({ code, out, err })); }); } function parseFrontendFailures(output) { const failures = new Set(); const lines = output.split("\n"); for (let i = 0; i < lines.length; i++) { const match = lines[i].match(/^test at (tests\/frontend\/(\S+\.test\.mjs)):\d+:\d+$/); if (match && lines[i + 1]?.startsWith("✖")) { const name = lines[i + 1].replace(/^✖\s*/, "").replace(/\s+\([\d.]+ms\)$/, ""); failures.add(`${match[2]} :: ${name}`); } } return failures; } function parseGoFailures(output) { const failures = new Set(); let pending = []; for (const line of output.split("\n")) { const fail = line.match(/^--- FAIL: (\w+)/); if (fail) { pending.push(fail[1]); continue; } const block = line.match(/^FAIL\t(\S+)\t/); if (block && pending.length > 0) { const pkg = block[1].replace(/^droboticflow\//, ""); for (const name of pending) failures.add(`${pkg} :: ${name}`); pending = []; } } return failures; } async function main() { const logOption = process.argv.indexOf("--log-dir"); if (logOption >= 0 && (!process.argv[logOption + 1] || process.argv[logOption + 1].startsWith("--"))) { console.error("--log-dir requires a directory path"); process.exit(2); } const baselineFile = JSON.parse(readFileSync(baselinePath, "utf8")); const known = new Set(baselineFile[suite] ?? []); if (known.size === 0 && !update) { console.error(`baseline file has no "${suite}" entries; run with --update first`); process.exit(2); } const result = suite === "frontend" ? await run("npm", ["run", "--silent", "test:frontend"]) : await run("npm", ["run", "--silent", "test"]); // Preserve every invocation, including crashes and runs which pass only // because failures are known. No automatic retries or baseline mutation. const option = logOption; const logDir = resolve( option >= 0 ? process.argv[option + 1] : join( repoRoot, "local-data/test-results", `${suite}-${new Date().toISOString().replaceAll(":", "-")}-${process.pid}`, ), ); mkdirSync(logDir, { recursive: true, mode: 0o700 }); writeFileSync(join(logDir, "stdout.log"), result.out, { mode: 0o600 }); writeFileSync(join(logDir, "stderr.log"), result.err, { mode: 0o600 }); console.log(`Full test output: ${logDir}`); const failures = suite === "frontend" ? parseFrontendFailures(result.out + result.err) : parseGoFailures(result.out + result.err); const fresh = [...failures].filter((id) => !known.has(id)); const fixed = [...known].filter((id) => !failures.has(id)); writeFileSync( join(logDir, "summary.json"), JSON.stringify( { suite, at: new Date().toISOString(), exitCode: result.code, failures: [...failures], newFailures: fresh, fixed, updateRequested: update, }, null, 2, ) + "\n", { mode: 0o600 }, ); const runnerCrashed = result.code === null; const noFailuresParsed = failures.size === 0 && result.code !== 0; if (runnerCrashed || noFailuresParsed) { console.error(`test runner did not produce parseable results (code=${result.code})`); console.error(result.err.slice(0, 2000)); process.exit(1); } if (update) { baselineFile[suite] = [...failures].sort(); writeFileSync(baselinePath, JSON.stringify(baselineFile, null, 2) + "\n"); console.log(`baseline updated: ${suite} = ${failures.size} known failures -> ${baselinePath}`); return; } if (fresh.length > 0) { console.error(`NEW ${suite} failures (${fresh.length}) not in tests/known-failures.json:`); for (const id of fresh) console.error(` + ${id}`); // Put a bounded diagnostic tail in CI output; full streams remain on disk. console.error((result.out + result.err).slice(-4000)); process.exit(1); } if (fixed.length > 0) { console.log(`fixed vs baseline (${fixed.length}) — prune with --update when confirmed:`); for (const id of fixed) console.log(` - ${id}`); } console.log(`${suite}: ${failures.size} failures, all in baseline (${known.size} known). OK`); } await main();