wishforge-ide/scripts/poc-project-check.mjs
eust-w a6b713f5df 🧪 Treat Go table cases with random hex names as the same baseline failure
A table test that names its case with a 16–32 hex id (an org, a token)
fails as a different string at the start commit and after the change.
The baseline then treats that case as new, so a frontend-only wish
cannot pass Go tests that were already red.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-11-02 11:23:52 +08:00

382 lines
15 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env node
// Protected host-side commands. Project settings use this file's absolute path; never copy it into the agent workspace.
import { spawn } from 'node:child_process';
import { access, mkdir, mkdtemp, readFile, readdir, realpath, rm, symlink } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
const APPROVED_ORIGINS = new Set([
'https://gitea.d-robotics.cc/longtao.wu/droboticflow.git',
'ssh://git@gitea.d-robotics.cc/longtao.wu/droboticflow.git',
'git@gitea.d-robotics.cc:longtao.wu/droboticflow.git',
]);
/** Fixed absolute tools, never a PATH lookup: the operator Mac's Homebrew, or the cluster image's. */
export const TOOLCHAINS = {
darwin: {
git: '/usr/bin/git',
node: '/opt/homebrew/bin/node',
npm: '/opt/homebrew/bin/npm',
go: '/opt/homebrew/bin/go',
path: '/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin',
},
linux: {
git: '/usr/bin/git',
node: '/usr/local/bin/node',
npm: '/usr/local/bin/npm',
go: '/usr/local/go/bin/go',
path: '/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin',
},
};
export function toolchain(platform = process.platform) {
const tools = TOOLCHAINS[platform];
if (!tools) throw new Error(`Protected checks have no approved toolchain on ${platform}`);
return tools;
}
const TOOLS = toolchain();
const CACHE = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../.wishforge/poc-check-cache');
export function parseCheckMode(argv) {
if (argv.length !== 1 || !['install', 'verify'].includes(argv[0]))
throw new Error('Usage: poc-project-check.mjs install|verify (no custom commands or targets)');
return argv[0];
}
export function checkEnvironment(cacheRoot = CACHE, inherited = process.env, network = false, tools = TOOLS) {
return {
PATH: tools.path,
HOME: path.join(cacheRoot, 'home'),
TMPDIR: path.join(cacheRoot, 'tmp'),
LANG: 'C.UTF-8',
CI: 'true',
GOPATH: path.join(cacheRoot, 'go'),
GOMODCACHE: path.join(cacheRoot, 'go/pkg/mod'),
GOCACHE: path.join(cacheRoot, 'go-build'),
GOTOOLCHAIN: 'local',
GOWORK: 'off',
GOFLAGS: '-mod=readonly',
GOPROXY: network ? 'https://proxy.golang.org,direct' : 'off',
GOSUMDB: 'sum.golang.org',
npm_config_cache: path.join(cacheRoot, 'npm'),
npm_config_userconfig: path.join(cacheRoot, 'home/user.npmrc'),
npm_config_globalconfig: path.join(cacheRoot, 'home/global.npmrc'),
npm_config_ignore_scripts: 'true',
GIT_CONFIG_NOSYSTEM: '1',
GIT_CONFIG_GLOBAL: '/dev/null',
GIT_TERMINAL_PROMPT: '0',
...(network
? Object.fromEntries(
['HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY'].filter(key => inherited[key]).map(key => [key, inherited[key]]),
)
: {}),
};
}
/** A Git command that may take a while, such as copying a commit; resolves to its exit code. */
function git(args, cwd) {
return new Promise(resolve => {
const child = spawn(TOOLS.git, ['-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args], {
cwd,
env: checkEnvironment(),
shell: false,
stdio: 'ignore',
});
const timer = setTimeout(() => child.kill('SIGKILL'), 5 * 60_000);
child.once('error', () => resolve(1));
child.once('close', code => {
clearTimeout(timer);
resolve(code ?? 1);
});
});
}
function capture(args, cwd) {
return new Promise((resolve, reject) => {
const child = spawn(TOOLS.git, ['-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args], {
cwd,
env: checkEnvironment(),
shell: false,
stdio: ['ignore', 'pipe', 'pipe'],
});
let output = '';
const timer = setTimeout(() => child.kill('SIGKILL'), 10000);
child.stdout.on('data', bytes => {
output += String(bytes);
if (output.length > 16000) child.kill('SIGKILL');
});
child.stderr.resume();
child.once('error', error => {
clearTimeout(timer);
reject(error);
});
child.once('close', code => {
clearTimeout(timer);
code === 0 ? resolve(output.trim()) : reject(new Error('The current directory is not an approved Git workspace'));
});
});
}
export async function assertApprovedWorkspace(cwd) {
const workspace = await realpath(cwd);
const manifest = JSON.parse(await readFile(path.join(workspace, 'package.json'), 'utf8'));
const goModule = await readFile(path.join(workspace, 'go.mod'), 'utf8');
if (manifest.name !== 'droboticflow' || !/^module droboticflow\s*$/m.test(goModule))
throw new Error('Protected checks require the droboticflow Node and Go project');
// verifyWorkspace exports the exact checkpoint without .git. The caller establishes
// repository/SHA authority before export; these checks are sanity checks, not a sandbox.
const hasGit = await access(path.join(workspace, '.git')).then(
() => true,
() => false,
);
if (!hasGit) return { workspace, head: null };
const top = await realpath(await capture(['rev-parse', '--show-toplevel'], workspace));
if (workspace !== top) throw new Error('Run the protected checks at the Git repository root');
const origins = (await capture(['config', '--local', '--get-all', 'remote.origin.url'], workspace)).split('\n');
if (origins.length !== 1 || !APPROVED_ORIGINS.has(origins[0]))
throw new Error('The Git origin must be the approved droboticflow repository');
const head = await capture(['rev-parse', '--verify', 'HEAD^{commit}'], workspace);
if (!/^[a-f0-9]{40}$/.test(head)) throw new Error('The Git workspace has no immutable HEAD');
return { workspace, head };
}
export function checkPlan(mode, workspace, frontendTests = []) {
if (mode === 'install')
return [
{
label: 'npm ci (lifecycle scripts disabled)',
executable: TOOLS.npm,
args: ['ci', '--ignore-scripts', '--no-audit', '--no-fund'],
},
{ label: 'Go module download', executable: TOOLS.go, args: ['mod', 'download'] },
];
if (mode !== 'verify' || !frontendTests.length)
throw new Error('Verification requires the protected frontend test suite');
return [
{
label: 'lint:web',
executable: TOOLS.node,
args: [path.join(workspace, 'node_modules/typescript/bin/tsc'), '--noEmit'],
},
// A test that never settles fails after two minutes instead of holding the whole suite.
{
label: 'test:frontend',
executable: TOOLS.node,
args: ['--test', `--test-timeout=${TEST_TIMEOUT_MS}`, ...frontendTests],
baseline: true,
},
{
label: 'build:web',
executable: TOOLS.node,
args: [path.join(workspace, 'node_modules/vite/bin/vite.js'), 'build'],
},
{
label: 'go test ./...',
executable: TOOLS.go,
args: ['test', './...'],
baseline: true,
failures: failedGoTests,
// At the start commit, only the packages that failed run again.
baselineArgs: failed => ['test', ...new Set(failed.map(test => test.split(' :: ')[0]))],
},
];
}
/** How long one frontend test may take before node --test fails it. */
export const TEST_TIMEOUT_MS = 120_000;
/** The commit a run started from, which the verifier places in the workspace's repository. */
export const BASE_REF = 'refs/wishforge/base';
const OUTPUT_KEPT = 4 * 1024 * 1024;
/**
* The tests node --test reported as failing, as "file :: name" (the form check-test-baseline.mjs
* keeps): read from the "failing tests" summary of its spec output.
*/
export function failedTests(output) {
const summary = output.lastIndexOf('✖ failing tests:');
if (summary < 0) return undefined;
const failures = new Set();
let file = '';
for (const line of output.slice(summary).split('\n').slice(1)) {
const location = /^test at (?:.*\/)?([^/:]+):\d+:\d+$/.exec(line.trim());
if (location) file = location[1];
const failed = /^✖ (.+?)(?: \([\d.]+m?s\))?$/.exec(line.trim());
if (failed) failures.add(`${file} :: ${failed[1]}`);
}
return [...failures].sort();
}
/**
* The tests go test reported as failing, as "package :: Test": a package that did not build or
* set up counts as one failure of its own.
*/
export function failedGoTests(output) {
const failures = new Set();
let pending = [];
for (const raw of output.split('\n')) {
const line = raw.trimEnd();
const test = /^\s*--- FAIL: (\S+)/.exec(line);
if (test) pending.push(test[1]);
const done = /^(FAIL|ok)\s+(\S+)(?:\s+(\[[^\]]+\]))?/.exec(line);
if (done && done[2] !== '') {
if (done[1] === 'FAIL') {
for (const name of pending) failures.add(`${done[2]} :: ${name}`);
if (!pending.length) failures.add(`${done[2]} :: ${done[3] ?? '[failed]'}`);
}
pending = [];
}
}
return failures.size ? [...failures].sort() : undefined;
}
/**
* A failure's identity without run-to-run IDs: Go table tests often name a case with a 16–32
* hex hash, so the same case is a different string at the start commit and after the change.
*/
export function comparableFailure(name) {
return name.replace(/[0-9a-f]{16,}/gi, '*');
}
/** Failures in `now` that were not already failing in `before`, after stripping those IDs. */
export function newlyFailing(now, before) {
const known = new Set(before.map(comparableFailure));
return now.filter(test => !known.has(comparableFailure(test)));
}
async function runCheck(check, cwd, env, quiet = false) {
const started = Date.now();
if (!quiet) console.log(`[poc-check] START ${check.label}`);
return new Promise(resolve => {
const child = spawn(check.executable, check.args, {
cwd,
env,
shell: false,
stdio: ['ignore', 'pipe', 'pipe'],
});
// The output streams on as before, and the end of it is kept to find the failed tests.
let output = '';
for (const [stream, target] of [
[child.stdout, process.stdout],
[child.stderr, process.stderr],
])
stream.on('data', bytes => {
if (!quiet) target.write(bytes);
output = (output + String(bytes)).slice(-OUTPUT_KEPT);
});
const heartbeat = setInterval(
() => console.log(`[poc-check] RUNNING ${check.label} (${Math.round((Date.now() - started) / 1000)}s)`),
30000,
);
const timer = setTimeout(() => child.kill('SIGKILL'), 9 * 60_000);
child.once('error', error => console.error(`[poc-check] ${error.message}`));
child.once('close', code => {
clearInterval(heartbeat);
clearTimeout(timer);
const result = { check: check.label, exitCode: code ?? 1, durationMs: Date.now() - started };
if (!quiet) console.log(`[poc-check] RESULT ${JSON.stringify(result)}`);
resolve({ ...result, output });
});
});
}
/**
* Whether every test that failed also fails at the commit the run started from: the same tests on
* that commit's source, with this workspace's dependencies. Those failures were there before this
* change, so they are not its to fix. Returns them, or undefined when a failure is new or cannot
* be told apart.
*/
async function alreadyFailing(check, workspace, env, output) {
const parse = check.failures ?? failedTests;
const failed = parse(output);
if (!failed?.length) return undefined;
const base = await capture(['rev-parse', '--verify', '--quiet', `${BASE_REF}^{commit}`], workspace).catch(() => '');
if (!/^[a-f0-9]{40}$/.test(base)) return undefined;
const directory = await mkdtemp(path.join(env.TMPDIR, 'base-'));
try {
// A repository of its own, like the workspace's: the start commit alone, checked out.
const origin = await capture(['config', '--get', 'remote.origin.url'], workspace).catch(() => '');
const steps = [
['init', '-q'],
[
'-c',
'protocol.file.allow=always',
'fetch',
'-q',
'--depth=1',
'--no-tags',
pathToFileURL(workspace).href,
BASE_REF,
],
['checkout', '-q', '--detach', 'FETCH_HEAD'],
...(origin ? [['remote', 'add', 'origin', origin]] : []),
];
for (const args of steps) if ((await git(args, directory)) !== 0) return undefined;
await symlink(path.join(workspace, 'node_modules'), path.join(directory, 'node_modules'));
const atBase = await runCheck(
check.baselineArgs ? { ...check, args: check.baselineArgs(failed) } : check,
directory,
env,
true,
);
const before = parse(atBase.output) ?? [];
const added = newlyFailing(failed, before);
console.log(
`[poc-check] BASELINE ${JSON.stringify({ check: check.label, baseCommit: base, failingBefore: before.length, failingNow: failed.length, newFailures: added })}`,
);
return added.length ? undefined : failed;
} finally {
await rm(directory, { recursive: true, force: true });
}
}
/**
* Runs the checks in order and stops at the first that fails. A check that compares with the
* commit the run started from passes when every test it failed was failing there already.
*/
export async function runPlan(plan, workspace, env, { quiet = false } = {}) {
const results = [];
for (const check of plan) {
const { output, ...result } = await runCheck(check, workspace, env, quiet);
if (result.exitCode !== 0 && check.baseline) {
const preexisting = await alreadyFailing(check, workspace, env, output);
if (preexisting) {
result.exitCode = 0;
result.preexistingFailures = preexisting.length;
}
}
results.push(result);
// Keep the failing command's diagnostics at the end of the evidence tail. Later
// successful build/test output would otherwise erase the error needed by repair.
if (result.exitCode !== 0) break;
}
return results;
}
export async function main(argv = process.argv.slice(2)) {
const mode = parseCheckMode(argv);
const { workspace, head } = await assertApprovedWorkspace(process.cwd());
const env = checkEnvironment(CACHE, process.env, mode === 'install');
for (const directory of [CACHE, env.HOME, env.TMPDIR, env.GOPATH, env.GOMODCACHE, env.GOCACHE, env.npm_config_cache])
await mkdir(directory, { recursive: true, mode: 0o700 });
const tests =
mode === 'verify'
? (await readdir(path.join(workspace, 'tests/frontend')))
.filter(file => /^[A-Za-z0-9_.-]+\.test\.mjs$/.test(file))
.sort()
.map(file => `tests/frontend/${file}`)
: [];
console.log(`[poc-check] ${JSON.stringify({ mode, sourceCommit: head, workspace })}`);
const results = await runPlan(checkPlan(mode, workspace, tests), workspace, env);
const passed = results.every(result => result.exitCode === 0);
console.log(`[poc-check] SUMMARY ${JSON.stringify({ mode, sourceCommit: head, passed, results })}`);
return passed ? 0 : 1;
}
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url)
main()
.then(code => {
process.exitCode = code;
})
.catch(error => {
console.error(`[poc-check] ${error.message}`);
process.exitCode = 2;
});