A table test that names its case with a 16–32 hex id (an org, a token) fails as a different string at the start commit and after the change. The baseline then treats that case as new, so a frontend-only wish cannot pass Go tests that were already red. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
382 lines
15 KiB
JavaScript
382 lines
15 KiB
JavaScript
#!/usr/bin/env node
|
||
// Protected host-side commands. Project settings use this file's absolute path; never copy it into the agent workspace.
|
||
import { spawn } from 'node:child_process';
|
||
import { access, mkdir, mkdtemp, readFile, readdir, realpath, rm, symlink } from 'node:fs/promises';
|
||
import path from 'node:path';
|
||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||
|
||
const APPROVED_ORIGINS = new Set([
|
||
'https://gitea.d-robotics.cc/longtao.wu/droboticflow.git',
|
||
'ssh://git@gitea.d-robotics.cc/longtao.wu/droboticflow.git',
|
||
'git@gitea.d-robotics.cc:longtao.wu/droboticflow.git',
|
||
]);
|
||
/** Fixed absolute tools, never a PATH lookup: the operator Mac's Homebrew, or the cluster image's. */
|
||
export const TOOLCHAINS = {
|
||
darwin: {
|
||
git: '/usr/bin/git',
|
||
node: '/opt/homebrew/bin/node',
|
||
npm: '/opt/homebrew/bin/npm',
|
||
go: '/opt/homebrew/bin/go',
|
||
path: '/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin',
|
||
},
|
||
linux: {
|
||
git: '/usr/bin/git',
|
||
node: '/usr/local/bin/node',
|
||
npm: '/usr/local/bin/npm',
|
||
go: '/usr/local/go/bin/go',
|
||
path: '/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin',
|
||
},
|
||
};
|
||
export function toolchain(platform = process.platform) {
|
||
const tools = TOOLCHAINS[platform];
|
||
if (!tools) throw new Error(`Protected checks have no approved toolchain on ${platform}`);
|
||
return tools;
|
||
}
|
||
const TOOLS = toolchain();
|
||
const CACHE = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../.wishforge/poc-check-cache');
|
||
|
||
export function parseCheckMode(argv) {
|
||
if (argv.length !== 1 || !['install', 'verify'].includes(argv[0]))
|
||
throw new Error('Usage: poc-project-check.mjs install|verify (no custom commands or targets)');
|
||
return argv[0];
|
||
}
|
||
|
||
export function checkEnvironment(cacheRoot = CACHE, inherited = process.env, network = false, tools = TOOLS) {
|
||
return {
|
||
PATH: tools.path,
|
||
HOME: path.join(cacheRoot, 'home'),
|
||
TMPDIR: path.join(cacheRoot, 'tmp'),
|
||
LANG: 'C.UTF-8',
|
||
CI: 'true',
|
||
GOPATH: path.join(cacheRoot, 'go'),
|
||
GOMODCACHE: path.join(cacheRoot, 'go/pkg/mod'),
|
||
GOCACHE: path.join(cacheRoot, 'go-build'),
|
||
GOTOOLCHAIN: 'local',
|
||
GOWORK: 'off',
|
||
GOFLAGS: '-mod=readonly',
|
||
GOPROXY: network ? 'https://proxy.golang.org,direct' : 'off',
|
||
GOSUMDB: 'sum.golang.org',
|
||
npm_config_cache: path.join(cacheRoot, 'npm'),
|
||
npm_config_userconfig: path.join(cacheRoot, 'home/user.npmrc'),
|
||
npm_config_globalconfig: path.join(cacheRoot, 'home/global.npmrc'),
|
||
npm_config_ignore_scripts: 'true',
|
||
GIT_CONFIG_NOSYSTEM: '1',
|
||
GIT_CONFIG_GLOBAL: '/dev/null',
|
||
GIT_TERMINAL_PROMPT: '0',
|
||
...(network
|
||
? Object.fromEntries(
|
||
['HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY'].filter(key => inherited[key]).map(key => [key, inherited[key]]),
|
||
)
|
||
: {}),
|
||
};
|
||
}
|
||
|
||
/** A Git command that may take a while, such as copying a commit; resolves to its exit code. */
|
||
function git(args, cwd) {
|
||
return new Promise(resolve => {
|
||
const child = spawn(TOOLS.git, ['-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args], {
|
||
cwd,
|
||
env: checkEnvironment(),
|
||
shell: false,
|
||
stdio: 'ignore',
|
||
});
|
||
const timer = setTimeout(() => child.kill('SIGKILL'), 5 * 60_000);
|
||
child.once('error', () => resolve(1));
|
||
child.once('close', code => {
|
||
clearTimeout(timer);
|
||
resolve(code ?? 1);
|
||
});
|
||
});
|
||
}
|
||
|
||
function capture(args, cwd) {
|
||
return new Promise((resolve, reject) => {
|
||
const child = spawn(TOOLS.git, ['-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args], {
|
||
cwd,
|
||
env: checkEnvironment(),
|
||
shell: false,
|
||
stdio: ['ignore', 'pipe', 'pipe'],
|
||
});
|
||
let output = '';
|
||
const timer = setTimeout(() => child.kill('SIGKILL'), 10000);
|
||
child.stdout.on('data', bytes => {
|
||
output += String(bytes);
|
||
if (output.length > 16000) child.kill('SIGKILL');
|
||
});
|
||
child.stderr.resume();
|
||
child.once('error', error => {
|
||
clearTimeout(timer);
|
||
reject(error);
|
||
});
|
||
child.once('close', code => {
|
||
clearTimeout(timer);
|
||
code === 0 ? resolve(output.trim()) : reject(new Error('The current directory is not an approved Git workspace'));
|
||
});
|
||
});
|
||
}
|
||
|
||
export async function assertApprovedWorkspace(cwd) {
|
||
const workspace = await realpath(cwd);
|
||
const manifest = JSON.parse(await readFile(path.join(workspace, 'package.json'), 'utf8'));
|
||
const goModule = await readFile(path.join(workspace, 'go.mod'), 'utf8');
|
||
if (manifest.name !== 'droboticflow' || !/^module droboticflow\s*$/m.test(goModule))
|
||
throw new Error('Protected checks require the droboticflow Node and Go project');
|
||
// verifyWorkspace exports the exact checkpoint without .git. The caller establishes
|
||
// repository/SHA authority before export; these checks are sanity checks, not a sandbox.
|
||
const hasGit = await access(path.join(workspace, '.git')).then(
|
||
() => true,
|
||
() => false,
|
||
);
|
||
if (!hasGit) return { workspace, head: null };
|
||
const top = await realpath(await capture(['rev-parse', '--show-toplevel'], workspace));
|
||
if (workspace !== top) throw new Error('Run the protected checks at the Git repository root');
|
||
const origins = (await capture(['config', '--local', '--get-all', 'remote.origin.url'], workspace)).split('\n');
|
||
if (origins.length !== 1 || !APPROVED_ORIGINS.has(origins[0]))
|
||
throw new Error('The Git origin must be the approved droboticflow repository');
|
||
const head = await capture(['rev-parse', '--verify', 'HEAD^{commit}'], workspace);
|
||
if (!/^[a-f0-9]{40}$/.test(head)) throw new Error('The Git workspace has no immutable HEAD');
|
||
return { workspace, head };
|
||
}
|
||
|
||
export function checkPlan(mode, workspace, frontendTests = []) {
|
||
if (mode === 'install')
|
||
return [
|
||
{
|
||
label: 'npm ci (lifecycle scripts disabled)',
|
||
executable: TOOLS.npm,
|
||
args: ['ci', '--ignore-scripts', '--no-audit', '--no-fund'],
|
||
},
|
||
{ label: 'Go module download', executable: TOOLS.go, args: ['mod', 'download'] },
|
||
];
|
||
if (mode !== 'verify' || !frontendTests.length)
|
||
throw new Error('Verification requires the protected frontend test suite');
|
||
return [
|
||
{
|
||
label: 'lint:web',
|
||
executable: TOOLS.node,
|
||
args: [path.join(workspace, 'node_modules/typescript/bin/tsc'), '--noEmit'],
|
||
},
|
||
// A test that never settles fails after two minutes instead of holding the whole suite.
|
||
{
|
||
label: 'test:frontend',
|
||
executable: TOOLS.node,
|
||
args: ['--test', `--test-timeout=${TEST_TIMEOUT_MS}`, ...frontendTests],
|
||
baseline: true,
|
||
},
|
||
{
|
||
label: 'build:web',
|
||
executable: TOOLS.node,
|
||
args: [path.join(workspace, 'node_modules/vite/bin/vite.js'), 'build'],
|
||
},
|
||
{
|
||
label: 'go test ./...',
|
||
executable: TOOLS.go,
|
||
args: ['test', './...'],
|
||
baseline: true,
|
||
failures: failedGoTests,
|
||
// At the start commit, only the packages that failed run again.
|
||
baselineArgs: failed => ['test', ...new Set(failed.map(test => test.split(' :: ')[0]))],
|
||
},
|
||
];
|
||
}
|
||
|
||
/** How long one frontend test may take before node --test fails it. */
|
||
export const TEST_TIMEOUT_MS = 120_000;
|
||
/** The commit a run started from, which the verifier places in the workspace's repository. */
|
||
export const BASE_REF = 'refs/wishforge/base';
|
||
const OUTPUT_KEPT = 4 * 1024 * 1024;
|
||
|
||
/**
|
||
* The tests node --test reported as failing, as "file :: name" (the form check-test-baseline.mjs
|
||
* keeps): read from the "failing tests" summary of its spec output.
|
||
*/
|
||
export function failedTests(output) {
|
||
const summary = output.lastIndexOf('✖ failing tests:');
|
||
if (summary < 0) return undefined;
|
||
const failures = new Set();
|
||
let file = '';
|
||
for (const line of output.slice(summary).split('\n').slice(1)) {
|
||
const location = /^test at (?:.*\/)?([^/:]+):\d+:\d+$/.exec(line.trim());
|
||
if (location) file = location[1];
|
||
const failed = /^✖ (.+?)(?: \([\d.]+m?s\))?$/.exec(line.trim());
|
||
if (failed) failures.add(`${file} :: ${failed[1]}`);
|
||
}
|
||
return [...failures].sort();
|
||
}
|
||
|
||
/**
|
||
* The tests go test reported as failing, as "package :: Test": a package that did not build or
|
||
* set up counts as one failure of its own.
|
||
*/
|
||
export function failedGoTests(output) {
|
||
const failures = new Set();
|
||
let pending = [];
|
||
for (const raw of output.split('\n')) {
|
||
const line = raw.trimEnd();
|
||
const test = /^\s*--- FAIL: (\S+)/.exec(line);
|
||
if (test) pending.push(test[1]);
|
||
const done = /^(FAIL|ok)\s+(\S+)(?:\s+(\[[^\]]+\]))?/.exec(line);
|
||
if (done && done[2] !== '') {
|
||
if (done[1] === 'FAIL') {
|
||
for (const name of pending) failures.add(`${done[2]} :: ${name}`);
|
||
if (!pending.length) failures.add(`${done[2]} :: ${done[3] ?? '[failed]'}`);
|
||
}
|
||
pending = [];
|
||
}
|
||
}
|
||
return failures.size ? [...failures].sort() : undefined;
|
||
}
|
||
|
||
/**
|
||
* A failure's identity without run-to-run IDs: Go table tests often name a case with a 16–32
|
||
* hex hash, so the same case is a different string at the start commit and after the change.
|
||
*/
|
||
export function comparableFailure(name) {
|
||
return name.replace(/[0-9a-f]{16,}/gi, '*');
|
||
}
|
||
|
||
/** Failures in `now` that were not already failing in `before`, after stripping those IDs. */
|
||
export function newlyFailing(now, before) {
|
||
const known = new Set(before.map(comparableFailure));
|
||
return now.filter(test => !known.has(comparableFailure(test)));
|
||
}
|
||
|
||
async function runCheck(check, cwd, env, quiet = false) {
|
||
const started = Date.now();
|
||
if (!quiet) console.log(`[poc-check] START ${check.label}`);
|
||
return new Promise(resolve => {
|
||
const child = spawn(check.executable, check.args, {
|
||
cwd,
|
||
env,
|
||
shell: false,
|
||
stdio: ['ignore', 'pipe', 'pipe'],
|
||
});
|
||
// The output streams on as before, and the end of it is kept to find the failed tests.
|
||
let output = '';
|
||
for (const [stream, target] of [
|
||
[child.stdout, process.stdout],
|
||
[child.stderr, process.stderr],
|
||
])
|
||
stream.on('data', bytes => {
|
||
if (!quiet) target.write(bytes);
|
||
output = (output + String(bytes)).slice(-OUTPUT_KEPT);
|
||
});
|
||
const heartbeat = setInterval(
|
||
() => console.log(`[poc-check] RUNNING ${check.label} (${Math.round((Date.now() - started) / 1000)}s)`),
|
||
30000,
|
||
);
|
||
const timer = setTimeout(() => child.kill('SIGKILL'), 9 * 60_000);
|
||
child.once('error', error => console.error(`[poc-check] ${error.message}`));
|
||
child.once('close', code => {
|
||
clearInterval(heartbeat);
|
||
clearTimeout(timer);
|
||
const result = { check: check.label, exitCode: code ?? 1, durationMs: Date.now() - started };
|
||
if (!quiet) console.log(`[poc-check] RESULT ${JSON.stringify(result)}`);
|
||
resolve({ ...result, output });
|
||
});
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Whether every test that failed also fails at the commit the run started from: the same tests on
|
||
* that commit's source, with this workspace's dependencies. Those failures were there before this
|
||
* change, so they are not its to fix. Returns them, or undefined when a failure is new or cannot
|
||
* be told apart.
|
||
*/
|
||
async function alreadyFailing(check, workspace, env, output) {
|
||
const parse = check.failures ?? failedTests;
|
||
const failed = parse(output);
|
||
if (!failed?.length) return undefined;
|
||
const base = await capture(['rev-parse', '--verify', '--quiet', `${BASE_REF}^{commit}`], workspace).catch(() => '');
|
||
if (!/^[a-f0-9]{40}$/.test(base)) return undefined;
|
||
const directory = await mkdtemp(path.join(env.TMPDIR, 'base-'));
|
||
try {
|
||
// A repository of its own, like the workspace's: the start commit alone, checked out.
|
||
const origin = await capture(['config', '--get', 'remote.origin.url'], workspace).catch(() => '');
|
||
const steps = [
|
||
['init', '-q'],
|
||
[
|
||
'-c',
|
||
'protocol.file.allow=always',
|
||
'fetch',
|
||
'-q',
|
||
'--depth=1',
|
||
'--no-tags',
|
||
pathToFileURL(workspace).href,
|
||
BASE_REF,
|
||
],
|
||
['checkout', '-q', '--detach', 'FETCH_HEAD'],
|
||
...(origin ? [['remote', 'add', 'origin', origin]] : []),
|
||
];
|
||
for (const args of steps) if ((await git(args, directory)) !== 0) return undefined;
|
||
await symlink(path.join(workspace, 'node_modules'), path.join(directory, 'node_modules'));
|
||
const atBase = await runCheck(
|
||
check.baselineArgs ? { ...check, args: check.baselineArgs(failed) } : check,
|
||
directory,
|
||
env,
|
||
true,
|
||
);
|
||
const before = parse(atBase.output) ?? [];
|
||
const added = newlyFailing(failed, before);
|
||
console.log(
|
||
`[poc-check] BASELINE ${JSON.stringify({ check: check.label, baseCommit: base, failingBefore: before.length, failingNow: failed.length, newFailures: added })}`,
|
||
);
|
||
return added.length ? undefined : failed;
|
||
} finally {
|
||
await rm(directory, { recursive: true, force: true });
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Runs the checks in order and stops at the first that fails. A check that compares with the
|
||
* commit the run started from passes when every test it failed was failing there already.
|
||
*/
|
||
export async function runPlan(plan, workspace, env, { quiet = false } = {}) {
|
||
const results = [];
|
||
for (const check of plan) {
|
||
const { output, ...result } = await runCheck(check, workspace, env, quiet);
|
||
if (result.exitCode !== 0 && check.baseline) {
|
||
const preexisting = await alreadyFailing(check, workspace, env, output);
|
||
if (preexisting) {
|
||
result.exitCode = 0;
|
||
result.preexistingFailures = preexisting.length;
|
||
}
|
||
}
|
||
results.push(result);
|
||
// Keep the failing command's diagnostics at the end of the evidence tail. Later
|
||
// successful build/test output would otherwise erase the error needed by repair.
|
||
if (result.exitCode !== 0) break;
|
||
}
|
||
return results;
|
||
}
|
||
|
||
export async function main(argv = process.argv.slice(2)) {
|
||
const mode = parseCheckMode(argv);
|
||
const { workspace, head } = await assertApprovedWorkspace(process.cwd());
|
||
const env = checkEnvironment(CACHE, process.env, mode === 'install');
|
||
for (const directory of [CACHE, env.HOME, env.TMPDIR, env.GOPATH, env.GOMODCACHE, env.GOCACHE, env.npm_config_cache])
|
||
await mkdir(directory, { recursive: true, mode: 0o700 });
|
||
const tests =
|
||
mode === 'verify'
|
||
? (await readdir(path.join(workspace, 'tests/frontend')))
|
||
.filter(file => /^[A-Za-z0-9_.-]+\.test\.mjs$/.test(file))
|
||
.sort()
|
||
.map(file => `tests/frontend/${file}`)
|
||
: [];
|
||
console.log(`[poc-check] ${JSON.stringify({ mode, sourceCommit: head, workspace })}`);
|
||
const results = await runPlan(checkPlan(mode, workspace, tests), workspace, env);
|
||
const passed = results.every(result => result.exitCode === 0);
|
||
console.log(`[poc-check] SUMMARY ${JSON.stringify({ mode, sourceCommit: head, passed, results })}`);
|
||
return passed ? 0 : 1;
|
||
}
|
||
|
||
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url)
|
||
main()
|
||
.then(code => {
|
||
process.exitCode = code;
|
||
})
|
||
.catch(error => {
|
||
console.error(`[poc-check] ${error.message}`);
|
||
process.exitCode = 2;
|
||
});
|