wishforge-ide/scripts/poc-project-check.test.mjs
eust-w a6b713f5df 🧪 Treat Go table cases with random hex names as the same baseline failure
A table test that names its case with a 16–32 hex id (an org, a token)
fails as a different string at the start commit and after the change.
The baseline then treats that case as new, so a frontend-only wish
cannot pass Go tests that were already red.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-11-02 11:23:52 +08:00

264 lines
11 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import test from 'node:test';
import assert from 'node:assert/strict';
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { spawnSync } from 'node:child_process';
import { existsSync } from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import {
assertApprovedWorkspace,
checkEnvironment,
checkPlan,
parseCheckMode,
TOOLCHAINS,
toolchain,
failedTests,
failedGoTests,
comparableFailure,
newlyFailing,
runPlan,
BASE_REF,
} from './poc-project-check.mjs';
test('each platform has one fixed absolute toolchain and its own PATH', () => {
for (const platform of ['darwin', 'linux']) {
const tools = toolchain(platform);
for (const name of ['git', 'node', 'npm', 'go']) assert.ok(path.isAbsolute(tools[name]), `${platform} ${name}`);
assert.equal(checkEnvironment('/private/cache', {}, false, tools).PATH, tools.path);
}
assert.equal(toolchain('linux').go, '/usr/local/go/bin/go');
assert.ok(!TOOLCHAINS.linux.path.includes('/opt/homebrew'));
assert.throws(() => toolchain('win32'), /no approved toolchain/);
});
test('only fixed install and verification modes are accepted', () => {
assert.equal(parseCheckMode(['install']), 'install');
assert.equal(parseCheckMode(['verify']), 'verify');
for (const args of [[], ['deploy'], ['verify', '--target', 'production'], ['sh', '-c', 'anything']])
assert.throws(() => parseCheckMode(args), /Usage/);
const plan = checkPlan('verify', '/owned/workspace', ['tests/frontend/check.test.mjs']);
assert.deepEqual(
plan.map(item => item.label),
['lint:web', 'test:frontend', 'build:web', 'go test ./...'],
);
assert.ok(plan.every(item => !item.args.includes('run')));
assert.ok(checkPlan('install', '/owned/workspace')[0].args.includes('--ignore-scripts'));
});
test('installation and verification share caches without inheriting host credentials', () => {
const inherited = {
HOME: '/operator',
KUBECONFIG: '/secret',
ANTHROPIC_AUTH_TOKEN: 'secret',
DATABASE_URL: 'secret',
NODE_OPTIONS: '--require evil',
HTTPS_PROXY: 'http://proxy.invalid',
};
const install = checkEnvironment('/private/cache', inherited, true);
const verify = checkEnvironment('/private/cache', inherited, false);
for (const name of ['HOME', 'GOPATH', 'GOMODCACHE', 'GOCACHE', 'npm_config_cache'])
assert.equal(install[name], verify[name]);
for (const name of ['KUBECONFIG', 'ANTHROPIC_AUTH_TOKEN', 'DATABASE_URL', 'NODE_OPTIONS'])
assert.equal(install[name], undefined);
assert.equal(verify.GOPROXY, 'off');
assert.equal(install.HTTPS_PROXY, inherited.HTTPS_PROXY);
assert.equal(verify.HTTPS_PROXY, undefined);
});
// The checks run the POC Mac's Homebrew toolchain; elsewhere, such as CI, there is none to run.
const skipWithoutPocToolchain = existsSync('/opt/homebrew/bin/node')
? false
: 'needs the POC host toolchain in /opt/homebrew';
test(
'stops verification at the first failure so repair evidence retains the actual diagnostic',
{ skip: skipWithoutPocToolchain },
async () => {
const directory = await mkdtemp(path.join(os.tmpdir(), 'poc-check-failure-'));
try {
await writeFile(path.join(directory, 'package.json'), JSON.stringify({ name: 'droboticflow' }));
await writeFile(path.join(directory, 'go.mod'), 'module droboticflow\n');
await mkdir(path.join(directory, 'node_modules/typescript/bin'), { recursive: true });
await mkdir(path.join(directory, 'tests/frontend'), { recursive: true });
await writeFile(
path.join(directory, 'node_modules/typescript/bin/tsc'),
'process.stderr.write("src/Layout.tsx(1,1): error TS2322: fixture type error\\n"); process.exit(2);',
);
await writeFile(
path.join(directory, 'tests/frontend/never-run.test.mjs'),
'throw new Error("Later checks must not run");',
);
const result = spawnSync(
'/opt/homebrew/bin/node',
[fileURLToPath(new URL('./poc-project-check.mjs', import.meta.url)), 'verify'],
{
cwd: directory,
encoding: 'utf8',
},
);
assert.equal(result.status, 1);
assert.match(result.stderr, /TS2322: fixture type error/);
assert.ok(!result.stdout.includes('START test:frontend'));
assert.ok(!result.stdout.includes('START build:web'));
assert.ok(!result.stdout.includes('START go test'));
assert.match(result.stdout, /"passed":false/);
} finally {
await rm(directory, { recursive: true, force: true });
}
},
);
test('accepts a verifier export, checks a repository origin when metadata exists, and rejects another project', async () => {
const directory = await mkdtemp(path.join(os.tmpdir(), 'poc-check-contract-'));
const git = args => {
const result = spawnSync('/usr/bin/git', ['-c', 'core.hooksPath=/dev/null', ...args], {
cwd: directory,
encoding: 'utf8',
});
assert.equal(result.status, 0, result.stderr);
};
try {
await writeFile(path.join(directory, 'package.json'), JSON.stringify({ name: 'droboticflow' }));
await writeFile(path.join(directory, 'go.mod'), 'module droboticflow\n\ngo 1.24.0\n');
assert.equal((await assertApprovedWorkspace(directory)).head, null);
git(['init']);
git(['-c', 'user.name=Fixture', '-c', 'user.email=fixture@invalid', 'commit', '--allow-empty', '-m', 'Fixture']);
git(['remote', 'add', 'origin', 'https://gitea.d-robotics.cc/longtao.wu/droboticflow.git']);
assert.match((await assertApprovedWorkspace(directory)).head, /^[a-f0-9]{40}$/);
git(['remote', 'set-url', 'origin', 'https://example.org/other.git']);
await assert.rejects(assertApprovedWorkspace(directory), /approved droboticflow/);
await writeFile(path.join(directory, 'package.json'), JSON.stringify({ name: 'other' }));
await assert.rejects(assertApprovedWorkspace(directory), /Node and Go project/);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test('reads the failed tests of node --test as "file :: name"', () => {
const output = [
'✔ fine (1ms)',
'✖ failing tests:',
'',
'test at tests/frontend/a.test.mjs:3:1',
'✖ old fails (0.5ms)',
' AssertionError [ERR_ASSERTION]: boom',
'test at /work/tests/frontend/b.test.mjs:9:1',
'✖ new fails (12.3ms)',
].join('\n');
assert.deepEqual(failedTests(output), ['a.test.mjs :: old fails', 'b.test.mjs :: new fails']);
assert.equal(failedTests('ℹ pass 3'), undefined);
});
/** A workspace as the verifier prepares it: the change checked out, its start at BASE_REF. */
async function workspaceWithBase(t, change) {
const root = await mkdtemp(path.join(os.tmpdir(), 'wf-poc-baseline-'));
t.after(() => rm(root, { recursive: true, force: true }));
const git = (cwd, ...args) => {
const result = spawnSync('git', ['-c', 'user.name=x', '-c', 'user.email=x@x', ...args], { cwd, encoding: 'utf8' });
assert.equal(result.status, 0, result.stderr);
return result.stdout.trim();
};
const source = path.join(root, 'source');
await mkdir(path.join(source, 'tests/frontend'), { recursive: true });
const testFile = (name, passing) =>
`import test from 'node:test';\nimport assert from 'node:assert';\ntest(${JSON.stringify(name)}, () => assert.ok(${passing}));\n`;
git(root, 'init', '-q', source);
await writeFile(path.join(source, 'tests/frontend/a.test.mjs'), testFile('old fails', false));
await writeFile(path.join(source, 'tests/frontend/ok.test.mjs'), testFile('passes', true));
git(source, 'add', '.');
git(source, 'commit', '-qm', 'base');
const base = git(source, 'rev-parse', 'HEAD');
await change(source, testFile);
git(source, 'add', '.');
git(source, 'commit', '-qm', 'change', '--allow-empty');
const head = git(source, 'rev-parse', 'HEAD');
const workspace = path.join(root, 'workspace');
await mkdir(workspace);
git(source, `--work-tree=${workspace}`, 'checkout', head, '--', '.');
git(workspace, 'init', '-q');
git(
workspace,
'-c',
'protocol.file.allow=always',
'fetch',
'-q',
'--depth=1',
`file://${source}`,
head,
`${base}:${BASE_REF}`,
);
git(workspace, 'update-ref', 'HEAD', head);
await mkdir(path.join(workspace, 'node_modules'));
const cache = path.join(root, 'cache');
const env = checkEnvironment(cache);
await mkdir(env.TMPDIR, { recursive: true });
const files = (await import('node:fs/promises')).readdir(path.join(workspace, 'tests/frontend'));
const plan = [
{
label: 'test:frontend',
executable: process.execPath,
args: ['--test', ...(await files).sort().map(file => `tests/frontend/${file}`)],
baseline: true,
},
];
return { workspace, env, plan, git };
}
test('a test that already failed where the run started does not fail the change', async t => {
const f = await workspaceWithBase(t, async (source, testFile) => {
await writeFile(path.join(source, 'tests/frontend/c.test.mjs'), testFile('added and passing', true));
});
const [result] = await runPlan(f.plan, f.workspace, f.env, { quiet: true });
assert.equal(result.exitCode, 0);
assert.equal(result.preexistingFailures, 1);
});
test('a test the change breaks fails it, however many were failing before', async t => {
const f = await workspaceWithBase(t, async (source, testFile) => {
await writeFile(path.join(source, 'tests/frontend/ok.test.mjs'), testFile('passes', false));
});
const [result] = await runPlan(f.plan, f.workspace, f.env, { quiet: true });
assert.notEqual(result.exitCode, 0);
assert.equal(result.preexistingFailures, undefined);
});
test('without the commit the run started from, every failure counts', async t => {
const f = await workspaceWithBase(t, async () => {});
f.git(f.workspace, 'update-ref', '-d', BASE_REF);
const [result] = await runPlan(f.plan, f.workspace, f.env, { quiet: true });
assert.notEqual(result.exitCode, 0);
});
test('reads the failed Go tests as "package :: Test", and a package that did not build as one failure', () => {
const output = [
'--- FAIL: TestRoute (0.78s)',
' --- FAIL: TestRoute/admin (0.01s)',
'FAIL',
'FAIL\tdroboticflow/internal/api\t82.919s',
'ok \tdroboticflow/internal/store\t1.2s',
'--- FAIL: TestNothingAfterOk (0.00s)',
'ok \tdroboticflow/internal/other\t0.1s',
'FAIL\tdroboticflow/internal/broken [build failed]',
].join('\n');
assert.deepEqual(failedGoTests(output), [
'droboticflow/internal/api :: TestRoute',
'droboticflow/internal/api :: TestRoute/admin',
'droboticflow/internal/broken :: [build failed]',
]);
assert.equal(failedGoTests('ok \tdroboticflow/internal/api\t1s'), undefined);
});
test('a Go table case named with a hex id is the same failure at the start commit', () => {
const now = [
'droboticflow/internal/api :: TestEmployeeSupplierFixedLifecycle/supplier-default-org-4bc9ed231f699499fa27768fb8ba5304',
];
const before = [
'droboticflow/internal/api :: TestEmployeeSupplierFixedLifecycle/supplier-default-org-d34742d770aec5c061e58d79d19b00f7',
];
assert.equal(comparableFailure(now[0]), comparableFailure(before[0]));
assert.deepEqual(newlyFailing(now, before), []);
assert.deepEqual(newlyFailing([...now, 'droboticflow/internal/api :: TestNew'], before), [
'droboticflow/internal/api :: TestNew',
]);
});