A table test that names its case with a 16–32 hex id (an org, a token) fails as a different string at the start commit and after the change. The baseline then treats that case as new, so a frontend-only wish cannot pass Go tests that were already red. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
264 lines
11 KiB
JavaScript
264 lines
11 KiB
JavaScript
import test from 'node:test';
|
||
import assert from 'node:assert/strict';
|
||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||
import { spawnSync } from 'node:child_process';
|
||
import { existsSync } from 'node:fs';
|
||
import os from 'node:os';
|
||
import path from 'node:path';
|
||
import { fileURLToPath } from 'node:url';
|
||
import {
|
||
assertApprovedWorkspace,
|
||
checkEnvironment,
|
||
checkPlan,
|
||
parseCheckMode,
|
||
TOOLCHAINS,
|
||
toolchain,
|
||
failedTests,
|
||
failedGoTests,
|
||
comparableFailure,
|
||
newlyFailing,
|
||
runPlan,
|
||
BASE_REF,
|
||
} from './poc-project-check.mjs';
|
||
|
||
test('each platform has one fixed absolute toolchain and its own PATH', () => {
|
||
for (const platform of ['darwin', 'linux']) {
|
||
const tools = toolchain(platform);
|
||
for (const name of ['git', 'node', 'npm', 'go']) assert.ok(path.isAbsolute(tools[name]), `${platform} ${name}`);
|
||
assert.equal(checkEnvironment('/private/cache', {}, false, tools).PATH, tools.path);
|
||
}
|
||
assert.equal(toolchain('linux').go, '/usr/local/go/bin/go');
|
||
assert.ok(!TOOLCHAINS.linux.path.includes('/opt/homebrew'));
|
||
assert.throws(() => toolchain('win32'), /no approved toolchain/);
|
||
});
|
||
|
||
test('only fixed install and verification modes are accepted', () => {
|
||
assert.equal(parseCheckMode(['install']), 'install');
|
||
assert.equal(parseCheckMode(['verify']), 'verify');
|
||
for (const args of [[], ['deploy'], ['verify', '--target', 'production'], ['sh', '-c', 'anything']])
|
||
assert.throws(() => parseCheckMode(args), /Usage/);
|
||
const plan = checkPlan('verify', '/owned/workspace', ['tests/frontend/check.test.mjs']);
|
||
assert.deepEqual(
|
||
plan.map(item => item.label),
|
||
['lint:web', 'test:frontend', 'build:web', 'go test ./...'],
|
||
);
|
||
assert.ok(plan.every(item => !item.args.includes('run')));
|
||
assert.ok(checkPlan('install', '/owned/workspace')[0].args.includes('--ignore-scripts'));
|
||
});
|
||
|
||
test('installation and verification share caches without inheriting host credentials', () => {
|
||
const inherited = {
|
||
HOME: '/operator',
|
||
KUBECONFIG: '/secret',
|
||
ANTHROPIC_AUTH_TOKEN: 'secret',
|
||
DATABASE_URL: 'secret',
|
||
NODE_OPTIONS: '--require evil',
|
||
HTTPS_PROXY: 'http://proxy.invalid',
|
||
};
|
||
const install = checkEnvironment('/private/cache', inherited, true);
|
||
const verify = checkEnvironment('/private/cache', inherited, false);
|
||
for (const name of ['HOME', 'GOPATH', 'GOMODCACHE', 'GOCACHE', 'npm_config_cache'])
|
||
assert.equal(install[name], verify[name]);
|
||
for (const name of ['KUBECONFIG', 'ANTHROPIC_AUTH_TOKEN', 'DATABASE_URL', 'NODE_OPTIONS'])
|
||
assert.equal(install[name], undefined);
|
||
assert.equal(verify.GOPROXY, 'off');
|
||
assert.equal(install.HTTPS_PROXY, inherited.HTTPS_PROXY);
|
||
assert.equal(verify.HTTPS_PROXY, undefined);
|
||
});
|
||
|
||
// The checks run the POC Mac's Homebrew toolchain; elsewhere, such as CI, there is none to run.
|
||
const skipWithoutPocToolchain = existsSync('/opt/homebrew/bin/node')
|
||
? false
|
||
: 'needs the POC host toolchain in /opt/homebrew';
|
||
test(
|
||
'stops verification at the first failure so repair evidence retains the actual diagnostic',
|
||
{ skip: skipWithoutPocToolchain },
|
||
async () => {
|
||
const directory = await mkdtemp(path.join(os.tmpdir(), 'poc-check-failure-'));
|
||
try {
|
||
await writeFile(path.join(directory, 'package.json'), JSON.stringify({ name: 'droboticflow' }));
|
||
await writeFile(path.join(directory, 'go.mod'), 'module droboticflow\n');
|
||
await mkdir(path.join(directory, 'node_modules/typescript/bin'), { recursive: true });
|
||
await mkdir(path.join(directory, 'tests/frontend'), { recursive: true });
|
||
await writeFile(
|
||
path.join(directory, 'node_modules/typescript/bin/tsc'),
|
||
'process.stderr.write("src/Layout.tsx(1,1): error TS2322: fixture type error\\n"); process.exit(2);',
|
||
);
|
||
await writeFile(
|
||
path.join(directory, 'tests/frontend/never-run.test.mjs'),
|
||
'throw new Error("Later checks must not run");',
|
||
);
|
||
const result = spawnSync(
|
||
'/opt/homebrew/bin/node',
|
||
[fileURLToPath(new URL('./poc-project-check.mjs', import.meta.url)), 'verify'],
|
||
{
|
||
cwd: directory,
|
||
encoding: 'utf8',
|
||
},
|
||
);
|
||
assert.equal(result.status, 1);
|
||
assert.match(result.stderr, /TS2322: fixture type error/);
|
||
assert.ok(!result.stdout.includes('START test:frontend'));
|
||
assert.ok(!result.stdout.includes('START build:web'));
|
||
assert.ok(!result.stdout.includes('START go test'));
|
||
assert.match(result.stdout, /"passed":false/);
|
||
} finally {
|
||
await rm(directory, { recursive: true, force: true });
|
||
}
|
||
},
|
||
);
|
||
|
||
test('accepts a verifier export, checks a repository origin when metadata exists, and rejects another project', async () => {
|
||
const directory = await mkdtemp(path.join(os.tmpdir(), 'poc-check-contract-'));
|
||
const git = args => {
|
||
const result = spawnSync('/usr/bin/git', ['-c', 'core.hooksPath=/dev/null', ...args], {
|
||
cwd: directory,
|
||
encoding: 'utf8',
|
||
});
|
||
assert.equal(result.status, 0, result.stderr);
|
||
};
|
||
try {
|
||
await writeFile(path.join(directory, 'package.json'), JSON.stringify({ name: 'droboticflow' }));
|
||
await writeFile(path.join(directory, 'go.mod'), 'module droboticflow\n\ngo 1.24.0\n');
|
||
assert.equal((await assertApprovedWorkspace(directory)).head, null);
|
||
git(['init']);
|
||
git(['-c', 'user.name=Fixture', '-c', 'user.email=fixture@invalid', 'commit', '--allow-empty', '-m', 'Fixture']);
|
||
git(['remote', 'add', 'origin', 'https://gitea.d-robotics.cc/longtao.wu/droboticflow.git']);
|
||
assert.match((await assertApprovedWorkspace(directory)).head, /^[a-f0-9]{40}$/);
|
||
git(['remote', 'set-url', 'origin', 'https://example.org/other.git']);
|
||
await assert.rejects(assertApprovedWorkspace(directory), /approved droboticflow/);
|
||
await writeFile(path.join(directory, 'package.json'), JSON.stringify({ name: 'other' }));
|
||
await assert.rejects(assertApprovedWorkspace(directory), /Node and Go project/);
|
||
} finally {
|
||
await rm(directory, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test('reads the failed tests of node --test as "file :: name"', () => {
|
||
const output = [
|
||
'✔ fine (1ms)',
|
||
'✖ failing tests:',
|
||
'',
|
||
'test at tests/frontend/a.test.mjs:3:1',
|
||
'✖ old fails (0.5ms)',
|
||
' AssertionError [ERR_ASSERTION]: boom',
|
||
'test at /work/tests/frontend/b.test.mjs:9:1',
|
||
'✖ new fails (12.3ms)',
|
||
].join('\n');
|
||
assert.deepEqual(failedTests(output), ['a.test.mjs :: old fails', 'b.test.mjs :: new fails']);
|
||
assert.equal(failedTests('ℹ pass 3'), undefined);
|
||
});
|
||
|
||
/** A workspace as the verifier prepares it: the change checked out, its start at BASE_REF. */
|
||
async function workspaceWithBase(t, change) {
|
||
const root = await mkdtemp(path.join(os.tmpdir(), 'wf-poc-baseline-'));
|
||
t.after(() => rm(root, { recursive: true, force: true }));
|
||
const git = (cwd, ...args) => {
|
||
const result = spawnSync('git', ['-c', 'user.name=x', '-c', 'user.email=x@x', ...args], { cwd, encoding: 'utf8' });
|
||
assert.equal(result.status, 0, result.stderr);
|
||
return result.stdout.trim();
|
||
};
|
||
const source = path.join(root, 'source');
|
||
await mkdir(path.join(source, 'tests/frontend'), { recursive: true });
|
||
const testFile = (name, passing) =>
|
||
`import test from 'node:test';\nimport assert from 'node:assert';\ntest(${JSON.stringify(name)}, () => assert.ok(${passing}));\n`;
|
||
git(root, 'init', '-q', source);
|
||
await writeFile(path.join(source, 'tests/frontend/a.test.mjs'), testFile('old fails', false));
|
||
await writeFile(path.join(source, 'tests/frontend/ok.test.mjs'), testFile('passes', true));
|
||
git(source, 'add', '.');
|
||
git(source, 'commit', '-qm', 'base');
|
||
const base = git(source, 'rev-parse', 'HEAD');
|
||
await change(source, testFile);
|
||
git(source, 'add', '.');
|
||
git(source, 'commit', '-qm', 'change', '--allow-empty');
|
||
const head = git(source, 'rev-parse', 'HEAD');
|
||
const workspace = path.join(root, 'workspace');
|
||
await mkdir(workspace);
|
||
git(source, `--work-tree=${workspace}`, 'checkout', head, '--', '.');
|
||
git(workspace, 'init', '-q');
|
||
git(
|
||
workspace,
|
||
'-c',
|
||
'protocol.file.allow=always',
|
||
'fetch',
|
||
'-q',
|
||
'--depth=1',
|
||
`file://${source}`,
|
||
head,
|
||
`${base}:${BASE_REF}`,
|
||
);
|
||
git(workspace, 'update-ref', 'HEAD', head);
|
||
await mkdir(path.join(workspace, 'node_modules'));
|
||
const cache = path.join(root, 'cache');
|
||
const env = checkEnvironment(cache);
|
||
await mkdir(env.TMPDIR, { recursive: true });
|
||
const files = (await import('node:fs/promises')).readdir(path.join(workspace, 'tests/frontend'));
|
||
const plan = [
|
||
{
|
||
label: 'test:frontend',
|
||
executable: process.execPath,
|
||
args: ['--test', ...(await files).sort().map(file => `tests/frontend/${file}`)],
|
||
baseline: true,
|
||
},
|
||
];
|
||
return { workspace, env, plan, git };
|
||
}
|
||
|
||
test('a test that already failed where the run started does not fail the change', async t => {
|
||
const f = await workspaceWithBase(t, async (source, testFile) => {
|
||
await writeFile(path.join(source, 'tests/frontend/c.test.mjs'), testFile('added and passing', true));
|
||
});
|
||
const [result] = await runPlan(f.plan, f.workspace, f.env, { quiet: true });
|
||
assert.equal(result.exitCode, 0);
|
||
assert.equal(result.preexistingFailures, 1);
|
||
});
|
||
|
||
test('a test the change breaks fails it, however many were failing before', async t => {
|
||
const f = await workspaceWithBase(t, async (source, testFile) => {
|
||
await writeFile(path.join(source, 'tests/frontend/ok.test.mjs'), testFile('passes', false));
|
||
});
|
||
const [result] = await runPlan(f.plan, f.workspace, f.env, { quiet: true });
|
||
assert.notEqual(result.exitCode, 0);
|
||
assert.equal(result.preexistingFailures, undefined);
|
||
});
|
||
|
||
test('without the commit the run started from, every failure counts', async t => {
|
||
const f = await workspaceWithBase(t, async () => {});
|
||
f.git(f.workspace, 'update-ref', '-d', BASE_REF);
|
||
const [result] = await runPlan(f.plan, f.workspace, f.env, { quiet: true });
|
||
assert.notEqual(result.exitCode, 0);
|
||
});
|
||
|
||
test('reads the failed Go tests as "package :: Test", and a package that did not build as one failure', () => {
|
||
const output = [
|
||
'--- FAIL: TestRoute (0.78s)',
|
||
' --- FAIL: TestRoute/admin (0.01s)',
|
||
'FAIL',
|
||
'FAIL\tdroboticflow/internal/api\t82.919s',
|
||
'ok \tdroboticflow/internal/store\t1.2s',
|
||
'--- FAIL: TestNothingAfterOk (0.00s)',
|
||
'ok \tdroboticflow/internal/other\t0.1s',
|
||
'FAIL\tdroboticflow/internal/broken [build failed]',
|
||
].join('\n');
|
||
assert.deepEqual(failedGoTests(output), [
|
||
'droboticflow/internal/api :: TestRoute',
|
||
'droboticflow/internal/api :: TestRoute/admin',
|
||
'droboticflow/internal/broken :: [build failed]',
|
||
]);
|
||
assert.equal(failedGoTests('ok \tdroboticflow/internal/api\t1s'), undefined);
|
||
});
|
||
|
||
test('a Go table case named with a hex id is the same failure at the start commit', () => {
|
||
const now = [
|
||
'droboticflow/internal/api :: TestEmployeeSupplierFixedLifecycle/supplier-default-org-4bc9ed231f699499fa27768fb8ba5304',
|
||
];
|
||
const before = [
|
||
'droboticflow/internal/api :: TestEmployeeSupplierFixedLifecycle/supplier-default-org-d34742d770aec5c061e58d79d19b00f7',
|
||
];
|
||
assert.equal(comparableFailure(now[0]), comparableFailure(before[0]));
|
||
assert.deepEqual(newlyFailing(now, before), []);
|
||
assert.deepEqual(newlyFailing([...now, 'droboticflow/internal/api :: TestNew'], before), [
|
||
'droboticflow/internal/api :: TestNew',
|
||
]);
|
||
});
|