- Checks run in a repository of their own holding only the commit (and the commit the run started from, as refs/wishforge/base), so code that asks Git which commit it is gets the true answer; the verifier's repository and configuration stay out of reach - A frontend test gets two minutes before it fails, instead of one that never settles holding the whole suite until the check is killed - When frontend or Go tests fail, the same tests run at the commit the run started from; only failures that were not there before fail the change Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
176 lines
8.2 KiB
TypeScript
176 lines
8.2 KiB
TypeScript
import { afterEach, describe, expect, it } from 'vitest';
|
|
import { access, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import {
|
|
executeCommand,
|
|
GitWorkspace,
|
|
isolatedVerifierExecutor,
|
|
verifyWorkspace,
|
|
TIMEOUT_EXIT_CODE,
|
|
type Command,
|
|
} from '../src/index.js';
|
|
|
|
const dirs: string[] = [];
|
|
afterEach(async () => {
|
|
await Promise.all(dirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })));
|
|
});
|
|
async function checkpointed() {
|
|
const root = await mkdtemp(path.join(os.tmpdir(), 'wishforge-verify-'));
|
|
dirs.push(root);
|
|
const git = new GitWorkspace();
|
|
const prepared = await git.create({ root, runId: 'run' });
|
|
await writeFile(path.join(prepared.workspace, 'hello.txt'), 'hello\n');
|
|
const checkpoint = await git.checkpoint(prepared.workspace, prepared.baseCommit, 'run', 'test');
|
|
return { root, git, workspace: prepared.workspace, sourceCommit: checkpoint.sourceCommit };
|
|
}
|
|
const node = (source: string) => [process.execPath, '-e', source];
|
|
|
|
describe('independent verification', () => {
|
|
it('runs checks in an exported tree, so planted Git configuration never reaches the verifier repository', async () => {
|
|
const f = await checkpointed();
|
|
const marker = path.join(f.root, 'filter-ran');
|
|
const plant = node(
|
|
[
|
|
"const fs = require('fs');",
|
|
"fs.mkdirSync('.git', { recursive: true });",
|
|
`fs.writeFileSync('.git/config', '[filter "x"]\\n\\tclean = touch ${marker}\\n');`,
|
|
"fs.writeFileSync('.gitattributes', '* filter=x\\n');",
|
|
"fs.writeFileSync('hello.txt', 'changed');",
|
|
].join(' '),
|
|
);
|
|
const report = await verifyWorkspace({ ...f, commands: [plant], isolation: 'host' });
|
|
expect(report.passed).toBe(true);
|
|
expect(await f.git.head(f.workspace)).toBe(f.sourceCommit);
|
|
expect(await readFile(path.join(f.workspace, 'hello.txt'), 'utf8')).toBe('hello\n');
|
|
const status = await executeCommand({ executable: 'git', args: ['status', '--porcelain'], cwd: f.workspace });
|
|
expect(status.stdout).toBe('');
|
|
await expect(access(marker)).rejects.toThrow();
|
|
});
|
|
|
|
it('gives the checks a repository of their own with just the commit, so they can ask Git which commit it is', async () => {
|
|
const f = await checkpointed();
|
|
// The verifier's repository carries configuration the checks must not inherit, and an origin
|
|
// with a credential.
|
|
const config = (key: string, value: string) =>
|
|
executeCommand({ executable: 'git', args: ['config', key, value], cwd: f.workspace });
|
|
await config('core.hooksPath', '/tmp/planted-hooks');
|
|
await config('remote.origin.url', 'https://token:secret@gitea.example/team/app.git');
|
|
const probe = node(
|
|
[
|
|
"const { execFileSync } = require('child_process');",
|
|
"const git = (...args) => execFileSync('git', args, { encoding: 'utf8' }).trim();",
|
|
"if (git('rev-parse', 'HEAD') !== process.argv[1]) process.exit(2);",
|
|
"if (git('status', '--porcelain') !== '') process.exit(3);",
|
|
"if (git('rev-list', '--count', 'HEAD') !== '1') process.exit(4);",
|
|
"if (git('config', '--get', 'remote.origin.url') !== 'https://gitea.example/team/app.git') process.exit(5);",
|
|
"try { git('config', '--get', 'core.hooksPath'); process.exit(6); } catch {}",
|
|
].join(' '),
|
|
);
|
|
const report = await verifyWorkspace({ ...f, commands: [[...probe, f.sourceCommit]], isolation: 'host' });
|
|
expect(report.commands.map(command => command.exitCode)).toEqual([0]);
|
|
expect(report.passed).toBe(true);
|
|
});
|
|
|
|
it('also holds the commit the run started from, for checks that compare with it', async () => {
|
|
const f = await checkpointed();
|
|
const base = (
|
|
await executeCommand({ executable: 'git', args: ['rev-parse', 'HEAD~1'], cwd: f.workspace })
|
|
).stdout.trim();
|
|
const probe = node(
|
|
[
|
|
"const { execFileSync } = require('child_process');",
|
|
"const base = execFileSync('git', ['rev-parse', 'refs/wishforge/base'], { encoding: 'utf8' }).trim();",
|
|
'if (base !== process.argv[1]) process.exit(2);',
|
|
"if (!execFileSync('git', ['archive', '--format=tar', base]).length) process.exit(3);",
|
|
].join(' '),
|
|
);
|
|
const report = await verifyWorkspace({ ...f, baseCommit: base, commands: [[...probe, base]], isolation: 'host' });
|
|
expect(report.commands.map(command => command.exitCode)).toEqual([0]);
|
|
});
|
|
|
|
it('records redacted output tails and stops at the first failing command', async () => {
|
|
const f = await checkpointed();
|
|
const report = await verifyWorkspace({
|
|
...f,
|
|
commands: [
|
|
node("console.log('api_key=abc123secret'); console.error('expected 2 rows, got 1'); process.exit(3)"),
|
|
node("require('fs').writeFileSync('should-not-run', '')"),
|
|
],
|
|
isolation: 'host',
|
|
});
|
|
expect(report.passed).toBe(false);
|
|
expect(report.commands).toHaveLength(1);
|
|
expect(report.commands[0]).toMatchObject({ exitCode: 3 });
|
|
expect(report.commands[0]!.outputTail).toContain('expected 2 rows, got 1');
|
|
expect(report.commands[0]!.outputTail).not.toContain('abc123secret');
|
|
expect(report.commands[0]!.outputDigest).toMatch(/^sha256:[0-9a-f]{64}$/);
|
|
});
|
|
|
|
it('records a time-out as a failed command instead of aborting verification', async () => {
|
|
const f = await checkpointed();
|
|
const report = await verifyWorkspace({
|
|
...f,
|
|
commands: [node('setTimeout(() => {}, 10_000)')],
|
|
timeoutMs: 300,
|
|
isolation: 'host',
|
|
});
|
|
expect(report.passed).toBe(false);
|
|
expect(report.commands[0]).toMatchObject({ exitCode: TIMEOUT_EXIT_CODE });
|
|
expect(report.commands[0]!.outputTail).toContain('stopped the command');
|
|
});
|
|
|
|
it('installs dependencies first and reports an install failure separately from test results', async () => {
|
|
const f = await checkpointed();
|
|
const installed = await verifyWorkspace({
|
|
...f,
|
|
installCommands: [node("require('fs').writeFileSync('installed', 'yes')")],
|
|
commands: [node("process.exit(require('fs').readFileSync('installed', 'utf8') === 'yes' ? 0 : 1)")],
|
|
isolation: 'host',
|
|
});
|
|
expect(installed.passed).toBe(true);
|
|
const broken = await verifyWorkspace({
|
|
...f,
|
|
installCommands: [node("console.error('registry unreachable'); process.exit(1)")],
|
|
commands: [node('0')],
|
|
isolation: 'host',
|
|
});
|
|
expect(broken).toMatchObject({ passed: false, commands: [] });
|
|
expect(broken.installFailure).toMatchObject({ exitCode: 1 });
|
|
expect(broken.installFailure!.outputTail).toContain('registry unreachable');
|
|
});
|
|
it('sends installations, and only them, through the configured egress proxy', async () => {
|
|
const f = await checkpointed();
|
|
const report = await verifyWorkspace({
|
|
...f,
|
|
installProxy: 'http://proxy.test:3128',
|
|
installCommands: [node("process.exit(process.env.HTTPS_PROXY === 'http://proxy.test:3128' ? 0 : 1)")],
|
|
commands: [node('process.exit(process.env.HTTPS_PROXY || process.env.https_proxy ? 1 : 0)')],
|
|
isolation: 'host',
|
|
});
|
|
expect(report).toMatchObject({ passed: true });
|
|
});
|
|
it('lets only a networked installation carry proxy settings into bubblewrap', async () => {
|
|
const platform = Object.getOwnPropertyDescriptor(process, 'platform')!;
|
|
Object.defineProperty(process, 'platform', { value: 'linux' });
|
|
try {
|
|
const seen: Command[] = [];
|
|
const sandboxed = isolatedVerifierExecutor(process.execPath, async command => {
|
|
seen.push(command);
|
|
return { exitCode: 0, stdout: '', stderr: '' };
|
|
});
|
|
const env = { PATH: '/usr/bin', HTTPS_PROXY: 'http://proxy.test:3128', SECRET_TOKEN: 'leak' };
|
|
await sandboxed({ executable: 'npm', args: ['ci'], cwd: '/tmp', env, network: true });
|
|
await sandboxed({ executable: 'npm', args: ['test'], cwd: '/tmp', env, network: false });
|
|
const [install, check] = seen.slice(1).map(command => command.args.join(' '));
|
|
expect(install).toContain('--share-net');
|
|
expect(install).toContain('--setenv HTTPS_PROXY http://proxy.test:3128');
|
|
expect(check).not.toContain('--share-net');
|
|
expect(check).not.toContain('HTTPS_PROXY');
|
|
expect(`${install} ${check}`).not.toContain('SECRET_TOKEN');
|
|
} finally {
|
|
Object.defineProperty(process, 'platform', platform);
|
|
}
|
|
});
|
|
});
|