eust-w 35c00f076d 🧪 Do not hold a change to failures already on its branch
- Checks run in a repository of their own holding only the commit (and
  the commit the run started from, as refs/wishforge/base), so code that
  asks Git which commit it is gets the true answer; the verifier's
  repository and configuration stay out of reach
- A frontend test gets two minutes before it fails, instead of one that
  never settles holding the whole suite until the check is killed
- When frontend or Go tests fail, the same tests run at the commit the
  run started from; only failures that were not there before fail the
  change

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 05:37:14 +08:00

176 lines
8.2 KiB
TypeScript

import { afterEach, describe, expect, it } from 'vitest';
import { access, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import {
executeCommand,
GitWorkspace,
isolatedVerifierExecutor,
verifyWorkspace,
TIMEOUT_EXIT_CODE,
type Command,
} from '../src/index.js';
const dirs: string[] = [];
afterEach(async () => {
await Promise.all(dirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })));
});
async function checkpointed() {
const root = await mkdtemp(path.join(os.tmpdir(), 'wishforge-verify-'));
dirs.push(root);
const git = new GitWorkspace();
const prepared = await git.create({ root, runId: 'run' });
await writeFile(path.join(prepared.workspace, 'hello.txt'), 'hello\n');
const checkpoint = await git.checkpoint(prepared.workspace, prepared.baseCommit, 'run', 'test');
return { root, git, workspace: prepared.workspace, sourceCommit: checkpoint.sourceCommit };
}
const node = (source: string) => [process.execPath, '-e', source];
describe('independent verification', () => {
it('runs checks in an exported tree, so planted Git configuration never reaches the verifier repository', async () => {
const f = await checkpointed();
const marker = path.join(f.root, 'filter-ran');
const plant = node(
[
"const fs = require('fs');",
"fs.mkdirSync('.git', { recursive: true });",
`fs.writeFileSync('.git/config', '[filter "x"]\\n\\tclean = touch ${marker}\\n');`,
"fs.writeFileSync('.gitattributes', '* filter=x\\n');",
"fs.writeFileSync('hello.txt', 'changed');",
].join(' '),
);
const report = await verifyWorkspace({ ...f, commands: [plant], isolation: 'host' });
expect(report.passed).toBe(true);
expect(await f.git.head(f.workspace)).toBe(f.sourceCommit);
expect(await readFile(path.join(f.workspace, 'hello.txt'), 'utf8')).toBe('hello\n');
const status = await executeCommand({ executable: 'git', args: ['status', '--porcelain'], cwd: f.workspace });
expect(status.stdout).toBe('');
await expect(access(marker)).rejects.toThrow();
});
it('gives the checks a repository of their own with just the commit, so they can ask Git which commit it is', async () => {
const f = await checkpointed();
// The verifier's repository carries configuration the checks must not inherit, and an origin
// with a credential.
const config = (key: string, value: string) =>
executeCommand({ executable: 'git', args: ['config', key, value], cwd: f.workspace });
await config('core.hooksPath', '/tmp/planted-hooks');
await config('remote.origin.url', 'https://token:secret@gitea.example/team/app.git');
const probe = node(
[
"const { execFileSync } = require('child_process');",
"const git = (...args) => execFileSync('git', args, { encoding: 'utf8' }).trim();",
"if (git('rev-parse', 'HEAD') !== process.argv[1]) process.exit(2);",
"if (git('status', '--porcelain') !== '') process.exit(3);",
"if (git('rev-list', '--count', 'HEAD') !== '1') process.exit(4);",
"if (git('config', '--get', 'remote.origin.url') !== 'https://gitea.example/team/app.git') process.exit(5);",
"try { git('config', '--get', 'core.hooksPath'); process.exit(6); } catch {}",
].join(' '),
);
const report = await verifyWorkspace({ ...f, commands: [[...probe, f.sourceCommit]], isolation: 'host' });
expect(report.commands.map(command => command.exitCode)).toEqual([0]);
expect(report.passed).toBe(true);
});
it('also holds the commit the run started from, for checks that compare with it', async () => {
const f = await checkpointed();
const base = (
await executeCommand({ executable: 'git', args: ['rev-parse', 'HEAD~1'], cwd: f.workspace })
).stdout.trim();
const probe = node(
[
"const { execFileSync } = require('child_process');",
"const base = execFileSync('git', ['rev-parse', 'refs/wishforge/base'], { encoding: 'utf8' }).trim();",
'if (base !== process.argv[1]) process.exit(2);',
"if (!execFileSync('git', ['archive', '--format=tar', base]).length) process.exit(3);",
].join(' '),
);
const report = await verifyWorkspace({ ...f, baseCommit: base, commands: [[...probe, base]], isolation: 'host' });
expect(report.commands.map(command => command.exitCode)).toEqual([0]);
});
it('records redacted output tails and stops at the first failing command', async () => {
const f = await checkpointed();
const report = await verifyWorkspace({
...f,
commands: [
node("console.log('api_key=abc123secret'); console.error('expected 2 rows, got 1'); process.exit(3)"),
node("require('fs').writeFileSync('should-not-run', '')"),
],
isolation: 'host',
});
expect(report.passed).toBe(false);
expect(report.commands).toHaveLength(1);
expect(report.commands[0]).toMatchObject({ exitCode: 3 });
expect(report.commands[0]!.outputTail).toContain('expected 2 rows, got 1');
expect(report.commands[0]!.outputTail).not.toContain('abc123secret');
expect(report.commands[0]!.outputDigest).toMatch(/^sha256:[0-9a-f]{64}$/);
});
it('records a time-out as a failed command instead of aborting verification', async () => {
const f = await checkpointed();
const report = await verifyWorkspace({
...f,
commands: [node('setTimeout(() => {}, 10_000)')],
timeoutMs: 300,
isolation: 'host',
});
expect(report.passed).toBe(false);
expect(report.commands[0]).toMatchObject({ exitCode: TIMEOUT_EXIT_CODE });
expect(report.commands[0]!.outputTail).toContain('stopped the command');
});
it('installs dependencies first and reports an install failure separately from test results', async () => {
const f = await checkpointed();
const installed = await verifyWorkspace({
...f,
installCommands: [node("require('fs').writeFileSync('installed', 'yes')")],
commands: [node("process.exit(require('fs').readFileSync('installed', 'utf8') === 'yes' ? 0 : 1)")],
isolation: 'host',
});
expect(installed.passed).toBe(true);
const broken = await verifyWorkspace({
...f,
installCommands: [node("console.error('registry unreachable'); process.exit(1)")],
commands: [node('0')],
isolation: 'host',
});
expect(broken).toMatchObject({ passed: false, commands: [] });
expect(broken.installFailure).toMatchObject({ exitCode: 1 });
expect(broken.installFailure!.outputTail).toContain('registry unreachable');
});
it('sends installations, and only them, through the configured egress proxy', async () => {
const f = await checkpointed();
const report = await verifyWorkspace({
...f,
installProxy: 'http://proxy.test:3128',
installCommands: [node("process.exit(process.env.HTTPS_PROXY === 'http://proxy.test:3128' ? 0 : 1)")],
commands: [node('process.exit(process.env.HTTPS_PROXY || process.env.https_proxy ? 1 : 0)')],
isolation: 'host',
});
expect(report).toMatchObject({ passed: true });
});
it('lets only a networked installation carry proxy settings into bubblewrap', async () => {
const platform = Object.getOwnPropertyDescriptor(process, 'platform')!;
Object.defineProperty(process, 'platform', { value: 'linux' });
try {
const seen: Command[] = [];
const sandboxed = isolatedVerifierExecutor(process.execPath, async command => {
seen.push(command);
return { exitCode: 0, stdout: '', stderr: '' };
});
const env = { PATH: '/usr/bin', HTTPS_PROXY: 'http://proxy.test:3128', SECRET_TOKEN: 'leak' };
await sandboxed({ executable: 'npm', args: ['ci'], cwd: '/tmp', env, network: true });
await sandboxed({ executable: 'npm', args: ['test'], cwd: '/tmp', env, network: false });
const [install, check] = seen.slice(1).map(command => command.args.join(' '));
expect(install).toContain('--share-net');
expect(install).toContain('--setenv HTTPS_PROXY http://proxy.test:3128');
expect(check).not.toContain('--share-net');
expect(check).not.toContain('HTTPS_PROXY');
expect(`${install} ${check}`).not.toContain('SECRET_TOKEN');
} finally {
Object.defineProperty(process, 'platform', platform);
}
});
});