wishforge-ide/packages/runtime/test/check-pods.test.ts
eust-w c5a6c6160d 🚀 Deploy WishForge to the POC cluster, with check Pods and daily backups
- cluster-poc.mjs: image (in-cluster Kaniko), bootstrap, secrets, apply,
  migrate --rehearsal|--final, public HTTPS entry, backup and restore
- One restricted Pod (platform, executor, Runner, PostgreSQL, Temporal,
  MinIO, Caddy, backup) listening only on loopback except Caddy
- Protected checks run in per-workspace check Pods; cancellation during
  start-up now always removes them
- Daily snapshots on the node (consistent dump with row counts, Temporal
  online backup), copied to the Mac by launchd with SHA-256 checks and a
  notification on failure; restore verified on the rehearsal copy
- Downloads and uploads survive the Mac's lossy link to the cluster
- Local public entry, evidence isolation labels, acceptance records

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-01 16:27:07 +08:00

533 lines
21 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { afterEach, describe, expect, it, vi } from 'vitest';
import { mkdir, mkdtemp, realpath, rm } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import {
CommandStopped,
RuntimeError,
checkPodExec,
checkPodExecutor,
checkPodManifest,
checkPodResult,
validateCheckPodConfig,
type CheckPodConfig,
type Command,
type CommandResult,
} from '../src/index.js';
const dirs: string[] = [];
afterEach(async () => {
await Promise.all(dirs.splice(0).map(directory => rm(directory, { recursive: true, force: true })));
});
const IMAGE = `registry.example/wishforge/cluster-poc@sha256:${'a'.repeat(64)}`;
async function volume() {
const root = await realpath(await mkdtemp(path.join(os.tmpdir(), 'wishforge-check-volume-')));
dirs.push(root);
const workspace = path.join(root, 'runs/demo/run-1/gen-1/verifier/workspace-tree');
await mkdir(workspace, { recursive: true });
const config: CheckPodConfig = {
kubectl: '/usr/local/bin/kubectl',
kubeconfig: '/etc/wishforge/kube/config',
namespace: 'wishforge-checks',
image: IMAGE,
claim: 'wishforge-exec',
mountRoot: root,
writable: [path.join(root, 'check-cache')],
uid: 10001,
gid: 10001,
owner: 'verifier-gen-1',
};
return { root, workspace, config };
}
/** A kubectl that creates, waits for and removes Pods and runs commands with the given results. */
type KubectlReply = (command: Command) => CommandResult | Promise<CommandResult>;
function fakeKubectl(run: KubectlReply, hooks: Partial<Record<'create' | 'wait' | 'delete', KubectlReply>> = {}) {
const calls: Command[] = [];
const execute = async (command: Command): Promise<CommandResult> => {
calls.push(command);
const verb = command.args[command.args.indexOf('-n') + 2];
const hook = hooks[verb as keyof typeof hooks];
if (hook) return hook(command);
if (verb === 'create') return { exitCode: 0, stdout: 'pod/created', stderr: '' };
if (verb === 'wait') return { exitCode: 0, stdout: '', stderr: '' };
if (verb === 'delete') return deleted(command);
return run(command);
};
const verbs = () => calls.map(call => call.args[call.args.indexOf('-n') + 2]);
const created = () =>
calls.filter(call => call.args.includes('create')).map(call => JSON.parse(call.input!) as Record<string, any>);
return { execute, calls, verbs, created };
}
function deferred<T>() {
let resolve!: (value: T) => void;
let reject!: (error: unknown) => void;
const promise = new Promise<T>((yes, no) => {
resolve = yes;
reject = no;
});
return { promise, resolve, reject };
}
const ok = () => ({ exitCode: 0, stdout: '', stderr: '' });
const deleted = (command: Command) => ({
...ok(),
stdout: `pod/${command.args[command.args.indexOf('pod') + 1]}\n`,
});
const reported = (code: number, stdout = 'ok\n', stderr = '') => ({
exitCode: code,
stdout,
stderr: `${stderr}\n__WISHFORGE_CHECK_EXIT__=${code}\n${code ? `command terminated with exit code ${code}\n` : ''}`,
});
describe('check Pods for cluster-hosted development checks', () => {
it('builds a credential-free Pod that mounts only the workspace, a read-only .git and the caches', async () => {
const { workspace, config } = await volume();
const manifest = checkPodManifest(config, 'wf-check-x', {
root: workspace,
rootSubPath: 'runs/demo/run-1/gen-1/verifier/workspace-tree',
git: 'directory',
writable: [{ path: path.join(config.mountRoot, 'check-cache'), subPath: 'check-cache' }],
});
const spec = manifest.spec;
expect(spec.automountServiceAccountToken).toBe(false);
expect(spec.enableServiceLinks).toBe(false);
expect(spec.securityContext).toEqual({
runAsNonRoot: true,
runAsUser: 10001,
runAsGroup: 10001,
seccompProfile: { type: 'RuntimeDefault' },
});
const [container] = spec.containers;
expect(container!.image).toBe(IMAGE);
expect(container!.securityContext).toEqual({
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
});
expect(container!.volumeMounts).toEqual([
{ name: 'run', mountPath: workspace, subPath: 'runs/demo/run-1/gen-1/verifier/workspace-tree' },
{
name: 'run',
mountPath: path.join(workspace, '.git'),
subPath: 'runs/demo/run-1/gen-1/verifier/workspace-tree/.git',
readOnly: true,
},
{ name: 'run', mountPath: path.join(config.mountRoot, 'check-cache'), subPath: 'check-cache' },
{ name: 'tmp', mountPath: '/tmp' },
]);
// Only the run volume and a scratch directory: never a Secret, a ConfigMap or a host path.
expect(spec.volumes.map(volume => Object.keys(volume).filter(key => key !== 'name'))).toEqual([
['persistentVolumeClaim'],
['emptyDir'],
]);
expect(spec.activeDeadlineSeconds).toBeGreaterThan(0);
});
it('refuses images by tag, root users and caches off the run volume', async () => {
const { config } = await volume();
expect(() => validateCheckPodConfig({ ...config, image: 'registry.example/check:latest' })).toThrow(/digest/);
expect(() => validateCheckPodConfig({ ...config, uid: 0 })).toThrow(/root/);
expect(() => validateCheckPodConfig({ ...config, writable: ['/etc'] })).toThrow(/run volume/);
expect(() => validateCheckPodConfig({ ...config, namespace: 'Bad_Name' })).toThrow(/Kubernetes names/);
});
it('runs the command with only its own variables and reports the exit status after the output', async () => {
const { workspace, config } = await volume();
const args = checkPodExec(
config,
'wf-check-x',
{
executable: '/usr/local/bin/node',
args: ['/opt/wishforge/scripts/poc-project-check.mjs', 'verify'],
cwd: workspace,
env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', CI: 'true' },
timeoutMs: 60_000,
},
{ home: '/tmp/command-1/home', temporary: '/tmp/command-1/tmp' },
'/tmp/kubectl-cache',
);
const after = args.slice(args.indexOf('--') + 1);
expect(after.slice(0, 2)).toEqual(['/usr/bin/env', '-i']);
expect(after).toContain('HOME=/tmp/command-1/home');
expect(after).toContain('TMPDIR=/tmp/command-1/tmp');
expect(after.join(' ')).not.toMatch(/TOKEN|KUBECONFIG|DATABASE/);
expect(after[after.indexOf('/usr/bin/timeout') + 3]).toBe('120');
expect(after.slice(-5)).toEqual([
'/tmp/command-1/home',
'/tmp/command-1/tmp',
'/usr/local/bin/node',
'/opt/wishforge/scripts/poc-project-check.mjs',
'verify',
]);
// No input: kubectl attaches no stdin.
expect(args.slice(0, args.indexOf('--'))).not.toContain('-i');
expect(() =>
checkPodExec(
config,
'x',
{ executable: 'a', args: [], cwd: workspace, env: { 'BAD-NAME': 'x' } },
{
home: '/tmp/h',
temporary: '/tmp/t',
},
'/tmp/c',
),
).toThrow(/variable/);
});
it('takes the reported status, never kubectl’s own failure, for the command’s', () => {
expect(checkPodResult(reported(3, 'out\n', 'error: tests failed'))).toEqual({
exitCode: 3,
stdout: 'out\n',
stderr: 'error: tests failed',
});
expect(checkPodResult(reported(0))).toEqual({ exitCode: 0, stdout: 'ok\n', stderr: '' });
// kubectl could not reach the Pod: no status, so no verdict on the check.
expect(
checkPodResult({ exitCode: 1, stdout: '', stderr: 'error: unable to upgrade connection: container not found' }),
).toBeUndefined();
});
it('reuses a workspace’s Pod, replaces it when the directory was recreated, and removes it when idle', async () => {
const { workspace, config } = await volume();
const fake = fakeKubectl(() => reported(0));
const executor = checkPodExecutor({ ...config, idleMs: 20 }, fake.execute);
const command = { executable: '/usr/bin/true', args: [], cwd: workspace, sandboxRoot: workspace, env: {} };
await executor(command);
await executor({ ...command, network: true });
expect(fake.verbs().filter(verb => verb === 'create')).toHaveLength(1);
// verifyWorkspace removes and recreates the exported tree for every verification.
await rm(workspace, { recursive: true });
await mkdir(workspace);
await executor(command);
expect(fake.verbs().filter(verb => verb === 'create')).toHaveLength(2);
await new Promise(resolve => setTimeout(resolve, 80));
expect(fake.verbs().filter(verb => verb === 'delete')).toHaveLength(2);
await executor.dispose();
});
it('removes the Pod when a command stops at its limit or is canceled, and keeps the stop', async () => {
const { workspace, config } = await volume();
const fake = fakeKubectl(() => {
throw new CommandStopped('TIMEOUT', 'Command exceeded its time limit', { stdout: 'partial', stderr: '' });
});
const executor = checkPodExecutor(config, fake.execute);
await expect(executor({ executable: '/usr/bin/true', args: [], cwd: workspace, env: {} })).rejects.toBeInstanceOf(
CommandStopped,
);
expect(fake.verbs()).toContain('delete');
await executor.dispose();
});
it('treats an exec that never reached the command as an infrastructure fault', async () => {
const { workspace, config } = await volume();
const fake = fakeKubectl(() => ({ exitCode: 1, stdout: '', stderr: 'error: dial tcp: i/o timeout' }));
const executor = checkPodExecutor(config, fake.execute);
const failure = await executor({ executable: '/usr/bin/true', args: [], cwd: workspace, env: {} }).catch(
error => error,
);
expect(failure).toBeInstanceOf(RuntimeError);
expect(failure.code).toBe('VERIFIER_ISOLATION_UNAVAILABLE');
await executor.dispose();
});
it('does not create a Pod for a command canceled before it starts', async () => {
const { workspace, config } = await volume();
const fake = fakeKubectl(() => reported(0));
const executor = checkPodExecutor(config, fake.execute);
try {
await expect(
executor({ executable: '/usr/bin/true', args: [], cwd: workspace, signal: AbortSignal.abort() }),
).rejects.toMatchObject({ code: 'CANCELED' });
expect(fake.calls).toHaveLength(0);
} finally {
await executor.dispose();
}
});
it('removes a Pod when cancellation interrupts its Ready wait', async () => {
const { workspace, config } = await volume();
const controller = new AbortController();
const started = deferred<Command>();
const ready = deferred<CommandResult>();
const fake = fakeKubectl(() => reported(0), {
wait: command => {
started.resolve(command);
return ready.promise;
},
});
const executor = checkPodExecutor(config, fake.execute);
const command = executor({ executable: '/usr/bin/true', args: [], cwd: workspace, signal: controller.signal });
const outcome = command.catch(error => error);
const wait = await started.promise;
controller.abort();
ready.reject(new RuntimeError('CANCELED', 'Ready wait canceled'));
try {
expect(await outcome).toMatchObject({ code: 'CANCELED' });
expect(wait.signal?.aborted).toBe(true);
expect(fake.verbs()).toContain('delete');
expect(fake.verbs()).not.toContain('exec');
const deletion = fake.calls.find(call => call.args.includes('delete'))!;
expect(deletion.signal?.aborted).not.toBe(true);
} finally {
await executor.dispose();
}
});
it('reclaims a possibly created Pod when the creation response is lost', async () => {
const { workspace, config } = await volume();
let live = false;
const fake = fakeKubectl(() => reported(0), {
create: () => {
live = true;
throw new Error('Creation response lost');
},
delete: command => {
live = false;
return deleted(command);
},
});
const executor = checkPodExecutor(config, fake.execute);
try {
await expect(executor({ executable: '/usr/bin/true', args: [], cwd: workspace })).rejects.toMatchObject({
code: 'VERIFIER_ISOLATION_UNAVAILABLE',
});
expect(live).toBe(false);
expect(fake.verbs()).not.toContain('exec');
} finally {
await executor.dispose();
}
});
it('classifies a startup timeout as infrastructure failure and deletes its Pod', async () => {
const { workspace, config } = await volume();
const fake = fakeKubectl(() => reported(0), {
wait: () => {
throw new CommandStopped('TIMEOUT', 'Ready wait timed out', { stdout: '', stderr: '' });
},
});
const executor = checkPodExecutor(config, fake.execute);
try {
await expect(executor({ executable: '/usr/bin/true', args: [], cwd: workspace })).rejects.toMatchObject({
code: 'VERIFIER_ISOLATION_UNAVAILABLE',
});
expect(fake.verbs()).toContain('delete');
expect(fake.verbs()).not.toContain('exec');
} finally {
await executor.dispose();
}
});
it('waits for a late creation before deleting it when disposed during startup', async () => {
const { workspace, config } = await volume();
const started = deferred<Command>();
const created = deferred<CommandResult>();
let live = false;
const fake = fakeKubectl(() => reported(0), {
create: async command => {
started.resolve(command);
const result = await created.promise;
live = true;
return result;
},
delete: command => {
live = false;
return deleted(command);
},
});
const executor = checkPodExecutor(config, fake.execute);
const outcome = executor({ executable: '/usr/bin/true', args: [], cwd: workspace }).catch(error => error);
const creation = await started.promise;
const disposed = executor.dispose();
expect(creation.signal?.aborted).toBe(true);
created.resolve(ok());
await disposed;
expect(await outcome).toMatchObject({ code: 'CANCELED' });
expect(live).toBe(false);
expect(fake.verbs()).not.toContain('exec');
await executor.dispose();
const count = fake.calls.length;
await expect(executor({ executable: '/usr/bin/true', args: [], cwd: workspace })).rejects.toMatchObject({
code: 'CANCELED',
});
expect(fake.calls).toHaveLength(count);
});
it('closes before asynchronous preparation can start a Pod', async () => {
const { workspace, config } = await volume();
const fake = fakeKubectl(() => reported(0));
const executor = checkPodExecutor(config, fake.execute);
const outcome = executor({ executable: '/usr/bin/true', args: [], cwd: workspace }).catch(error => error);
await executor.dispose();
expect(await outcome).toMatchObject({ code: 'CANCELED' });
expect(fake.calls).toHaveLength(0);
});
it.each(['exit', 'throw'] as const)('keeps failed %s deletions available for a later dispose', async failure => {
const { workspace, config } = await volume();
let fail = true;
const fake = fakeKubectl(() => reported(0), {
delete: () => {
if (!fail) return ok();
if (failure === 'throw') throw new Error('Delete connection failed');
return { exitCode: 1, stdout: '', stderr: 'Deletion refused' };
},
});
const executor = checkPodExecutor(config, fake.execute);
await executor({ executable: '/usr/bin/true', args: [], cwd: workspace });
try {
await expect(executor.dispose()).rejects.toMatchObject({ code: 'VERIFIER_ISOLATION_UNAVAILABLE' });
const attempts = fake.verbs().filter(verb => verb === 'delete').length;
fail = false;
await executor.dispose();
expect(fake.verbs().filter(verb => verb === 'delete')).toHaveLength(attempts + 1);
expect(fake.calls.filter(call => call.args.includes('delete')).every(call => call.cwd === config.mountRoot)).toBe(
true,
);
} finally {
fail = false;
await executor.dispose();
}
});
it('preserves a canceled command when deletion fails and retries its Pod at disposal', async () => {
const { workspace, config } = await volume();
const warning = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
let fail = true;
const canceled = new RuntimeError('CANCELED', 'Check canceled');
const fake = fakeKubectl(
() => {
throw canceled;
},
{
delete: () => (fail ? { exitCode: 1, stdout: '', stderr: 'Unavailable' } : ok()),
},
);
const executor = checkPodExecutor(config, fake.execute);
try {
await expect(executor({ executable: '/usr/bin/true', args: [], cwd: workspace })).rejects.toBe(canceled);
expect(warning).toHaveBeenCalled();
fail = false;
await executor.dispose();
expect(fake.verbs().filter(verb => verb === 'delete').length).toBeGreaterThan(1);
} finally {
fail = false;
await executor.dispose();
warning.mockRestore();
}
});
it('retains an ambiguous creation after NotFound and can remove it when it appears later', async () => {
const { workspace, config } = await volume();
const warning = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
let live = false;
const fake = fakeKubectl(() => reported(0), {
create: () => {
throw new Error('Create request outcome is unknown');
},
delete: command => {
if (!live) return ok(); // --ignore-not-found: nothing was observed yet.
live = false;
return deleted(command);
},
});
const executor = checkPodExecutor(config, fake.execute);
try {
await expect(executor({ executable: '/usr/bin/true', args: [], cwd: workspace })).rejects.toMatchObject({
code: 'VERIFIER_ISOLATION_UNAVAILABLE',
});
await expect(executor.dispose()).rejects.toMatchObject({ code: 'VERIFIER_ISOLATION_UNAVAILABLE' });
live = true;
await executor.dispose();
expect(live).toBe(false);
} finally {
live = true;
await executor.dispose();
warning.mockRestore();
}
});
it('bounds disposal without forgetting a creation that finishes after the close deadline', async () => {
const { workspace, config } = await volume();
const started = deferred<Command>();
const created = deferred<CommandResult>();
let live = false;
const fake = fakeKubectl(() => reported(0), {
create: async command => {
started.resolve(command);
const result = await created.promise;
live = true;
return result;
},
delete: command => {
live = false;
return deleted(command);
},
});
const executor = checkPodExecutor(config, fake.execute);
const outcome = executor({ executable: '/usr/bin/true', args: [], cwd: workspace }).catch(error => error);
await started.promise;
vi.useFakeTimers();
try {
const disposal = executor.dispose().catch(error => error);
await vi.advanceTimersByTimeAsync(2000);
expect(await disposal).toMatchObject({ code: 'VERIFIER_ISOLATION_UNAVAILABLE' });
expect(fake.verbs()).not.toContain('delete');
created.resolve(ok());
expect(await outcome).toMatchObject({ code: 'CANCELED' });
await executor.dispose();
expect(live).toBe(false);
expect(fake.verbs()).not.toContain('exec');
} finally {
created.resolve(ok());
await outcome;
await executor.dispose();
vi.useRealTimers();
}
});
it('reuses one Pod for concurrent commands and coalesces concurrent disposal', async () => {
const { workspace, config } = await volume();
const deleting = deferred<void>();
const deletedPod = deferred<CommandResult>();
const fake = fakeKubectl(() => reported(0), {
delete: () => {
deleting.resolve();
return deletedPod.promise;
},
});
const executor = checkPodExecutor(config, fake.execute);
const command = { executable: '/usr/bin/true', args: [], cwd: workspace };
await Promise.all([executor(command), executor(command)]);
expect(fake.created()).toHaveLength(1);
const first = executor.dispose();
const second = executor.dispose();
expect(first).toBe(second);
await deleting.promise;
deletedPod.resolve(ok());
await first;
await executor.dispose();
expect(fake.verbs().filter(verb => verb === 'delete')).toHaveLength(1);
});
it('keeps a home inside the workspace and gives the others a private one, and refuses paths off the volume', async () => {
const { root, workspace, config } = await volume();
const fake = fakeKubectl(() => reported(0));
const executor = checkPodExecutor(config, fake.execute);
await executor({ executable: '/usr/bin/true', args: [], cwd: workspace, env: { HOME: workspace } });
await executor({ executable: '/usr/bin/true', args: [], cwd: workspace, env: { HOME: '/home/operator' } });
const execs = fake.calls.filter(call => call.args.includes('exec'));
expect(execs[0]!.args).toContain(`HOME=${workspace}`);
expect(execs[1]!.args.find(arg => arg.startsWith('HOME='))).toMatch(/^HOME=\/tmp\/command-\d+\/home$/);
const outside = await realpath(await mkdtemp(path.join(os.tmpdir(), 'wishforge-outside-')));
dirs.push(outside);
await expect(executor({ executable: '/usr/bin/true', args: [], cwd: outside, env: {} })).rejects.toThrow(
/run volume/,
);
await expect(executor({ executable: '/usr/bin/true', args: [], cwd: root, env: {} })).rejects.toThrow(/run volume/);
await executor.dispose();
});
});