- cluster-poc.mjs: image (in-cluster Kaniko), bootstrap, secrets, apply, migrate --rehearsal|--final, public HTTPS entry, backup and restore - One restricted Pod (platform, executor, Runner, PostgreSQL, Temporal, MinIO, Caddy, backup) listening only on loopback except Caddy - Protected checks run in per-workspace check Pods; cancellation during start-up now always removes them - Daily snapshots on the node (consistent dump with row counts, Temporal online backup), copied to the Mac by launchd with SHA-256 checks and a notification on failure; restore verified on the rehearsal copy - Downloads and uploads survive the Mac's lossy link to the cluster - Local public entry, evidence isolation labels, acceptance records Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
209 lines
9.7 KiB
TypeScript
209 lines
9.7 KiB
TypeScript
import { mkdir, mkdtemp, rm } from 'node:fs/promises';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
import type { CommandExecutor, HostCheckSandbox } from '@wishforge/runtime';
|
|
import { validateCheckPodConfig } from '@wishforge/runtime';
|
|
import {
|
|
closeHostChecks,
|
|
hostCheckExecutor,
|
|
hostCheckPods,
|
|
hostCheckSandbox,
|
|
type HostCheckExecutor,
|
|
} from '../apps/worker/src/host-checks.js';
|
|
import type { WorkerConfig } from '../apps/worker/src/common.js';
|
|
|
|
describe('operator opt-in to sandboxed host checks', () => {
|
|
it('reads the paths, loopback and blocked ports, and always blocks the control plane and Temporal', () => {
|
|
expect(hostCheckSandbox({})).toBeUndefined();
|
|
expect(hostCheckSandbox({ WF_HOST_CHECK_SANDBOX: 'false', WF_HOST_CHECK_READ_PATHS: 'relative' })).toBeUndefined();
|
|
expect(
|
|
hostCheckSandbox({
|
|
WF_HOST_CHECK_SANDBOX: 'true',
|
|
WF_HOST_CHECK_READ_PATHS: ['/opt/check.mjs', '/opt/tools'].join(path.delimiter),
|
|
WF_HOST_CHECK_WRITE_PATHS: '/opt/cache',
|
|
WF_HOST_CHECK_LOOPBACK: 'true',
|
|
WF_HOST_CHECK_BLOCKED_PORTS: '55432,3000',
|
|
WF_CONTROL_PLANE_URL: 'http://127.0.0.1:4100',
|
|
TEMPORAL_ADDRESS: '127.0.0.1:7233',
|
|
}),
|
|
).toEqual({
|
|
readable: ['/opt/check.mjs', '/opt/tools'],
|
|
writable: ['/opt/cache'],
|
|
loopback: true,
|
|
blockedPorts: [3000, 4100, 7233, 55432],
|
|
});
|
|
expect(hostCheckSandbox({ WF_HOST_CHECK_SANDBOX: 'true' })).toMatchObject({
|
|
loopback: false,
|
|
blockedPorts: [4000, 7233],
|
|
});
|
|
expect(() => hostCheckSandbox({ WF_HOST_CHECK_SANDBOX: 'yes' })).toThrow(/true or false/);
|
|
expect(() => hostCheckSandbox({ WF_HOST_CHECK_SANDBOX: 'true', WF_HOST_CHECK_READ_PATHS: 'relative' })).toThrow(
|
|
/absolute paths/,
|
|
);
|
|
expect(() => hostCheckSandbox({ WF_HOST_CHECK_SANDBOX: 'true', WF_HOST_CHECK_WRITE_PATHS: '/a"b' })).toThrow(
|
|
/absolute paths/,
|
|
);
|
|
expect(() => hostCheckSandbox({ WF_HOST_CHECK_SANDBOX: 'true', WF_HOST_CHECK_BLOCKED_PORTS: '99999' })).toThrow(
|
|
/TCP ports/,
|
|
);
|
|
});
|
|
it('sandboxes only the checks of the development host mode', () => {
|
|
const execute: CommandExecutor = async () => ({ exitCode: 0, stdout: '', stderr: '' });
|
|
const hostCheckSandbox: HostCheckSandbox = { readable: [], writable: [] };
|
|
expect(hostCheckExecutor({ sandboxMode: 'host' } as WorkerConfig, execute)).toBe(execute);
|
|
expect(hostCheckExecutor({ sandboxMode: 'kata', hostCheckSandbox } as WorkerConfig, execute)).toBe(execute);
|
|
expect(hostCheckExecutor({ sandboxMode: 'host', hostCheckSandbox } as WorkerConfig, execute)).not.toBe(execute);
|
|
});
|
|
});
|
|
|
|
describe('operator opt-in to check Pods on a cluster-hosted development host', () => {
|
|
const pods = {
|
|
WF_HOST_CHECK_POD_NAMESPACE: 'wishforge-checks',
|
|
WF_HOST_CHECK_POD_IMAGE: `registry.example/wishforge/cluster-poc@sha256:${'b'.repeat(64)}`,
|
|
WF_HOST_CHECK_POD_CLAIM: 'wishforge-exec',
|
|
WF_HOST_CHECK_POD_ROOT: '/var/lib/wishforge/exec',
|
|
WF_HOST_CHECK_KUBECONFIG: '/etc/wishforge/kube/checks.config',
|
|
WF_HOST_CHECK_WRITE_PATHS: '/var/lib/wishforge/exec/check-cache',
|
|
WF_WORKER_ID: 'verifier-1234',
|
|
WF_SANDBOX_RUN_ID: 'run-1',
|
|
};
|
|
it('reads the namespace, image, claim, volume root and kubeconfig, and runs as the worker user', () => {
|
|
expect(hostCheckPods({})).toBeUndefined();
|
|
expect(hostCheckPods(pods)).toMatchObject({
|
|
namespace: 'wishforge-checks',
|
|
claim: 'wishforge-exec',
|
|
mountRoot: '/var/lib/wishforge/exec',
|
|
kubectl: '/usr/local/bin/kubectl',
|
|
writable: ['/var/lib/wishforge/exec/check-cache'],
|
|
uid: process.getuid!(),
|
|
gid: process.getgid!(),
|
|
owner: 'verifier-1234:run-1',
|
|
});
|
|
});
|
|
it('refuses a missing setting, an image by tag, caches off the volume and a second sandbox', () => {
|
|
expect(() => hostCheckPods({ ...pods, WF_HOST_CHECK_POD_CLAIM: '' })).toThrow(/WF_HOST_CHECK_POD_CLAIM/);
|
|
expect(() => hostCheckPods({ ...pods, WF_HOST_CHECK_POD_IMAGE: 'check:latest' })).toThrow(/digest/);
|
|
expect(() => hostCheckPods({ ...pods, WF_HOST_CHECK_WRITE_PATHS: '/tmp/cache' })).toThrow(/run volume/);
|
|
expect(() => hostCheckPods({ ...pods, WF_HOST_CHECK_SANDBOX: 'true' })).toThrow(/alternatives/);
|
|
});
|
|
it('uses check Pods only in the development host mode', () => {
|
|
const execute: CommandExecutor = async () => ({ exitCode: 0, stdout: '', stderr: '' });
|
|
const hostCheckPodsConfig = hostCheckPods(pods)!;
|
|
expect(
|
|
hostCheckExecutor({ sandboxMode: 'kata', hostCheckPods: hostCheckPodsConfig } as WorkerConfig, execute),
|
|
).toBe(execute);
|
|
expect(
|
|
hostCheckExecutor({ sandboxMode: 'host', hostCheckPods: hostCheckPodsConfig } as WorkerConfig, execute),
|
|
).not.toBe(execute);
|
|
});
|
|
});
|
|
|
|
describe('hostCheckExecutor binds cancellation only at the command boundary', () => {
|
|
it('composes the activity signal with a command signal, and uses the activity signal alone otherwise', async () => {
|
|
const seen: (AbortSignal | undefined)[] = [];
|
|
const execute: CommandExecutor = async command => {
|
|
seen.push(command.signal);
|
|
return { exitCode: 0, stdout: '', stderr: '' };
|
|
};
|
|
const activity = new AbortController();
|
|
const bound = hostCheckExecutor({ sandboxMode: 'kata' } as WorkerConfig, execute, activity.signal);
|
|
await bound({ executable: 'true', args: [], cwd: '/work' });
|
|
expect(seen[0]).toBe(activity.signal);
|
|
const perCommand = new AbortController();
|
|
await bound({ executable: 'true', args: [], cwd: '/work', signal: perCommand.signal });
|
|
const composed = seen[1]!;
|
|
expect(composed).not.toBe(activity.signal);
|
|
expect(composed).not.toBe(perCommand.signal);
|
|
expect(composed.aborted).toBe(false);
|
|
perCommand.abort();
|
|
expect(composed.aborted).toBe(true);
|
|
});
|
|
it('returns the raw executor unchanged when no activity signal is given, even with check Pods configured', () => {
|
|
const execute: CommandExecutor = async () => ({ exitCode: 0, stdout: '', stderr: '' });
|
|
expect(hostCheckExecutor({ sandboxMode: 'kata' } as WorkerConfig, execute)).toBe(execute);
|
|
});
|
|
});
|
|
|
|
describe('check Pod cleanup never inherits the activity signal', () => {
|
|
const directories: string[] = [];
|
|
afterEach(async () => {
|
|
await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true })));
|
|
});
|
|
async function checkPodsConfig(mountRoot: string) {
|
|
return validateCheckPodConfig({
|
|
kubectl: '/usr/bin/kubectl',
|
|
kubeconfig: '/etc/wishforge/kube/checks.config',
|
|
namespace: 'wishforge-checks',
|
|
image: `registry.example/wishforge/cluster-poc@sha256:${'d'.repeat(64)}`,
|
|
claim: 'wishforge-exec',
|
|
mountRoot,
|
|
writable: [],
|
|
uid: process.getuid!(),
|
|
gid: process.getgid!(),
|
|
owner: 'fixture:run',
|
|
});
|
|
}
|
|
function fakeKubectl(kubectl: string, onDelete: () => void): CommandExecutor {
|
|
return async command => {
|
|
// A raw transport must never see an already-canceled activity signal on its own calls.
|
|
if (command.signal?.aborted) throw new Error('kubectl invoked with an already-aborted signal');
|
|
if (command.executable !== kubectl) return { exitCode: 0, stdout: '', stderr: '' };
|
|
const verb = command.args[command.args.indexOf('-n') + 2];
|
|
if (verb === 'create') return { exitCode: 0, stdout: 'pod/wf-check-fixture', stderr: '' };
|
|
if (verb === 'wait') return { exitCode: 0, stdout: '', stderr: '' };
|
|
if (verb === 'exec') return { exitCode: 0, stdout: '', stderr: '\n__WISHFORGE_CHECK_EXIT__=0\n' };
|
|
if (verb === 'delete') {
|
|
onDelete();
|
|
return { exitCode: 0, stdout: 'pod/wf-check-fixture\n', stderr: '' };
|
|
}
|
|
throw new Error(`Unexpected kubectl verb in fixture: ${verb}`);
|
|
};
|
|
}
|
|
it('still deletes a check Pod through dispose once the activity signal that ran its command has aborted', async () => {
|
|
const root = await mkdtemp(path.join(os.tmpdir(), 'wishforge-host-checks-'));
|
|
directories.push(root);
|
|
const workspace = path.join(root, 'workspace');
|
|
await mkdir(workspace, { recursive: true });
|
|
const deleted = vi.fn();
|
|
const execute = fakeKubectl('/usr/bin/kubectl', deleted);
|
|
const config = { sandboxMode: 'host', hostCheckPods: await checkPodsConfig(root) } as WorkerConfig;
|
|
const activity = new AbortController();
|
|
const bound: HostCheckExecutor = hostCheckExecutor(config, execute, activity.signal);
|
|
await bound({
|
|
executable: 'true',
|
|
args: [],
|
|
cwd: workspace,
|
|
sandboxRoot: workspace,
|
|
env: {},
|
|
timeoutMs: 1000,
|
|
});
|
|
activity.abort();
|
|
await expect(bound.dispose!()).resolves.toBeUndefined();
|
|
expect(deleted).toHaveBeenCalledTimes(1);
|
|
});
|
|
});
|
|
|
|
describe('closeHostChecks reports whether cleanup is confirmed, without throwing', () => {
|
|
it('resolves true when there is nothing to close', async () => {
|
|
await expect(closeHostChecks(undefined)).resolves.toBe(true);
|
|
});
|
|
it('resolves true once dispose succeeds', async () => {
|
|
const dispose = vi.fn().mockResolvedValue(undefined);
|
|
const checks = Object.assign((async () => ({ exitCode: 0, stdout: '', stderr: '' })) as CommandExecutor, {
|
|
dispose,
|
|
});
|
|
await expect(closeHostChecks(checks)).resolves.toBe(true);
|
|
expect(dispose).toHaveBeenCalledTimes(1);
|
|
});
|
|
it('resolves false and warns, without throwing, when dispose cannot confirm cleanup', async () => {
|
|
const warning = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
|
const checks = Object.assign((async () => ({ exitCode: 0, stdout: '', stderr: '' })) as CommandExecutor, {
|
|
dispose: vi.fn().mockRejectedValue(new Error('Check Pod cleanup remains pending')),
|
|
});
|
|
await expect(closeHostChecks(checks)).resolves.toBe(false);
|
|
expect(warning).toHaveBeenCalled();
|
|
warning.mockRestore();
|
|
});
|
|
});
|