wishforge-ide/scripts/check-host-check-sandbox.ts
eust-w 3623d318af ✨ Sandbox the local POC's protected checks and installations on macOS, by operator opt-in
In the development host mode, protected checks and dependency
installations ran directly as the operator. They run the agent's code:
a test it wrote could read the home directory's secrets, reach the
network, or plant Git configuration that the trusted Git commands after
it would execute.

With WF_HOST_CHECK_SANDBOX=true they run under Seatbelt instead. File
contents in the home directory are unreadable except the command's own
directory and the paths the operator lists; it may write only there,
in the caches the operator lists and in a temporary directory of its
own, never in a .git directory. Only installations reach the network;
checks may use loopback ports by opt-in, for test suites that start
local servers. The platform's own ports (control plane, Temporal and
its web UI, database, object storage, provisioner, Git service, IDE,
console) stay blocked in any case: the local start lists them from
their configured addresses. Checks get no answer from the keychain,
and a command whose home is the executor's (holding its credentials)
gets a private one. A sandbox that cannot start leaves the run waiting
on infrastructure; verification evidence records checkSandbox.

scripts/check-host-check-sandbox.ts checks all of this on the target
Mac, from outside the sandbox, then runs the POC's installation and
checks sandboxed on a recent droboticflow source. Enabled on the local
POC, a real wish passed through both, including a repair round for the
agent's own failing tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 10:10:47 +08:00

237 lines
11 KiB
TypeScript

/**
* Checks the macOS sandbox for protected host checks (WF_HOST_CHECK_SANDBOX) on this machine,
* before it is switched on: fixed probes run sandboxed and their effects are checked here, not
* taken from their own account; then the POC's protected installation and checks run sandboxed on
* a droboticflow source archive. No model is called and nothing is deployed:
*
* node --env-file=.env.local --import tsx scripts/check-host-check-sandbox.ts [source.tar.gz]
*
* Without an archive, the source of the newest POC build is used.
*/
import { execFileSync } from 'node:child_process';
import { randomBytes } from 'node:crypto';
import { access, mkdir, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
import http from 'node:http';
import type { AddressInfo } from 'node:net';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { hostCheckSandbox } from '../apps/worker/src/host-checks.js';
import { platformPorts } from './local-services.mjs';
import {
executeCommand,
outputTail,
seatbeltCheckExecutor,
type CommandResult,
} from '../packages/runtime/src/index.js';
const repository = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const pocCheck = path.join(repository, 'scripts/poc-project-check.mjs');
const pocCache = path.join(repository, '.wishforge/poc-check-cache');
// As the local start configures the workers: the platform's ports from its addresses.
const configured = hostCheckSandbox({
...process.env,
WF_HOST_CHECK_SANDBOX: 'true',
WF_HOST_CHECK_BLOCKED_PORTS: platformPorts(process.env).join(','),
})!;
// Without the operator's settings, check what the POC needs, and say so: its Go tests start local servers.
const sandbox = {
...configured,
readable: configured.readable.length ? configured.readable : [pocCheck],
// droboticflow's Go tests write under /tmp/workspace.
writable: configured.writable.length ? configured.writable : [pocCache, '/tmp/workspace'],
loopback: process.env.WF_HOST_CHECK_LOOPBACK === undefined ? true : configured.loopback,
};
console.log(
`Checking WF_HOST_CHECK_READ_PATHS=${sandbox.readable.join(path.delimiter)} WF_HOST_CHECK_WRITE_PATHS=${sandbox.writable.join(path.delimiter)} WF_HOST_CHECK_LOOPBACK=${sandbox.loopback}, blocked ports ${sandbox.blockedPorts?.join(',')}\n`,
);
const secret = () => randomBytes(12).toString('hex');
const canary = { file: `file-${secret()}`, service: `service-${secret()}` };
// Like a run directory: in the home directory, outside the WishForge source.
const root = path.join(os.homedir(), '.local/share/wishforge/sandbox-check', `host-${secret()}`);
const tree = path.join(root, 'tree');
const outside = path.join(root, 'outside');
await mkdir(path.join(tree, '.git'), { recursive: true, mode: 0o700 });
await mkdir(outside, { recursive: true, mode: 0o700 });
await writeFile(path.join(outside, 'canary.txt'), canary.file);
await writeFile(path.join(tree, '.git/config'), '[core]\n\tbare = false\n');
await writeFile(path.join(tree, 'inside.txt'), 'inside');
await writeFile(
path.join(tree, 'sum.test.mjs'),
"import test from 'node:test';\nimport assert from 'node:assert/strict';\ntest('adds', () => assert.equal(2 + 3, 5));\n",
);
const server = http.createServer((_request, response) => response.end(canary.service));
await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));
const port = (server.address() as AddressInfo).port;
const execute = seatbeltCheckExecutor(sandbox, executeCommand);
const env = { PATH: '/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin', LANG: 'C.UTF-8', CI: 'true', HOME: tree };
const node = (script: string, network = false) =>
execute({ executable: process.execPath, args: ['-e', script], cwd: tree, env, timeoutMs: 60_000, network });
const text = (result: CommandResult) => `${result.stdout}\n${result.stderr}`;
const exists = (location: string) =>
access(location).then(
() => true,
() => false,
);
type Verdict = 'pass' | 'FAIL' | 'warn';
const results: Array<[string, Verdict, string]> = [];
const record = (label: string, verdict: Verdict, detail: string) => results.push([label, verdict, detail]);
const first = (result: CommandResult) => text(result).trim().split('\n')[0]!.slice(0, 120);
try {
const tests = await execute({
executable: process.execPath,
args: ['--test', 'sum.test.mjs'],
cwd: tree,
env,
timeoutMs: 60_000,
});
record('Tests run inside the sandbox', tests.exitCode === 0 ? 'pass' : 'FAIL', first(tests));
const read = await node(
`process.stdout.write(require('fs').readFileSync(${JSON.stringify(path.join(outside, 'canary.txt'))}, 'utf8'))`,
);
record(
'Files elsewhere in the home directory stay unreadable',
text(read).includes(canary.file) ? 'FAIL' : 'pass',
first(read),
);
const own = await node(`process.stdout.write(require('fs').readFileSync('inside.txt', 'utf8'))`);
record('Its own files are readable', own.stdout === 'inside' ? 'pass' : 'FAIL', first(own));
const written = path.join(outside, 'written.txt');
const write = await node(`require('fs').writeFileSync(${JSON.stringify(written)}, 'x')`);
record('Writes outside its directory are refused', (await exists(written)) ? 'FAIL' : 'pass', first(write));
const git = await node(`require('fs').writeFileSync('.git/config', '[core]\\n\\tfsmonitor = evil\\n')`);
const gitConfig = await readFile(path.join(tree, '.git/config'), 'utf8');
record('Its .git directory stays unwritable', gitConfig.includes('evil') ? 'FAIL' : 'pass', first(git));
const temporary = await node(
`const fs = require('fs'); const file = require('path').join(require('os').tmpdir(), 'probe.txt'); fs.writeFileSync(file, 'x'); process.stdout.write(file)`,
);
record(
'A private temporary directory is writable, and removed afterwards',
temporary.exitCode === 0 && !(await exists(temporary.stdout)) ? 'pass' : 'FAIL',
temporary.stdout || first(temporary),
);
const network = await node(
`fetch('https://registry.npmjs.org/', { method: 'HEAD' }).then(r => console.log('status', r.status), e => console.log(String(e.cause ?? e)))`,
);
record('Checks have no network', /status \d+/.test(network.stdout) ? 'FAIL' : 'pass', first(network));
const installing = await node(
`fetch('https://registry.npmjs.org/', { method: 'HEAD' }).then(r => console.log('status', r.status), e => console.log(String(e.cause ?? e)))`,
true,
);
record(
'Installations reach the network',
/status \d+/.test(installing.stdout) ? 'pass' : 'warn',
`${first(installing)} (needs this machine's own network)`,
);
const local = await node(
`require('http').get('http://127.0.0.1:${port}/', r => { r.setEncoding('utf8'); r.on('data', d => process.stdout.write(d)); }).on('error', e => console.log(String(e)))`,
);
const reached = text(local).includes(canary.service);
record(
sandbox.loopback ? 'Local ports are reachable (WF_HOST_CHECK_LOOPBACK)' : 'Local ports stay unreachable',
reached === Boolean(sandbox.loopback) ? 'pass' : 'FAIL',
first(local),
);
const api = new URL(process.env.WF_API_URL ?? 'http://127.0.0.1:4000');
const platform = await node(
`require('http').get('http://127.0.0.1:${api.port || 80}/health', r => console.log('status', r.statusCode)).on('error', e => console.log(String(e)))`,
);
record(
`The platform's own ports stay unreachable (API ${api.port})`,
/status \d+/.test(platform.stdout) ? 'FAIL' : 'pass',
first(platform),
);
// A throwaway keychain with an item the security tool may read: outside the sandbox it answers.
const keychainFile = path.join(tree, 'check.keychain-db');
const item = ['find-generic-password', '-s', 'wishforge-sandbox-check', '-w', keychainFile];
const security = (args: string[]) =>
executeCommand({ executable: '/usr/bin/security', args, env: { PATH: '/usr/bin:/bin' }, timeoutMs: 30_000 });
const password = secret();
try {
await security(['create-keychain', '-p', password, keychainFile]);
await security([
'add-generic-password',
'-s',
'wishforge-sandbox-check',
'-a',
'check',
'-w',
canary.file,
'-T',
'/usr/bin/security',
keychainFile,
]);
const control = await security(item);
const sandboxed = await execute({ executable: '/usr/bin/security', args: item, cwd: tree, env, timeoutMs: 30_000 });
record(
'The keychain does not answer checks',
!control.stdout.includes(canary.file) ? 'warn' : sandboxed.stdout.includes(canary.file) ? 'FAIL' : 'pass',
control.stdout.includes(canary.file) ? first(sandboxed) : `the control read failed: ${first(control)}`,
);
} finally {
await security(['delete-keychain', keychainFile]);
}
for (const location of sandbox.readable) {
const readable = await node(
`require('fs').statSync(${JSON.stringify(location)}).isDirectory() ? require('fs').readdirSync(${JSON.stringify(location)}) : require('fs').readFileSync(${JSON.stringify(location)})`,
);
record(`Allowed to read ${location}`, readable.exitCode === 0 ? 'pass' : 'FAIL', first(readable));
}
} finally {
server.close();
}
// The POC's own protected installation and checks, on real droboticflow source.
let archive: string | undefined = process.argv[2];
if (!archive) {
const builds = path.join(os.homedir(), '.local/share/wishforge/poc/builds');
const candidates = await Promise.all(
(await readdir(builds).catch(() => [])).map(async name => {
const file = path.join(builds, name, 'source.tar.gz');
return stat(file).then(
info => ({ file, time: info.mtimeMs }),
() => undefined,
);
}),
);
archive = candidates.filter(Boolean).sort((a, b) => b!.time - a!.time)[0]?.file;
}
if (!archive) record('POC checks', 'warn', 'no droboticflow source archive found; pass one as an argument');
else {
const poc = path.join(root, 'poc');
await mkdir(poc, { recursive: true, mode: 0o700 });
execFileSync('tar', ['-xzf', archive, '-C', poc]);
for (const mode of ['install', 'verify'] as const) {
const started = Date.now();
const result = await execute({
executable: process.execPath,
args: [pocCheck, mode],
cwd: poc,
env,
timeoutMs: (mode === 'install' ? 15 : 30) * 60_000,
network: mode === 'install',
});
const seconds = Math.round((Date.now() - started) / 1000);
record(
`POC ${mode} runs sandboxed`,
result.exitCode === 0 ? 'pass' : 'FAIL',
result.exitCode === 0
? `${seconds} s`
: `exit ${result.exitCode} after ${seconds} s:\n${outputTail(result, 2000)}`,
);
if (result.exitCode !== 0) break;
}
}
for (const [label, verdict, detail] of results) console.log(`${verdict.padEnd(5)} ${label}: ${detail}`);
const passed = results.every(([, verdict]) => verdict !== 'FAIL');
if (passed) await rm(root, { recursive: true, force: true });
else console.log(`\nKept for inspection: ${root}`);
process.exitCode = passed ? 0 : 1;