doctor only looked at the local services. A POC run also needs the cluster API to build and deploy, the Gitea branch to start from and push to, the public application to accept a release on, the Claude CLI and room on the disk; when one of them was down, doctor still said ready, and a wish found out minutes later, at its build. doctor now checks them read only, without a model call: the cluster's readiness, the branch head as builds read it and as integration reaches it over the operator's SSH login, the build commit the application reports and whether it is the branch head, the CLI's version and the free space where runs live. A failing one is named with the first line of its error and makes the report degraded. start does not wait on them, so a network problem cannot hold up starting the service. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
316 lines
12 KiB
JavaScript
316 lines
12 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { createServer } from 'node:http';
|
|
import { homedir } from 'node:os';
|
|
import path from 'node:path';
|
|
import {
|
|
checkLanOrigins,
|
|
checkRunnerHealth,
|
|
inspectLocalPoc,
|
|
inspectPocDependencies,
|
|
liveChildPids,
|
|
probeEndpoint,
|
|
waitForReadiness,
|
|
} from './local-poc-doctor.mjs';
|
|
|
|
const now = Date.parse('2026-10-25T00:00:00Z');
|
|
const recent = new Date(now - 1000).toISOString();
|
|
const supervisor = {
|
|
state: 'running',
|
|
pid: 10,
|
|
workspace: '/owned/wishforge',
|
|
children: [
|
|
{ name: 'platform', pid: 11 },
|
|
{ name: 'runner', pid: 12 },
|
|
],
|
|
};
|
|
const processList =
|
|
'11 10 node /owned/wishforge/scripts/dev.mjs\n12 10 node /owned/wishforge/apps/runner/src/main.ts\n';
|
|
const health = { schemaVersion: 1, pid: 12, status: 'idle', updatedAt: recent, lastPollSuccessAt: recent };
|
|
const interfaces = { en0: [{ family: 'IPv4', address: '192.168.6.207', internal: false }] };
|
|
const healthyFetch = async url => {
|
|
if (url.endsWith('/login')) return new Response('<title>WishForge</title><input name="username">');
|
|
if (url.endsWith('/readyz')) return Response.json({ status: 'ready' });
|
|
if (url.includes(':4000')) return Response.json({ status: 'ok', service: 'wishforge-control-plane' });
|
|
if (url.includes(':4450')) return Response.json({ service: 'wishforge-provisioner' });
|
|
return Response.json({ service: 'git-storage' });
|
|
};
|
|
|
|
test('launchd running is insufficient when the API is unhealthy', async () => {
|
|
const report = await inspectLocalPoc({
|
|
root: '/unused',
|
|
supervisor,
|
|
processList,
|
|
health,
|
|
env: {},
|
|
interfaces: {},
|
|
now,
|
|
fetcher: async url => (url.endsWith('/readyz') ? new Response('', { status: 503 }) : healthyFetch(url)),
|
|
});
|
|
assert.equal(report.ready, false);
|
|
assert.equal(report.services.database.httpStatus, 503);
|
|
});
|
|
|
|
test('a correct body identity is required and redirects are not followed', async () => {
|
|
assert.equal(
|
|
(
|
|
await probeEndpoint('http://localhost/health', { service: 'wishforge' }, async (_url, options) => {
|
|
assert.equal(options.redirect, 'manual');
|
|
return Response.json({ service: 'other-process' });
|
|
})
|
|
).status,
|
|
'unavailable',
|
|
);
|
|
assert.equal(
|
|
(await probeEndpoint('http://localhost/health', {}, async () => new Response('', { status: 302 }))).httpStatus,
|
|
302,
|
|
);
|
|
});
|
|
|
|
test('runner readiness requires a matching live process and successful recent authenticated polling', () => {
|
|
assert.equal(checkRunnerHealth(undefined, 12, now).status, 'unverified');
|
|
assert.equal(checkRunnerHealth(health, null, now).status, 'unavailable');
|
|
assert.equal(checkRunnerHealth(health, 99, now).status, 'unverified');
|
|
assert.equal(checkRunnerHealth({ ...health, lastPollSuccessAt: undefined }, 12, now).status, 'unverified');
|
|
assert.equal(checkRunnerHealth({ ...health, status: 'poll_error' }, 12, now).status, 'unavailable');
|
|
assert.equal(checkRunnerHealth(health, 12, now + 60000).status, 'unavailable');
|
|
assert.equal(checkRunnerHealth(health, 12, now).status, 'ready');
|
|
assert.equal(
|
|
checkRunnerHealth(
|
|
{ ...health, status: 'executing', lastPollSuccessAt: new Date(now - 600000).toISOString() },
|
|
12,
|
|
now,
|
|
).status,
|
|
'busy',
|
|
);
|
|
assert.equal(liveChildPids(supervisor, processList.replace('12 10', '12 99')).runner, null);
|
|
});
|
|
|
|
test('reports the changed LAN origin without authorizing or sending requests to it', async () => {
|
|
const env = {
|
|
WF_WEB_HOST: '0.0.0.0',
|
|
WF_LOCAL_ALLOWED_ORIGINS: 'http://172.20.10.5:3000',
|
|
WF_DEV_AUTH_TOKEN: 'never-print-fixture',
|
|
};
|
|
const lan = checkLanOrigins(env, interfaces);
|
|
assert.equal(lan.status, 'mismatch');
|
|
assert.deepEqual(lan.missingOrigins, ['http://192.168.6.207:3000']);
|
|
const requests = [];
|
|
const report = await inspectLocalPoc({
|
|
root: '/unused',
|
|
supervisor,
|
|
processList,
|
|
health,
|
|
env,
|
|
interfaces,
|
|
now,
|
|
fetcher: url => {
|
|
requests.push(url);
|
|
return healthyFetch(url);
|
|
},
|
|
});
|
|
assert.equal(report.status, 'degraded');
|
|
assert.equal(report.ready, true);
|
|
assert.ok(requests.every(url => url.startsWith('http://127.0.0.1:')));
|
|
assert.ok(!requests.some(url => url.includes('/runner/tasks')));
|
|
assert.ok(!JSON.stringify(report).includes('never-print-fixture'));
|
|
});
|
|
|
|
test('readiness waits for evidence and returns the last diagnosis at a bounded deadline', async () => {
|
|
let time = 0,
|
|
calls = 0;
|
|
const ready = await waitForReadiness(async () => ({ ready: ++calls === 3 }), {
|
|
timeoutMs: 30,
|
|
intervalMs: 10,
|
|
now: () => time,
|
|
sleep: async ms => {
|
|
time += ms;
|
|
},
|
|
});
|
|
assert.equal(ready.ready, true);
|
|
assert.equal(time, 20);
|
|
const failed = await waitForReadiness(async () => ({ ready: false, reason: 'runner unavailable' }), {
|
|
timeoutMs: 30,
|
|
intervalMs: 10,
|
|
now: () => time,
|
|
sleep: async ms => {
|
|
time += ms;
|
|
},
|
|
});
|
|
assert.equal(failed.reason, 'runner unavailable');
|
|
assert.equal(time, 50);
|
|
});
|
|
|
|
test('an approved current LAN interface is sufficient without approving VPN addresses', () => {
|
|
const env = { WF_WEB_HOST: '0.0.0.0', WF_LOCAL_ALLOWED_ORIGINS: 'http://192.168.6.207:3000' };
|
|
const lan = checkLanOrigins(env, {
|
|
...interfaces,
|
|
utun0: [{ family: 'IPv4', address: '10.251.1.1', internal: false }],
|
|
});
|
|
assert.equal(lan.status, 'configured');
|
|
assert.deepEqual(lan.currentOrigins, ['http://192.168.6.207:3000']);
|
|
});
|
|
|
|
test('detects runtime origin drift via a loopback request with the approved local Host', async () => {
|
|
const env = { WF_WEB_HOST: '0.0.0.0', WF_LOCAL_ALLOWED_ORIGINS: 'http://192.168.6.207:3000' };
|
|
const calls = [];
|
|
const report = await inspectLocalPoc({
|
|
root: '/unused',
|
|
supervisor,
|
|
processList,
|
|
health,
|
|
env,
|
|
interfaces,
|
|
now,
|
|
fetcher: (url, options) => {
|
|
calls.push({ url, host: options.headers?.Host });
|
|
return options.headers?.Host ? new Response('Forbidden', { status: 403 }) : healthyFetch(url);
|
|
},
|
|
});
|
|
assert.equal(report.lan.status, 'runtime-mismatch');
|
|
assert.equal(report.lan.runtime.httpStatus, 403);
|
|
assert.equal(report.status, 'degraded');
|
|
assert.ok(calls.some(call => call.host === '192.168.6.207:3000'));
|
|
assert.ok(calls.every(call => call.url.startsWith('http://127.0.0.1:')));
|
|
});
|
|
|
|
test('canonicalizes port 80 and respects an explicit interface binding', () => {
|
|
const env = { WF_WEB_HOST: '192.168.6.207', WF_WEB_PORT: '80', WF_LOCAL_ALLOWED_ORIGINS: 'http://192.168.6.207' };
|
|
assert.equal(checkLanOrigins(env, interfaces).status, 'configured');
|
|
assert.deepEqual(checkLanOrigins({ ...env, WF_WEB_HOST: '192.168.0.10' }, interfaces).approvedOrigins, []);
|
|
});
|
|
|
|
test('the real HTTP probe preserves Host, unlike Node fetch, and does not follow redirects', async () => {
|
|
const hosts = [];
|
|
const server = createServer((request, response) => {
|
|
hosts.push(request.headers.host);
|
|
response.writeHead(403);
|
|
response.end();
|
|
});
|
|
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
|
try {
|
|
const response = await probeEndpoint(`http://127.0.0.1:${server.address().port}/login`, 'web-login', fetch, 1000, {
|
|
Host: '192.168.6.207:3000',
|
|
});
|
|
assert.equal(response.httpStatus, 403);
|
|
assert.deepEqual(hosts, ['192.168.6.207:3000']);
|
|
} finally {
|
|
await new Promise(resolve => server.close(resolve));
|
|
}
|
|
});
|
|
|
|
const binding = { targetBranch: 'poc', repositoryUrl: 'https://gitea.example/team/app.git', gitHome: '/operator' };
|
|
const pocProfile = {
|
|
kubeconfig: '/operator/poc.config',
|
|
clusterServer: 'https://cluster.test:6443',
|
|
repository: 'git@gitea.example:team/app.git',
|
|
applicationUrl: 'http://poc.test',
|
|
};
|
|
|
|
test('checks what POC runs need beyond this machine, and only reads', async () => {
|
|
const head = 'a'.repeat(40);
|
|
const commands = [];
|
|
let pushEnv;
|
|
const run = async (executable, args, { env }) => {
|
|
commands.push([executable, ...args]);
|
|
if (executable === 'kubectl') return { exitCode: 0, stdout: 'ok', stderr: '' };
|
|
if (executable === 'git') {
|
|
if (args[1].startsWith('git@')) pushEnv = env;
|
|
return { exitCode: 0, stdout: `${head}\trefs/heads/poc\n`, stderr: '' };
|
|
}
|
|
return { exitCode: 0, stdout: '2.1.0 (Claude Code)\n', stderr: '' };
|
|
};
|
|
const fetcher = async url => {
|
|
assert.equal(String(url), 'http://poc.test/api/health');
|
|
return Response.json({ status: 'ok', buildCommit: head });
|
|
};
|
|
const report = await inspectPocDependencies({
|
|
env: { WF_CLAUDE_BIN: '/opt/claude', WF_LOCAL_RUN_ROOT: '/runs' },
|
|
binding,
|
|
profile: pocProfile,
|
|
run,
|
|
fetcher,
|
|
freeBytes: async () => 50 * 1024 ** 3,
|
|
});
|
|
assert.deepEqual(Object.fromEntries(Object.entries(report).map(([name, item]) => [name, item.status])), {
|
|
cluster: 'ready',
|
|
repository: 'ready',
|
|
application: 'ready',
|
|
executor: 'ready',
|
|
disk: 'ready',
|
|
});
|
|
assert.equal(report.application.runsBranchHead, true);
|
|
assert.equal(report.executor.version, '2.1.0 (Claude Code)');
|
|
// The cluster's readiness, the branch head as builds and as pushes reach it, and the CLI's version.
|
|
assert.deepEqual(commands, [
|
|
['kubectl', '--kubeconfig', '/operator/poc.config', '--request-timeout=5s', 'get', '--raw', '/readyz'],
|
|
['git', 'ls-remote', 'https://gitea.example/team/app.git', 'refs/heads/poc'],
|
|
['git', 'ls-remote', 'git@gitea.example:team/app.git', 'refs/heads/poc'],
|
|
['/opt/claude', '--version'],
|
|
]);
|
|
assert.equal(pushEnv.HOME, '/operator');
|
|
assert.equal(pushEnv.GIT_CONFIG_GLOBAL, '/dev/null');
|
|
assert.match(pushEnv.GIT_SSH_COMMAND, /BatchMode=yes/);
|
|
});
|
|
|
|
test('names the dependency that fails, and an application behind its branch', async () => {
|
|
const run = async (executable, args) => {
|
|
if (executable === 'kubectl')
|
|
return {
|
|
exitCode: 1,
|
|
stdout: '',
|
|
stderr: 'Unable to connect to the server: dial tcp 10.0.0.1:6443: i/o timeout\n',
|
|
};
|
|
if (executable === 'git' && args[1].startsWith('git@'))
|
|
return {
|
|
exitCode: 128,
|
|
stdout: '',
|
|
stderr: 'git@gitea.example: Permission denied (publickey).\nfatal: Could not read from remote repository.',
|
|
};
|
|
if (executable === 'git') return { exitCode: 0, stdout: `${'b'.repeat(40)}\trefs/heads/poc\n`, stderr: '' };
|
|
return { exitCode: 0, stdout: '2.1.0\n', stderr: '' };
|
|
};
|
|
const report = await inspectPocDependencies({
|
|
env: { WF_CLAUDE_BIN: '/opt/claude' },
|
|
binding,
|
|
profile: pocProfile,
|
|
run,
|
|
// People pushed to the branch after the last deployment.
|
|
fetcher: async () => Response.json({ status: 'ok', buildCommit: 'a'.repeat(40) }),
|
|
freeBytes: async () => 2 * 1024 ** 3,
|
|
});
|
|
assert.deepEqual(report.cluster, {
|
|
status: 'unavailable',
|
|
server: 'https://cluster.test:6443',
|
|
reason: 'Unable to connect to the server: dial tcp 10.0.0.1:6443: i/o timeout',
|
|
});
|
|
assert.deepEqual(report.repository, {
|
|
status: 'unavailable',
|
|
branch: 'poc',
|
|
head: 'b'.repeat(40),
|
|
read: 'ready',
|
|
push: 'git@gitea.example: Permission denied (publickey).',
|
|
});
|
|
assert.equal(report.application.runsBranchHead, false);
|
|
assert.deepEqual(report.disk, {
|
|
status: 'low',
|
|
path: path.join(homedir(), '.local/share/wishforge/runs'),
|
|
freeGiB: 2,
|
|
});
|
|
// An application that answers without its build commit is not taken for ready.
|
|
const html = await inspectPocDependencies({
|
|
env: {},
|
|
binding,
|
|
profile: pocProfile,
|
|
run,
|
|
fetcher: async () => new Response('<html>Bad gateway</html>', { status: 502 }),
|
|
freeBytes: async () => 50 * 1024 ** 3,
|
|
});
|
|
assert.deepEqual(html.application, {
|
|
status: 'unavailable',
|
|
url: 'http://poc.test',
|
|
reason: 'Health returned 502 without a build commit',
|
|
});
|
|
assert.deepEqual(html.executor, { status: 'unavailable', reason: 'WF_CLAUDE_BIN is not set' });
|
|
});
|