doctor only looked at the local services. A POC run also needs the cluster API to build and deploy, the Gitea branch to start from and push to, the public application to accept a release on, the Claude CLI and room on the disk; when one of them was down, doctor still said ready, and a wish found out minutes later, at its build. doctor now checks them read only, without a model call: the cluster's readiness, the branch head as builds read it and as integration reaches it over the operator's SSH login, the build commit the application reports and whether it is the branch head, the CLI's version and the free space where runs live. A failing one is named with the first line of its error and makes the report degraded. start does not wait on them, so a network problem cannot hold up starting the service. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
390 lines
16 KiB
JavaScript
390 lines
16 KiB
JavaScript
// Read-only diagnostics. Never poll /internal/runner/tasks: that endpoint claims deployment work.
|
|
import { readFile, stat, statfs } from 'node:fs/promises';
|
|
import { homedir, networkInterfaces } from 'node:os';
|
|
import path from 'node:path';
|
|
import { parseEnv } from 'node:util';
|
|
import { execFile, spawnSync } from 'node:child_process';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { request as httpRequest } from 'node:http';
|
|
import { Readable } from 'node:stream';
|
|
|
|
export function checkRunnerHealth(health, pid, now = Date.now()) {
|
|
if (!pid) return { status: 'unavailable', reason: 'No matching live runner process' };
|
|
if (!health || health.schemaVersion !== 1 || health.pid !== pid)
|
|
return { status: 'unverified', reason: 'No health snapshot for the current runner process' };
|
|
const updated = Date.parse(health.updatedAt),
|
|
polled = Date.parse(health.lastPollSuccessAt);
|
|
if (!Number.isFinite(updated) || updated > now + 5000 || now - updated > 30000)
|
|
return { status: 'unavailable', reason: 'Runner health snapshot is stale' };
|
|
if (health.status === 'poll_error')
|
|
return { status: 'unavailable', reason: 'Runner cannot complete authenticated polling' };
|
|
if (health.status === 'task_error')
|
|
return { status: 'unavailable', reason: 'Runner task or receipt failed; reconciliation is required' };
|
|
if (!Number.isFinite(polled) || polled > now + 5000)
|
|
return { status: 'unverified', reason: 'Runner has not completed authenticated polling yet' };
|
|
if (health.status === 'executing') return { status: 'busy', lastPollSuccessAt: health.lastPollSuccessAt };
|
|
if (health.status !== 'idle' || now - polled > 30000)
|
|
return { status: 'unavailable', reason: 'Runner has no recent successful authenticated poll' };
|
|
return { status: 'ready', lastPollSuccessAt: health.lastPollSuccessAt };
|
|
}
|
|
|
|
export function checkLanOrigins(env, interfaces = networkInterfaces()) {
|
|
const port = String(env.WF_WEB_PORT || '3000');
|
|
const binding = env.WF_WEB_HOST || '127.0.0.1';
|
|
const addresses = [
|
|
...new Set(
|
|
Object.entries(interfaces)
|
|
.filter(([name]) => !/^(utun|tun|tap|wg|ppp|ipsec)/i.test(name))
|
|
.flatMap(([, items]) =>
|
|
(items ?? [])
|
|
.filter(
|
|
item =>
|
|
!item.internal &&
|
|
item.family === 'IPv4' &&
|
|
/^(10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.)/.test(item.address),
|
|
)
|
|
.map(item => item.address),
|
|
),
|
|
),
|
|
]
|
|
.filter(address => ['0.0.0.0', '::'].includes(binding) || address === binding)
|
|
.sort();
|
|
const allowed = (env.WF_LOCAL_ALLOWED_ORIGINS ?? '')
|
|
.split(',')
|
|
.map(item => item.trim())
|
|
.filter(Boolean);
|
|
const origins = addresses.map(address => new URL(`http://${address}:${port}`).origin);
|
|
const missingOrigins = origins.filter(origin => !allowed.includes(origin));
|
|
const approvedOrigins = origins.filter(origin => allowed.includes(origin));
|
|
const bindsLan = !['127.0.0.1', 'localhost', '::1'].includes(env.WF_WEB_HOST || '127.0.0.1');
|
|
return {
|
|
status: !bindsLan
|
|
? 'loopback-only'
|
|
: approvedOrigins.length
|
|
? 'configured'
|
|
: origins.length
|
|
? 'mismatch'
|
|
: 'unverified',
|
|
currentOrigins: origins,
|
|
approvedOrigins,
|
|
missingOrigins,
|
|
note: 'VPN interfaces excluded; one approved current LAN address suffices. External reachability still requires a client check',
|
|
};
|
|
}
|
|
|
|
export function liveChildPids(supervisor, processList) {
|
|
const records = processList.split('\n').flatMap(line => {
|
|
const match = line.trim().match(/^(\d+)\s+(\d+)\s+(.+)$/);
|
|
return match ? [{ pid: Number(match[1]), parent: Number(match[2]), command: match[3] }] : [];
|
|
});
|
|
return Object.fromEntries(
|
|
['platform', 'runner'].map(name => {
|
|
const recorded = supervisor.children?.find(child => child.name === name);
|
|
const expected = path.join(
|
|
supervisor.workspace,
|
|
name === 'platform' ? 'scripts/dev.mjs' : 'apps/runner/src/main.ts',
|
|
);
|
|
const live =
|
|
supervisor.state === 'running' &&
|
|
recorded &&
|
|
records.some(
|
|
record =>
|
|
record.pid === recorded.pid && record.parent === supervisor.pid && record.command.includes(expected),
|
|
);
|
|
return [name, live ? recorded.pid : null];
|
|
}),
|
|
);
|
|
}
|
|
|
|
async function bodyText(response) {
|
|
if (!response.body) return '';
|
|
const reader = response.body.getReader();
|
|
const chunks = [];
|
|
let bytes = 0;
|
|
try {
|
|
while (true) {
|
|
const next = await reader.read();
|
|
if (next.done) break;
|
|
bytes += next.value.length;
|
|
if (bytes > 128 * 1024) throw new Error('Oversized health response');
|
|
chunks.push(Buffer.from(next.value));
|
|
}
|
|
return Buffer.concat(chunks).toString('utf8');
|
|
} finally {
|
|
await reader.cancel().catch(() => undefined);
|
|
}
|
|
}
|
|
|
|
export async function probeEndpoint(url, expected, fetcher = fetch, timeoutMs = 1500, headers = {}) {
|
|
try {
|
|
// Node's fetch replaces custom Host headers. Use the HTTP client for this local
|
|
// request so the diagnostic exercises the same origin check as a LAN browser.
|
|
const request = headers.Host && fetcher === fetch ? loopbackHostRequest : fetcher;
|
|
const response = await request(url, { redirect: 'manual', signal: AbortSignal.timeout(timeoutMs), headers });
|
|
if (response.status !== 200) {
|
|
await response.body?.cancel().catch(() => undefined);
|
|
return { status: 'unavailable', httpStatus: response.status };
|
|
}
|
|
const text = await bodyText(response);
|
|
const matches =
|
|
expected === 'web-login'
|
|
? text.includes('WishForge') && text.includes('name="username"')
|
|
: Object.entries(expected).every(([key, value]) => JSON.parse(text)[key] === value);
|
|
return matches
|
|
? { status: 'ready', httpStatus: 200 }
|
|
: { status: 'unavailable', reason: 'Response identity or readiness does not match' };
|
|
} catch {
|
|
return { status: 'unavailable', reason: 'Request failed, timed out or returned an invalid health response' };
|
|
}
|
|
}
|
|
|
|
export function loopbackHostRequest(url, options) {
|
|
return new Promise((resolve, reject) => {
|
|
const target = new URL(url);
|
|
if (target.protocol !== 'http:' || target.hostname !== '127.0.0.1' || target.username || target.password)
|
|
return reject(new Error('LAN origin diagnostics only connect to the IPv4 loopback listener'));
|
|
const request = httpRequest(
|
|
target,
|
|
{ method: 'GET', headers: options.headers, signal: options.signal },
|
|
response => {
|
|
resolve(new Response(Readable.toWeb(response), { status: response.statusCode }));
|
|
},
|
|
);
|
|
request.once('error', reject);
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
export async function inspectLocalPoc({
|
|
root,
|
|
supervisor,
|
|
env,
|
|
runnerEnv,
|
|
processList,
|
|
health,
|
|
interfaces,
|
|
fetcher = fetch,
|
|
now = Date.now(),
|
|
}) {
|
|
const port = Number(env.WF_WEB_PORT || 3000);
|
|
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error('Invalid local web port');
|
|
const children = liveChildPids(supervisor, processList);
|
|
const targets = {
|
|
web: [`http://127.0.0.1:${port}/login`, 'web-login'],
|
|
api: ['http://127.0.0.1:4000/health', { service: 'wishforge-control-plane', status: 'ok' }],
|
|
database: ['http://127.0.0.1:4000/readyz', { status: 'ready' }],
|
|
provisioner: ['http://127.0.0.1:4450/healthz', { service: 'wishforge-provisioner' }],
|
|
git: ['http://127.0.0.1:4504/health', { service: 'git-storage' }],
|
|
};
|
|
const services = Object.fromEntries(
|
|
await Promise.all(
|
|
Object.entries(targets).map(async ([name, [url, expected]]) => [
|
|
name,
|
|
await probeEndpoint(url, expected, fetcher),
|
|
]),
|
|
),
|
|
);
|
|
const runner = checkRunnerHealth(health, children.runner, now);
|
|
let build = { status: 'not-required' };
|
|
if (env.WF_WEB_SERVE_MODE === 'production') {
|
|
const buildId = await readFile(path.join(root, 'apps/web/.next/BUILD_ID'), 'utf8').catch(() => '');
|
|
build = buildId.trim()
|
|
? { status: 'ready', buildId: buildId.trim().slice(0, 200) }
|
|
: { status: 'unavailable', reason: 'No completed production web build' };
|
|
}
|
|
const coreReady =
|
|
supervisor.state === 'running' &&
|
|
Boolean(children.platform) &&
|
|
Object.values(services).every(service => service.status === 'ready') &&
|
|
build.status !== 'unavailable';
|
|
const ready = coreReady && ['ready', 'busy'].includes(runner.status);
|
|
const lan = checkLanOrigins(env, interfaces);
|
|
if (lan.status === 'configured') {
|
|
lan.runtimeOrigin = lan.approvedOrigins[0];
|
|
lan.runtime = await probeEndpoint(`http://127.0.0.1:${port}/login`, 'web-login', fetcher, 1500, {
|
|
Host: new URL(lan.runtimeOrigin).host,
|
|
});
|
|
if (lan.runtime.status !== 'ready') lan.status = 'runtime-mismatch';
|
|
}
|
|
return {
|
|
checkedAt: new Date(now).toISOString(),
|
|
ready,
|
|
status: ready ? (['mismatch', 'runtime-mismatch'].includes(lan.status) ? 'degraded' : 'ready') : 'not-ready',
|
|
supervisor,
|
|
children,
|
|
services,
|
|
runner,
|
|
build,
|
|
lan,
|
|
note: 'Read-only checks; no task is claimed and no model or deployment is started',
|
|
};
|
|
}
|
|
|
|
/** A read-only command with a time limit and no shell. Its failure is reported, never thrown. */
|
|
export function runReadOnly(executable, args, { env, timeoutMs }) {
|
|
return new Promise(resolve => {
|
|
execFile(
|
|
executable,
|
|
args,
|
|
{ env, timeout: timeoutMs, maxBuffer: 256 * 1024, encoding: 'utf8' },
|
|
(error, stdout, stderr) =>
|
|
resolve({
|
|
exitCode: error ? (typeof error.code === 'number' ? error.code : 1) : 0,
|
|
stdout: stdout ?? '',
|
|
stderr: stderr || (error ? error.message : ''),
|
|
}),
|
|
);
|
|
});
|
|
}
|
|
|
|
const firstLine = text =>
|
|
String(text ?? '')
|
|
.trim()
|
|
.split('\n')[0]
|
|
.slice(0, 300);
|
|
/** Below this, a few builds and their installed dependencies may not fit. */
|
|
const LOW_DISK_BYTES = 5 * 1024 ** 3;
|
|
|
|
/**
|
|
* What POC runs need beyond this machine, checked read only and without a model call: the
|
|
* cluster API that builds and deploys, the Gitea branch as builds read it and as integration
|
|
* reaches it over the operator's SSH login, the public application releases are accepted on,
|
|
* the Claude CLI, and room on the disk that holds runs. Also says whether the application runs
|
|
* the branch head: it does not once people pushed commits that no run has deployed yet.
|
|
*/
|
|
export async function inspectPocDependencies({
|
|
env,
|
|
binding,
|
|
profile,
|
|
run = runReadOnly,
|
|
fetcher = fetch,
|
|
freeBytes = async directory => {
|
|
const info = await statfs(directory);
|
|
return info.bavail * info.bsize;
|
|
},
|
|
}) {
|
|
const ref = `refs/heads/${binding.targetBranch}`;
|
|
const shell = { PATH: process.env.PATH, HOME: homedir(), LANG: 'C.UTF-8' };
|
|
const git = { ...shell, GIT_TERMINAL_PROMPT: '0' };
|
|
const headOf = result => (result.exitCode === 0 ? /^([0-9a-f]{40})\s/.exec(result.stdout)?.[1] : undefined);
|
|
const runRoot = env.WF_LOCAL_RUN_ROOT || path.join(homedir(), '.local/share/wishforge/runs');
|
|
const [cluster, read, push, health, claude, free] = await Promise.all([
|
|
run('kubectl', ['--kubeconfig', profile.kubeconfig, '--request-timeout=5s', 'get', '--raw', '/readyz'], {
|
|
env: shell,
|
|
timeoutMs: 15_000,
|
|
}),
|
|
// As builds read the branch: the operator's Git configuration and credentials.
|
|
run('git', ['ls-remote', binding.repositoryUrl, ref], { env: git, timeoutMs: 30_000 }),
|
|
run('git', ['ls-remote', profile.repository, ref], {
|
|
// As integration pushes: the operator's SSH login, without the global Git config.
|
|
env: {
|
|
...git,
|
|
HOME: binding.gitHome,
|
|
GIT_CONFIG_GLOBAL: '/dev/null',
|
|
GIT_CONFIG_NOSYSTEM: '1',
|
|
GIT_SSH_COMMAND: 'ssh -oBatchMode=yes -oStrictHostKeyChecking=yes',
|
|
},
|
|
timeoutMs: 30_000,
|
|
}),
|
|
fetcher(new URL('/api/health', profile.applicationUrl), { redirect: 'error', signal: AbortSignal.timeout(10_000) })
|
|
.then(async response => {
|
|
const text = await bodyText(response).catch(() => '');
|
|
let body;
|
|
try {
|
|
body = JSON.parse(text);
|
|
} catch {
|
|
body = undefined;
|
|
}
|
|
return { httpStatus: response.status, body };
|
|
})
|
|
.catch(() => undefined),
|
|
env.WF_CLAUDE_BIN ? run(env.WF_CLAUDE_BIN, ['--version'], { env: shell, timeoutMs: 15_000 }) : undefined,
|
|
freeBytes(runRoot).catch(() => undefined),
|
|
]);
|
|
const head = headOf(read);
|
|
const buildCommit = /^[0-9a-f]{40}$/.test(health?.body?.buildCommit ?? '') ? health.body.buildCommit : undefined;
|
|
return {
|
|
cluster:
|
|
cluster.exitCode === 0 && cluster.stdout.trim() === 'ok'
|
|
? { status: 'ready', server: profile.clusterServer }
|
|
: { status: 'unavailable', server: profile.clusterServer, reason: firstLine(cluster.stderr) || 'No answer' },
|
|
repository: {
|
|
status: head && headOf(push) ? 'ready' : 'unavailable',
|
|
branch: binding.targetBranch,
|
|
...(head ? { head } : {}),
|
|
read: head ? 'ready' : firstLine(read.stderr) || 'unavailable',
|
|
push: headOf(push) ? 'ready' : firstLine(push.stderr) || 'unavailable',
|
|
},
|
|
application: buildCommit
|
|
? {
|
|
status: 'ready',
|
|
url: profile.applicationUrl,
|
|
buildCommit,
|
|
...(head ? { runsBranchHead: buildCommit === head } : {}),
|
|
}
|
|
: {
|
|
status: 'unavailable',
|
|
url: profile.applicationUrl,
|
|
reason: health ? `Health returned ${health.httpStatus} without a build commit` : 'No answer',
|
|
},
|
|
executor: !env.WF_CLAUDE_BIN
|
|
? { status: 'unavailable', reason: 'WF_CLAUDE_BIN is not set' }
|
|
: claude.exitCode === 0
|
|
? { status: 'ready', version: firstLine(claude.stdout) }
|
|
: { status: 'unavailable', reason: firstLine(claude.stderr) || 'The Claude CLI did not start' },
|
|
disk:
|
|
free === undefined
|
|
? { status: 'unavailable', path: runRoot, reason: 'Free space unknown' }
|
|
: { status: free >= LOW_DISK_BYTES ? 'ready' : 'low', path: runRoot, freeGiB: Math.floor(free / 1024 ** 3) },
|
|
};
|
|
}
|
|
|
|
export async function localPocDoctor(root, supervisor, options = {}) {
|
|
const env = parseEnv(await readFile(path.join(root, '.env.local'), 'utf8'));
|
|
const runnerEnv = parseEnv(await readFile(path.join(root, '.wishforge/local-poc/runner.env'), 'utf8'));
|
|
let health;
|
|
if (path.isAbsolute(runnerEnv.WF_RUNNER_STATE_DIR || '')) {
|
|
const file = path.join(runnerEnv.WF_RUNNER_STATE_DIR, 'health.json');
|
|
const info = await stat(file).catch(() => undefined);
|
|
if (info?.isFile() && info.size <= 4096 && info.uid === process.getuid() && !(info.mode & 0o077))
|
|
health = await readFile(file, 'utf8')
|
|
.then(JSON.parse)
|
|
.catch(() => undefined);
|
|
}
|
|
const processes = spawnSync('/bin/ps', ['-axo', 'pid=,ppid=,command='], { encoding: 'utf8', timeout: 3000 });
|
|
const { dependencies, ...inspect } = options;
|
|
const report = await inspectLocalPoc({
|
|
root,
|
|
supervisor,
|
|
env,
|
|
runnerEnv,
|
|
health,
|
|
processList: processes.status === 0 ? processes.stdout : '',
|
|
...inspect,
|
|
});
|
|
if (!dependencies) return report;
|
|
const binding = JSON.parse(
|
|
await readFile(env.WF_LOCAL_POC_BINDING_FILE || path.join(root, '.wishforge/local-poc/project.json'), 'utf8'),
|
|
);
|
|
const profile = JSON.parse(await readFile(binding.profileFile, 'utf8'));
|
|
report.dependencies = await inspectPocDependencies({ env, binding, profile });
|
|
if (report.status === 'ready' && Object.values(report.dependencies).some(item => item.status !== 'ready'))
|
|
report.status = 'degraded';
|
|
return report;
|
|
}
|
|
|
|
export async function waitForReadiness(
|
|
inspect,
|
|
{ timeoutMs = 90000, intervalMs = 1000, now = Date.now, sleep = delay } = {},
|
|
) {
|
|
const deadline = now() + timeoutMs;
|
|
let report;
|
|
do {
|
|
report = await inspect();
|
|
if (report.ready) return report;
|
|
if (now() >= deadline) return report;
|
|
await sleep(Math.min(intervalMs, Math.max(0, deadline - now())));
|
|
} while (now() <= deadline);
|
|
return report;
|
|
}
|