wishforge-ide/scripts/k3s-local.mjs
eust-w 71cc6de5e2 fix: keep the local cluster from filling the Mac's disk
Repeated builds grew the VM's disk to 63 GB, until the Mac's disk was
full. Writes inside the VM then failed and its file systems broke.

- The service image installed dependencies after copying every source
  file, so each source change rebuilt a 750 MB layer, five images at a
  time. It now installs from the workspace manifests first, and a test
  keeps every workspace listed.
- The VM's disk defaults to 40 GiB instead of 80.
- `up` refuses to create the VM when the Mac has less free space than
  that, and `build` refuses below 15 GiB free.
- After building, superseded images are pruned and the build cache is
  capped at 10 GB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 08:03:41 +08:00

537 lines
23 KiB
JavaScript

// Runs WishForge on a local k3s cluster (a Colima VM) for development, end to end: wishes are
// executed by the Kubernetes provisioner in per-run pods, verified and merged.
//
// node scripts/k3s-local.mjs up start the colima-wishforge VM with k3s and a local registry
// node scripts/k3s-local.mjs build build the service images and push them to the local registry
// (`build web worker` rebuilds only those)
// node scripts/k3s-local.mjs deploy render infra/k8s/local and apply it (needs WF_CODEX_DEFAULT_MODEL)
// node scripts/k3s-local.mjs model store WF_TEST_MODEL_API_KEY as the tenant's Codex key
// node scripts/k3s-local.mjs smoke create a project and run one wish to the end
// node scripts/k3s-local.mjs web forward the workbench (http://localhost:3100) and the IDE
// gateway (http://localhost:3101) to the host
// node scripts/k3s-local.mjs status show the platform and sandbox pods
// node scripts/k3s-local.mjs down stop the VM (data stays; `up` resumes it)
//
// Every kubectl and docker call names the colima-wishforge context explicitly, so the current
// context (which may be a shared cluster) is never touched.
import { spawn, spawnSync } from 'node:child_process';
import { createHash, randomBytes } from 'node:crypto';
import { mkdir, readFile, statfs, symlink, writeFile } from 'node:fs/promises';
import net from 'node:net';
import os from 'node:os';
import path from 'node:path';
const PROFILE = 'wishforge';
const CONTEXT = `colima-${PROFILE}`;
const REGISTRY = 'localhost:5000';
const STATE = '.wishforge/k3s';
const DOCKER_CONFIG = path.resolve(STATE, 'docker');
const SERVICES = 'infra/images/runtime.Dockerfile';
/** How each image is built; the collaboration image builds on a local base image first. */
const IMAGES = {
runtime: { file: SERVICES, context: '.', target: 'runtime' },
worker: { file: SERVICES, context: '.', target: 'worker' },
'git-storage': { file: SERVICES, context: '.', target: 'git-storage' },
provisioner: { file: SERVICES, context: '.', target: 'provisioner' },
// Development sign-in exists only in a development build of the web.
web: { file: SERVICES, context: '.', target: 'web', args: { WEB_BUILD: 'development' } },
ide: { file: 'apps/ide/Dockerfile', context: 'apps/ide', target: 'theia' },
'ide-gateway': { file: 'apps/ide/Dockerfile', context: 'apps/ide', target: 'gateway' },
'oct-workspace': {
file: 'apps/ide/collaboration/Dockerfile',
context: 'apps/ide/collaboration',
base: { file: 'infra/oct/Dockerfile', context: 'infra/oct', tag: 'wishforge/oct:0.3.1-wf1' },
},
};
const TARGETS = Object.keys(IMAGES);
const BASE_IMAGES = {
NODE_IMAGE: process.env.WF_K3S_NODE_IMAGE ?? 'node:22-bookworm',
NODE_SLIM_IMAGE: process.env.WF_K3S_NODE_SLIM_IMAGE ?? 'node:22-bookworm-slim',
DOCKER_CLI_IMAGE: process.env.WF_K3S_DOCKER_CLI_IMAGE ?? 'docker:28-cli',
KUBECTL_IMAGE: process.env.WF_K3S_KUBECTL_IMAGE ?? 'rancher/kubectl:v1.31.4',
KUBECTL_PATH: process.env.WF_K3S_KUBECTL_PATH ?? '/bin/kubectl',
};
function run(command, args, options = {}) {
const shown = args.map(arg => (arg.includes('\n') ? '<program>' : arg.replace(/(\/\/)[^@/\s]+@/, '$1***@')));
console.log(`$ ${command} ${shown.join(' ')}`);
const result = spawnSync(command, args, {
stdio: options.input ? ['pipe', 'inherit', 'inherit'] : 'inherit',
...options,
});
if (result.status !== 0) throw new Error(`${command} exited ${result.status ?? result.signal}`);
}
function capture(command, args, options = {}) {
const result = spawnSync(command, args, { encoding: 'utf8', ...options });
return { ok: result.status === 0, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
}
const kubectl = (args, options) => run('kubectl', ['--context', CONTEXT, ...args], options);
const docker = (args, options) => run('docker', ['--context', CONTEXT, ...args], { env: dockerEnv(), ...options });
const dockerEnv = () => ({ ...process.env, DOCKER_CONFIG });
/**
* Docker runs with a client configuration of its own: a credential helper left in the user's
* (Docker Desktop's `desktop` after uninstalling it, for one) breaks BuildKit's pulls, and nothing
* here needs registry credentials. The contexts, CLI plugins and other settings stay the user's.
*/
async function prepareDocker() {
const home = process.env.DOCKER_CONFIG ?? path.join(os.homedir(), '.docker');
const settings = JSON.parse(await readFile(path.join(home, 'config.json'), 'utf8').catch(() => '{}'));
for (const key of ['auths', 'credsStore', 'credHelpers', 'currentContext']) delete settings[key];
await mkdir(DOCKER_CONFIG, { recursive: true, mode: 0o700 });
await writeFile(path.join(DOCKER_CONFIG, 'config.json'), `${JSON.stringify(settings, null, 2)}\n`, { mode: 0o600 });
for (const name of ['contexts', 'cli-plugins'])
await symlink(path.join(home, name), path.join(DOCKER_CONFIG, name)).catch(error => {
if (error.code !== 'EEXIST') throw error;
});
}
async function readJson(name) {
return JSON.parse(await readFile(`${STATE}/${name}`, 'utf8').catch(() => '{}'));
}
async function writeJson(name, value) {
await mkdir(STATE, { recursive: true, mode: 0o700 });
await writeFile(`${STATE}/${name}`, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
}
function requireCluster() {
if (!capture('kubectl', ['config', 'get-contexts', CONTEXT]).ok)
throw new Error(`The ${CONTEXT} context does not exist yet: run \`node scripts/k3s-local.mjs up\` first`);
}
/** The profile's status in `colima list` (Running, Stopped, Broken), or undefined before it exists. */
function profileStatus() {
for (const line of capture('colima', ['list', '--json']).stdout.split('\n').filter(Boolean)) {
const profile = JSON.parse(line);
if (profile.name === PROFILE) return profile.status;
}
}
/**
* Colima hands a proxy in the host environment (a local Clash, for one) to the VM's Docker and k3s
* but excludes none of the VM's own addresses: the API server then reaches the kubelet through
* the proxy and `kubectl logs` and `exec` fail. Exclude the VM network (Colima's default
* 192.168.5.0/24), the pod and service ranges and the cluster domain.
*/
function colimaEnvironment() {
if (!['HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY'].some(name => process.env[name] || process.env[name.toLowerCase()]))
return process.env;
const excluded = new Set((process.env.NO_PROXY ?? process.env.no_proxy ?? '').split(',').filter(Boolean));
for (const entry of [
'localhost',
'127.0.0.1',
'192.168.5.0/24',
'10.42.0.0/16',
'10.43.0.0/16',
'.svc',
'.cluster.local',
])
excluded.add(entry);
const noProxy = [...excluded].join(',');
return { ...process.env, NO_PROXY: noProxy, no_proxy: noProxy };
}
/** The user's current kubectl and Docker contexts (with the user's own Docker configuration). */
const currentContexts = () => ({
kubectl: capture('kubectl', ['config', 'current-context']).stdout.trim(),
docker: capture('docker', ['context', 'show']).stdout.trim(),
});
/**
* Free space on this Mac's disk, where Colima keeps the VM's disks. They grow with every image
* built; if the Mac's disk fills up, writes inside the VM fail and its file systems break.
*/
async function freeGiB() {
const stats = await statfs(os.homedir());
return (stats.bavail * stats.bsize) / 2 ** 30;
}
const DISK_GIB = Number(process.env.WF_K3S_DISK_GIB ?? '40');
const MIN_FREE_TO_BUILD_GIB = 15;
async function up() {
const previous = currentContexts();
if (!profileStatus()) {
const free = await freeGiB();
if (free < DISK_GIB)
throw new Error(
`The VM's disk may grow to ${DISK_GIB} GiB, but only ${free.toFixed(1)} GiB are free on this Mac. Free some space or set WF_K3S_DISK_GIB lower.`,
);
}
if (!capture('colima', ['status', '--profile', PROFILE]).ok) {
// A start that failed while installing k3s (often an interrupted GitHub download) leaves the VM
// running without a runtime; stop it so that this start provisions again.
if (profileStatus() === 'Running') run('colima', ['stop', '--profile', PROFILE]);
run(
'colima',
[
'start',
'--profile',
PROFILE,
'--runtime',
'docker',
'--kubernetes',
'--cpu',
process.env.WF_K3S_CPUS ?? '6',
'--memory',
process.env.WF_K3S_MEMORY_GIB ?? '12',
'--disk',
String(DISK_GIB),
],
{ env: colimaEnvironment() },
);
}
requireCluster();
await prepareDocker();
// k3s runs on the VM's Docker, which pulls localhost:5000 from this registry.
if (!capture('docker', ['--context', CONTEXT, 'inspect', 'wishforge-registry'], { env: dockerEnv() }).ok)
docker([
'run',
'-d',
'--restart=always',
'-p',
'127.0.0.1:5000:5000',
'--name',
'wishforge-registry',
'registry:2',
]);
kubectl(['wait', '--for=condition=Ready', 'node', '--all', '--timeout=300s']);
console.log(`k3s is ready in the ${CONTEXT} context.`);
// Colima makes its VM the current kubectl and Docker context on every start. This script names
// its context on every call and never relies on that; switching back is the user's call.
const current = currentContexts();
if (previous.kubectl && current.kubectl !== previous.kubectl)
console.log(
`Colima switched the current kubectl context from ${previous.kubectl} to ${current.kubectl}. To switch back:\n kubectl config use-context ${previous.kubectl}`,
);
if (previous.docker && current.docker !== previous.docker)
console.log(
`Colima switched the current Docker context from ${previous.docker} to ${current.docker}. To switch back:\n docker context use ${previous.docker}`,
);
}
/**
* Build steps reach the network straight from the VM. Behind a proxy in the host environment they
* use it too, at the address the VM sees the host at, as Colima arranges for Docker and k3s.
* Docker leaves these predefined arguments out of the image history and the build cache.
*/
function proxyBuildArgs() {
const args = {};
for (const name of ['HTTP_PROXY', 'HTTPS_PROXY']) {
const value = process.env[name] ?? process.env[name.toLowerCase()];
if (!value) continue;
const url = new URL(value);
if (['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname)) url.hostname = '192.168.5.2';
args[name] = args[name.toLowerCase()] = url.href.replace(/\/$/, '');
}
if (Object.keys(args).length) args.NO_PROXY = args.no_proxy = 'localhost,127.0.0.1';
return args;
}
async function build() {
requireCluster();
const selected = process.argv.slice(3);
const unknown = selected.filter(target => !TARGETS.includes(target));
if (unknown.length) throw new Error(`Unknown images ${unknown.join(', ')}; choose from ${TARGETS.join(', ')}`);
const free = await freeGiB();
if (free < MIN_FREE_TO_BUILD_GIB)
throw new Error(
`Only ${free.toFixed(1)} GiB are free on this Mac; free at least ${MIN_FREE_TO_BUILD_GIB} GiB before building, since the VM's disk grows with every build.`,
);
await prepareDocker();
const buildArgs = (extra = {}) =>
Object.entries({ ...BASE_IMAGES, ...proxyBuildArgs(), ...extra }).flatMap(([name, value]) => [
'--build-arg',
`${name}=${value}`,
]);
// Images not rebuilt keep the digests they were deployed with.
const images = await readJson('images.json');
for (const target of selected.length ? selected : TARGETS) {
const image = IMAGES[target];
const tag = `${REGISTRY}/wishforge/${target}:dev`;
if (image.base)
docker(['build', '--file', image.base.file, ...buildArgs(), '--tag', image.base.tag, image.base.context]);
docker([
'build',
'--file',
image.file,
...(image.target ? ['--target', image.target] : []),
...buildArgs(image.args),
'--tag',
tag,
image.context,
]);
docker(['push', tag]);
const digest = capture(
'docker',
['--context', CONTEXT, 'image', 'inspect', '--format', '{{json .RepoDigests}}', tag],
{ env: dockerEnv() },
);
const reference = JSON.parse(digest.stdout || '[]').find(value =>
value.startsWith(`${REGISTRY}/wishforge/${target}@`),
);
if (!reference) throw new Error(`No registry digest for ${tag}`);
images[target] = reference;
}
await writeJson('images.json', images);
console.log(images);
// Superseded images no container uses, and build cache beyond 10 GB, would only grow the disk.
for (const prune of [
['image', 'prune', '--force'],
['builder', 'prune', '--force', '--max-used-space', '10GB'],
])
if (!capture('docker', ['--context', CONTEXT, ...prune], { env: dockerEnv() }).ok)
console.log(`Could not run docker ${prune.join(' ')}; the VM's disk keeps the old layers.`);
}
async function deploy() {
requireCluster();
const images = await readJson('images.json');
if (TARGETS.some(target => !images[target]))
throw new Error('Build the images first: node scripts/k3s-local.mjs build');
const model = process.env.WF_CODEX_DEFAULT_MODEL;
if (!model) throw new Error('Set WF_CODEX_DEFAULT_MODEL to a model your gateway serves (GET <gateway>/v1/models)');
// Clear of the local development stack (npm run dev:all uses 3000 and 4400), so both can run.
const ports = { web: process.env.WF_K3S_WEB_PORT ?? '3100', ide: process.env.WF_K3S_IDE_PORT ?? '3101' };
if (!Object.values(ports).every(port => /^\d{2,5}$/.test(port)))
throw new Error('WF_K3S_WEB_PORT and WF_K3S_IDE_PORT must be port numbers');
// Generated once and kept outside Git, so redeploying keeps the database password and tokens.
const secrets = await readJson('secrets.json');
const secret = () => randomBytes(32).toString('hex');
for (const name of [
'POSTGRES_PASSWORD',
'APP_PASSWORD',
'WF_DEV_AUTH_TOKEN',
'WF_WORKER_TOKEN',
'WF_DEPLOYER_TOKEN',
'WF_VAULT_KEY',
'WF_INTERNAL_TOKEN_KEY',
'WF_RESOURCE_GRANT_SECRET',
'WF_PROVISIONER_TOKEN',
'WF_GIT_ADMIN_TOKEN',
'WF_GIT_TOKEN_SECRET',
'WF_PROVIDER_PROXY_SIGNING_KEY',
'WF_WORKSPACE_GATEWAY_TOKEN',
])
secrets[name] ??= secret();
await writeJson('secrets.json', secrets);
const values = {
...secrets,
RUNTIME_IMAGE: images.runtime,
WORKER_IMAGE: images.worker,
GIT_STORAGE_IMAGE: images['git-storage'],
PROVISIONER_IMAGE: images.provisioner,
WEB_IMAGE: images.web,
IDE_IMAGE: images.ide,
IDE_GATEWAY_IMAGE: images['ide-gateway'],
OCT_IMAGE: images['oct-workspace'],
// `web` forwards these ports; the workbench and the gateway accept only these origins.
WEB_PUBLIC_URL: `http://localhost:${ports.web}`,
IDE_PUBLIC_URL: `http://localhost:${ports.ide}`,
MODEL: model,
OPENAI_BASE_URL: process.env.WF_OPENAI_BASE_URL ?? '',
};
const template = await readFile('infra/k8s/local/wishforge.yaml', 'utf8');
const render = digest =>
template.replace(/\{\{([A-Z_]+)\}\}/g, (_, name) => {
const value = name === 'SETTINGS_DIGEST' ? digest : values[name];
if (value === undefined) throw new Error(`No value for {{${name}}}`);
if (/['\n]/.test(value)) throw new Error(`${name} cannot contain quotes or line breaks`);
return value;
});
// Pods read the settings and secrets when they start; a digest of both in their templates
// rolls them whenever either changes.
const settings = render('')
.split(/^---$/m)
.filter(
document =>
/^kind: (?:ConfigMap|Secret)$/m.test(document) && /^ {2}name: wishforge-(?:settings|local)$/m.test(document),
);
if (settings.length !== 2) throw new Error('The manifest lost its settings or secrets');
kubectl(['apply', '-f', '-'], {
input: render(createHash('sha256').update(settings.join('---')).digest('hex').slice(0, 16)),
});
for (const name of [
'postgres',
'temporal',
'git-storage',
'api',
'provisioner',
'worker-coordinator',
'worker-deployer',
'web',
'ide-gateway',
])
kubectl(['--namespace', 'wishforge-control', 'rollout', 'status', `deployment/${name}`, '--timeout=600s']);
console.log(
'WishForge is running. Next: node scripts/k3s-local.mjs model, then smoke; `web` opens the workbench and IDE.',
);
}
/** Runs a small Node program inside the API pod, against the API on its own loopback. */
function insideApi(program, input) {
kubectl(
[
'--namespace',
'wishforge-control',
'exec',
'-i',
'deployment/api',
'-c',
'api',
'--',
'node',
'--input-type=module',
'-e',
program,
],
{ input: input ?? '' },
);
}
const API_CLIENT = `
const token = process.env.WF_DEV_AUTH_TOKEN;
const api = async (method, path, body, key) => {
const response = await fetch('http://127.0.0.1:4000/v1' + path, {
method,
headers: { authorization: 'Bearer ' + token, 'content-type': 'application/json', ...(key ? { 'idempotency-key': key } : {}) },
...(body ? { body: JSON.stringify(body) } : {}),
});
const payload = await response.json().catch(() => ({}));
if (!response.ok) throw new Error(method + ' ' + path + ' -> ' + response.status + ' ' + JSON.stringify(payload));
return payload.data;
};
const stdin = async () => { let text = ''; for await (const chunk of process.stdin) text += chunk; return text; };
`;
async function model() {
requireCluster();
const apiKey = process.env.WF_TEST_MODEL_API_KEY;
if (!apiKey) throw new Error('Set WF_TEST_MODEL_API_KEY to the gateway key');
// The key travels on stdin, never on a command line.
insideApi(
`${API_CLIENT}
const { apiKey } = JSON.parse(await stdin());
const existing = (await api('GET', '/provider-connections')).filter(c => c.provider === 'codex' && c.status === 'active');
for (const connection of existing) await api('DELETE', '/provider-connections/' + connection.id);
const created = await api('POST', '/provider-connections', { provider: 'codex', label: 'model gateway', apiKey });
console.log('Codex key stored as connection ' + created.id);
console.log((await api('GET', '/providers')).map(p => p.id + ': ' + (p.enabled ? 'enabled' : p.reason)).join('\\n'));`,
JSON.stringify({ apiKey }),
);
}
async function smoke() {
requireCluster();
insideApi(`${API_CLIENT}
const stamp = Date.now().toString(36);
const project = await api('POST', '/projects', { name: 'k3s smoke ' + stamp });
console.log('project', project.id, project.repositoryStatus, project.repositoryUrl);
await api('PATCH', '/projects/' + project.id, {
verificationCommands: [['node', '-e', "process.exit(require('fs').readFileSync('greeting.txt','utf8').trim()==='hello world'?0:1)"]],
});
const wish = await api('POST', '/projects/' + project.id + '/wishes', {
goal: 'Create a file named greeting.txt whose whole content is the line "hello world".',
acceptanceCriteria: ['greeting.txt contains exactly "hello world"'],
provider: 'codex',
budget: { maxTokens: 150000, maxMinutes: 20, maxRepairRounds: 2 },
}, 'k3s-smoke-' + stamp);
console.log('run', wish.run_id);
let after = 0;
for (let i = 0; i < 240; i++) {
for (const event of await api('GET', '/runs/' + wish.run_id + '/events?after=' + after)) {
after = event.seq;
console.log(event.time.slice(11, 19), event.type, event.message.slice(0, 160));
}
const run = await api('GET', '/runs/' + wish.run_id);
// The dispatcher retries a provisioner that is not ready yet; every other wait needs a person.
if (['succeeded', 'failed', 'canceled'].includes(run.status) || (run.status === 'waiting' && run.waitingReason !== 'sandbox_provisioner')) {
console.log('\\nfinal:', run.status, run.stage, run.waitingReason ?? '', run.waitingDetail ?? '');
console.log('integrated commit:', run.integratedCommit ?? '-', ' tokens used:', run.usedTokens);
process.exit(run.status === 'succeeded' ? 0 : 1);
}
await new Promise(resolve => setTimeout(resolve, 5000));
}
console.log('Timed out after 20 minutes');
process.exit(1);`);
}
/** The browser-facing origin a deployment was configured with (it refuses requests from others). */
function deployedOrigin(deployment, variable) {
const deployed = capture('kubectl', [
'--context',
CONTEXT,
'--namespace',
'wishforge-control',
'get',
`deployment/${deployment}`,
'--output',
`jsonpath={.spec.template.spec.containers[0].env[?(@.name=="${variable}")].value}`,
]);
if (!deployed.ok || !deployed.stdout.trim()) throw new Error('Deploy first: node scripts/k3s-local.mjs deploy');
return new URL(deployed.stdout.trim());
}
/** Whether a local port is free: kubectl, finding it taken on 127.0.0.1, would listen on ::1 only. */
const portFree = port =>
new Promise(resolve => {
const server = net.createServer();
server.once('error', () => resolve(false));
server.listen(Number(port), '127.0.0.1', () => server.close(() => resolve(true)));
});
/** Forwards the workbench and the IDE gateway to the ports they were deployed for. */
async function web() {
requireCluster();
const forwards = [
{ service: 'web', port: 3000, url: deployedOrigin('web', 'WF_PUBLIC_URL') },
{ service: 'ide-gateway', port: 4400, url: deployedOrigin('ide-gateway', 'WF_IDE_PUBLIC_URL') },
];
for (const { url } of forwards)
if (!(await portFree(url.port)))
throw new Error(
`Port ${url.port} is taken on this machine (a local dev server?). Deploy with free WF_K3S_WEB_PORT and WF_K3S_IDE_PORT, then run web again.`,
);
console.log(`Workbench ${forwards[0].url.origin}, IDE ${forwards[1].url.origin} (exactly these addresses).`);
console.log('Ctrl-C stops forwarding; a deploy replaces the pods and ends it, so run `web` again then.');
const children = forwards.map(({ service, port, url }) =>
spawn(
'kubectl',
[
'--context',
CONTEXT,
'--namespace',
'wishforge-control',
'port-forward',
`service/${service}`,
`${url.port}:${port}`,
],
{ stdio: 'inherit' },
),
);
// Ctrl-C reaches kubectl too; a plain SIGTERM (from a supervisor) reaches only this process.
for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP'])
process.once(signal, () => {
for (const child of children) child.kill();
process.exit(1);
});
const code = await Promise.race(children.map(child => new Promise(resolve => child.on('exit', resolve))));
for (const child of children) child.kill();
if (code) throw new Error(`kubectl port-forward exited ${code}`);
}
async function status() {
requireCluster();
kubectl(['get', 'pods,jobs,pvc', '--namespace', 'wishforge-control']);
kubectl(['get', 'pods,jobs,pvc,networkpolicies', '--namespace', 'wishforge-tenant-demo']);
}
async function down() {
run('colima', ['stop', '--profile', PROFILE]);
}
const commands = { up, build, deploy, model, smoke, web, status, down };
const command = commands[process.argv[2]];
if (!command) {
console.error(`Usage: node scripts/k3s-local.mjs ${Object.keys(commands).join('|')}`);
process.exit(2);
}
await command().catch(error => {
console.error(error.message);
process.exit(1);
});