// Runs WishForge on a local k3s cluster (a Colima VM) for development, end to end: wishes are // executed by the Kubernetes provisioner in per-run pods, verified and merged. // // node scripts/k3s-local.mjs up start the colima-wishforge VM with k3s and a local registry // node scripts/k3s-local.mjs build build the service images and push them to the local registry // (`build web worker` rebuilds only those) // node scripts/k3s-local.mjs deploy render infra/k8s/local and apply it (needs WF_CODEX_DEFAULT_MODEL) // node scripts/k3s-local.mjs model store WF_TEST_MODEL_API_KEY as the tenant's Codex key // node scripts/k3s-local.mjs smoke create a project and run one wish to the end // node scripts/k3s-local.mjs web forward the workbench (http://localhost:3100) to the host // node scripts/k3s-local.mjs status show the platform and sandbox pods // node scripts/k3s-local.mjs down stop the VM (data stays; `up` resumes it) // // Every kubectl and docker call names the colima-wishforge context explicitly, so the current // context (which may be a shared cluster) is never touched. import { spawn, spawnSync } from 'node:child_process'; import { createHash, randomBytes } from 'node:crypto'; import { mkdir, readFile, statfs, symlink, writeFile } from 'node:fs/promises'; import net from 'node:net'; import os from 'node:os'; import path from 'node:path'; const PROFILE = 'wishforge'; const CONTEXT = `colima-${PROFILE}`; const REGISTRY = 'localhost:5000'; const STATE = '.wishforge/k3s'; const DOCKER_CONFIG = path.resolve(STATE, 'docker'); const SERVICES = 'infra/images/runtime.Dockerfile'; /** How each image is built. */ const IMAGES = { runtime: { file: SERVICES, context: '.', target: 'runtime' }, worker: { file: SERVICES, context: '.', target: 'worker' }, 'git-storage': { file: SERVICES, context: '.', target: 'git-storage' }, provisioner: { file: SERVICES, context: '.', target: 'provisioner' }, // Development sign-in exists only in a development build of the web. web: { file: SERVICES, context: '.', target: 'web', args: { WEB_BUILD: 'development' } }, }; const TARGETS = Object.keys(IMAGES); const BASE_IMAGES = { NODE_IMAGE: process.env.WF_K3S_NODE_IMAGE ?? 'node:22-bookworm', NODE_SLIM_IMAGE: process.env.WF_K3S_NODE_SLIM_IMAGE ?? 'node:22-bookworm-slim', DOCKER_CLI_IMAGE: process.env.WF_K3S_DOCKER_CLI_IMAGE ?? 'docker:28-cli', KUBECTL_IMAGE: process.env.WF_K3S_KUBECTL_IMAGE ?? 'rancher/kubectl:v1.31.4', KUBECTL_PATH: process.env.WF_K3S_KUBECTL_PATH ?? '/bin/kubectl', }; function run(command, args, options = {}) { const shown = args.map(arg => (arg.includes('\n') ? '' : arg.replace(/(\/\/)[^@/\s]+@/, '$1***@'))); console.log(`$ ${command} ${shown.join(' ')}`); const result = spawnSync(command, args, { stdio: options.input ? ['pipe', 'inherit', 'inherit'] : 'inherit', ...options, }); if (result.status !== 0) throw new Error(`${command} exited ${result.status ?? result.signal}`); } function capture(command, args, options = {}) { const result = spawnSync(command, args, { encoding: 'utf8', ...options }); return { ok: result.status === 0, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }; } const kubectl = (args, options) => run('kubectl', ['--context', CONTEXT, ...args], options); const docker = (args, options) => run('docker', ['--context', CONTEXT, ...args], { env: dockerEnv(), ...options }); const dockerEnv = () => ({ ...process.env, DOCKER_CONFIG }); /** * Docker runs with a client configuration of its own: a credential helper left in the user's * (Docker Desktop's `desktop` after uninstalling it, for one) breaks BuildKit's pulls, and nothing * here needs registry credentials. The contexts, CLI plugins and other settings stay the user's. */ async function prepareDocker() { const home = process.env.DOCKER_CONFIG ?? path.join(os.homedir(), '.docker'); const settings = JSON.parse(await readFile(path.join(home, 'config.json'), 'utf8').catch(() => '{}')); for (const key of ['auths', 'credsStore', 'credHelpers', 'currentContext']) delete settings[key]; await mkdir(DOCKER_CONFIG, { recursive: true, mode: 0o700 }); await writeFile(path.join(DOCKER_CONFIG, 'config.json'), `${JSON.stringify(settings, null, 2)}\n`, { mode: 0o600 }); for (const name of ['contexts', 'cli-plugins']) await symlink(path.join(home, name), path.join(DOCKER_CONFIG, name)).catch(error => { if (error.code !== 'EEXIST') throw error; }); } async function readJson(name) { return JSON.parse(await readFile(`${STATE}/${name}`, 'utf8').catch(() => '{}')); } async function writeJson(name, value) { await mkdir(STATE, { recursive: true, mode: 0o700 }); await writeFile(`${STATE}/${name}`, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); } function requireCluster() { if (!capture('kubectl', ['config', 'get-contexts', CONTEXT]).ok) throw new Error(`The ${CONTEXT} context does not exist yet: run \`node scripts/k3s-local.mjs up\` first`); } /** The profile's status in `colima list` (Running, Stopped, Broken), or undefined before it exists. */ function profileStatus() { for (const line of capture('colima', ['list', '--json']).stdout.split('\n').filter(Boolean)) { const profile = JSON.parse(line); if (profile.name === PROFILE) return profile.status; } } /** * Colima hands a proxy in the host environment (a local Clash, for one) to the VM's Docker and k3s * but excludes none of the VM's own addresses: the API server then reaches the kubelet through * the proxy and `kubectl logs` and `exec` fail. Exclude the VM network (Colima's default * 192.168.5.0/24), the pod and service ranges and the cluster domain. */ function colimaEnvironment() { if (!['HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY'].some(name => process.env[name] || process.env[name.toLowerCase()])) return process.env; const excluded = new Set((process.env.NO_PROXY ?? process.env.no_proxy ?? '').split(',').filter(Boolean)); for (const entry of [ 'localhost', '127.0.0.1', '192.168.5.0/24', '10.42.0.0/16', '10.43.0.0/16', '.svc', '.cluster.local', ]) excluded.add(entry); const noProxy = [...excluded].join(','); return { ...process.env, NO_PROXY: noProxy, no_proxy: noProxy }; } /** The user's current kubectl and Docker contexts (with the user's own Docker configuration). */ const currentContexts = () => ({ kubectl: capture('kubectl', ['config', 'current-context']).stdout.trim(), docker: capture('docker', ['context', 'show']).stdout.trim(), }); /** * Free space on this Mac's disk, where Colima keeps the VM's disks. They grow with every image * built; if the Mac's disk fills up, writes inside the VM fail and its file systems break. */ async function freeGiB() { const stats = await statfs(os.homedir()); return (stats.bavail * stats.bsize) / 2 ** 30; } const DISK_GIB = Number(process.env.WF_K3S_DISK_GIB ?? '40'); const MIN_FREE_TO_BUILD_GIB = 15; async function up() { const previous = currentContexts(); if (!profileStatus()) { const free = await freeGiB(); if (free < DISK_GIB) throw new Error( `The VM's disk may grow to ${DISK_GIB} GiB, but only ${free.toFixed(1)} GiB are free on this Mac. Free some space or set WF_K3S_DISK_GIB lower.`, ); } if (!capture('colima', ['status', '--profile', PROFILE]).ok) { // A start that failed while installing k3s (often an interrupted GitHub download) leaves the VM // running without a runtime; stop it so that this start provisions again. if (profileStatus() === 'Running') run('colima', ['stop', '--profile', PROFILE]); run( 'colima', [ 'start', '--profile', PROFILE, '--runtime', 'docker', '--kubernetes', '--cpu', process.env.WF_K3S_CPUS ?? '6', '--memory', process.env.WF_K3S_MEMORY_GIB ?? '12', '--disk', String(DISK_GIB), ], { env: colimaEnvironment() }, ); } requireCluster(); await prepareDocker(); // k3s runs on the VM's Docker, which pulls localhost:5000 from this registry. if (!capture('docker', ['--context', CONTEXT, 'inspect', 'wishforge-registry'], { env: dockerEnv() }).ok) docker([ 'run', '-d', '--restart=always', '-p', '127.0.0.1:5000:5000', '--name', 'wishforge-registry', 'registry:2', ]); kubectl(['wait', '--for=condition=Ready', 'node', '--all', '--timeout=300s']); console.log(`k3s is ready in the ${CONTEXT} context.`); // Colima makes its VM the current kubectl and Docker context on every start. This script names // its context on every call and never relies on that; switching back is the user's call. const current = currentContexts(); if (previous.kubectl && current.kubectl !== previous.kubectl) console.log( `Colima switched the current kubectl context from ${previous.kubectl} to ${current.kubectl}. To switch back:\n kubectl config use-context ${previous.kubectl}`, ); if (previous.docker && current.docker !== previous.docker) console.log( `Colima switched the current Docker context from ${previous.docker} to ${current.docker}. To switch back:\n docker context use ${previous.docker}`, ); } /** * Build steps reach the network straight from the VM. Behind a proxy in the host environment they * use it too, at the address the VM sees the host at, as Colima arranges for Docker and k3s. * Docker leaves these predefined arguments out of the image history and the build cache. */ function proxyBuildArgs() { const args = {}; for (const name of ['HTTP_PROXY', 'HTTPS_PROXY']) { const value = process.env[name] ?? process.env[name.toLowerCase()]; if (!value) continue; const url = new URL(value); if (['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname)) url.hostname = '192.168.5.2'; args[name] = args[name.toLowerCase()] = url.href.replace(/\/$/, ''); } if (Object.keys(args).length) args.NO_PROXY = args.no_proxy = 'localhost,127.0.0.1'; return args; } async function build() { requireCluster(); const selected = process.argv.slice(3); const unknown = selected.filter(target => !TARGETS.includes(target)); if (unknown.length) throw new Error(`Unknown images ${unknown.join(', ')}; choose from ${TARGETS.join(', ')}`); const free = await freeGiB(); if (free < MIN_FREE_TO_BUILD_GIB) throw new Error( `Only ${free.toFixed(1)} GiB are free on this Mac; free at least ${MIN_FREE_TO_BUILD_GIB} GiB before building, since the VM's disk grows with every build.`, ); await prepareDocker(); const buildArgs = (extra = {}) => Object.entries({ ...BASE_IMAGES, ...proxyBuildArgs(), ...extra }).flatMap(([name, value]) => [ '--build-arg', `${name}=${value}`, ]); // Images not rebuilt keep the digests they were deployed with. const images = await readJson('images.json'); for (const target of selected.length ? selected : TARGETS) { const image = IMAGES[target]; const tag = `${REGISTRY}/wishforge/${target}:dev`; docker([ 'build', '--file', image.file, ...(image.target ? ['--target', image.target] : []), ...buildArgs(image.args), '--tag', tag, image.context, ]); docker(['push', tag]); const digest = capture( 'docker', ['--context', CONTEXT, 'image', 'inspect', '--format', '{{json .RepoDigests}}', tag], { env: dockerEnv() }, ); const reference = JSON.parse(digest.stdout || '[]').find(value => value.startsWith(`${REGISTRY}/wishforge/${target}@`), ); if (!reference) throw new Error(`No registry digest for ${tag}`); images[target] = reference; } await writeJson('images.json', images); console.log(images); // Superseded images no container uses, and build cache beyond 10 GB, would only grow the disk. for (const prune of [ ['image', 'prune', '--force'], ['builder', 'prune', '--force', '--max-used-space', '10GB'], ]) if (!capture('docker', ['--context', CONTEXT, ...prune], { env: dockerEnv() }).ok) console.log(`Could not run docker ${prune.join(' ')}; the VM's disk keeps the old layers.`); } async function deploy() { requireCluster(); const images = await readJson('images.json'); if (TARGETS.some(target => !images[target])) throw new Error('Build the images first: node scripts/k3s-local.mjs build'); const model = process.env.WF_CODEX_DEFAULT_MODEL; if (!model) throw new Error('Set WF_CODEX_DEFAULT_MODEL to a model your gateway serves (GET /v1/models)'); // Clear of the local development stack (npm run dev:all uses 3000), so both can run. const webPort = process.env.WF_K3S_WEB_PORT ?? '3100'; if (!/^\d{2,5}$/.test(webPort)) throw new Error('WF_K3S_WEB_PORT must be a port number'); // Generated once and kept outside Git, so redeploying keeps the database password and tokens. const secrets = await readJson('secrets.json'); const secret = () => randomBytes(32).toString('hex'); for (const name of [ 'POSTGRES_PASSWORD', 'APP_PASSWORD', 'WF_DEV_AUTH_TOKEN', 'WF_WORKER_TOKEN', 'WF_DEPLOYER_TOKEN', 'WF_VAULT_KEY', 'WF_INTERNAL_TOKEN_KEY', 'WF_RESOURCE_GRANT_SECRET', 'WF_PROVISIONER_TOKEN', 'WF_GIT_ADMIN_TOKEN', 'WF_GIT_TOKEN_SECRET', 'WF_PROVIDER_PROXY_SIGNING_KEY', ]) secrets[name] ??= secret(); await writeJson('secrets.json', secrets); const values = { ...secrets, RUNTIME_IMAGE: images.runtime, WORKER_IMAGE: images.worker, GIT_STORAGE_IMAGE: images['git-storage'], PROVISIONER_IMAGE: images.provisioner, WEB_IMAGE: images.web, // `web` forwards this port; the workbench accepts only this origin. WEB_PUBLIC_URL: `http://localhost:${webPort}`, MODEL: model, OPENAI_BASE_URL: process.env.WF_OPENAI_BASE_URL ?? '', }; const template = await readFile('infra/k8s/local/wishforge.yaml', 'utf8'); const render = digest => template.replace(/\{\{([A-Z_]+)\}\}/g, (_, name) => { const value = name === 'SETTINGS_DIGEST' ? digest : values[name]; if (value === undefined) throw new Error(`No value for {{${name}}}`); if (/['\n]/.test(value)) throw new Error(`${name} cannot contain quotes or line breaks`); return value; }); // Pods read the settings and secrets when they start; a digest of both in their templates // rolls them whenever either changes. const settings = render('') .split(/^---$/m) .filter( document => /^kind: (?:ConfigMap|Secret)$/m.test(document) && /^ {2}name: wishforge-(?:settings|local)$/m.test(document), ); if (settings.length !== 2) throw new Error('The manifest lost its settings or secrets'); kubectl(['apply', '-f', '-'], { input: render(createHash('sha256').update(settings.join('---')).digest('hex').slice(0, 16)), }); for (const name of [ 'postgres', 'temporal', 'git-storage', 'api', 'provisioner', 'worker-coordinator', 'worker-deployer', 'web', ]) kubectl(['--namespace', 'wishforge-control', 'rollout', 'status', `deployment/${name}`, '--timeout=600s']); console.log('WishForge is running. Next: node scripts/k3s-local.mjs model, then smoke; `web` opens the workbench.'); } /** Runs a small Node program inside the API pod, against the API on its own loopback. */ function insideApi(program, input) { kubectl( [ '--namespace', 'wishforge-control', 'exec', '-i', 'deployment/api', '-c', 'api', '--', 'node', '--input-type=module', '-e', program, ], { input: input ?? '' }, ); } const API_CLIENT = ` const token = process.env.WF_DEV_AUTH_TOKEN; const api = async (method, path, body, key) => { const response = await fetch('http://127.0.0.1:4000/v1' + path, { method, headers: { authorization: 'Bearer ' + token, 'content-type': 'application/json', ...(key ? { 'idempotency-key': key } : {}) }, ...(body ? { body: JSON.stringify(body) } : {}), }); const payload = await response.json().catch(() => ({})); if (!response.ok) throw new Error(method + ' ' + path + ' -> ' + response.status + ' ' + JSON.stringify(payload)); return payload.data; }; const stdin = async () => { let text = ''; for await (const chunk of process.stdin) text += chunk; return text; }; `; async function model() { requireCluster(); const apiKey = process.env.WF_TEST_MODEL_API_KEY; if (!apiKey) throw new Error('Set WF_TEST_MODEL_API_KEY to the gateway key'); // The key travels on stdin, never on a command line. insideApi( `${API_CLIENT} const { apiKey } = JSON.parse(await stdin()); const existing = (await api('GET', '/provider-connections')).filter(c => c.provider === 'codex' && c.status === 'active'); for (const connection of existing) await api('DELETE', '/provider-connections/' + connection.id); const created = await api('POST', '/provider-connections', { provider: 'codex', label: 'model gateway', apiKey }); console.log('Codex key stored as connection ' + created.id); console.log((await api('GET', '/providers')).map(p => p.id + ': ' + (p.enabled ? 'enabled' : p.reason)).join('\\n'));`, JSON.stringify({ apiKey }), ); } async function smoke() { requireCluster(); insideApi(`${API_CLIENT} const stamp = Date.now().toString(36); const project = await api('POST', '/projects', { name: 'k3s smoke ' + stamp }); console.log('project', project.id, project.repositoryStatus, project.repositoryUrl); await api('PATCH', '/projects/' + project.id, { verificationCommands: [['node', '-e', "process.exit(require('fs').readFileSync('greeting.txt','utf8').trim()==='hello world'?0:1)"]], }); const wish = await api('POST', '/projects/' + project.id + '/wishes', { goal: 'Create a file named greeting.txt whose whole content is the line "hello world".', acceptanceCriteria: ['greeting.txt contains exactly "hello world"'], provider: 'codex', budget: { maxTokens: 150000, maxMinutes: 20, maxRepairRounds: 2 }, }, 'k3s-smoke-' + stamp); console.log('run', wish.run_id); let after = 0; for (let i = 0; i < 240; i++) { for (const event of await api('GET', '/runs/' + wish.run_id + '/events?after=' + after)) { after = event.seq; console.log(event.time.slice(11, 19), event.type, event.message.slice(0, 160)); } const run = await api('GET', '/runs/' + wish.run_id); // The dispatcher retries a provisioner that is not ready yet; every other wait needs a person. if (['succeeded', 'failed', 'canceled'].includes(run.status) || (run.status === 'waiting' && run.waitingReason !== 'sandbox_provisioner')) { console.log('\\nfinal:', run.status, run.stage, run.waitingReason ?? '', run.waitingDetail ?? ''); console.log('integrated commit:', run.integratedCommit ?? '-', ' tokens used:', run.usedTokens); process.exit(run.status === 'succeeded' ? 0 : 1); } await new Promise(resolve => setTimeout(resolve, 5000)); } console.log('Timed out after 20 minutes'); process.exit(1);`); } /** The browser-facing origin a deployment was configured with (it refuses requests from others). */ function deployedOrigin(deployment, variable) { const deployed = capture('kubectl', [ '--context', CONTEXT, '--namespace', 'wishforge-control', 'get', `deployment/${deployment}`, '--output', `jsonpath={.spec.template.spec.containers[0].env[?(@.name=="${variable}")].value}`, ]); if (!deployed.ok || !deployed.stdout.trim()) throw new Error('Deploy first: node scripts/k3s-local.mjs deploy'); return new URL(deployed.stdout.trim()); } /** Whether a local port is free: kubectl, finding it taken on 127.0.0.1, would listen on ::1 only. */ const portFree = port => new Promise(resolve => { const server = net.createServer(); server.once('error', () => resolve(false)); server.listen(Number(port), '127.0.0.1', () => server.close(() => resolve(true))); }); /** Forwards the workbench to the port it was deployed for. */ async function web() { requireCluster(); const forwards = [{ service: 'web', port: 3000, url: deployedOrigin('web', 'WF_PUBLIC_URL') }]; for (const { url } of forwards) if (!(await portFree(url.port))) throw new Error( `Port ${url.port} is taken on this machine (a local dev server?). Deploy with a free WF_K3S_WEB_PORT, then run web again.`, ); console.log(`Workbench ${forwards[0].url.origin} (exactly this address).`); console.log('Ctrl-C stops forwarding; a deploy replaces the pods and ends it, so run `web` again then.'); const children = forwards.map(({ service, port, url }) => spawn( 'kubectl', [ '--context', CONTEXT, '--namespace', 'wishforge-control', 'port-forward', `service/${service}`, `${url.port}:${port}`, ], { stdio: 'inherit' }, ), ); // Ctrl-C reaches kubectl too; a plain SIGTERM (from a supervisor) reaches only this process. for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.once(signal, () => { for (const child of children) child.kill(); process.exit(1); }); const code = await Promise.race(children.map(child => new Promise(resolve => child.on('exit', resolve)))); for (const child of children) child.kill(); if (code) throw new Error(`kubectl port-forward exited ${code}`); } async function status() { requireCluster(); kubectl(['get', 'pods,jobs,pvc', '--namespace', 'wishforge-control']); kubectl(['get', 'pods,jobs,pvc,networkpolicies', '--namespace', 'wishforge-tenant-demo']); } async function down() { run('colima', ['stop', '--profile', PROFILE]); } const commands = { up, build, deploy, model, smoke, web, status, down }; const command = commands[process.argv[2]]; if (!command) { console.error(`Usage: node scripts/k3s-local.mjs ${Object.keys(commands).join('|')}`); process.exit(2); } await command().catch(error => { console.error(error.message); process.exit(1); });