wishforge-ide/scripts/k3s-local.mjs
eust-w da1cfcea13 🔥 Remove the cloud IDE
The cluster never ran Theia or the IDE gateway, so every workspace stayed
"未配置" forever. The feature is gone instead of half-present:

- API: workspace, launch and lifecycle endpoints, the gateway identity and
  its internal routes, the dispatcher's workspace provisioning and the
  lifecycle worker. Migration 003 deletes stored workspaces and settles
  their pending outbox items.
- Provisioner: workspace provisioning and lifecycle for both backends.
- Web: the 云 IDE panel, the launch route, the start-branch picker that
  listed IDE branches, and the workspace count on usage.
- apps/ide, infra/oct, dev:ide and the CI job; IDE parts of local k3s,
  network policies, RBAC, docs and the architecture diagram.

A wish can still start from a pushed branch through the API (sourceBranch).
Plan files and stored entitlements may still carry maxWorkspaces; it is
accepted and ignored.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-11-04 12:43:02 +08:00

515 lines
22 KiB
JavaScript

// Runs WishForge on a local k3s cluster (a Colima VM) for development, end to end: wishes are
// executed by the Kubernetes provisioner in per-run pods, verified and merged.
//
// node scripts/k3s-local.mjs up start the colima-wishforge VM with k3s and a local registry
// node scripts/k3s-local.mjs build build the service images and push them to the local registry
// (`build web worker` rebuilds only those)
// node scripts/k3s-local.mjs deploy render infra/k8s/local and apply it (needs WF_CODEX_DEFAULT_MODEL)
// node scripts/k3s-local.mjs model store WF_TEST_MODEL_API_KEY as the tenant's Codex key
// node scripts/k3s-local.mjs smoke create a project and run one wish to the end
// node scripts/k3s-local.mjs web forward the workbench (http://localhost:3100) to the host
// node scripts/k3s-local.mjs status show the platform and sandbox pods
// node scripts/k3s-local.mjs down stop the VM (data stays; `up` resumes it)
//
// Every kubectl and docker call names the colima-wishforge context explicitly, so the current
// context (which may be a shared cluster) is never touched.
import { spawn, spawnSync } from 'node:child_process';
import { createHash, randomBytes } from 'node:crypto';
import { mkdir, readFile, statfs, symlink, writeFile } from 'node:fs/promises';
import net from 'node:net';
import os from 'node:os';
import path from 'node:path';
const PROFILE = 'wishforge';
const CONTEXT = `colima-${PROFILE}`;
const REGISTRY = 'localhost:5000';
const STATE = '.wishforge/k3s';
const DOCKER_CONFIG = path.resolve(STATE, 'docker');
const SERVICES = 'infra/images/runtime.Dockerfile';
/** How each image is built. */
const IMAGES = {
runtime: { file: SERVICES, context: '.', target: 'runtime' },
worker: { file: SERVICES, context: '.', target: 'worker' },
'git-storage': { file: SERVICES, context: '.', target: 'git-storage' },
provisioner: { file: SERVICES, context: '.', target: 'provisioner' },
// Development sign-in exists only in a development build of the web.
web: { file: SERVICES, context: '.', target: 'web', args: { WEB_BUILD: 'development' } },
};
const TARGETS = Object.keys(IMAGES);
const BASE_IMAGES = {
NODE_IMAGE: process.env.WF_K3S_NODE_IMAGE ?? 'node:22-bookworm',
NODE_SLIM_IMAGE: process.env.WF_K3S_NODE_SLIM_IMAGE ?? 'node:22-bookworm-slim',
DOCKER_CLI_IMAGE: process.env.WF_K3S_DOCKER_CLI_IMAGE ?? 'docker:28-cli',
KUBECTL_IMAGE: process.env.WF_K3S_KUBECTL_IMAGE ?? 'rancher/kubectl:v1.31.4',
KUBECTL_PATH: process.env.WF_K3S_KUBECTL_PATH ?? '/bin/kubectl',
};
function run(command, args, options = {}) {
const shown = args.map(arg => (arg.includes('\n') ? '<program>' : arg.replace(/(\/\/)[^@/\s]+@/, '$1***@')));
console.log(`$ ${command} ${shown.join(' ')}`);
const result = spawnSync(command, args, {
stdio: options.input ? ['pipe', 'inherit', 'inherit'] : 'inherit',
...options,
});
if (result.status !== 0) throw new Error(`${command} exited ${result.status ?? result.signal}`);
}
function capture(command, args, options = {}) {
const result = spawnSync(command, args, { encoding: 'utf8', ...options });
return { ok: result.status === 0, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
}
const kubectl = (args, options) => run('kubectl', ['--context', CONTEXT, ...args], options);
const docker = (args, options) => run('docker', ['--context', CONTEXT, ...args], { env: dockerEnv(), ...options });
const dockerEnv = () => ({ ...process.env, DOCKER_CONFIG });
/**
* Docker runs with a client configuration of its own: a credential helper left in the user's
* (Docker Desktop's `desktop` after uninstalling it, for one) breaks BuildKit's pulls, and nothing
* here needs registry credentials. The contexts, CLI plugins and other settings stay the user's.
*/
async function prepareDocker() {
const home = process.env.DOCKER_CONFIG ?? path.join(os.homedir(), '.docker');
const settings = JSON.parse(await readFile(path.join(home, 'config.json'), 'utf8').catch(() => '{}'));
for (const key of ['auths', 'credsStore', 'credHelpers', 'currentContext']) delete settings[key];
await mkdir(DOCKER_CONFIG, { recursive: true, mode: 0o700 });
await writeFile(path.join(DOCKER_CONFIG, 'config.json'), `${JSON.stringify(settings, null, 2)}\n`, { mode: 0o600 });
for (const name of ['contexts', 'cli-plugins'])
await symlink(path.join(home, name), path.join(DOCKER_CONFIG, name)).catch(error => {
if (error.code !== 'EEXIST') throw error;
});
}
async function readJson(name) {
return JSON.parse(await readFile(`${STATE}/${name}`, 'utf8').catch(() => '{}'));
}
async function writeJson(name, value) {
await mkdir(STATE, { recursive: true, mode: 0o700 });
await writeFile(`${STATE}/${name}`, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
}
function requireCluster() {
if (!capture('kubectl', ['config', 'get-contexts', CONTEXT]).ok)
throw new Error(`The ${CONTEXT} context does not exist yet: run \`node scripts/k3s-local.mjs up\` first`);
}
/** The profile's status in `colima list` (Running, Stopped, Broken), or undefined before it exists. */
function profileStatus() {
for (const line of capture('colima', ['list', '--json']).stdout.split('\n').filter(Boolean)) {
const profile = JSON.parse(line);
if (profile.name === PROFILE) return profile.status;
}
}
/**
* Colima hands a proxy in the host environment (a local Clash, for one) to the VM's Docker and k3s
* but excludes none of the VM's own addresses: the API server then reaches the kubelet through
* the proxy and `kubectl logs` and `exec` fail. Exclude the VM network (Colima's default
* 192.168.5.0/24), the pod and service ranges and the cluster domain.
*/
function colimaEnvironment() {
if (!['HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY'].some(name => process.env[name] || process.env[name.toLowerCase()]))
return process.env;
const excluded = new Set((process.env.NO_PROXY ?? process.env.no_proxy ?? '').split(',').filter(Boolean));
for (const entry of [
'localhost',
'127.0.0.1',
'192.168.5.0/24',
'10.42.0.0/16',
'10.43.0.0/16',
'.svc',
'.cluster.local',
])
excluded.add(entry);
const noProxy = [...excluded].join(',');
return { ...process.env, NO_PROXY: noProxy, no_proxy: noProxy };
}
/** The user's current kubectl and Docker contexts (with the user's own Docker configuration). */
const currentContexts = () => ({
kubectl: capture('kubectl', ['config', 'current-context']).stdout.trim(),
docker: capture('docker', ['context', 'show']).stdout.trim(),
});
/**
* Free space on this Mac's disk, where Colima keeps the VM's disks. They grow with every image
* built; if the Mac's disk fills up, writes inside the VM fail and its file systems break.
*/
async function freeGiB() {
const stats = await statfs(os.homedir());
return (stats.bavail * stats.bsize) / 2 ** 30;
}
const DISK_GIB = Number(process.env.WF_K3S_DISK_GIB ?? '40');
const MIN_FREE_TO_BUILD_GIB = 15;
async function up() {
const previous = currentContexts();
if (!profileStatus()) {
const free = await freeGiB();
if (free < DISK_GIB)
throw new Error(
`The VM's disk may grow to ${DISK_GIB} GiB, but only ${free.toFixed(1)} GiB are free on this Mac. Free some space or set WF_K3S_DISK_GIB lower.`,
);
}
if (!capture('colima', ['status', '--profile', PROFILE]).ok) {
// A start that failed while installing k3s (often an interrupted GitHub download) leaves the VM
// running without a runtime; stop it so that this start provisions again.
if (profileStatus() === 'Running') run('colima', ['stop', '--profile', PROFILE]);
run(
'colima',
[
'start',
'--profile',
PROFILE,
'--runtime',
'docker',
'--kubernetes',
'--cpu',
process.env.WF_K3S_CPUS ?? '6',
'--memory',
process.env.WF_K3S_MEMORY_GIB ?? '12',
'--disk',
String(DISK_GIB),
],
{ env: colimaEnvironment() },
);
}
requireCluster();
await prepareDocker();
// k3s runs on the VM's Docker, which pulls localhost:5000 from this registry.
if (!capture('docker', ['--context', CONTEXT, 'inspect', 'wishforge-registry'], { env: dockerEnv() }).ok)
docker([
'run',
'-d',
'--restart=always',
'-p',
'127.0.0.1:5000:5000',
'--name',
'wishforge-registry',
'registry:2',
]);
kubectl(['wait', '--for=condition=Ready', 'node', '--all', '--timeout=300s']);
console.log(`k3s is ready in the ${CONTEXT} context.`);
// Colima makes its VM the current kubectl and Docker context on every start. This script names
// its context on every call and never relies on that; switching back is the user's call.
const current = currentContexts();
if (previous.kubectl && current.kubectl !== previous.kubectl)
console.log(
`Colima switched the current kubectl context from ${previous.kubectl} to ${current.kubectl}. To switch back:\n kubectl config use-context ${previous.kubectl}`,
);
if (previous.docker && current.docker !== previous.docker)
console.log(
`Colima switched the current Docker context from ${previous.docker} to ${current.docker}. To switch back:\n docker context use ${previous.docker}`,
);
}
/**
* Build steps reach the network straight from the VM. Behind a proxy in the host environment they
* use it too, at the address the VM sees the host at, as Colima arranges for Docker and k3s.
* Docker leaves these predefined arguments out of the image history and the build cache.
*/
function proxyBuildArgs() {
const args = {};
for (const name of ['HTTP_PROXY', 'HTTPS_PROXY']) {
const value = process.env[name] ?? process.env[name.toLowerCase()];
if (!value) continue;
const url = new URL(value);
if (['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname)) url.hostname = '192.168.5.2';
args[name] = args[name.toLowerCase()] = url.href.replace(/\/$/, '');
}
if (Object.keys(args).length) args.NO_PROXY = args.no_proxy = 'localhost,127.0.0.1';
return args;
}
async function build() {
requireCluster();
const selected = process.argv.slice(3);
const unknown = selected.filter(target => !TARGETS.includes(target));
if (unknown.length) throw new Error(`Unknown images ${unknown.join(', ')}; choose from ${TARGETS.join(', ')}`);
const free = await freeGiB();
if (free < MIN_FREE_TO_BUILD_GIB)
throw new Error(
`Only ${free.toFixed(1)} GiB are free on this Mac; free at least ${MIN_FREE_TO_BUILD_GIB} GiB before building, since the VM's disk grows with every build.`,
);
await prepareDocker();
const buildArgs = (extra = {}) =>
Object.entries({ ...BASE_IMAGES, ...proxyBuildArgs(), ...extra }).flatMap(([name, value]) => [
'--build-arg',
`${name}=${value}`,
]);
// Images not rebuilt keep the digests they were deployed with.
const images = await readJson('images.json');
for (const target of selected.length ? selected : TARGETS) {
const image = IMAGES[target];
const tag = `${REGISTRY}/wishforge/${target}:dev`;
docker([
'build',
'--file',
image.file,
...(image.target ? ['--target', image.target] : []),
...buildArgs(image.args),
'--tag',
tag,
image.context,
]);
docker(['push', tag]);
const digest = capture(
'docker',
['--context', CONTEXT, 'image', 'inspect', '--format', '{{json .RepoDigests}}', tag],
{ env: dockerEnv() },
);
const reference = JSON.parse(digest.stdout || '[]').find(value =>
value.startsWith(`${REGISTRY}/wishforge/${target}@`),
);
if (!reference) throw new Error(`No registry digest for ${tag}`);
images[target] = reference;
}
await writeJson('images.json', images);
console.log(images);
// Superseded images no container uses, and build cache beyond 10 GB, would only grow the disk.
for (const prune of [
['image', 'prune', '--force'],
['builder', 'prune', '--force', '--max-used-space', '10GB'],
])
if (!capture('docker', ['--context', CONTEXT, ...prune], { env: dockerEnv() }).ok)
console.log(`Could not run docker ${prune.join(' ')}; the VM's disk keeps the old layers.`);
}
async function deploy() {
requireCluster();
const images = await readJson('images.json');
if (TARGETS.some(target => !images[target]))
throw new Error('Build the images first: node scripts/k3s-local.mjs build');
const model = process.env.WF_CODEX_DEFAULT_MODEL;
if (!model) throw new Error('Set WF_CODEX_DEFAULT_MODEL to a model your gateway serves (GET <gateway>/v1/models)');
// Clear of the local development stack (npm run dev:all uses 3000), so both can run.
const webPort = process.env.WF_K3S_WEB_PORT ?? '3100';
if (!/^\d{2,5}$/.test(webPort)) throw new Error('WF_K3S_WEB_PORT must be a port number');
// Generated once and kept outside Git, so redeploying keeps the database password and tokens.
const secrets = await readJson('secrets.json');
const secret = () => randomBytes(32).toString('hex');
for (const name of [
'POSTGRES_PASSWORD',
'APP_PASSWORD',
'WF_DEV_AUTH_TOKEN',
'WF_WORKER_TOKEN',
'WF_DEPLOYER_TOKEN',
'WF_VAULT_KEY',
'WF_INTERNAL_TOKEN_KEY',
'WF_RESOURCE_GRANT_SECRET',
'WF_PROVISIONER_TOKEN',
'WF_GIT_ADMIN_TOKEN',
'WF_GIT_TOKEN_SECRET',
'WF_PROVIDER_PROXY_SIGNING_KEY',
])
secrets[name] ??= secret();
await writeJson('secrets.json', secrets);
const values = {
...secrets,
RUNTIME_IMAGE: images.runtime,
WORKER_IMAGE: images.worker,
GIT_STORAGE_IMAGE: images['git-storage'],
PROVISIONER_IMAGE: images.provisioner,
WEB_IMAGE: images.web,
// `web` forwards this port; the workbench accepts only this origin.
WEB_PUBLIC_URL: `http://localhost:${webPort}`,
MODEL: model,
OPENAI_BASE_URL: process.env.WF_OPENAI_BASE_URL ?? '',
};
const template = await readFile('infra/k8s/local/wishforge.yaml', 'utf8');
const render = digest =>
template.replace(/\{\{([A-Z_]+)\}\}/g, (_, name) => {
const value = name === 'SETTINGS_DIGEST' ? digest : values[name];
if (value === undefined) throw new Error(`No value for {{${name}}}`);
if (/['\n]/.test(value)) throw new Error(`${name} cannot contain quotes or line breaks`);
return value;
});
// Pods read the settings and secrets when they start; a digest of both in their templates
// rolls them whenever either changes.
const settings = render('')
.split(/^---$/m)
.filter(
document =>
/^kind: (?:ConfigMap|Secret)$/m.test(document) && /^ {2}name: wishforge-(?:settings|local)$/m.test(document),
);
if (settings.length !== 2) throw new Error('The manifest lost its settings or secrets');
kubectl(['apply', '-f', '-'], {
input: render(createHash('sha256').update(settings.join('---')).digest('hex').slice(0, 16)),
});
for (const name of [
'postgres',
'temporal',
'git-storage',
'api',
'provisioner',
'worker-coordinator',
'worker-deployer',
'web',
])
kubectl(['--namespace', 'wishforge-control', 'rollout', 'status', `deployment/${name}`, '--timeout=600s']);
console.log('WishForge is running. Next: node scripts/k3s-local.mjs model, then smoke; `web` opens the workbench.');
}
/** Runs a small Node program inside the API pod, against the API on its own loopback. */
function insideApi(program, input) {
kubectl(
[
'--namespace',
'wishforge-control',
'exec',
'-i',
'deployment/api',
'-c',
'api',
'--',
'node',
'--input-type=module',
'-e',
program,
],
{ input: input ?? '' },
);
}
const API_CLIENT = `
const token = process.env.WF_DEV_AUTH_TOKEN;
const api = async (method, path, body, key) => {
const response = await fetch('http://127.0.0.1:4000/v1' + path, {
method,
headers: { authorization: 'Bearer ' + token, 'content-type': 'application/json', ...(key ? { 'idempotency-key': key } : {}) },
...(body ? { body: JSON.stringify(body) } : {}),
});
const payload = await response.json().catch(() => ({}));
if (!response.ok) throw new Error(method + ' ' + path + ' -> ' + response.status + ' ' + JSON.stringify(payload));
return payload.data;
};
const stdin = async () => { let text = ''; for await (const chunk of process.stdin) text += chunk; return text; };
`;
async function model() {
requireCluster();
const apiKey = process.env.WF_TEST_MODEL_API_KEY;
if (!apiKey) throw new Error('Set WF_TEST_MODEL_API_KEY to the gateway key');
// The key travels on stdin, never on a command line.
insideApi(
`${API_CLIENT}
const { apiKey } = JSON.parse(await stdin());
const existing = (await api('GET', '/provider-connections')).filter(c => c.provider === 'codex' && c.status === 'active');
for (const connection of existing) await api('DELETE', '/provider-connections/' + connection.id);
const created = await api('POST', '/provider-connections', { provider: 'codex', label: 'model gateway', apiKey });
console.log('Codex key stored as connection ' + created.id);
console.log((await api('GET', '/providers')).map(p => p.id + ': ' + (p.enabled ? 'enabled' : p.reason)).join('\\n'));`,
JSON.stringify({ apiKey }),
);
}
async function smoke() {
requireCluster();
insideApi(`${API_CLIENT}
const stamp = Date.now().toString(36);
const project = await api('POST', '/projects', { name: 'k3s smoke ' + stamp });
console.log('project', project.id, project.repositoryStatus, project.repositoryUrl);
await api('PATCH', '/projects/' + project.id, {
verificationCommands: [['node', '-e', "process.exit(require('fs').readFileSync('greeting.txt','utf8').trim()==='hello world'?0:1)"]],
});
const wish = await api('POST', '/projects/' + project.id + '/wishes', {
goal: 'Create a file named greeting.txt whose whole content is the line "hello world".',
acceptanceCriteria: ['greeting.txt contains exactly "hello world"'],
provider: 'codex',
budget: { maxTokens: 150000, maxMinutes: 20, maxRepairRounds: 2 },
}, 'k3s-smoke-' + stamp);
console.log('run', wish.run_id);
let after = 0;
for (let i = 0; i < 240; i++) {
for (const event of await api('GET', '/runs/' + wish.run_id + '/events?after=' + after)) {
after = event.seq;
console.log(event.time.slice(11, 19), event.type, event.message.slice(0, 160));
}
const run = await api('GET', '/runs/' + wish.run_id);
// The dispatcher retries a provisioner that is not ready yet; every other wait needs a person.
if (['succeeded', 'failed', 'canceled'].includes(run.status) || (run.status === 'waiting' && run.waitingReason !== 'sandbox_provisioner')) {
console.log('\\nfinal:', run.status, run.stage, run.waitingReason ?? '', run.waitingDetail ?? '');
console.log('integrated commit:', run.integratedCommit ?? '-', ' tokens used:', run.usedTokens);
process.exit(run.status === 'succeeded' ? 0 : 1);
}
await new Promise(resolve => setTimeout(resolve, 5000));
}
console.log('Timed out after 20 minutes');
process.exit(1);`);
}
/** The browser-facing origin a deployment was configured with (it refuses requests from others). */
function deployedOrigin(deployment, variable) {
const deployed = capture('kubectl', [
'--context',
CONTEXT,
'--namespace',
'wishforge-control',
'get',
`deployment/${deployment}`,
'--output',
`jsonpath={.spec.template.spec.containers[0].env[?(@.name=="${variable}")].value}`,
]);
if (!deployed.ok || !deployed.stdout.trim()) throw new Error('Deploy first: node scripts/k3s-local.mjs deploy');
return new URL(deployed.stdout.trim());
}
/** Whether a local port is free: kubectl, finding it taken on 127.0.0.1, would listen on ::1 only. */
const portFree = port =>
new Promise(resolve => {
const server = net.createServer();
server.once('error', () => resolve(false));
server.listen(Number(port), '127.0.0.1', () => server.close(() => resolve(true)));
});
/** Forwards the workbench to the port it was deployed for. */
async function web() {
requireCluster();
const forwards = [{ service: 'web', port: 3000, url: deployedOrigin('web', 'WF_PUBLIC_URL') }];
for (const { url } of forwards)
if (!(await portFree(url.port)))
throw new Error(
`Port ${url.port} is taken on this machine (a local dev server?). Deploy with a free WF_K3S_WEB_PORT, then run web again.`,
);
console.log(`Workbench ${forwards[0].url.origin} (exactly this address).`);
console.log('Ctrl-C stops forwarding; a deploy replaces the pods and ends it, so run `web` again then.');
const children = forwards.map(({ service, port, url }) =>
spawn(
'kubectl',
[
'--context',
CONTEXT,
'--namespace',
'wishforge-control',
'port-forward',
`service/${service}`,
`${url.port}:${port}`,
],
{ stdio: 'inherit' },
),
);
// Ctrl-C reaches kubectl too; a plain SIGTERM (from a supervisor) reaches only this process.
for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP'])
process.once(signal, () => {
for (const child of children) child.kill();
process.exit(1);
});
const code = await Promise.race(children.map(child => new Promise(resolve => child.on('exit', resolve))));
for (const child of children) child.kill();
if (code) throw new Error(`kubectl port-forward exited ${code}`);
}
async function status() {
requireCluster();
kubectl(['get', 'pods,jobs,pvc', '--namespace', 'wishforge-control']);
kubectl(['get', 'pods,jobs,pvc,networkpolicies', '--namespace', 'wishforge-tenant-demo']);
}
async function down() {
run('colima', ['stop', '--profile', PROFILE]);
}
const commands = { up, build, deploy, model, smoke, web, status, down };
const command = commands[process.argv[2]];
if (!command) {
console.error(`Usage: node scripts/k3s-local.mjs ${Object.keys(commands).join('|')}`);
process.exit(2);
}
await command().catch(error => {
console.error(error.message);
process.exit(1);
});