#!/usr/bin/env node // Operator CLI of WishForge's deployment in the POC cluster (docs/cluster-poc-deployment.md). // Every command names its cluster explicitly and prints no secret. Importing it starts nothing. // WishForge runs in the cluster only; this machine keeps the operator's inputs (STATE) and nothing else. // // node scripts/cluster-poc.mjs image build and push the image in the cluster (Kaniko) // node scripts/cluster-poc.mjs bootstrap namespaces, storage, access rules // node scripts/cluster-poc.mjs deploy-key create the Gitea deploy key; prints its public half // node scripts/cluster-poc.mjs gateway-key [--stdin] the model gateway key, from the clipboard or stdin // node scripts/cluster-poc.mjs secrets Secrets and ConfigMaps from this Mac's settings // node scripts/cluster-poc.mjs apply [--maintenance] [--runner] the WishForge Pod // node scripts/cluster-poc.mjs public [--production] the public HTTPS entry (billed load balancer) // node scripts/cluster-poc.mjs fanyu-access --kubeconfig WishForge's identity in Fanyu's cluster // node scripts/cluster-poc.mjs projects --replace delete every project; create the POC and Fanyu projects // node scripts/cluster-poc.mjs restore [--runner] replace the cluster's data with a node snapshot // node scripts/cluster-poc.mjs status show the deployment's state import { spawn } from 'node:child_process'; import { createHash, randomBytes } from 'node:crypto'; import { createReadStream, createWriteStream } from 'node:fs'; import { chmod, mkdir, mkdtemp, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { Resolver, getServers, resolve4 } from 'node:dns/promises'; import { isIP } from 'node:net'; import { setTimeout as delay } from 'node:timers/promises'; import { ARCHIVES, COUNTS_SQL } from './cluster-poc-backup.mjs'; export const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); export const STATE = path.join(ROOT, '.wishforge/cluster-poc'); export const KUBECONFIG = path.join(os.homedir(), '.config/droboticflow/kubeconfigs/poc.config'); /** * The operator's inputs to the cluster's settings, kept private here: the keys and tokens the data * depends on (carried.env, see CARRIED), the Runners' signing key, the plan catalog, the POC's * profile and Gitea's pinned host keys. */ export const INPUTS = { carried: path.join(STATE, 'carried.env'), signingKey: path.join(STATE, 'runner-ed25519.pem'), plans: path.join(STATE, 'plans.json'), knownHosts: path.join(STATE, 'known_hosts'), }; export const POC_PROFILE = path.join(STATE, 'poc-profile.json'); export const CLUSTER = 'https://120.48.181.105:6443'; export const NAMESPACE = 'wishforge'; export const CHECK_NAMESPACE = 'wishforge-checks'; /** Fanyu's cluster and namespace, which its project delivers to (fanyu-access.yaml). */ export const FANYU = { cluster: 'https://180.76.138.88:6443', namespace: 'droboticflow-fanyu' }; /** Pushes use the registry's public address; the cluster pulls through its VPC address. */ export const PULL_REGISTRY = 'ccr-29eug8s3-vpc.cnc.bj.baidubce.com/dataflow-poc'; export const IMAGE_REPOSITORY = 'wishforge-cluster-poc'; const POSTGRES_IMAGE = 'docker.io/library/postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3'; /** Every input of the image, pinned: base images by digest, sources by commit, binaries by checksum. */ export const IMAGE_INPUTS = { NODE_IMAGE: 'docker.io/library/node:26-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9', GO_IMAGE: 'docker.io/library/golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac', CADDY_IMAGE: 'docker.io/library/caddy:2.11.7-alpine@sha256:eb4d35b7452d1ab6171fbdbe8d9275b2301dad6dad92115d91e19fb9b24d510f', // The image's pg_dump and pg_restore are the database's own (the backup). POSTGRES_IMAGE, MINIO_RELEASE: 'RELEASE.2025-07-23T15-54-02Z', MINIO_COMMIT: '7ced9663e6a791fef9dc6be798ff24cda9c730ac', KUBECTL_VERSION: 'v1.34.2', KUBECTL_SHA256: '9591f3d75e1581f3f7392e6ad119aab2f28ae7d6c6e083dc5d22469667f27253', CLAUDE_CODE_VERSION: '2.1.292', }; /** Third-party service images, pinned by digest; the cluster pulls them from Docker Hub. */ export const SERVICE_IMAGES = { postgres: POSTGRES_IMAGE, temporal: 'docker.io/temporalio/temporal:1.4.1@sha256:9844298258519ebece82b3253f1ee1fc4eb8d87b7abcc264cfb676be342c74a5', }; /** Paths never sent to the build, whatever Git says: local agent settings and private state. */ const EXCLUDED = [/^\.claude\//, /^\.wishforge\//, /(^|\/)\.env(\..*)?$/]; function capture(executable, args, options = {}) { return new Promise((resolve, reject) => { const child = spawn(executable, args, { cwd: ROOT, ...options, stdio: ['pipe', 'pipe', 'pipe'] }); const out = []; const err = []; child.stdout.on('data', chunk => out.push(chunk)); child.stderr.on('data', chunk => err.push(chunk)); child.once('error', reject); child.once('close', code => resolve({ code, stdout: Buffer.concat(out).toString('utf8'), stderr: Buffer.concat(err).toString('utf8') }), ); child.stdin.end(options.input); }); } export async function kubectl(args, { input, timeoutSeconds = 60, allowFailure = false } = {}) { const result = await capture( 'kubectl', ['--kubeconfig', KUBECONFIG, `--request-timeout=${timeoutSeconds}s`, ...args], { input, env: { PATH: process.env.PATH, HOME: os.homedir() } }, ); if (result.code !== 0 && !allowFailure) throw new Error(`kubectl ${args.slice(0, 3).join(' ')} failed: ${result.stderr.trim().slice(0, 800)}`); return result; } /** Refuses to act unless the kubeconfig names the POC cluster. */ export async function assertCluster() { const view = JSON.parse((await kubectl(['config', 'view', '--minify', '-o', 'json'])).stdout); if (view.clusters?.length !== 1 || view.clusters[0].cluster.server !== CLUSTER) throw new Error(`The kubeconfig does not name the POC cluster ${CLUSTER}`); } const sha256File = file => new Promise((resolve, reject) => { const hash = createHash('sha256'); createReadStream(file) .on('data', chunk => hash.update(chunk)) .on('error', reject) .on('end', () => resolve(hash.digest('hex'))); }); /** kubectl for the image, downloaded once and kept only if it matches the published checksum. */ async function kubectlBinary() { const file = path.join(STATE, `kubectl-${IMAGE_INPUTS.KUBECTL_VERSION}-linux-amd64`); if ( await stat(file).then( () => true, () => false, ) ) { if ((await sha256File(file)) === IMAGE_INPUTS.KUBECTL_SHA256) return file; await rm(file, { force: true }); } const url = `https://dl.k8s.io/release/${IMAGE_INPUTS.KUBECTL_VERSION}/bin/linux/amd64/kubectl`; const response = await fetch(url); if (!response.ok) throw new Error(`Cannot download kubectl (${response.status}); set HTTPS_PROXY if needed`); const temporary = `${file}.${randomBytes(4).toString('hex')}.tmp`; await writeFile(temporary, Buffer.from(await response.arrayBuffer()), { mode: 0o600 }); if ((await sha256File(temporary)) !== IMAGE_INPUTS.KUBECTL_SHA256) { await rm(temporary, { force: true }); throw new Error('The downloaded kubectl does not match its published SHA-256'); } await rename(temporary, file); return file; } /** The files of the build: what Git tracks or would track, minus {@link EXCLUDED}. */ export function buildFiles(listing) { return listing .split('\0') .filter(Boolean) .filter(file => !EXCLUDED.some(pattern => pattern.test(file))); } async function sourceArchive(directory) { const listing = await capture('git', ['ls-files', '-z', '--cached', '--others', '--exclude-standard']); if (listing.code !== 0) throw new Error(`git ls-files failed: ${listing.stderr}`); const files = []; for (const file of buildFiles(listing.stdout)) if ( await stat(path.join(ROOT, file)).then( info => info.isFile(), () => false, ) ) files.push(file); const list = path.join(directory, 'files'); await writeFile(list, `${files.join('\0')}\0`); const extra = path.join(directory, 'extra'); await mkdir(path.join(extra, '.cluster-poc-build'), { recursive: true }); await writeFile(path.join(extra, '.cluster-poc-build/kubectl'), await readFile(await kubectlBinary())); const tarball = path.join(directory, 'source.tar'); const env = { PATH: process.env.PATH, COPYFILE_DISABLE: '1' }; for (const args of [ ['--no-mac-metadata', '--null', '-cf', tarball, '-T', list], ['--no-mac-metadata', '-rf', tarball, '-C', extra, '.cluster-poc-build/kubectl'], ]) { const result = await capture('tar', args, { env }); if (result.code !== 0) throw new Error(`tar failed: ${result.stderr}`); } const gzip = await capture('gzip', ['-n', '-9', tarball], { env }); if (gzip.code !== 0) throw new Error(`gzip failed: ${gzip.stderr}`); return { file: `${tarball}.gz`, count: files.length }; } const UNPACK = [ 'set -e', 'sum=$(sha256sum /context/source.tar.gz)', '[ "${sum%% *}" = "$1" ] || exit 76', 'mkdir -p /context/source && tar -xzf /context/source.tar.gz -C /context/source && touch /context/ready', ].join('\n'); /** The Kaniko Pod of one image build, in the POC namespace where the push credential lives. */ export function imageBuildPod(profile, input) { return { apiVersion: 'v1', kind: 'Pod', metadata: { name: input.name, namespace: profile.namespace, labels: { 'app.kubernetes.io/managed-by': 'wishforge-cluster-poc', 'app.kubernetes.io/name': 'wishforge-image' }, }, spec: { automountServiceAccountToken: false, restartPolicy: 'Never', activeDeadlineSeconds: 5400, nodeSelector: { 'kubernetes.io/arch': 'amd64', 'kubernetes.io/os': 'linux' }, containers: [ { name: 'kaniko', image: profile.kanikoImage, command: ['/busybox/sh', '-c'], args: [ 'until [ -f /context/ready ]; do /busybox/sleep 1; done; exec /kaniko/executor "$@"', '--', '--context=dir:///context/source', '--dockerfile=/context/source/infra/images/cluster-poc.Dockerfile', `--destination=${profile.registry}/${IMAGE_REPOSITORY}:${input.tag}`, '--digest-file=/dev/termination-log', ...Object.entries(IMAGE_INPUTS).map(([key, value]) => `--build-arg=${key}=${value}`), ...profile.registryMirrors.map(mirror => `--registry-mirror=${mirror}`), ...(profile.registryMirrors.length ? ['--skip-default-registry-fallback'] : []), '--snapshot-mode=redo', '--verbosity=info', ], resources: { requests: { cpu: '1', memory: '2Gi' }, limits: { cpu: '6', memory: '12Gi', 'ephemeral-storage': '30Gi' }, }, volumeMounts: [ { name: 'context', mountPath: '/context' }, { name: 'registry', mountPath: '/kaniko/.docker', readOnly: true }, ], terminationMessagePolicy: 'File', }, ], volumes: [ { name: 'context', emptyDir: { sizeLimit: '4Gi' } }, { name: 'registry', secret: { secretName: profile.registrySecret, items: [{ key: '.dockerconfigjson', path: 'config.json' }] }, }, ], }, }; } async function image() { await assertCluster(); const profile = JSON.parse(await readFile(POC_PROFILE, 'utf8')); await mkdir(STATE, { recursive: true, mode: 0o700 }); await chmod(STATE, 0o700); const head = (await capture('git', ['rev-parse', 'HEAD'])).stdout.trim(); const dirty = (await capture('git', ['status', '--porcelain'])).stdout.trim().length > 0; const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..*$/, '').replace('T', ''); const tag = `${head.slice(0, 12)}${dirty ? '-dirty' : ''}-${stamp}`; const directory = await mkdtemp(path.join(os.tmpdir(), 'wishforge-image-')); const name = `wishforge-image-${randomBytes(4).toString('hex')}`; try { const archive = await sourceArchive(directory); const sha = await sha256File(archive.file); console.log( `Source: ${archive.count} files, ${((await stat(archive.file)).size / 1e6).toFixed(1)} MB, sha256 ${sha.slice(0, 16)}…`, ); await kubectl(['create', '-f', '-'], { input: JSON.stringify(imageBuildPod(profile, { name, tag })) }); await kubectl(['-n', profile.namespace, 'wait', '--for=condition=Ready', `pod/${name}`, '--timeout=600s'], { timeoutSeconds: 620, }); await kubectl(['-n', profile.namespace, 'cp', archive.file, `${name}:/context/source.tar.gz`, '-c', 'kaniko'], { timeoutSeconds: 600, }); await kubectl([ '-n', profile.namespace, 'exec', name, '-c', 'kaniko', '--', '/busybox/sh', '-c', UNPACK, 'unpack', sha, ]); console.log(`Building ${profile.registry}/${IMAGE_REPOSITORY}:${tag} in pod ${name} …`); let pod; for (const until = Date.now() + 90 * 60_000; Date.now() < until; await delay(15_000)) { const result = await kubectl(['-n', profile.namespace, 'get', 'pod', name, '-o', 'json'], { allowFailure: true }); if (result.code !== 0) continue; pod = JSON.parse(result.stdout); if (['Succeeded', 'Failed'].includes(pod.status?.phase)) break; } const terminated = pod?.status?.containerStatuses?.[0]?.state?.terminated; if (pod?.status?.phase !== 'Succeeded' || !/^sha256:[a-f0-9]{64}$/.test(terminated?.message?.trim() ?? '')) { const logs = await kubectl(['-n', profile.namespace, 'logs', name, '-c', 'kaniko', '--tail=80'], { allowFailure: true, }); throw new Error(`The image build failed (${pod?.status?.phase ?? 'unknown'}):\n${logs.stdout.slice(-6000)}`); } const digest = terminated.message.trim(); const record = { repository: `${PULL_REGISTRY}/${IMAGE_REPOSITORY}`, pushedAs: `${profile.registry}/${IMAGE_REPOSITORY}:${tag}`, digest, reference: `${PULL_REGISTRY}/${IMAGE_REPOSITORY}@${digest}`, sourceCommit: head, dirty, sourceSha256: sha, inputs: IMAGE_INPUTS, builtAt: new Date().toISOString(), }; await writeFile(path.join(STATE, 'image.json'), `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 }); console.log(`Image: ${record.reference}`); } finally { await kubectl(['-n', profile.namespace, 'delete', 'pod', name, '--ignore-not-found', '--wait=false'], { allowFailure: true, }); await rm(directory, { recursive: true, force: true }); } } // ------------------------------------------------------------------ configuration of the Pod /** Where each container keeps its files: the paths infra/k8s/cluster-poc/platform.yaml mounts. */ export const PATHS = { gitRoot: '/var/lib/wishforge/git', localAccess: '/var/lib/wishforge/platform-state/local-access.json', pocState: '/var/lib/wishforge/poc-state', runnerState: '/var/lib/wishforge/runner-state', execRoot: '/var/lib/wishforge/exec', claudeHome: '/var/lib/wishforge/claude-home', claudeHomeApi: '/var/lib/wishforge/claude-home-api', // OpenSSH finds ~/.ssh through the user database, not $HOME: this is the image user's home. gitHome: '/home/wishforge', config: '/etc/wishforge/config', signingKey: '/etc/wishforge/keys/runner-ed25519.pem', kubeToken: '/var/run/wishforge/kube/token', fanyuKubeconfig: '/etc/wishforge/fanyu/kubeconfig', }; /** * The Mac's settings the cluster keeps as they are: the keys and tokens that persisted data, issued * grants and the Runner's registration rely on. Everything else is the cluster's own. */ export const CARRIED = [ 'WF_TENANT_ID', 'WF_REGION', 'WF_DISPATCH_TENANTS', 'WF_DEV_AUTH_TOKEN', 'WF_WORKER_TOKEN', 'WF_VERIFIER_TOKEN', 'WF_DEPLOYER_TOKEN', 'WF_EVALUATOR_TOKEN', 'WF_VAULT_KEY', 'WF_SESSION_SECRET', 'WF_INTERNAL_TOKEN_KEY', 'WF_PROVISIONER_TOKEN', 'WF_RESOURCE_GRANT_SECRET', 'WF_PROVIDER_PROXY_SIGNING_KEY', 'WF_METRICS_TOKEN', 'WF_GIT_ADMIN_TOKEN', 'WF_GIT_TOKEN_SECRET', 'S3_BUCKET', 'S3_REGION', 'TEMPORAL_NAMESPACE', ]; const RUNNER_CARRIED = ['WF_RUNNER_TOKEN', 'WF_TENANT_ID', 'WF_ENVIRONMENT_ID', 'WF_RUNNER_PUBLIC_KEYS']; const DOMAIN = /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/; const PUBLIC_ORIGIN = new RegExp(`^https?://(?:\\d{1,3}(?:\\.\\d{1,3}){3}|${DOMAIN.source.slice(1, -1)})$`); /** Public resolvers that show what the internet, Let's Encrypt included, sees of a domain. */ const PUBLIC_RESOLVERS = ['223.5.5.5', '119.29.29.29']; /** A publicly routable IPv4 address: never a URL, private, shared or documentation address. */ export function publicIPv4(value) { if (isIP(value) !== 4) throw new Error('A public IPv4 address is required'); const [a, b, c] = value.split('.').map(Number); if ( a === 0 || a === 10 || a === 127 || a >= 224 || (a === 100 && b >= 64 && b <= 127) || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) || (a === 192 && (b === 168 || b === 0 || (b === 88 && c === 99))) || (a === 198 && (b === 18 || b === 19 || (b === 51 && c === 100))) || (a === 203 && b === 0 && c === 113) ) throw new Error('The entry must use a publicly routable IPv4 address'); return value; } /** A domain name for the public entry, as the operator's DNS points it at the load balancer. */ export function publicDomain(name) { const domain = String(name ?? '') .trim() .toLowerCase(); if (!DOMAIN.test(domain)) throw new Error(`Not a domain name: ${name}`); return domain; } /** The console's address through a port-forward to the Pod: the Mac's own console keeps port 3000. */ export const PORT_FORWARD_ORIGIN = 'http://127.0.0.1:13000'; /** The settings files of the platform, executor and Runner containers; see {@link CARRIED}. */ export function clusterEnvironments({ local, runner, runnerFanyu, image, generated, origins = [] }) { const missing = [ ...CARRIED.filter(key => !local[key]).map(key => `carried.env ${key}`), ...RUNNER_CARRIED.filter(key => !runner[key]).map(key => `runner.env ${key}`), ...(runnerFanyu ? RUNNER_CARRIED.filter(key => !runnerFanyu[key]).map(key => `Fanyu's Runner ${key}`) : []), ]; if (missing.length) throw new Error(`Missing settings: ${missing.join(', ')}`); for (const origin of origins) if (!PUBLIC_ORIGIN.test(origin)) throw new Error(`A public origin is http(s)://: ${origin}`); const poc = { WF_LOCAL_POC_BINDING_FILE: `${PATHS.pocState}/project.json`, WF_POC_PROFILE_FILE: `${PATHS.pocState}/poc-profile.json`, }; const platform = { WF_DEPLOYMENT_MODE: 'development', ...Object.fromEntries(CARRIED.map(key => [key, local[key]])), DATABASE_ADMIN_URL: `postgresql://wishforge:${encodeURIComponent(generated.postgresPassword)}@127.0.0.1:5432/wishforge`, DATABASE_URL: `postgresql://wishforge_app:${encodeURIComponent(generated.appPassword)}@127.0.0.1:5432/wishforge`, WF_DATABASE_APP_PASSWORD: generated.appPassword, S3_ENDPOINT: 'http://127.0.0.1:9000', S3_ACCESS_KEY_ID: generated.objectStorageUser, S3_SECRET_ACCESS_KEY: generated.objectStoragePassword, PORT: '4000', HOST: '127.0.0.1', WF_API_URL: 'http://127.0.0.1:4000', WF_CONTROL_PLANE_URL: 'http://127.0.0.1:4000', WF_PROVISIONER_URL: 'http://127.0.0.1:4450', WF_PROVISIONER_BACKEND: 'local', TEMPORAL_ADDRESS: '127.0.0.1:7233', WF_GIT_PORT: '4504', WF_GIT_SERVICE_URL: 'http://127.0.0.1:4504', WF_GIT_PUBLIC_BASE_URL: 'http://127.0.0.1:4504', WF_GIT_ALLOW_LOCAL_HTTP: 'true', WF_GIT_ROOT: PATHS.gitRoot, WF_LOCAL_ACCESS: 'true', WF_LOCAL_ACCESS_FILE: PATHS.localAccess, // The web launcher passes only this token name, not WF_DEV_AUTH_TOKEN, to the console. WF_LOCAL_API_TOKEN: local.WF_DEV_AUTH_TOKEN, // In the Pod, and through `kubectl port-forward pod/wishforge-0 13000:3000` from an operator's machine. WF_LOCAL_ALLOWED_ORIGINS: ['http://127.0.0.1:3000', 'http://localhost:3000', PORT_FORWARD_ORIGIN, ...origins].join( ',', ), WF_PUBLIC_URL: origins[0] ?? 'http://127.0.0.1:3000', WF_WEB_SERVE_MODE: 'production', WF_WEB_HOST: '127.0.0.1', WF_WEB_PORT: '3000', WF_RUNNER_SIGNING_KEY_FILE: PATHS.signingKey, WF_PLAN_CATALOG_FILE: `${PATHS.config}/plans.json`, WF_CLAUDE_BIN: '/usr/local/bin/claude', WF_CLAUDE_LOCAL_AUTH_HOME: PATHS.claudeHomeApi, WF_CLAUDE_ASK_MODEL: 'deepseek-flash@latest', WF_CLAUDE_ASK_FALLBACK_MODEL: 'deepseek-pro@latest', ...poc, }; const executor = { WF_DEPLOYMENT_MODE: 'development', WF_TENANT_ID: local.WF_TENANT_ID, WF_RESOURCE_GRANT_SECRET: local.WF_RESOURCE_GRANT_SECRET, WF_PROVISIONER_TOKEN: local.WF_PROVISIONER_TOKEN, WF_PROVISIONER_BACKEND: 'local', WF_CONTROL_PLANE_URL: 'http://127.0.0.1:4000', TEMPORAL_ADDRESS: '127.0.0.1:7233', TEMPORAL_NAMESPACE: local.TEMPORAL_NAMESPACE, WF_LOCAL_RUN_ROOT: `${PATHS.execRoot}/runs`, WF_GIT_ALLOW_LOCAL_HTTP: 'true', WF_CLAUDE_BIN: '/usr/local/bin/claude', WF_CLAUDE_LOCAL_AUTH_HOME: PATHS.claudeHome, WF_HOST_CHECK_POD_NAMESPACE: CHECK_NAMESPACE, WF_HOST_CHECK_POD_IMAGE: image, WF_HOST_CHECK_POD_CLAIM: 'wishforge-exec', WF_HOST_CHECK_POD_ROOT: PATHS.execRoot, WF_HOST_CHECK_KUBECONFIG: `${PATHS.config}/kubeconfig`, WF_HOST_CHECK_WRITE_PATHS: `${PATHS.execRoot}/check-cache`, WF_HOST_CHECK_POD_CPU: '4', WF_HOST_CHECK_POD_MEMORY: '8Gi', ...poc, }; // Each Runner keeps its state in its own directory, mounted at the same path. const runnerEnv = (values, keys, profile) => ({ WF_DEPLOYMENT_MODE: 'development', WF_CONTROL_PLANE_URL: 'http://127.0.0.1:4000', WF_RUNNER_ALLOW_LOCAL_HTTP: 'true', ...Object.fromEntries(RUNNER_CARRIED.map(key => [key, values[key]])), // The Mac's runner.env names a file on the Mac; its content travels in the ConfigMap. WF_RUNNER_PUBLIC_KEYS: `${PATHS.config}/${keys}`, WF_RUNNER_EXECUTE: values.WF_RUNNER_EXECUTE === 'true' ? 'true' : 'false', WF_RUNNER_STATE_DIR: PATHS.runnerState, WF_POC_PROFILE_FILE: `${PATHS.pocState}/${profile}`, }); return { platform, executor, runner: runnerEnv(runner, 'runner-public.json', 'poc-profile.json'), ...(runnerFanyu ? { runnerFanyu: runnerEnv(runnerFanyu, 'runner-fanyu-public.json', 'fanyu-profile.json') } : {}), }; } /** A settings file Node's --env-file and the container entry both read; one value per line. */ export function envFile(values) { const lines = Object.entries(values).map(([key, value]) => { if (!/^[A-Z_][A-Z0-9_]*$/.test(key) || typeof value !== 'string' || /[\r\n\0']/.test(value)) throw new Error(`Cannot write ${key} as a single-line setting`); return /^[A-Za-z0-9_@%:/.,+=?&[\]-]*$/.test(value) ? `${key}=${value}` : `${key}='${value}'`; }); return `${lines.join('\n')}\n`; } /** WishForge's identity at the POC cluster's address, by its short-lived projected token. */ export function clusterKubeconfig(certificateAuthorityData) { if (!/^[A-Za-z0-9+/=]+$/.test(certificateAuthorityData)) throw new Error('Invalid cluster CA data'); return JSON.stringify( { apiVersion: 'v1', kind: 'Config', clusters: [{ name: 'poc', cluster: { server: CLUSTER, 'certificate-authority-data': certificateAuthorityData } }], users: [{ name: 'wishforge', user: { tokenFile: PATHS.kubeToken } }], contexts: [{ name: 'poc', context: { cluster: 'poc', user: 'wishforge' } }], 'current-context': 'poc', }, null, 2, ); } /** Until the public address exists: HTTP only, answering that the entry is not configured. */ export const PENDING_CADDYFILE = `{ admin off auto_https off } :8080 { respond "WishForge's public address is not configured yet." 503 } `; /** * The public HTTPS entry at the load balancer's address: a Let's Encrypt certificate for the IP * (short-lived profile, HTTP-01 only), HTTP GET/HEAD redirected to the standard HTTPS port, and the * console behind it on loopback with forwarding headers it can trust. */ export function clusterCaddyfile(address, { staging = false, domain } = {}) { const ip = publicIPv4(address); const hosts = [...(domain ? [publicDomain(domain)] : []), ip]; const site = (scheme, port) => hosts.map(host => `${scheme}://${host}:${port}`).join(', '); const ca = `https://acme${staging ? '-staging' : ''}-v02.api.letsencrypt.org/directory`; return `{ admin off skip_install_trust http_port 8080 https_port 8443 default_sni ${ip} auto_https disable_redirects servers { protocols h1 h2 } } ${site('http', 8080)} { @safe method GET HEAD redir @safe https://{host}{uri} 308 respond 405 } ${site('https', 8443)} { tls { issuer acme { dir ${ca} test_dir ${ca} profile shortlived disable_tlsalpn_challenge alt_http_port 8080 } } reverse_proxy 127.0.0.1:3000 { header_up X-Forwarded-Proto https header_up -Forwarded } } `; } /** * The public plain-HTTP entry at the load balancer's address, which the operator chose over HTTPS * (`public --http`): the console behind it on loopback. Passwords and sessions travel unencrypted. */ export function clusterHttpCaddyfile(address, { domain } = {}) { const ip = publicIPv4(address); const site = [...(domain ? [publicDomain(domain)] : []), ip].map(host => `http://${host}:8080`).join(', '); return `{ admin off auto_https off http_port 8080 servers { protocols h1 } } ${site} { reverse_proxy 127.0.0.1:3000 { header_up X-Forwarded-Proto http header_up -Forwarded } } `; } /** public-service.yaml as applied: a plain-HTTP entry has no HTTPS port. */ export function publicService(template, { http = false } = {}) { const https = ' - { name: https, protocol: TCP, port: 443, targetPort: https }\n'; if (!template.includes(https)) throw new Error('public-service.yaml has no HTTPS port'); return http ? template.replace(https, '') : template; } const IMAGE_REFERENCE = /^ccr-29eug8s3-vpc\.cnc\.bj\.baidubce\.com\/dataflow-poc\/wishforge-cluster-poc@sha256:[a-f0-9]{64}$/; const PLACEHOLDER = name => `\${${name}}`; /** * platform.yaml with its image and switches filled in; nothing left unfilled. `settings` is the * digest of the settings the containers read when they start, so a change to them rolls the Pod. */ export function renderPlatform(template, { image, runnerDisabled, maintenance, settings }) { if (!IMAGE_REFERENCE.test(image)) throw new Error('The WishForge image must be the pinned cluster image'); if (!/^[a-f0-9]{32}$/.test(settings ?? '')) throw new Error('The settings digest is missing'); const rendered = template .replaceAll(PLACEHOLDER('WISHFORGE_IMAGE'), image) .replaceAll(PLACEHOLDER('SETTINGS_DIGEST'), settings) .replaceAll(PLACEHOLDER('RUNNER_DISABLED'), String(Boolean(runnerDisabled))) .replaceAll(PLACEHOLDER('MAINTENANCE'), String(Boolean(maintenance))); if (rendered.includes('${')) throw new Error('platform.yaml has an unfilled placeholder'); return rendered; } // ------------------------------------------------------------------------------- commands const MANIFESTS = path.join(ROOT, 'infra/k8s/cluster-poc'); const GENERATED = path.join(STATE, 'generated.json'); /** Written by fanyu-access: Fanyu's profile, naming WishForge's kubeconfig for Fanyu's cluster. */ const FANYU_PROFILE = path.join(STATE, 'fanyu-profile.json'); /** Written by projects --replace: the cluster's projects' bindings and their Runners' registrations. */ const PROJECTS_FILE = path.join(STATE, 'projects.json'); const secretValue = () => randomBytes(32).toString('base64url'); async function readEnv(file) { const values = {}; for (const line of (await readFile(file, 'utf8')).split('\n')) { const match = /^([A-Z_][A-Z0-9_]*)=(.*)$/.exec(line); if (match) values[match[1]] = match[2].replace(/^(['"])(.*)\1$/, '$2'); } return values; } async function privateState() { await mkdir(STATE, { recursive: true, mode: 0o700 }); await chmod(STATE, 0o700); } async function writePrivate(file, content) { await privateState(); const temporary = `${file}.${randomBytes(4).toString('hex')}.tmp`; await writeFile(temporary, content, { mode: 0o600 }); await rename(temporary, file); } /** Passwords the cluster's own services use; made once, kept only in this Mac's private state. */ async function generatedSecrets() { try { return JSON.parse(await readFile(GENERATED, 'utf8')); } catch (error) { if (error.code !== 'ENOENT') throw error; } const generated = { postgresPassword: secretValue(), appPassword: secretValue(), objectStorageUser: `wishforge-${randomBytes(6).toString('hex')}`, objectStoragePassword: secretValue(), }; await writePrivate(GENERATED, `${JSON.stringify(generated, null, 2)}\n`); return generated; } async function imageRecord() { try { return JSON.parse(await readFile(path.join(STATE, 'image.json'), 'utf8')); } catch { throw new Error('No image yet; run: node scripts/cluster-poc.mjs image'); } } /** Applies Kubernetes objects from JSON on stdin, so no value appears in a command line. */ const applyObjects = objects => kubectl(['apply', '-f', '-'], { input: JSON.stringify({ apiVersion: 'v1', kind: 'List', items: objects }) }); const labels = { 'app.kubernetes.io/part-of': 'wishforge-cluster-poc' }; const secret = (name, stringData) => ({ apiVersion: 'v1', kind: 'Secret', type: 'Opaque', metadata: { name, namespace: NAMESPACE, labels }, stringData, }); const configMap = (name, data) => ({ apiVersion: 'v1', kind: 'ConfigMap', metadata: { name, namespace: NAMESPACE, labels }, data, }); async function exists(kind, name, namespace) { const scope = namespace ? ['-n', namespace] : []; return (await kubectl([...scope, 'get', kind, name, '--ignore-not-found', '-o', 'name'])).stdout.trim() !== ''; } async function bootstrap() { await assertCluster(); await kubectl(['apply', '-f', path.join(MANIFESTS, 'namespaces.yaml')]); if (!(await exists('pv', 'wishforge-data'))) { const directories = path.join(MANIFESTS, 'bootstrap/directories.yaml'); await kubectl(['apply', '-f', directories]); let phase = ''; for (const until = Date.now() + 5 * 60_000; Date.now() < until && !['Succeeded', 'Failed'].includes(phase);) { await delay(3000); phase = ( await kubectl(['-n', NAMESPACE, 'get', 'pod', 'wishforge-bootstrap', '-o', 'jsonpath={.status.phase}'], { allowFailure: true, }) ).stdout.trim(); } await kubectl(['delete', '-f', directories, '--wait=true'], { timeoutSeconds: 120 }); if (phase !== 'Succeeded') throw new Error(`Creating the data directory did not succeed (${phase || 'no phase'})`); console.log('Data directory created on 192.168.80.249.'); } for (const file of ['storage.yaml', 'rbac.yaml', 'admission.yaml']) await kubectl(['apply', '-f', path.join(MANIFESTS, file)]); console.log('Namespaces, storage, access rules and admission policies are in place.'); } /** The Gitea deploy key: its private half goes to the cluster only; the public half is printed. */ async function deployKey(flag) { await assertCluster(); const publicKey = path.join(STATE, 'deploy-key.pub'); if (flag !== '--rotate' && (await exists('secret', 'wishforge-git', NAMESPACE))) { console.log((await readFile(publicKey, 'utf8')).trim()); return; } const directory = await mkdtemp(path.join(os.tmpdir(), 'wishforge-deploy-key-')); try { const key = path.join(directory, 'id_ed25519'); const made = await capture('ssh-keygen', [ '-q', '-t', 'ed25519', '-N', '', '-C', 'wishforge-cluster-poc', '-f', key, ]); if (made.code !== 0) throw new Error(`ssh-keygen failed: ${made.stderr}`); await applyObjects([secret('wishforge-git', { id_ed25519: await readFile(key, 'utf8') })]); await writePrivate(publicKey, await readFile(`${key}.pub`, 'utf8')); console.log((await readFile(`${key}.pub`, 'utf8')).trim()); } finally { await rm(directory, { recursive: true, force: true }); } } /** * The Claude CLI's settings in the cluster: the company's Anthropic-protocol gateway, its models * with their 1M context, and no traffic to services the gateway does not replace. */ export function claudeSettings(key) { return { env: { ANTHROPIC_BASE_URL: 'https://ai-api.d-robotics.cc', ...(key ? { ANTHROPIC_API_KEY: key } : {}), ANTHROPIC_MODEL: 'deepseek-pro@latest', ANTHROPIC_DEFAULT_OPUS_MODEL: 'deepseek-pro@latest', ANTHROPIC_DEFAULT_SONNET_MODEL: 'deepseek-pro@latest', ANTHROPIC_DEFAULT_HAIKU_MODEL: 'deepseek-flash@latest', CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1', DISABLE_AUTOUPDATER: '1', DISABLE_TELEMETRY: '1', DISABLE_ERROR_REPORTING: '1', }, }; } const readStdin = () => new Promise((resolve, reject) => { const chunks = []; process.stdin .on('data', chunk => chunks.push(chunk)) .once('end', () => resolve(Buffer.concat(chunks).toString('utf8'))) .once('error', reject); }); /** The model gateway's key, from the clipboard or with --stdin from standard input, into the cluster only. */ async function gatewayKey(flag) { await assertCluster(); if (flag !== undefined && flag !== '--stdin') throw new Error('Usage: cluster-poc.mjs gateway-key [--stdin]'); const source = flag === '--stdin' ? 'standard input' : 'clipboard'; const key = (flag === '--stdin' ? await readStdin() : (await capture('pbpaste', [])).stdout).trim(); if (!/^sk-[A-Za-z0-9_-]{16,200}$/.test(key)) throw new Error( `The ${source} holds no gateway key (sk-…, ${key.length} characters read); provide the key and run this again`, ); await applyObjects([ secret('wishforge-claude', { 'settings.json': `${JSON.stringify(claudeSettings(key), null, 2)}\n` }), ]); console.log(`The gateway key is in the cluster (Secret wishforge-claude)${flag ? '.' : '; clear your clipboard.'}`); } /** The projects the cluster delivers and their Runners (projects --replace). */ async function clusterProjects() { const pocProfile = { ...JSON.parse(await readFile(POC_PROFILE, 'utf8')), kubeconfig: `${PATHS.config}/kubeconfig` }; const fanyuProfile = await readFile(FANYU_PROFILE, 'utf8').then(JSON.parse, () => undefined); const profiles = { 'poc-profile.json': pocProfile, ...(fanyuProfile ? { 'fanyu-profile.json': fanyuProfile } : {}), }; const replaced = JSON.parse(await readFile(PROJECTS_FILE, 'utf8')); for (const binding of replaced.bindings) if (!profiles[path.basename(binding.profileFile)]) throw new Error(`No ${path.basename(binding.profileFile)} for project ${binding.projectId}; run fanyu-access`); const values = registration => ({ WF_RUNNER_TOKEN: registration.token, WF_TENANT_ID: replaced.tenantId, WF_ENVIRONMENT_ID: registration.environmentId, WF_RUNNER_PUBLIC_KEYS: 'in the ConfigMap', WF_RUNNER_EXECUTE: 'true', }); const { poc, fanyu } = replaced.runners; return { profiles, bindings: replaced.bindings, runner: values(poc), runnerFanyu: fanyu ? values(fanyu) : undefined, keys: { 'runner-public.json': `${JSON.stringify(poc.publicKeys, null, 2)}\n`, ...(fanyu ? { 'runner-fanyu-public.json': `${JSON.stringify(fanyu.publicKeys, null, 2)}\n` } : {}), }, }; } async function secrets() { await assertCluster(); const local = await readEnv(INPUTS.carried); const projects = await clusterProjects(); const { runner, runnerFanyu } = projects; const image = (await imageRecord()).reference; const generated = await generatedSecrets(); const http = generated.publicScheme === 'http'; const domain = generated.publicDomain; const origins = generated.publicAddress ? [...(domain ? [domain] : []), generated.publicAddress].map(host => `${http ? 'http' : 'https'}://${host}`) : []; const environments = clusterEnvironments({ local, runner, runnerFanyu, image, generated, origins }); const ca = ( await kubectl([ 'config', 'view', '--raw', '--minify', '-o', 'jsonpath={.clusters[0].cluster.certificate-authority-data}', ]) ).stdout.trim(); const hosts = (await readFile(INPUTS.knownHosts, 'utf8')).split('\n').filter(line => line && !line.startsWith('#')); if (!hosts.length) throw new Error(`No pinned host key for gitea.d-robotics.cc in ${INPUTS.knownHosts}`); // The Pod mounts the Claude settings; until gateway-key stores the key, they name the gateway only. if (!(await exists('secret', 'wishforge-claude', NAMESPACE))) { await applyObjects([ secret('wishforge-claude', { 'settings.json': `${JSON.stringify(claudeSettings(), null, 2)}\n` }), ]); console.log('No gateway key yet: Claude runs and AI optimization stay unavailable until gateway-key.'); } const objects = [ secret('wishforge-postgres', { password: generated.postgresPassword }), secret('wishforge-minio', { user: generated.objectStorageUser, password: generated.objectStoragePassword }), secret('wishforge-env', { 'platform.env': envFile(environments.platform), 'executor.env': envFile(environments.executor), 'runner.env': envFile(environments.runner), ...(environments.runnerFanyu ? { 'runner-fanyu.env': envFile(environments.runnerFanyu) } : {}), }), secret('wishforge-keys', { 'runner-ed25519.pem': await readFile(INPUTS.signingKey, 'utf8') }), configMap('wishforge-config', { 'plans.json': await readFile(INPUTS.plans, 'utf8'), ...projects.keys, ...Object.fromEntries( Object.entries(projects.profiles).map(([name, profile]) => [name, `${JSON.stringify(profile, null, 2)}\n`]), ), 'project.json': `${JSON.stringify(projects.bindings, null, 2)}\n`, known_hosts: `${hosts.join('\n')}\n`, kubeconfig: clusterKubeconfig(ca), }), configMap('wishforge-caddy', { Caddyfile: !generated.publicAddress ? PENDING_CADDYFILE : http ? clusterHttpCaddyfile(generated.publicAddress, { domain }) : clusterCaddyfile(generated.publicAddress, { staging: generated.acme !== 'production', domain }), }), ]; await applyObjects(objects); const digest = createHash('sha256').update(JSON.stringify(objects)).digest('hex').slice(0, 32); console.log(`Secrets and ConfigMaps are in place: settings, keys, POC files and the Caddyfile (settings ${digest}).`); return digest; } async function apply(...flags) { await assertCluster(); // The containers read their settings, which name the check Pods' image too, only when they // start: refresh them, and roll the Pod when they changed. const settings = await secrets(); const rendered = renderPlatform(await readFile(path.join(MANIFESTS, 'platform.yaml'), 'utf8'), { image: (await imageRecord()).reference, settings, runnerDisabled: !flags.includes('--runner'), maintenance: flags.includes('--maintenance'), }); await kubectl(['apply', '-f', '-'], { input: rendered }); // A StatefulSet rolls a Pod that is not Ready (one in maintenance, say) only once it is deleted. let replaced = false; await waitFor('the WishForge Pod to take the new revision', async () => { const set = JSON.parse((await kubectl(['-n', NAMESPACE, 'get', 'statefulset', 'wishforge', '-o', 'json'])).stdout); const pod = await kubectl(['-n', NAMESPACE, 'get', 'pod', 'wishforge-0', '-o', 'json'], { allowFailure: true }); if (pod.code !== 0 || set.metadata.generation !== set.status?.observedGeneration) return false; if (JSON.parse(pod.stdout).metadata.labels?.['controller-revision-hash'] === set.status.updateRevision) return true; if (!replaced) { await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-0', '--wait=true'], { timeoutSeconds: 180 }); replaced = true; } return false; }); console.log(`Applied the WishForge Pod${flags.length ? ` (${flags.join(' ')})` : ''}.`); } async function status() { await assertCluster(); for (const [label, args] of [ ['Pod', ['-n', NAMESPACE, 'get', 'pods', '-o', 'wide']], ['Storage', ['get', 'pv', 'wishforge-data', 'wishforge-checks-exec', '--ignore-not-found']], ['Check Pods', ['-n', CHECK_NAMESPACE, 'get', 'pods']], ['Public entry', ['-n', NAMESPACE, 'get', 'service', 'wishforge-public', '--ignore-not-found']], ]) { const result = await kubectl(args, { allowFailure: true }); console.log(`== ${label} ==\n${result.stdout.trim() || result.stderr.trim() || '(none)'}`); } const pod = await kubectl(['-n', NAMESPACE, 'get', 'pod', 'wishforge-0', '-o', 'json'], { allowFailure: true }); if (pod.code === 0) for (const container of JSON.parse(pod.stdout).status?.containerStatuses ?? []) console.log( `${container.name.padEnd(10)} ${container.ready ? 'ready' : 'not ready'}, restarts ${container.restartCount}${ container.state?.waiting ? `, ${container.state.waiting.reason}` : '' }`, ); const backups = await kubectl(backupExec(['node', 'scripts/cluster-poc-backup.mjs', 'latest']), { allowFailure: true, }); if (backups.code === 0) { const { snapshots, manifest, last } = JSON.parse(backups.stdout); console.log( `== Backups ==\n${snapshots.length} on the node, newest ${manifest?.takenAt ?? 'none'}; last attempt ${ last ? `${last.ok ? 'succeeded' : `failed (${last.error})`} at ${last.at}` : 'none yet' }`, ); } } // ------------------------------------------------------------------------------- restore /** What the cluster is still doing: projects are replaced and data restored only when nothing runs. */ export const UNFINISHED_SQL = `select kind || ' ' || id || ' ' || coalesce(data->>'status', '') from entities where (kind = 'run' and data->>'status' in ('queued', 'running')) or (kind = 'runner_task' and data->>'status' in ('pending', 'dispatching', 'unknown')) or (kind = 'release' and data->>'status' in ('ready', 'deploying', 'unknown'))`; async function waitFor(description, check, minutes = 10) { for (const until = Date.now() + minutes * 60_000; Date.now() < until; await delay(5000)) if (await check()) return; throw new Error(`Timed out waiting for ${description}`); } const podGone = async name => ( await kubectl(['-n', NAMESPACE, 'get', 'pod', name, '--ignore-not-found', '-o', 'name'], { allowFailure: true }) ).stdout.trim() === ''; /** A restricted Pod with the data volume, to put a snapshot's files in place while the WishForge Pod is stopped. */ const restorePod = image => ({ apiVersion: 'v1', kind: 'Pod', metadata: { name: 'wishforge-restore', namespace: NAMESPACE, labels }, spec: { restartPolicy: 'Never', automountServiceAccountToken: false, enableServiceLinks: false, imagePullSecrets: [{ name: 'cce-image-credential-ccr-29eug8s3-vpc.cnc.bj.baidubce.com' }], activeDeadlineSeconds: 3600, securityContext: { runAsNonRoot: true, runAsUser: 10001, runAsGroup: 10001, fsGroup: 10001, fsGroupChangePolicy: 'OnRootMismatch', seccompProfile: { type: 'RuntimeDefault' }, }, containers: [ { name: 'restore', image, command: ['/bin/sh', '-c', 'sleep 3600'], securityContext: { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: { drop: ['ALL'] }, }, resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '1', memory: '1Gi' } }, volumeMounts: [ { name: 'data', mountPath: '/mnt/data' }, { name: 'tmp', mountPath: '/tmp' }, ], }, ], volumes: [ { name: 'data', persistentVolumeClaim: { claimName: 'wishforge-data' } }, { name: 'tmp', emptyDir: { sizeLimit: '4Gi' } }, ], }, }); const SNAPSHOT = /^\d{8}T\d{6}Z$/; /** * Puts snapshot `name`'s files in place, in the restore Pod: each file is checked against the * manifest first, and only then are the directories emptied and unpacked. `archives` are the * directories the snapshot holds; the others are emptied. /tmp/verified and /tmp/restored mark how * far it got, since kubectl can lose an exec's status. */ export function restoreScript(name, archives = ARCHIVES) { if (!SNAPSHOT.test(name)) throw new Error(`Not a snapshot name: ${name}`); if (archives.some(archive => !ARCHIVES.includes(archive))) throw new Error('Unknown archive'); const snapshot = `/mnt/data/backups/${name}`; const reset = ['postgres', 'temporal', 'exec/runs', ...ARCHIVES]; return [ 'set -e', `node /opt/wishforge/scripts/cluster-poc-backup.mjs verify '${snapshot}'`, 'touch /tmp/verified', 'cd /mnt/data', ...reset.map(dir => `mkdir -p '${dir}' && find '${dir}' -mindepth 1 -delete && chmod 0700 '${dir}'`), ...archives.map(dir => `tar -xzf '${snapshot}/${dir}.tgz' -C '${dir}' --no-same-owner --no-same-permissions`), `cp '${snapshot}/temporal.db' temporal/temporal.db && chmod 0600 temporal/temporal.db`, 'touch /tmp/restored', 'echo restored: $(find minio -type f | wc -l) object storage files, $(find git -type f | wc -l) Git files', ].join('\n'); } /** * Replaces the cluster's data with a snapshot on the node, with the WishForge Pod stopped. The * database is then restored into a fresh PostgreSQL started with nothing else running, by the * backup container, which reads the snapshot and reaches the database on loopback; the Pod starts * only if every table's row count is `counts`. */ async function replaceData({ image, snapshot, archives, counts, runner }) { const script = restoreScript(snapshot, archives); if (await exists('statefulset', 'wishforge', NAMESPACE)) { await kubectl(['-n', NAMESPACE, 'scale', 'statefulset', 'wishforge', '--replicas=0']); await waitFor('the WishForge Pod to stop', () => podGone('wishforge-0')); } await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-restore', '--ignore-not-found', '--wait=true']); await kubectl(['create', '-f', '-'], { input: JSON.stringify(restorePod(image)) }); const inRestorePod = (command, options) => kubectl(['-n', NAMESPACE, 'exec', 'wishforge-restore', '-c', 'restore', '--', '/bin/sh', '-c', command], options); try { await kubectl(['-n', NAMESPACE, 'wait', '--for=condition=Ready', 'pod/wishforge-restore', '--timeout=600s'], { timeoutSeconds: 620, }); const restored = await inRestorePod(script, { timeoutSeconds: 1200, allowFailure: true }); const marks = (await inRestorePod('ls /tmp', { allowFailure: true })).stdout; if (!/\brestored\b/.test(marks)) { if (/\bverified\b/.test(marks)) throw new Error( `Unpacking snapshot ${snapshot} did not finish (${restored.stderr.trim().slice(0, 400)}); the Pod stays stopped, run this again`, ); await kubectl(['-n', NAMESPACE, 'scale', 'statefulset', 'wishforge', '--replicas=1'], { allowFailure: true }); throw new Error( `Snapshot ${snapshot} did not pass its check (${restored.stderr.trim().slice(0, 400)}); nothing was changed and the WishForge Pod is starting again`, ); } console.log(restored.stdout.trim() || 'Files restored.'); } finally { await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-restore', '--ignore-not-found', '--wait=true'], { allowFailure: true, }); } await apply('--maintenance'); await kubectl(['-n', NAMESPACE, 'scale', 'statefulset', 'wishforge', '--replicas=1']); await waitFor('PostgreSQL in the cluster', async () => { const ready = await kubectl( [ '-n', NAMESPACE, 'exec', 'wishforge-0', '-c', 'postgres', '--', 'pg_isready', '-h', '127.0.0.1', '-U', 'wishforge', '-d', 'wishforge', ], { allowFailure: true }, ); return ready.code === 0; }); // kubectl can lose an exec's status: the row counts say whether the restore took. const restored = await kubectl( backupExec([ 'pg_restore', '-h', '127.0.0.1', '-U', 'wishforge', '-d', 'wishforge', '--no-owner', '--no-privileges', '--exit-on-error', `/var/lib/wishforge/backups/${snapshot}/postgres.dump`, ]), { timeoutSeconds: 1800, allowFailure: true }, ); const clusterCounts = ( await kubectl([ '-n', NAMESPACE, 'exec', 'wishforge-0', '-c', 'postgres', '--', 'psql', '-U', 'wishforge', '-d', 'wishforge', '-At', '-c', COUNTS_SQL, ]) ).stdout.trim(); if (clusterCounts !== counts) throw new Error( `Row counts differ after the restore (${restored.stderr.trim().slice(0, 800)}):\nSnapshot: ${counts}\nCluster: ${clusterCounts}`, ); console.log(`Database restored; row counts match: ${counts}`); await apply(...(runner ? ['--runner'] : [])); } // ------------------------------------------------------------------------------ public entry /** The load balancer's public IPv4, once CCE has assigned it; private addresses it also lists are skipped. */ export function publicAddressOf(service) { for (const ingress of service?.status?.loadBalancer?.ingress ?? []) { try { return publicIPv4(ingress.ip); } catch { // A VPC address of the load balancer, not the public one. } } } /** * The public entry: WishForge's own load balancer and elastic IP (billed while they exist). HTTPS * with a certificate for that IP from Let's Encrypt's staging CA first, and with --production the * trusted one; or, with --http, plain HTTP on port 80 only. `--domain ` adds a domain the * operator's DNS points at the load balancer, which becomes the console's address (the IP keeps * working); later runs keep it. Each step restarts the WishForge Pod so the console accepts the * new origin. */ async function publicEntry(...args) { await assertCluster(); let flag, domain; for (let i = 0; i < args.length; i++) if (args[i] === '--domain' && args[i + 1] && !domain) domain = publicDomain(args[++i]); else if (['--production', '--http'].includes(args[i]) && !flag) flag = args[i]; else throw new Error('Usage: cluster-poc.mjs public [--production|--http] [--domain ]'); if (!(await exists('statefulset', 'wishforge', NAMESPACE))) throw new Error('Deploy the WishForge Pod first'); const http = flag === '--http'; await kubectl(['apply', '-f', '-'], { input: publicService(await readFile(path.join(MANIFESTS, 'public-service.yaml'), 'utf8'), { http }), }); let address; await waitFor( 'the load balancer address', async () => { const service = await kubectl(['-n', NAMESPACE, 'get', 'service', 'wishforge-public', '-o', 'json']); address = publicAddressOf(JSON.parse(service.stdout)); return Boolean(address); }, 15, ); const generated = { ...(await generatedSecrets()), publicAddress: address, publicScheme: http ? 'http' : 'https' }; if (http) delete generated.acme; else generated.acme = flag === '--production' ? 'production' : 'staging'; if (domain) { // Browsers and Let's Encrypt find the entry through public DNS: it must already point here. const resolver = new Resolver({ timeout: 5000, tries: 2 }); resolver.setServers(PUBLIC_RESOLVERS); const published = await resolver.resolve4(domain).catch(() => []); if (!published.includes(address)) throw new Error( `${domain} resolves publicly to ${published.join(', ') || 'nothing'}, not ${address}: point its A record here first`, ); // A company DNS with its own copy of the zone answers its users from that copy instead. if (!(await resolve4(domain).catch(() => [])).includes(address)) console.warn( `This machine's DNS (${getServers().join(', ')}) does not resolve ${domain} to ${address}: people using ` + 'it cannot open the domain until that DNS has the record too.', ); generated.publicDomain = domain; } const host = generated.publicDomain ?? address; await writePrivate(GENERATED, `${JSON.stringify(generated, null, 2)}\n`); await secrets(); await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-0', '--wait=true'], { timeoutSeconds: 180 }); console.log( http ? `Public address ${address}: plain HTTP, port 80 only; the console will answer at http://${host} ` + 'once the Pod is ready. Passwords and sessions travel unencrypted.' : `Public address ${address}: ${flag === '--production' ? 'trusted' : 'staging (untrusted)'} certificate ` + `requested; the console will answer at https://${host} once the Pod is ready.`, ); } // ------------------------------------------------------------------------------- snapshots const backupExec = args => ['-n', NAMESPACE, 'exec', 'wishforge-0', '-c', 'backup', '--', ...args]; /** * Copies one file out of one of the WishForge Pod's containers. kubectl drops whatever of an exec's output is * still in flight when the remote command exits, so the remote side sends the file and then waits * for this end to close its input, which happens only once every byte has arrived. */ async function download(remote, local, bytes, container = 'backup') { for (let attempt = 1; ; attempt++) { const result = await new Promise((resolve, reject) => { const child = spawn( 'kubectl', [ '--kubeconfig', KUBECONFIG, '-n', NAMESPACE, 'exec', '-i', 'wishforge-0', '-c', container, '--', '/bin/sh', '-c', 'cat "$0" || exit 1; read -r _', remote, ], { cwd: ROOT, env: { PATH: process.env.PATH, HOME: os.homedir() }, stdio: ['pipe', 'pipe', 'pipe'] }, ); const err = []; let received = 0; // A link at a few dozen kB/s still finishes; a stalled one is given up. const timer = setTimeout(() => child.kill(), 120_000 + bytes / 20); const file = createWriteStream(local, { mode: 0o600 }); const written = new Promise((done, fail) => file.on('finish', done).on('error', fail)); if (bytes === 0) child.stdin.end('\n'); child.stdout.on('data', chunk => { received += chunk.length; if (received >= bytes) child.stdin.end('\n'); }); child.stdout.pipe(file); child.stderr.on('data', chunk => err.push(chunk)); child.stdin.on('error', () => undefined); child.once('error', reject); child.once('close', () => { clearTimeout(timer); written.then(() => resolve({ received, stderr: Buffer.concat(err).toString('utf8') }), reject); }); }); if (result.received === bytes) return; if (attempt === 5) throw new Error( `Copying ${path.basename(remote)} failed after ${attempt} attempts (${result.received} of ${bytes} bytes): ${result.stderr.trim().slice(-400)}`, ); await delay(3000 * attempt); } } /** * Replaces everything the cluster holds with one of the node's daily snapshots (`status` lists * them): the database, Temporal, Git, object storage and the platform's, POC's and Runners' * records. Starts the Runners only with --runner. The node keeps its snapshots. */ async function restore(name, ...flags) { if (!name || name.startsWith('--') || flags.some(flag => flag !== '--runner')) throw new Error('Usage: cluster-poc.mjs restore [--runner]'); restoreScript(name); await assertCluster(); const image = (await imageRecord()).reference; const { snapshots } = JSON.parse( (await kubectl(backupExec(['node', 'scripts/cluster-poc-backup.mjs', 'latest']))).stdout, ); if (!snapshots.includes(name)) throw new Error(`The node has no snapshot ${name}; it has ${snapshots.join(', ') || 'none'}`); const remote = `/var/lib/wishforge/backups/${name}/manifest.json`; const work = await mkdtemp(path.join(os.tmpdir(), 'wishforge-restore-')); try { const size = (await kubectl(backupExec(['/bin/sh', '-c', `wc -c < ${remote}`]))).stdout.trim(); await download(remote, path.join(work, 'manifest.json'), Number(size)); const manifest = JSON.parse(await readFile(path.join(work, 'manifest.json'), 'utf8')); if (!manifest.database?.counts) throw new Error('The snapshot records no row counts to check the restore against'); for (const file of ['postgres.dump', 'temporal.db']) if (!manifest.files[file]) throw new Error(`The snapshot has no ${file}`); const archives = ARCHIVES.filter(archive => manifest.files[`${archive}.tgz`]); console.log(`Restoring snapshot ${name} (taken ${manifest.takenAt}) …`); await replaceData({ image, snapshot: name, archives, counts: manifest.database.counts, runner: flags.includes('--runner'), }); console.log(`Restored snapshot ${name}${flags.includes('--runner') ? '' : '; the Runners stay idle'}.`); } finally { await rm(work, { recursive: true, force: true }); } } // ------------------------------------------------------------------------------ Fanyu's project const FANYU_DEPLOYMENTS = { api: 'droboticflow-api', worker: 'droboticflow-worker', coordinator: 'drosync-coordinator', 'annotation-processor': 'droboticflow-annotation-processor', 'annotation-validator': 'droboticflow-annotation-validator', }; /** WishForge's kubeconfig for Fanyu's cluster: its Service Account's token, nothing else. */ export function fanyuKubeconfig({ certificateAuthorityData, token }) { if (!/^[A-Za-z0-9+/=]+$/.test(certificateAuthorityData)) throw new Error('Invalid Fanyu cluster CA data'); if (!/^[A-Za-z0-9._-]+$/.test(token)) throw new Error('Invalid Fanyu Service Account token'); return JSON.stringify( { apiVersion: 'v1', kind: 'Config', clusters: [ { name: 'fanyu', cluster: { server: FANYU.cluster, 'certificate-authority-data': certificateAuthorityData } }, ], users: [{ name: 'wishforge', user: { token } }], contexts: [{ name: 'fanyu', context: { cluster: 'fanyu', user: 'wishforge', namespace: FANYU.namespace } }], 'current-context': 'fanyu', }, null, 2, ); } /** Fanyu's profile, as WishForge's containers read it; the POC's pins the same Kaniko and mirrors. */ export function fanyuProfile({ poc, namespaceUid, deployments }) { return { schemaVersion: 1, id: 'droboticflow-fanyu', repository: poc.repository, branch: 'dev', kubeconfig: PATHS.fanyuKubeconfig, clusterServer: FANYU.cluster, namespace: FANYU.namespace, namespaceUid, deployments, registry: 'ccr-29eug8s3-pub.cnc.bj.baidubce.com/fanyu', registrySecret: 'droboticflow-registry', applicationUrl: 'http://120.48.180.80', kanikoImage: poc.kanikoImage, registryMirrors: poc.registryMirrors, rolloutTimeoutSeconds: poc.rolloutTimeoutSeconds ?? 600, buildTimeoutSeconds: poc.buildTimeoutSeconds ?? 1800, }; } /** * WishForge's identity in Fanyu's cluster (fanyu-access.yaml), applied with Fanyu's administrator * kubeconfig, which stays on this Mac: its token goes into the WishForge cluster's Secret * wishforge-fanyu-kube only, after checking it can do what the delivery needs and no more. */ async function fanyuAccess(...args) { if (args[0] !== '--kubeconfig' || !args[1] || args.length !== 2) throw new Error('Usage: cluster-poc.mjs fanyu-access --kubeconfig '); await assertCluster(); const admin = path.resolve(args[1]); const env = { PATH: process.env.PATH, HOME: os.homedir() }; const fanyu = async (kubeconfig, command, options = {}) => { const result = await capture('kubectl', ['--kubeconfig', kubeconfig, '--request-timeout=60s', ...command], { env, input: options.input, }); if (result.code !== 0 && !options.allowFailure) throw new Error(`kubectl ${command.slice(0, 3).join(' ')} (Fanyu) failed: ${result.stderr.trim().slice(0, 600)}`); return result; }; const view = JSON.parse((await fanyu(admin, ['config', 'view', '--minify', '-o', 'json'])).stdout); if (view.clusters?.length !== 1 || view.clusters[0].cluster.server !== FANYU.cluster) throw new Error(`The kubeconfig does not name Fanyu's cluster ${FANYU.cluster}`); await fanyu(admin, ['apply', '-f', path.join(MANIFESTS, 'fanyu-access.yaml')]); let token; await waitFor( 'the Fanyu Service Account token', async () => { token = JSON.parse( (await fanyu(admin, ['-n', FANYU.namespace, 'get', 'secret', 'wishforge-token', '-o', 'json'])).stdout, ).data; return Boolean(token?.token && token['ca.crt']); }, 2, ); const kubeconfig = fanyuKubeconfig({ certificateAuthorityData: token['ca.crt'], token: Buffer.from(token.token, 'base64').toString('utf8'), }); const directory = await mkdtemp(path.join(os.tmpdir(), 'wishforge-fanyu-')); try { const file = path.join(directory, 'kubeconfig'); await writeFile(file, kubeconfig, { mode: 0o600 }); const may = async (...check) => (await fanyu(file, ['auth', 'can-i', ...check, '-n', FANYU.namespace], { allowFailure: true })).stdout.trim(); const expected = [ [['patch', 'deployment/droboticflow-annotation-validator'], 'yes'], [['create', 'pods'], 'yes'], [['get', 'secrets'], 'no'], [['get', 'configmaps'], 'no'], [['patch', 'deployment/droboticflow-headpose'], 'no'], [['delete', 'deployments'], 'no'], ]; for (const [check, answer] of expected) if ((await may(...check)) !== answer) throw new Error(`WishForge's Fanyu identity: can-i ${check.join(' ')} is not ${answer}`); } finally { await rm(directory, { recursive: true, force: true }); } const namespaceUid = ( await fanyu(admin, ['get', 'namespace', FANYU.namespace, '-o', 'jsonpath={.metadata.uid}']) ).stdout.trim(); const deployments = {}; for (const [name, deployment] of Object.entries(FANYU_DEPLOYMENTS)) deployments[name] = ( await fanyu(admin, ['-n', FANYU.namespace, 'get', 'deployment', deployment, '-o', 'jsonpath={.metadata.uid}']) ).stdout.trim(); const profile = fanyuProfile({ poc: JSON.parse(await readFile(POC_PROFILE, 'utf8')), namespaceUid, deployments }); const parsed = await capture( process.execPath, [ '--import', 'tsx', '-e', "import('./packages/runtime/src/poc-profile.ts').then(m => m.pocProfileSchema.parse(JSON.parse(process.argv[1])))", JSON.stringify(profile), ], { env: process.env }, ); if (parsed.code !== 0) throw new Error(`Fanyu's profile is not valid: ${parsed.stderr.trim().slice(0, 800)}`); await writePrivate(FANYU_PROFILE, `${JSON.stringify(profile, null, 2)}\n`); await applyObjects([secret('wishforge-fanyu-kube', { kubeconfig })]); console.log( `WishForge's identity in Fanyu's cluster is in place (Secret wishforge-fanyu-kube); profile: ${FANYU_PROFILE}`, ); } const inPod = (container, command, options = {}) => kubectl(['-n', NAMESPACE, 'exec', 'wishforge-0', '-c', container, '--', ...command], { timeoutSeconds: 600, ...options, }); /** * Deletes every project of the cluster's WishForge with all it holds, and creates the POC and * Fanyu projects in their place (cluster-poc-projects.ts), with nothing running. The node takes a * snapshot first. Their Runners' registrations come to this Mac (STATE/projects.json); `apply` * then gives both Runners their settings. */ async function projectsCommand(flag) { if (flag !== '--replace') throw new Error('Usage: cluster-poc.mjs projects --replace'); await assertCluster(); if (!(await stat(FANYU_PROFILE).catch(() => undefined))) throw new Error('Run fanyu-access first'); const unfinished = ( await inPod('postgres', ['psql', '-U', 'wishforge', '-d', 'wishforge', '-At', '-c', UNFINISHED_SQL]) ).stdout.trim(); if (unfinished) throw new Error(`The cluster is still working; wait until these finish:\n${unfinished}`); const before = Date.now(); await inPod('backup', ['node', 'scripts/cluster-poc-backup.mjs', 'once'], { allowFailure: true }); const { last } = JSON.parse((await inPod('backup', ['node', 'scripts/cluster-poc-backup.mjs', 'latest'])).stdout); if (!(last?.ok && Date.parse(last.at) >= before)) throw new Error(`The snapshot before replacing the projects failed${last?.error ? `: ${last.error}` : ''}`); console.log(`Snapshot ${last.name} taken on the node.`); const output = '/var/lib/wishforge/platform-state/projects-setup.json'; await inPod('platform', ['rm', '-f', output]); // kubectl can lose an exec's output and status: the file it writes says whether it finished. const replaced = await inPod( 'platform', [ 'node', '--env-file=/etc/wishforge/env/platform.env', '--import', 'tsx', 'scripts/cluster-poc-projects.ts', '--replace', ], { allowFailure: true }, ); const size = await inPod('platform', ['/bin/sh', '-c', `wc -c < ${output}`], { allowFailure: true }); if (size.code !== 0 || !/^\d+$/.test(size.stdout.trim())) throw new Error(`Replacing the projects did not finish: ${replaced.stderr.trim().slice(-1500)}`); if (replaced.stdout.trim()) console.log(replaced.stdout.trim()); await privateState(); const partial = `${PROJECTS_FILE}.partial`; await download(output, partial, Number(size.stdout.trim()), 'platform'); JSON.parse(await readFile(partial, 'utf8')); await rename(partial, PROJECTS_FILE); await inPod('platform', ['rm', '-f', output]); // Workspaces of the runs that are gone, and the Runner's records of their tasks. await inPod('executor', [ '/bin/sh', '-c', 'find /var/lib/wishforge/exec/runs -mindepth 1 -maxdepth 1 -exec rm -rf {} +', ]); await inPod('runner', [ '/bin/sh', '-c', 'find /var/lib/wishforge/runner-state -mindepth 1 -maxdepth 1 -exec rm -rf {} +', ]); const disabled = ( await kubectl([ '-n', NAMESPACE, 'get', 'statefulset', 'wishforge', '-o', 'jsonpath={.spec.template.spec.containers[?(@.name=="runner")].env[?(@.name=="WF_RUNNER_DISABLED")].value}', ]) ).stdout.trim(); await apply(...(disabled === 'true' ? [] : ['--runner'])); console.log(`The cluster now has the POC and Fanyu projects; their Runners' registrations are in ${PROJECTS_FILE}.`); } const COMMANDS = { image, public: publicEntry, bootstrap, 'deploy-key': deployKey, 'gateway-key': gatewayKey, secrets, apply, restore, 'fanyu-access': fanyuAccess, projects: projectsCommand, status, }; export async function main(argv = process.argv.slice(2)) { const command = COMMANDS[argv[0]]; if (!command) throw new Error(`Usage: cluster-poc.mjs ${Object.keys(COMMANDS).join('|')}`); await command(...argv.slice(1)); } if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) main().catch(error => { console.error(error.message); process.exitCode = 1; });