wishforge-ide/scripts/cluster-poc.mjs

1566 lines
66 KiB
JavaScript

#!/usr/bin/env node
// Operator CLI of WishForge's deployment in the POC cluster (docs/cluster-poc-deployment.md).
// Every command names its cluster explicitly and prints no secret. Importing it starts nothing.
// WishForge runs in the cluster only; this machine keeps the operator's inputs (STATE) and nothing else.
//
// node scripts/cluster-poc.mjs image build and push the image in the cluster (Kaniko)
// node scripts/cluster-poc.mjs bootstrap namespaces, storage, access rules
// node scripts/cluster-poc.mjs deploy-key create the Gitea deploy key; prints its public half
// node scripts/cluster-poc.mjs gateway-key [--stdin] the model gateway key, from the clipboard or stdin
// node scripts/cluster-poc.mjs secrets Secrets and ConfigMaps from this Mac's settings
// node scripts/cluster-poc.mjs apply [--maintenance] [--runner] the WishForge Pod
// node scripts/cluster-poc.mjs public [--production] the public HTTPS entry (billed load balancer)
// node scripts/cluster-poc.mjs fanyu-access --kubeconfig <Fanyu admin kubeconfig> WishForge's identity in Fanyu's cluster
// node scripts/cluster-poc.mjs projects --replace delete every project; create the POC and Fanyu projects
// node scripts/cluster-poc.mjs restore <snapshot> [--runner] replace the cluster's data with a node snapshot
// node scripts/cluster-poc.mjs status show the deployment's state
import { spawn } from 'node:child_process';
import { createHash, randomBytes } from 'node:crypto';
import { createReadStream, createWriteStream } from 'node:fs';
import { chmod, mkdir, mkdtemp, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { Resolver, getServers, resolve4 } from 'node:dns/promises';
import { isIP } from 'node:net';
import { setTimeout as delay } from 'node:timers/promises';
import { ARCHIVES, COUNTS_SQL } from './cluster-poc-backup.mjs';
export const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
export const STATE = path.join(ROOT, '.wishforge/cluster-poc');
export const KUBECONFIG = path.join(os.homedir(), '.config/droboticflow/kubeconfigs/poc.config');
/**
* The operator's inputs to the cluster's settings, kept private here: the keys and tokens the data
* depends on (carried.env, see CARRIED), the Runners' signing key, the plan catalog, the POC's
* profile and Gitea's pinned host keys.
*/
export const INPUTS = {
carried: path.join(STATE, 'carried.env'),
signingKey: path.join(STATE, 'runner-ed25519.pem'),
plans: path.join(STATE, 'plans.json'),
knownHosts: path.join(STATE, 'known_hosts'),
};
export const POC_PROFILE = path.join(STATE, 'poc-profile.json');
export const CLUSTER = 'https://120.48.181.105:6443';
export const NAMESPACE = 'wishforge';
export const CHECK_NAMESPACE = 'wishforge-checks';
/** Fanyu's cluster and namespace, which its project delivers to (fanyu-access.yaml). */
export const FANYU = { cluster: 'https://180.76.138.88:6443', namespace: 'droboticflow-fanyu' };
/** Pushes use the registry's public address; the cluster pulls through its VPC address. */
export const PULL_REGISTRY = 'ccr-29eug8s3-vpc.cnc.bj.baidubce.com/dataflow-poc';
export const IMAGE_REPOSITORY = 'wishforge-cluster-poc';
const POSTGRES_IMAGE =
'docker.io/library/postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3';
/** Every input of the image, pinned: base images by digest, sources by commit, binaries by checksum. */
export const IMAGE_INPUTS = {
NODE_IMAGE:
'docker.io/library/node:26-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9',
GO_IMAGE:
'docker.io/library/golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac',
CADDY_IMAGE:
'docker.io/library/caddy:2.11.7-alpine@sha256:eb4d35b7452d1ab6171fbdbe8d9275b2301dad6dad92115d91e19fb9b24d510f',
// The image's pg_dump and pg_restore are the database's own (the backup).
POSTGRES_IMAGE,
MINIO_RELEASE: 'RELEASE.2025-07-23T15-54-02Z',
MINIO_COMMIT: '7ced9663e6a791fef9dc6be798ff24cda9c730ac',
KUBECTL_VERSION: 'v1.34.2',
KUBECTL_SHA256: '9591f3d75e1581f3f7392e6ad119aab2f28ae7d6c6e083dc5d22469667f27253',
CLAUDE_CODE_VERSION: '2.1.292',
};
/** Third-party service images, pinned by digest; the cluster pulls them from Docker Hub. */
export const SERVICE_IMAGES = {
postgres: POSTGRES_IMAGE,
temporal:
'docker.io/temporalio/temporal:1.4.1@sha256:9844298258519ebece82b3253f1ee1fc4eb8d87b7abcc264cfb676be342c74a5',
};
/** Paths never sent to the build, whatever Git says: local agent settings and private state. */
const EXCLUDED = [/^\.claude\//, /^\.wishforge\//, /(^|\/)\.env(\..*)?$/, /^apps\/ide\//, /^infra\/oct\//];
function capture(executable, args, options = {}) {
return new Promise((resolve, reject) => {
const child = spawn(executable, args, { cwd: ROOT, ...options, stdio: ['pipe', 'pipe', 'pipe'] });
const out = [];
const err = [];
child.stdout.on('data', chunk => out.push(chunk));
child.stderr.on('data', chunk => err.push(chunk));
child.once('error', reject);
child.once('close', code =>
resolve({ code, stdout: Buffer.concat(out).toString('utf8'), stderr: Buffer.concat(err).toString('utf8') }),
);
child.stdin.end(options.input);
});
}
export async function kubectl(args, { input, timeoutSeconds = 60, allowFailure = false } = {}) {
const result = await capture(
'kubectl',
['--kubeconfig', KUBECONFIG, `--request-timeout=${timeoutSeconds}s`, ...args],
{ input, env: { PATH: process.env.PATH, HOME: os.homedir() } },
);
if (result.code !== 0 && !allowFailure)
throw new Error(`kubectl ${args.slice(0, 3).join(' ')} failed: ${result.stderr.trim().slice(0, 800)}`);
return result;
}
/** Refuses to act unless the kubeconfig names the POC cluster. */
export async function assertCluster() {
const view = JSON.parse((await kubectl(['config', 'view', '--minify', '-o', 'json'])).stdout);
if (view.clusters?.length !== 1 || view.clusters[0].cluster.server !== CLUSTER)
throw new Error(`The kubeconfig does not name the POC cluster ${CLUSTER}`);
}
const sha256File = file =>
new Promise((resolve, reject) => {
const hash = createHash('sha256');
createReadStream(file)
.on('data', chunk => hash.update(chunk))
.on('error', reject)
.on('end', () => resolve(hash.digest('hex')));
});
/** kubectl for the image, downloaded once and kept only if it matches the published checksum. */
async function kubectlBinary() {
const file = path.join(STATE, `kubectl-${IMAGE_INPUTS.KUBECTL_VERSION}-linux-amd64`);
if (
await stat(file).then(
() => true,
() => false,
)
) {
if ((await sha256File(file)) === IMAGE_INPUTS.KUBECTL_SHA256) return file;
await rm(file, { force: true });
}
const url = `https://dl.k8s.io/release/${IMAGE_INPUTS.KUBECTL_VERSION}/bin/linux/amd64/kubectl`;
const response = await fetch(url);
if (!response.ok) throw new Error(`Cannot download kubectl (${response.status}); set HTTPS_PROXY if needed`);
const temporary = `${file}.${randomBytes(4).toString('hex')}.tmp`;
await writeFile(temporary, Buffer.from(await response.arrayBuffer()), { mode: 0o600 });
if ((await sha256File(temporary)) !== IMAGE_INPUTS.KUBECTL_SHA256) {
await rm(temporary, { force: true });
throw new Error('The downloaded kubectl does not match its published SHA-256');
}
await rename(temporary, file);
return file;
}
/** The files of the build: what Git tracks or would track, minus {@link EXCLUDED}. */
export function buildFiles(listing) {
return listing
.split('\0')
.filter(Boolean)
.filter(file => !EXCLUDED.some(pattern => pattern.test(file)));
}
async function sourceArchive(directory) {
const listing = await capture('git', ['ls-files', '-z', '--cached', '--others', '--exclude-standard']);
if (listing.code !== 0) throw new Error(`git ls-files failed: ${listing.stderr}`);
const files = [];
for (const file of buildFiles(listing.stdout))
if (
await stat(path.join(ROOT, file)).then(
info => info.isFile(),
() => false,
)
)
files.push(file);
const list = path.join(directory, 'files');
await writeFile(list, `${files.join('\0')}\0`);
const extra = path.join(directory, 'extra');
await mkdir(path.join(extra, '.cluster-poc-build'), { recursive: true });
await writeFile(path.join(extra, '.cluster-poc-build/kubectl'), await readFile(await kubectlBinary()));
const tarball = path.join(directory, 'source.tar');
const env = { PATH: process.env.PATH, COPYFILE_DISABLE: '1' };
for (const args of [
['--no-mac-metadata', '--null', '-cf', tarball, '-T', list],
['--no-mac-metadata', '-rf', tarball, '-C', extra, '.cluster-poc-build/kubectl'],
]) {
const result = await capture('tar', args, { env });
if (result.code !== 0) throw new Error(`tar failed: ${result.stderr}`);
}
const gzip = await capture('gzip', ['-n', '-9', tarball], { env });
if (gzip.code !== 0) throw new Error(`gzip failed: ${gzip.stderr}`);
return { file: `${tarball}.gz`, count: files.length };
}
const UNPACK = [
'set -e',
'sum=$(sha256sum /context/source.tar.gz)',
'[ "${sum%% *}" = "$1" ] || exit 76',
'mkdir -p /context/source && tar -xzf /context/source.tar.gz -C /context/source && touch /context/ready',
].join('\n');
/** The Kaniko Pod of one image build, in the POC namespace where the push credential lives. */
export function imageBuildPod(profile, input) {
return {
apiVersion: 'v1',
kind: 'Pod',
metadata: {
name: input.name,
namespace: profile.namespace,
labels: { 'app.kubernetes.io/managed-by': 'wishforge-cluster-poc', 'app.kubernetes.io/name': 'wishforge-image' },
},
spec: {
automountServiceAccountToken: false,
restartPolicy: 'Never',
activeDeadlineSeconds: 5400,
nodeSelector: { 'kubernetes.io/arch': 'amd64', 'kubernetes.io/os': 'linux' },
containers: [
{
name: 'kaniko',
image: profile.kanikoImage,
command: ['/busybox/sh', '-c'],
args: [
'until [ -f /context/ready ]; do /busybox/sleep 1; done; exec /kaniko/executor "$@"',
'--',
'--context=dir:///context/source',
'--dockerfile=/context/source/infra/images/cluster-poc.Dockerfile',
`--destination=${profile.registry}/${IMAGE_REPOSITORY}:${input.tag}`,
'--digest-file=/dev/termination-log',
...Object.entries(IMAGE_INPUTS).map(([key, value]) => `--build-arg=${key}=${value}`),
...profile.registryMirrors.map(mirror => `--registry-mirror=${mirror}`),
...(profile.registryMirrors.length ? ['--skip-default-registry-fallback'] : []),
'--snapshot-mode=redo',
'--verbosity=info',
],
resources: {
requests: { cpu: '1', memory: '2Gi' },
limits: { cpu: '6', memory: '12Gi', 'ephemeral-storage': '30Gi' },
},
volumeMounts: [
{ name: 'context', mountPath: '/context' },
{ name: 'registry', mountPath: '/kaniko/.docker', readOnly: true },
],
terminationMessagePolicy: 'File',
},
],
volumes: [
{ name: 'context', emptyDir: { sizeLimit: '4Gi' } },
{
name: 'registry',
secret: { secretName: profile.registrySecret, items: [{ key: '.dockerconfigjson', path: 'config.json' }] },
},
],
},
};
}
async function image() {
await assertCluster();
const profile = JSON.parse(await readFile(POC_PROFILE, 'utf8'));
await mkdir(STATE, { recursive: true, mode: 0o700 });
await chmod(STATE, 0o700);
const head = (await capture('git', ['rev-parse', 'HEAD'])).stdout.trim();
const dirty = (await capture('git', ['status', '--porcelain'])).stdout.trim().length > 0;
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..*$/, '').replace('T', '');
const tag = `${head.slice(0, 12)}${dirty ? '-dirty' : ''}-${stamp}`;
const directory = await mkdtemp(path.join(os.tmpdir(), 'wishforge-image-'));
const name = `wishforge-image-${randomBytes(4).toString('hex')}`;
try {
const archive = await sourceArchive(directory);
const sha = await sha256File(archive.file);
console.log(
`Source: ${archive.count} files, ${((await stat(archive.file)).size / 1e6).toFixed(1)} MB, sha256 ${sha.slice(0, 16)}…`,
);
await kubectl(['create', '-f', '-'], { input: JSON.stringify(imageBuildPod(profile, { name, tag })) });
await kubectl(['-n', profile.namespace, 'wait', '--for=condition=Ready', `pod/${name}`, '--timeout=600s'], {
timeoutSeconds: 620,
});
await kubectl(['-n', profile.namespace, 'cp', archive.file, `${name}:/context/source.tar.gz`, '-c', 'kaniko'], {
timeoutSeconds: 600,
});
await kubectl([
'-n',
profile.namespace,
'exec',
name,
'-c',
'kaniko',
'--',
'/busybox/sh',
'-c',
UNPACK,
'unpack',
sha,
]);
console.log(`Building ${profile.registry}/${IMAGE_REPOSITORY}:${tag} in pod ${name} …`);
let pod;
for (const until = Date.now() + 90 * 60_000; Date.now() < until; await delay(15_000)) {
const result = await kubectl(['-n', profile.namespace, 'get', 'pod', name, '-o', 'json'], { allowFailure: true });
if (result.code !== 0) continue;
pod = JSON.parse(result.stdout);
if (['Succeeded', 'Failed'].includes(pod.status?.phase)) break;
}
const terminated = pod?.status?.containerStatuses?.[0]?.state?.terminated;
if (pod?.status?.phase !== 'Succeeded' || !/^sha256:[a-f0-9]{64}$/.test(terminated?.message?.trim() ?? '')) {
const logs = await kubectl(['-n', profile.namespace, 'logs', name, '-c', 'kaniko', '--tail=80'], {
allowFailure: true,
});
throw new Error(`The image build failed (${pod?.status?.phase ?? 'unknown'}):\n${logs.stdout.slice(-6000)}`);
}
const digest = terminated.message.trim();
const record = {
repository: `${PULL_REGISTRY}/${IMAGE_REPOSITORY}`,
pushedAs: `${profile.registry}/${IMAGE_REPOSITORY}:${tag}`,
digest,
reference: `${PULL_REGISTRY}/${IMAGE_REPOSITORY}@${digest}`,
sourceCommit: head,
dirty,
sourceSha256: sha,
inputs: IMAGE_INPUTS,
builtAt: new Date().toISOString(),
};
await writeFile(path.join(STATE, 'image.json'), `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
console.log(`Image: ${record.reference}`);
} finally {
await kubectl(['-n', profile.namespace, 'delete', 'pod', name, '--ignore-not-found', '--wait=false'], {
allowFailure: true,
});
await rm(directory, { recursive: true, force: true });
}
}
// ------------------------------------------------------------------ configuration of the Pod
/** Where each container keeps its files: the paths infra/k8s/cluster-poc/platform.yaml mounts. */
export const PATHS = {
gitRoot: '/var/lib/wishforge/git',
localAccess: '/var/lib/wishforge/platform-state/local-access.json',
pocState: '/var/lib/wishforge/poc-state',
runnerState: '/var/lib/wishforge/runner-state',
execRoot: '/var/lib/wishforge/exec',
claudeHome: '/var/lib/wishforge/claude-home',
claudeHomeApi: '/var/lib/wishforge/claude-home-api',
// OpenSSH finds ~/.ssh through the user database, not $HOME: this is the image user's home.
gitHome: '/home/wishforge',
config: '/etc/wishforge/config',
signingKey: '/etc/wishforge/keys/runner-ed25519.pem',
kubeToken: '/var/run/wishforge/kube/token',
fanyuKubeconfig: '/etc/wishforge/fanyu/kubeconfig',
};
/**
* The Mac's settings the cluster keeps as they are: the keys and tokens that persisted data, issued
* grants and the Runner's registration rely on. Everything else is the cluster's own.
*/
export const CARRIED = [
'WF_TENANT_ID',
'WF_REGION',
'WF_DISPATCH_TENANTS',
'WF_DEV_AUTH_TOKEN',
'WF_WORKER_TOKEN',
'WF_VERIFIER_TOKEN',
'WF_DEPLOYER_TOKEN',
'WF_EVALUATOR_TOKEN',
'WF_VAULT_KEY',
'WF_SESSION_SECRET',
'WF_INTERNAL_TOKEN_KEY',
'WF_PROVISIONER_TOKEN',
'WF_RESOURCE_GRANT_SECRET',
'WF_PROVIDER_PROXY_SIGNING_KEY',
'WF_METRICS_TOKEN',
'WF_GIT_ADMIN_TOKEN',
'WF_GIT_TOKEN_SECRET',
'S3_BUCKET',
'S3_REGION',
'TEMPORAL_NAMESPACE',
];
const RUNNER_CARRIED = ['WF_RUNNER_TOKEN', 'WF_TENANT_ID', 'WF_ENVIRONMENT_ID', 'WF_RUNNER_PUBLIC_KEYS'];
const DOMAIN = /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/;
const PUBLIC_ORIGIN = new RegExp(`^https?://(?:\\d{1,3}(?:\\.\\d{1,3}){3}|${DOMAIN.source.slice(1, -1)})$`);
/** Public resolvers that show what the internet, Let's Encrypt included, sees of a domain. */
const PUBLIC_RESOLVERS = ['223.5.5.5', '119.29.29.29'];
/** A publicly routable IPv4 address: never a URL, private, shared or documentation address. */
export function publicIPv4(value) {
if (isIP(value) !== 4) throw new Error('A public IPv4 address is required');
const [a, b, c] = value.split('.').map(Number);
if (
a === 0 ||
a === 10 ||
a === 127 ||
a >= 224 ||
(a === 100 && b >= 64 && b <= 127) ||
(a === 169 && b === 254) ||
(a === 172 && b >= 16 && b <= 31) ||
(a === 192 && (b === 168 || b === 0 || (b === 88 && c === 99))) ||
(a === 198 && (b === 18 || b === 19 || (b === 51 && c === 100))) ||
(a === 203 && b === 0 && c === 113)
)
throw new Error('The entry must use a publicly routable IPv4 address');
return value;
}
/** A domain name for the public entry, as the operator's DNS points it at the load balancer. */
export function publicDomain(name) {
const domain = String(name ?? '')
.trim()
.toLowerCase();
if (!DOMAIN.test(domain)) throw new Error(`Not a domain name: ${name}`);
return domain;
}
/** The console's address through a port-forward to the Pod: the Mac's own console keeps port 3000. */
export const PORT_FORWARD_ORIGIN = 'http://127.0.0.1:13000';
/** The settings files of the platform, executor and Runner containers; see {@link CARRIED}. */
export function clusterEnvironments({ local, runner, runnerFanyu, image, generated, origins = [] }) {
const missing = [
...CARRIED.filter(key => !local[key]).map(key => `carried.env ${key}`),
...RUNNER_CARRIED.filter(key => !runner[key]).map(key => `runner.env ${key}`),
...(runnerFanyu ? RUNNER_CARRIED.filter(key => !runnerFanyu[key]).map(key => `Fanyu's Runner ${key}`) : []),
];
if (missing.length) throw new Error(`Missing settings: ${missing.join(', ')}`);
for (const origin of origins)
if (!PUBLIC_ORIGIN.test(origin)) throw new Error(`A public origin is http(s)://<IPv4 or domain>: ${origin}`);
const poc = {
WF_LOCAL_POC_BINDING_FILE: `${PATHS.pocState}/project.json`,
WF_POC_PROFILE_FILE: `${PATHS.pocState}/poc-profile.json`,
};
const platform = {
WF_DEPLOYMENT_MODE: 'development',
...Object.fromEntries(CARRIED.map(key => [key, local[key]])),
DATABASE_ADMIN_URL: `postgresql://wishforge:${encodeURIComponent(generated.postgresPassword)}@127.0.0.1:5432/wishforge`,
DATABASE_URL: `postgresql://wishforge_app:${encodeURIComponent(generated.appPassword)}@127.0.0.1:5432/wishforge`,
WF_DATABASE_APP_PASSWORD: generated.appPassword,
S3_ENDPOINT: 'http://127.0.0.1:9000',
S3_ACCESS_KEY_ID: generated.objectStorageUser,
S3_SECRET_ACCESS_KEY: generated.objectStoragePassword,
PORT: '4000',
HOST: '127.0.0.1',
WF_API_URL: 'http://127.0.0.1:4000',
WF_CONTROL_PLANE_URL: 'http://127.0.0.1:4000',
WF_PROVISIONER_URL: 'http://127.0.0.1:4450',
WF_PROVISIONER_BACKEND: 'local',
TEMPORAL_ADDRESS: '127.0.0.1:7233',
WF_GIT_PORT: '4504',
WF_GIT_SERVICE_URL: 'http://127.0.0.1:4504',
WF_GIT_PUBLIC_BASE_URL: 'http://127.0.0.1:4504',
WF_GIT_ALLOW_LOCAL_HTTP: 'true',
WF_GIT_ROOT: PATHS.gitRoot,
WF_LOCAL_ACCESS: 'true',
WF_LOCAL_ACCESS_FILE: PATHS.localAccess,
// The web launcher passes only this token name, not WF_DEV_AUTH_TOKEN, to the console.
WF_LOCAL_API_TOKEN: local.WF_DEV_AUTH_TOKEN,
// In the Pod, and through `kubectl port-forward pod/wishforge-0 13000:3000` from an operator's machine.
WF_LOCAL_ALLOWED_ORIGINS: ['http://127.0.0.1:3000', 'http://localhost:3000', PORT_FORWARD_ORIGIN, ...origins].join(
',',
),
WF_PUBLIC_URL: origins[0] ?? 'http://127.0.0.1:3000',
WF_WEB_SERVE_MODE: 'production',
WF_WEB_HOST: '127.0.0.1',
WF_WEB_PORT: '3000',
WF_RUNNER_SIGNING_KEY_FILE: PATHS.signingKey,
WF_PLAN_CATALOG_FILE: `${PATHS.config}/plans.json`,
WF_CLAUDE_BIN: '/usr/local/bin/claude',
WF_CLAUDE_LOCAL_AUTH_HOME: PATHS.claudeHomeApi,
WF_CLAUDE_ASK_MODEL: 'deepseek-flash@latest',
WF_CLAUDE_ASK_FALLBACK_MODEL: 'deepseek-pro@latest',
...poc,
};
const executor = {
WF_DEPLOYMENT_MODE: 'development',
WF_TENANT_ID: local.WF_TENANT_ID,
WF_RESOURCE_GRANT_SECRET: local.WF_RESOURCE_GRANT_SECRET,
WF_PROVISIONER_TOKEN: local.WF_PROVISIONER_TOKEN,
WF_PROVISIONER_BACKEND: 'local',
WF_CONTROL_PLANE_URL: 'http://127.0.0.1:4000',
TEMPORAL_ADDRESS: '127.0.0.1:7233',
TEMPORAL_NAMESPACE: local.TEMPORAL_NAMESPACE,
WF_LOCAL_RUN_ROOT: `${PATHS.execRoot}/runs`,
WF_GIT_ALLOW_LOCAL_HTTP: 'true',
WF_CLAUDE_BIN: '/usr/local/bin/claude',
WF_CLAUDE_LOCAL_AUTH_HOME: PATHS.claudeHome,
WF_HOST_CHECK_POD_NAMESPACE: CHECK_NAMESPACE,
WF_HOST_CHECK_POD_IMAGE: image,
WF_HOST_CHECK_POD_CLAIM: 'wishforge-exec',
WF_HOST_CHECK_POD_ROOT: PATHS.execRoot,
WF_HOST_CHECK_KUBECONFIG: `${PATHS.config}/kubeconfig`,
WF_HOST_CHECK_WRITE_PATHS: `${PATHS.execRoot}/check-cache`,
WF_HOST_CHECK_POD_CPU: '4',
WF_HOST_CHECK_POD_MEMORY: '8Gi',
...poc,
};
// Each Runner keeps its state in its own directory, mounted at the same path.
const runnerEnv = (values, keys, profile) => ({
WF_DEPLOYMENT_MODE: 'development',
WF_CONTROL_PLANE_URL: 'http://127.0.0.1:4000',
WF_RUNNER_ALLOW_LOCAL_HTTP: 'true',
...Object.fromEntries(RUNNER_CARRIED.map(key => [key, values[key]])),
// The Mac's runner.env names a file on the Mac; its content travels in the ConfigMap.
WF_RUNNER_PUBLIC_KEYS: `${PATHS.config}/${keys}`,
WF_RUNNER_EXECUTE: values.WF_RUNNER_EXECUTE === 'true' ? 'true' : 'false',
WF_RUNNER_STATE_DIR: PATHS.runnerState,
WF_POC_PROFILE_FILE: `${PATHS.pocState}/${profile}`,
});
return {
platform,
executor,
runner: runnerEnv(runner, 'runner-public.json', 'poc-profile.json'),
...(runnerFanyu ? { runnerFanyu: runnerEnv(runnerFanyu, 'runner-fanyu-public.json', 'fanyu-profile.json') } : {}),
};
}
/** A settings file Node's --env-file and the container entry both read; one value per line. */
export function envFile(values) {
const lines = Object.entries(values).map(([key, value]) => {
if (!/^[A-Z_][A-Z0-9_]*$/.test(key) || typeof value !== 'string' || /[\r\n\0']/.test(value))
throw new Error(`Cannot write ${key} as a single-line setting`);
return /^[A-Za-z0-9_@%:/.,+=?&[\]-]*$/.test(value) ? `${key}=${value}` : `${key}='${value}'`;
});
return `${lines.join('\n')}\n`;
}
/** WishForge's identity at the POC cluster's address, by its short-lived projected token. */
export function clusterKubeconfig(certificateAuthorityData) {
if (!/^[A-Za-z0-9+/=]+$/.test(certificateAuthorityData)) throw new Error('Invalid cluster CA data');
return JSON.stringify(
{
apiVersion: 'v1',
kind: 'Config',
clusters: [{ name: 'poc', cluster: { server: CLUSTER, 'certificate-authority-data': certificateAuthorityData } }],
users: [{ name: 'wishforge', user: { tokenFile: PATHS.kubeToken } }],
contexts: [{ name: 'poc', context: { cluster: 'poc', user: 'wishforge' } }],
'current-context': 'poc',
},
null,
2,
);
}
/** Until the public address exists: HTTP only, answering that the entry is not configured. */
export const PENDING_CADDYFILE = `{
admin off
auto_https off
}
:8080 {
respond "WishForge's public address is not configured yet." 503
}
`;
/**
* The public HTTPS entry at the load balancer's address: a Let's Encrypt certificate for the IP
* (short-lived profile, HTTP-01 only), HTTP GET/HEAD redirected to the standard HTTPS port, and the
* console behind it on loopback with forwarding headers it can trust.
*/
export function clusterCaddyfile(address, { staging = false, domain } = {}) {
const ip = publicIPv4(address);
const hosts = [...(domain ? [publicDomain(domain)] : []), ip];
const site = (scheme, port) => hosts.map(host => `${scheme}://${host}:${port}`).join(', ');
const ca = `https://acme${staging ? '-staging' : ''}-v02.api.letsencrypt.org/directory`;
return `{
admin off
skip_install_trust
http_port 8080
https_port 8443
default_sni ${ip}
auto_https disable_redirects
servers {
protocols h1 h2
}
}
${site('http', 8080)} {
@safe method GET HEAD
redir @safe https://{host}{uri} 308
respond 405
}
${site('https', 8443)} {
tls {
issuer acme {
dir ${ca}
test_dir ${ca}
profile shortlived
disable_tlsalpn_challenge
alt_http_port 8080
}
}
reverse_proxy 127.0.0.1:3000 {
header_up X-Forwarded-Proto https
header_up -Forwarded
}
}
`;
}
/**
* The public plain-HTTP entry at the load balancer's address, which the operator chose over HTTPS
* (`public --http`): the console behind it on loopback. Passwords and sessions travel unencrypted.
*/
export function clusterHttpCaddyfile(address, { domain } = {}) {
const ip = publicIPv4(address);
const site = [...(domain ? [publicDomain(domain)] : []), ip].map(host => `http://${host}:8080`).join(', ');
return `{
admin off
auto_https off
http_port 8080
servers {
protocols h1
}
}
${site} {
reverse_proxy 127.0.0.1:3000 {
header_up X-Forwarded-Proto http
header_up -Forwarded
}
}
`;
}
/** public-service.yaml as applied: a plain-HTTP entry has no HTTPS port. */
export function publicService(template, { http = false } = {}) {
const https = ' - { name: https, protocol: TCP, port: 443, targetPort: https }\n';
if (!template.includes(https)) throw new Error('public-service.yaml has no HTTPS port');
return http ? template.replace(https, '') : template;
}
const IMAGE_REFERENCE =
/^ccr-29eug8s3-vpc\.cnc\.bj\.baidubce\.com\/dataflow-poc\/wishforge-cluster-poc@sha256:[a-f0-9]{64}$/;
const PLACEHOLDER = name => `\${${name}}`;
/**
* platform.yaml with its image and switches filled in; nothing left unfilled. `settings` is the
* digest of the settings the containers read when they start, so a change to them rolls the Pod.
*/
export function renderPlatform(template, { image, runnerDisabled, maintenance, settings }) {
if (!IMAGE_REFERENCE.test(image)) throw new Error('The WishForge image must be the pinned cluster image');
if (!/^[a-f0-9]{32}$/.test(settings ?? '')) throw new Error('The settings digest is missing');
const rendered = template
.replaceAll(PLACEHOLDER('WISHFORGE_IMAGE'), image)
.replaceAll(PLACEHOLDER('SETTINGS_DIGEST'), settings)
.replaceAll(PLACEHOLDER('RUNNER_DISABLED'), String(Boolean(runnerDisabled)))
.replaceAll(PLACEHOLDER('MAINTENANCE'), String(Boolean(maintenance)));
if (rendered.includes('${')) throw new Error('platform.yaml has an unfilled placeholder');
return rendered;
}
// ------------------------------------------------------------------------------- commands
const MANIFESTS = path.join(ROOT, 'infra/k8s/cluster-poc');
const GENERATED = path.join(STATE, 'generated.json');
/** Written by fanyu-access: Fanyu's profile, naming WishForge's kubeconfig for Fanyu's cluster. */
const FANYU_PROFILE = path.join(STATE, 'fanyu-profile.json');
/** Written by projects --replace: the cluster's projects' bindings and their Runners' registrations. */
const PROJECTS_FILE = path.join(STATE, 'projects.json');
const secretValue = () => randomBytes(32).toString('base64url');
async function readEnv(file) {
const values = {};
for (const line of (await readFile(file, 'utf8')).split('\n')) {
const match = /^([A-Z_][A-Z0-9_]*)=(.*)$/.exec(line);
if (match) values[match[1]] = match[2].replace(/^(['"])(.*)\1$/, '$2');
}
return values;
}
async function privateState() {
await mkdir(STATE, { recursive: true, mode: 0o700 });
await chmod(STATE, 0o700);
}
async function writePrivate(file, content) {
await privateState();
const temporary = `${file}.${randomBytes(4).toString('hex')}.tmp`;
await writeFile(temporary, content, { mode: 0o600 });
await rename(temporary, file);
}
/** Passwords the cluster's own services use; made once, kept only in this Mac's private state. */
async function generatedSecrets() {
try {
return JSON.parse(await readFile(GENERATED, 'utf8'));
} catch (error) {
if (error.code !== 'ENOENT') throw error;
}
const generated = {
postgresPassword: secretValue(),
appPassword: secretValue(),
objectStorageUser: `wishforge-${randomBytes(6).toString('hex')}`,
objectStoragePassword: secretValue(),
};
await writePrivate(GENERATED, `${JSON.stringify(generated, null, 2)}\n`);
return generated;
}
async function imageRecord() {
try {
return JSON.parse(await readFile(path.join(STATE, 'image.json'), 'utf8'));
} catch {
throw new Error('No image yet; run: node scripts/cluster-poc.mjs image');
}
}
/** Applies Kubernetes objects from JSON on stdin, so no value appears in a command line. */
const applyObjects = objects =>
kubectl(['apply', '-f', '-'], { input: JSON.stringify({ apiVersion: 'v1', kind: 'List', items: objects }) });
const labels = { 'app.kubernetes.io/part-of': 'wishforge-cluster-poc' };
const secret = (name, stringData) => ({
apiVersion: 'v1',
kind: 'Secret',
type: 'Opaque',
metadata: { name, namespace: NAMESPACE, labels },
stringData,
});
const configMap = (name, data) => ({
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: { name, namespace: NAMESPACE, labels },
data,
});
async function exists(kind, name, namespace) {
const scope = namespace ? ['-n', namespace] : [];
return (await kubectl([...scope, 'get', kind, name, '--ignore-not-found', '-o', 'name'])).stdout.trim() !== '';
}
async function bootstrap() {
await assertCluster();
await kubectl(['apply', '-f', path.join(MANIFESTS, 'namespaces.yaml')]);
if (!(await exists('pv', 'wishforge-data'))) {
const directories = path.join(MANIFESTS, 'bootstrap/directories.yaml');
await kubectl(['apply', '-f', directories]);
let phase = '';
for (const until = Date.now() + 5 * 60_000; Date.now() < until && !['Succeeded', 'Failed'].includes(phase);) {
await delay(3000);
phase = (
await kubectl(['-n', NAMESPACE, 'get', 'pod', 'wishforge-bootstrap', '-o', 'jsonpath={.status.phase}'], {
allowFailure: true,
})
).stdout.trim();
}
await kubectl(['delete', '-f', directories, '--wait=true'], { timeoutSeconds: 120 });
if (phase !== 'Succeeded') throw new Error(`Creating the data directory did not succeed (${phase || 'no phase'})`);
console.log('Data directory created on 192.168.80.249.');
}
for (const file of ['storage.yaml', 'rbac.yaml', 'admission.yaml'])
await kubectl(['apply', '-f', path.join(MANIFESTS, file)]);
console.log('Namespaces, storage, access rules and admission policies are in place.');
}
/** The Gitea deploy key: its private half goes to the cluster only; the public half is printed. */
async function deployKey(flag) {
await assertCluster();
const publicKey = path.join(STATE, 'deploy-key.pub');
if (flag !== '--rotate' && (await exists('secret', 'wishforge-git', NAMESPACE))) {
console.log((await readFile(publicKey, 'utf8')).trim());
return;
}
const directory = await mkdtemp(path.join(os.tmpdir(), 'wishforge-deploy-key-'));
try {
const key = path.join(directory, 'id_ed25519');
const made = await capture('ssh-keygen', [
'-q',
'-t',
'ed25519',
'-N',
'',
'-C',
'wishforge-cluster-poc',
'-f',
key,
]);
if (made.code !== 0) throw new Error(`ssh-keygen failed: ${made.stderr}`);
await applyObjects([secret('wishforge-git', { id_ed25519: await readFile(key, 'utf8') })]);
await writePrivate(publicKey, await readFile(`${key}.pub`, 'utf8'));
console.log((await readFile(`${key}.pub`, 'utf8')).trim());
} finally {
await rm(directory, { recursive: true, force: true });
}
}
/**
* The Claude CLI's settings in the cluster: the company's Anthropic-protocol gateway, its models
* with their 1M context, and no traffic to services the gateway does not replace.
*/
export function claudeSettings(key) {
return {
env: {
ANTHROPIC_BASE_URL: 'https://ai-api.d-robotics.cc',
...(key ? { ANTHROPIC_API_KEY: key } : {}),
ANTHROPIC_MODEL: 'deepseek-pro@latest',
ANTHROPIC_DEFAULT_OPUS_MODEL: 'deepseek-pro@latest',
ANTHROPIC_DEFAULT_SONNET_MODEL: 'deepseek-pro@latest',
ANTHROPIC_DEFAULT_HAIKU_MODEL: 'deepseek-flash@latest',
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1',
DISABLE_AUTOUPDATER: '1',
DISABLE_TELEMETRY: '1',
DISABLE_ERROR_REPORTING: '1',
},
};
}
const readStdin = () =>
new Promise((resolve, reject) => {
const chunks = [];
process.stdin
.on('data', chunk => chunks.push(chunk))
.once('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
.once('error', reject);
});
/** The model gateway's key, from the clipboard or with --stdin from standard input, into the cluster only. */
async function gatewayKey(flag) {
await assertCluster();
if (flag !== undefined && flag !== '--stdin') throw new Error('Usage: cluster-poc.mjs gateway-key [--stdin]');
const source = flag === '--stdin' ? 'standard input' : 'clipboard';
const key = (flag === '--stdin' ? await readStdin() : (await capture('pbpaste', [])).stdout).trim();
if (!/^sk-[A-Za-z0-9_-]{16,200}$/.test(key))
throw new Error(
`The ${source} holds no gateway key (sk-…, ${key.length} characters read); provide the key and run this again`,
);
await applyObjects([
secret('wishforge-claude', { 'settings.json': `${JSON.stringify(claudeSettings(key), null, 2)}\n` }),
]);
console.log(`The gateway key is in the cluster (Secret wishforge-claude)${flag ? '.' : '; clear your clipboard.'}`);
}
/** The projects the cluster delivers and their Runners (projects --replace). */
async function clusterProjects() {
const pocProfile = { ...JSON.parse(await readFile(POC_PROFILE, 'utf8')), kubeconfig: `${PATHS.config}/kubeconfig` };
const fanyuProfile = await readFile(FANYU_PROFILE, 'utf8').then(JSON.parse, () => undefined);
const profiles = {
'poc-profile.json': pocProfile,
...(fanyuProfile ? { 'fanyu-profile.json': fanyuProfile } : {}),
};
const replaced = JSON.parse(await readFile(PROJECTS_FILE, 'utf8'));
for (const binding of replaced.bindings)
if (!profiles[path.basename(binding.profileFile)])
throw new Error(`No ${path.basename(binding.profileFile)} for project ${binding.projectId}; run fanyu-access`);
const values = registration => ({
WF_RUNNER_TOKEN: registration.token,
WF_TENANT_ID: replaced.tenantId,
WF_ENVIRONMENT_ID: registration.environmentId,
WF_RUNNER_PUBLIC_KEYS: 'in the ConfigMap',
WF_RUNNER_EXECUTE: 'true',
});
const { poc, fanyu } = replaced.runners;
return {
profiles,
bindings: replaced.bindings,
runner: values(poc),
runnerFanyu: fanyu ? values(fanyu) : undefined,
keys: {
'runner-public.json': `${JSON.stringify(poc.publicKeys, null, 2)}\n`,
...(fanyu ? { 'runner-fanyu-public.json': `${JSON.stringify(fanyu.publicKeys, null, 2)}\n` } : {}),
},
};
}
async function secrets() {
await assertCluster();
const local = await readEnv(INPUTS.carried);
const projects = await clusterProjects();
const { runner, runnerFanyu } = projects;
const image = (await imageRecord()).reference;
const generated = await generatedSecrets();
const http = generated.publicScheme === 'http';
const domain = generated.publicDomain;
const origins = generated.publicAddress
? [...(domain ? [domain] : []), generated.publicAddress].map(host => `${http ? 'http' : 'https'}://${host}`)
: [];
const environments = clusterEnvironments({ local, runner, runnerFanyu, image, generated, origins });
const ca = (
await kubectl([
'config',
'view',
'--raw',
'--minify',
'-o',
'jsonpath={.clusters[0].cluster.certificate-authority-data}',
])
).stdout.trim();
const hosts = (await readFile(INPUTS.knownHosts, 'utf8')).split('\n').filter(line => line && !line.startsWith('#'));
if (!hosts.length) throw new Error(`No pinned host key for gitea.d-robotics.cc in ${INPUTS.knownHosts}`);
// The Pod mounts the Claude settings; until gateway-key stores the key, they name the gateway only.
if (!(await exists('secret', 'wishforge-claude', NAMESPACE))) {
await applyObjects([
secret('wishforge-claude', { 'settings.json': `${JSON.stringify(claudeSettings(), null, 2)}\n` }),
]);
console.log('No gateway key yet: Claude runs and AI optimization stay unavailable until gateway-key.');
}
const objects = [
secret('wishforge-postgres', { password: generated.postgresPassword }),
secret('wishforge-minio', { user: generated.objectStorageUser, password: generated.objectStoragePassword }),
secret('wishforge-env', {
'platform.env': envFile(environments.platform),
'executor.env': envFile(environments.executor),
'runner.env': envFile(environments.runner),
...(environments.runnerFanyu ? { 'runner-fanyu.env': envFile(environments.runnerFanyu) } : {}),
}),
secret('wishforge-keys', { 'runner-ed25519.pem': await readFile(INPUTS.signingKey, 'utf8') }),
configMap('wishforge-config', {
'plans.json': await readFile(INPUTS.plans, 'utf8'),
...projects.keys,
...Object.fromEntries(
Object.entries(projects.profiles).map(([name, profile]) => [name, `${JSON.stringify(profile, null, 2)}\n`]),
),
'project.json': `${JSON.stringify(projects.bindings, null, 2)}\n`,
known_hosts: `${hosts.join('\n')}\n`,
kubeconfig: clusterKubeconfig(ca),
}),
configMap('wishforge-caddy', {
Caddyfile: !generated.publicAddress
? PENDING_CADDYFILE
: http
? clusterHttpCaddyfile(generated.publicAddress, { domain })
: clusterCaddyfile(generated.publicAddress, { staging: generated.acme !== 'production', domain }),
}),
];
await applyObjects(objects);
const digest = createHash('sha256').update(JSON.stringify(objects)).digest('hex').slice(0, 32);
console.log(`Secrets and ConfigMaps are in place: settings, keys, POC files and the Caddyfile (settings ${digest}).`);
return digest;
}
async function apply(...flags) {
await assertCluster();
// The containers read their settings, which name the check Pods' image too, only when they
// start: refresh them, and roll the Pod when they changed.
const settings = await secrets();
const rendered = renderPlatform(await readFile(path.join(MANIFESTS, 'platform.yaml'), 'utf8'), {
image: (await imageRecord()).reference,
settings,
runnerDisabled: !flags.includes('--runner'),
maintenance: flags.includes('--maintenance'),
});
await kubectl(['apply', '-f', '-'], { input: rendered });
// A StatefulSet rolls a Pod that is not Ready (one in maintenance, say) only once it is deleted.
let replaced = false;
await waitFor('the WishForge Pod to take the new revision', async () => {
const set = JSON.parse((await kubectl(['-n', NAMESPACE, 'get', 'statefulset', 'wishforge', '-o', 'json'])).stdout);
const pod = await kubectl(['-n', NAMESPACE, 'get', 'pod', 'wishforge-0', '-o', 'json'], { allowFailure: true });
if (pod.code !== 0 || set.metadata.generation !== set.status?.observedGeneration) return false;
if (JSON.parse(pod.stdout).metadata.labels?.['controller-revision-hash'] === set.status.updateRevision) return true;
if (!replaced) {
await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-0', '--wait=true'], { timeoutSeconds: 180 });
replaced = true;
}
return false;
});
console.log(`Applied the WishForge Pod${flags.length ? ` (${flags.join(' ')})` : ''}.`);
}
async function status() {
await assertCluster();
for (const [label, args] of [
['Pod', ['-n', NAMESPACE, 'get', 'pods', '-o', 'wide']],
['Storage', ['get', 'pv', 'wishforge-data', 'wishforge-checks-exec', '--ignore-not-found']],
['Check Pods', ['-n', CHECK_NAMESPACE, 'get', 'pods']],
['Public entry', ['-n', NAMESPACE, 'get', 'service', 'wishforge-public', '--ignore-not-found']],
]) {
const result = await kubectl(args, { allowFailure: true });
console.log(`== ${label} ==\n${result.stdout.trim() || result.stderr.trim() || '(none)'}`);
}
const pod = await kubectl(['-n', NAMESPACE, 'get', 'pod', 'wishforge-0', '-o', 'json'], { allowFailure: true });
if (pod.code === 0)
for (const container of JSON.parse(pod.stdout).status?.containerStatuses ?? [])
console.log(
`${container.name.padEnd(10)} ${container.ready ? 'ready' : 'not ready'}, restarts ${container.restartCount}${
container.state?.waiting ? `, ${container.state.waiting.reason}` : ''
}`,
);
const backups = await kubectl(backupExec(['node', 'scripts/cluster-poc-backup.mjs', 'latest']), {
allowFailure: true,
});
if (backups.code === 0) {
const { snapshots, manifest, last } = JSON.parse(backups.stdout);
console.log(
`== Backups ==\n${snapshots.length} on the node, newest ${manifest?.takenAt ?? 'none'}; last attempt ${
last ? `${last.ok ? 'succeeded' : `failed (${last.error})`} at ${last.at}` : 'none yet'
}`,
);
}
}
// ------------------------------------------------------------------------------- restore
/** What the cluster is still doing: projects are replaced and data restored only when nothing runs. */
export const UNFINISHED_SQL = `select kind || ' ' || id || ' ' || coalesce(data->>'status', '') from entities where
(kind = 'run' and data->>'status' in ('queued', 'running')) or
(kind = 'runner_task' and data->>'status' in ('pending', 'dispatching', 'unknown')) or
(kind = 'release' and data->>'status' in ('ready', 'deploying', 'unknown'))`;
async function waitFor(description, check, minutes = 10) {
for (const until = Date.now() + minutes * 60_000; Date.now() < until; await delay(5000)) if (await check()) return;
throw new Error(`Timed out waiting for ${description}`);
}
const podGone = async name =>
(
await kubectl(['-n', NAMESPACE, 'get', 'pod', name, '--ignore-not-found', '-o', 'name'], { allowFailure: true })
).stdout.trim() === '';
/** A restricted Pod with the data volume, to put a snapshot's files in place while the WishForge Pod is stopped. */
const restorePod = image => ({
apiVersion: 'v1',
kind: 'Pod',
metadata: { name: 'wishforge-restore', namespace: NAMESPACE, labels },
spec: {
restartPolicy: 'Never',
automountServiceAccountToken: false,
enableServiceLinks: false,
imagePullSecrets: [{ name: 'cce-image-credential-ccr-29eug8s3-vpc.cnc.bj.baidubce.com' }],
activeDeadlineSeconds: 3600,
securityContext: {
runAsNonRoot: true,
runAsUser: 10001,
runAsGroup: 10001,
fsGroup: 10001,
fsGroupChangePolicy: 'OnRootMismatch',
seccompProfile: { type: 'RuntimeDefault' },
},
containers: [
{
name: 'restore',
image,
command: ['/bin/sh', '-c', 'sleep 3600'],
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '1', memory: '1Gi' } },
volumeMounts: [
{ name: 'data', mountPath: '/mnt/data' },
{ name: 'tmp', mountPath: '/tmp' },
],
},
],
volumes: [
{ name: 'data', persistentVolumeClaim: { claimName: 'wishforge-data' } },
{ name: 'tmp', emptyDir: { sizeLimit: '4Gi' } },
],
},
});
const SNAPSHOT = /^\d{8}T\d{6}Z$/;
/**
* Puts snapshot `name`'s files in place, in the restore Pod: each file is checked against the
* manifest first, and only then are the directories emptied and unpacked. `archives` are the
* directories the snapshot holds; the others are emptied. /tmp/verified and /tmp/restored mark how
* far it got, since kubectl can lose an exec's status.
*/
export function restoreScript(name, archives = ARCHIVES) {
if (!SNAPSHOT.test(name)) throw new Error(`Not a snapshot name: ${name}`);
if (archives.some(archive => !ARCHIVES.includes(archive))) throw new Error('Unknown archive');
const snapshot = `/mnt/data/backups/${name}`;
const reset = ['postgres', 'temporal', 'exec/runs', ...ARCHIVES];
return [
'set -e',
`node /opt/wishforge/scripts/cluster-poc-backup.mjs verify '${snapshot}'`,
'touch /tmp/verified',
'cd /mnt/data',
...reset.map(dir => `mkdir -p '${dir}' && find '${dir}' -mindepth 1 -delete && chmod 0700 '${dir}'`),
...archives.map(dir => `tar -xzf '${snapshot}/${dir}.tgz' -C '${dir}' --no-same-owner --no-same-permissions`),
`cp '${snapshot}/temporal.db' temporal/temporal.db && chmod 0600 temporal/temporal.db`,
'touch /tmp/restored',
'echo restored: $(find minio -type f | wc -l) object storage files, $(find git -type f | wc -l) Git files',
].join('\n');
}
/**
* Replaces the cluster's data with a snapshot on the node, with the WishForge Pod stopped. The
* database is then restored into a fresh PostgreSQL started with nothing else running, by the
* backup container, which reads the snapshot and reaches the database on loopback; the Pod starts
* only if every table's row count is `counts`.
*/
async function replaceData({ image, snapshot, archives, counts, runner }) {
const script = restoreScript(snapshot, archives);
if (await exists('statefulset', 'wishforge', NAMESPACE)) {
await kubectl(['-n', NAMESPACE, 'scale', 'statefulset', 'wishforge', '--replicas=0']);
await waitFor('the WishForge Pod to stop', () => podGone('wishforge-0'));
}
await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-restore', '--ignore-not-found', '--wait=true']);
await kubectl(['create', '-f', '-'], { input: JSON.stringify(restorePod(image)) });
const inRestorePod = (command, options) =>
kubectl(['-n', NAMESPACE, 'exec', 'wishforge-restore', '-c', 'restore', '--', '/bin/sh', '-c', command], options);
try {
await kubectl(['-n', NAMESPACE, 'wait', '--for=condition=Ready', 'pod/wishforge-restore', '--timeout=600s'], {
timeoutSeconds: 620,
});
const restored = await inRestorePod(script, { timeoutSeconds: 1200, allowFailure: true });
const marks = (await inRestorePod('ls /tmp', { allowFailure: true })).stdout;
if (!/\brestored\b/.test(marks)) {
if (/\bverified\b/.test(marks))
throw new Error(
`Unpacking snapshot ${snapshot} did not finish (${restored.stderr.trim().slice(0, 400)}); the Pod stays stopped, run this again`,
);
await kubectl(['-n', NAMESPACE, 'scale', 'statefulset', 'wishforge', '--replicas=1'], { allowFailure: true });
throw new Error(
`Snapshot ${snapshot} did not pass its check (${restored.stderr.trim().slice(0, 400)}); nothing was changed and the WishForge Pod is starting again`,
);
}
console.log(restored.stdout.trim() || 'Files restored.');
} finally {
await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-restore', '--ignore-not-found', '--wait=true'], {
allowFailure: true,
});
}
await apply('--maintenance');
await kubectl(['-n', NAMESPACE, 'scale', 'statefulset', 'wishforge', '--replicas=1']);
await waitFor('PostgreSQL in the cluster', async () => {
const ready = await kubectl(
[
'-n',
NAMESPACE,
'exec',
'wishforge-0',
'-c',
'postgres',
'--',
'pg_isready',
'-h',
'127.0.0.1',
'-U',
'wishforge',
'-d',
'wishforge',
],
{ allowFailure: true },
);
return ready.code === 0;
});
// kubectl can lose an exec's status: the row counts say whether the restore took.
const restored = await kubectl(
backupExec([
'pg_restore',
'-h',
'127.0.0.1',
'-U',
'wishforge',
'-d',
'wishforge',
'--no-owner',
'--no-privileges',
'--exit-on-error',
`/var/lib/wishforge/backups/${snapshot}/postgres.dump`,
]),
{ timeoutSeconds: 1800, allowFailure: true },
);
const clusterCounts = (
await kubectl([
'-n',
NAMESPACE,
'exec',
'wishforge-0',
'-c',
'postgres',
'--',
'psql',
'-U',
'wishforge',
'-d',
'wishforge',
'-At',
'-c',
COUNTS_SQL,
])
).stdout.trim();
if (clusterCounts !== counts)
throw new Error(
`Row counts differ after the restore (${restored.stderr.trim().slice(0, 800)}):\nSnapshot: ${counts}\nCluster: ${clusterCounts}`,
);
console.log(`Database restored; row counts match: ${counts}`);
await apply(...(runner ? ['--runner'] : []));
}
// ------------------------------------------------------------------------------ public entry
/** The load balancer's public IPv4, once CCE has assigned it; private addresses it also lists are skipped. */
export function publicAddressOf(service) {
for (const ingress of service?.status?.loadBalancer?.ingress ?? []) {
try {
return publicIPv4(ingress.ip);
} catch {
// A VPC address of the load balancer, not the public one.
}
}
}
/**
* The public entry: WishForge's own load balancer and elastic IP (billed while they exist). HTTPS
* with a certificate for that IP from Let's Encrypt's staging CA first, and with --production the
* trusted one; or, with --http, plain HTTP on port 80 only. `--domain <name>` adds a domain the
* operator's DNS points at the load balancer, which becomes the console's address (the IP keeps
* working); later runs keep it. Each step restarts the WishForge Pod so the console accepts the
* new origin.
*/
async function publicEntry(...args) {
await assertCluster();
let flag, domain;
for (let i = 0; i < args.length; i++)
if (args[i] === '--domain' && args[i + 1] && !domain) domain = publicDomain(args[++i]);
else if (['--production', '--http'].includes(args[i]) && !flag) flag = args[i];
else throw new Error('Usage: cluster-poc.mjs public [--production|--http] [--domain <name>]');
if (!(await exists('statefulset', 'wishforge', NAMESPACE))) throw new Error('Deploy the WishForge Pod first');
const http = flag === '--http';
await kubectl(['apply', '-f', '-'], {
input: publicService(await readFile(path.join(MANIFESTS, 'public-service.yaml'), 'utf8'), { http }),
});
let address;
await waitFor(
'the load balancer address',
async () => {
const service = await kubectl(['-n', NAMESPACE, 'get', 'service', 'wishforge-public', '-o', 'json']);
address = publicAddressOf(JSON.parse(service.stdout));
return Boolean(address);
},
15,
);
const generated = { ...(await generatedSecrets()), publicAddress: address, publicScheme: http ? 'http' : 'https' };
if (http) delete generated.acme;
else generated.acme = flag === '--production' ? 'production' : 'staging';
if (domain) {
// Browsers and Let's Encrypt find the entry through public DNS: it must already point here.
const resolver = new Resolver({ timeout: 5000, tries: 2 });
resolver.setServers(PUBLIC_RESOLVERS);
const published = await resolver.resolve4(domain).catch(() => []);
if (!published.includes(address))
throw new Error(
`${domain} resolves publicly to ${published.join(', ') || 'nothing'}, not ${address}: point its A record here first`,
);
// A company DNS with its own copy of the zone answers its users from that copy instead.
if (!(await resolve4(domain).catch(() => [])).includes(address))
console.warn(
`This machine's DNS (${getServers().join(', ')}) does not resolve ${domain} to ${address}: people using ` +
'it cannot open the domain until that DNS has the record too.',
);
generated.publicDomain = domain;
}
const host = generated.publicDomain ?? address;
await writePrivate(GENERATED, `${JSON.stringify(generated, null, 2)}\n`);
await secrets();
await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-0', '--wait=true'], { timeoutSeconds: 180 });
console.log(
http
? `Public address ${address}: plain HTTP, port 80 only; the console will answer at http://${host} ` +
'once the Pod is ready. Passwords and sessions travel unencrypted.'
: `Public address ${address}: ${flag === '--production' ? 'trusted' : 'staging (untrusted)'} certificate ` +
`requested; the console will answer at https://${host} once the Pod is ready.`,
);
}
// ------------------------------------------------------------------------------- snapshots
const backupExec = args => ['-n', NAMESPACE, 'exec', 'wishforge-0', '-c', 'backup', '--', ...args];
/**
* Copies one file out of one of the WishForge Pod's containers. kubectl drops whatever of an exec's output is
* still in flight when the remote command exits, so the remote side sends the file and then waits
* for this end to close its input, which happens only once every byte has arrived.
*/
async function download(remote, local, bytes, container = 'backup') {
for (let attempt = 1; ; attempt++) {
const result = await new Promise((resolve, reject) => {
const child = spawn(
'kubectl',
[
'--kubeconfig',
KUBECONFIG,
'-n',
NAMESPACE,
'exec',
'-i',
'wishforge-0',
'-c',
container,
'--',
'/bin/sh',
'-c',
'cat "$0" || exit 1; read -r _',
remote,
],
{ cwd: ROOT, env: { PATH: process.env.PATH, HOME: os.homedir() }, stdio: ['pipe', 'pipe', 'pipe'] },
);
const err = [];
let received = 0;
// A link at a few dozen kB/s still finishes; a stalled one is given up.
const timer = setTimeout(() => child.kill(), 120_000 + bytes / 20);
const file = createWriteStream(local, { mode: 0o600 });
const written = new Promise((done, fail) => file.on('finish', done).on('error', fail));
if (bytes === 0) child.stdin.end('\n');
child.stdout.on('data', chunk => {
received += chunk.length;
if (received >= bytes) child.stdin.end('\n');
});
child.stdout.pipe(file);
child.stderr.on('data', chunk => err.push(chunk));
child.stdin.on('error', () => undefined);
child.once('error', reject);
child.once('close', () => {
clearTimeout(timer);
written.then(() => resolve({ received, stderr: Buffer.concat(err).toString('utf8') }), reject);
});
});
if (result.received === bytes) return;
if (attempt === 5)
throw new Error(
`Copying ${path.basename(remote)} failed after ${attempt} attempts (${result.received} of ${bytes} bytes): ${result.stderr.trim().slice(-400)}`,
);
await delay(3000 * attempt);
}
}
/**
* Replaces everything the cluster holds with one of the node's daily snapshots (`status` lists
* them): the database, Temporal, Git, object storage and the platform's, POC's and Runners'
* records. Starts the Runners only with --runner. The node keeps its snapshots.
*/
async function restore(name, ...flags) {
if (!name || name.startsWith('--') || flags.some(flag => flag !== '--runner'))
throw new Error('Usage: cluster-poc.mjs restore <snapshot, e.g. 20261102T192000Z> [--runner]');
restoreScript(name);
await assertCluster();
const image = (await imageRecord()).reference;
const { snapshots } = JSON.parse(
(await kubectl(backupExec(['node', 'scripts/cluster-poc-backup.mjs', 'latest']))).stdout,
);
if (!snapshots.includes(name))
throw new Error(`The node has no snapshot ${name}; it has ${snapshots.join(', ') || 'none'}`);
const remote = `/var/lib/wishforge/backups/${name}/manifest.json`;
const work = await mkdtemp(path.join(os.tmpdir(), 'wishforge-restore-'));
try {
const size = (await kubectl(backupExec(['/bin/sh', '-c', `wc -c < ${remote}`]))).stdout.trim();
await download(remote, path.join(work, 'manifest.json'), Number(size));
const manifest = JSON.parse(await readFile(path.join(work, 'manifest.json'), 'utf8'));
if (!manifest.database?.counts) throw new Error('The snapshot records no row counts to check the restore against');
for (const file of ['postgres.dump', 'temporal.db'])
if (!manifest.files[file]) throw new Error(`The snapshot has no ${file}`);
const archives = ARCHIVES.filter(archive => manifest.files[`${archive}.tgz`]);
console.log(`Restoring snapshot ${name} (taken ${manifest.takenAt}) …`);
await replaceData({
image,
snapshot: name,
archives,
counts: manifest.database.counts,
runner: flags.includes('--runner'),
});
console.log(`Restored snapshot ${name}${flags.includes('--runner') ? '' : '; the Runners stay idle'}.`);
} finally {
await rm(work, { recursive: true, force: true });
}
}
// ------------------------------------------------------------------------------ Fanyu's project
const FANYU_DEPLOYMENTS = {
api: 'droboticflow-api',
worker: 'droboticflow-worker',
coordinator: 'drosync-coordinator',
'annotation-processor': 'droboticflow-annotation-processor',
'annotation-validator': 'droboticflow-annotation-validator',
};
/** WishForge's kubeconfig for Fanyu's cluster: its Service Account's token, nothing else. */
export function fanyuKubeconfig({ certificateAuthorityData, token }) {
if (!/^[A-Za-z0-9+/=]+$/.test(certificateAuthorityData)) throw new Error('Invalid Fanyu cluster CA data');
if (!/^[A-Za-z0-9._-]+$/.test(token)) throw new Error('Invalid Fanyu Service Account token');
return JSON.stringify(
{
apiVersion: 'v1',
kind: 'Config',
clusters: [
{ name: 'fanyu', cluster: { server: FANYU.cluster, 'certificate-authority-data': certificateAuthorityData } },
],
users: [{ name: 'wishforge', user: { token } }],
contexts: [{ name: 'fanyu', context: { cluster: 'fanyu', user: 'wishforge', namespace: FANYU.namespace } }],
'current-context': 'fanyu',
},
null,
2,
);
}
/** Fanyu's profile, as WishForge's containers read it; the POC's pins the same Kaniko and mirrors. */
export function fanyuProfile({ poc, namespaceUid, deployments }) {
return {
schemaVersion: 1,
id: 'droboticflow-fanyu',
repository: poc.repository,
branch: 'dev',
kubeconfig: PATHS.fanyuKubeconfig,
clusterServer: FANYU.cluster,
namespace: FANYU.namespace,
namespaceUid,
deployments,
registry: 'ccr-29eug8s3-pub.cnc.bj.baidubce.com/fanyu',
registrySecret: 'droboticflow-registry',
applicationUrl: 'http://120.48.180.80',
kanikoImage: poc.kanikoImage,
registryMirrors: poc.registryMirrors,
rolloutTimeoutSeconds: poc.rolloutTimeoutSeconds ?? 600,
buildTimeoutSeconds: poc.buildTimeoutSeconds ?? 1800,
};
}
/**
* WishForge's identity in Fanyu's cluster (fanyu-access.yaml), applied with Fanyu's administrator
* kubeconfig, which stays on this Mac: its token goes into the WishForge cluster's Secret
* wishforge-fanyu-kube only, after checking it can do what the delivery needs and no more.
*/
async function fanyuAccess(...args) {
if (args[0] !== '--kubeconfig' || !args[1] || args.length !== 2)
throw new Error('Usage: cluster-poc.mjs fanyu-access --kubeconfig <Fanyu administrator kubeconfig>');
await assertCluster();
const admin = path.resolve(args[1]);
const env = { PATH: process.env.PATH, HOME: os.homedir() };
const fanyu = async (kubeconfig, command, options = {}) => {
const result = await capture('kubectl', ['--kubeconfig', kubeconfig, '--request-timeout=60s', ...command], {
env,
input: options.input,
});
if (result.code !== 0 && !options.allowFailure)
throw new Error(`kubectl ${command.slice(0, 3).join(' ')} (Fanyu) failed: ${result.stderr.trim().slice(0, 600)}`);
return result;
};
const view = JSON.parse((await fanyu(admin, ['config', 'view', '--minify', '-o', 'json'])).stdout);
if (view.clusters?.length !== 1 || view.clusters[0].cluster.server !== FANYU.cluster)
throw new Error(`The kubeconfig does not name Fanyu's cluster ${FANYU.cluster}`);
await fanyu(admin, ['apply', '-f', path.join(MANIFESTS, 'fanyu-access.yaml')]);
let token;
await waitFor(
'the Fanyu Service Account token',
async () => {
token = JSON.parse(
(await fanyu(admin, ['-n', FANYU.namespace, 'get', 'secret', 'wishforge-token', '-o', 'json'])).stdout,
).data;
return Boolean(token?.token && token['ca.crt']);
},
2,
);
const kubeconfig = fanyuKubeconfig({
certificateAuthorityData: token['ca.crt'],
token: Buffer.from(token.token, 'base64').toString('utf8'),
});
const directory = await mkdtemp(path.join(os.tmpdir(), 'wishforge-fanyu-'));
try {
const file = path.join(directory, 'kubeconfig');
await writeFile(file, kubeconfig, { mode: 0o600 });
const may = async (...check) =>
(await fanyu(file, ['auth', 'can-i', ...check, '-n', FANYU.namespace], { allowFailure: true })).stdout.trim();
const expected = [
[['patch', 'deployment/droboticflow-annotation-validator'], 'yes'],
[['create', 'pods'], 'yes'],
[['get', 'secrets'], 'no'],
[['get', 'configmaps'], 'no'],
[['patch', 'deployment/droboticflow-headpose'], 'no'],
[['delete', 'deployments'], 'no'],
];
for (const [check, answer] of expected)
if ((await may(...check)) !== answer)
throw new Error(`WishForge's Fanyu identity: can-i ${check.join(' ')} is not ${answer}`);
} finally {
await rm(directory, { recursive: true, force: true });
}
const namespaceUid = (
await fanyu(admin, ['get', 'namespace', FANYU.namespace, '-o', 'jsonpath={.metadata.uid}'])
).stdout.trim();
const deployments = {};
for (const [name, deployment] of Object.entries(FANYU_DEPLOYMENTS))
deployments[name] = (
await fanyu(admin, ['-n', FANYU.namespace, 'get', 'deployment', deployment, '-o', 'jsonpath={.metadata.uid}'])
).stdout.trim();
const profile = fanyuProfile({ poc: JSON.parse(await readFile(POC_PROFILE, 'utf8')), namespaceUid, deployments });
const parsed = await capture(
process.execPath,
[
'--import',
'tsx',
'-e',
"import('./packages/runtime/src/poc-profile.ts').then(m => m.pocProfileSchema.parse(JSON.parse(process.argv[1])))",
JSON.stringify(profile),
],
{ env: process.env },
);
if (parsed.code !== 0) throw new Error(`Fanyu's profile is not valid: ${parsed.stderr.trim().slice(0, 800)}`);
await writePrivate(FANYU_PROFILE, `${JSON.stringify(profile, null, 2)}\n`);
await applyObjects([secret('wishforge-fanyu-kube', { kubeconfig })]);
console.log(
`WishForge's identity in Fanyu's cluster is in place (Secret wishforge-fanyu-kube); profile: ${FANYU_PROFILE}`,
);
}
const inPod = (container, command, options = {}) =>
kubectl(['-n', NAMESPACE, 'exec', 'wishforge-0', '-c', container, '--', ...command], {
timeoutSeconds: 600,
...options,
});
/**
* Deletes every project of the cluster's WishForge with all it holds, and creates the POC and
* Fanyu projects in their place (cluster-poc-projects.ts), with nothing running. The node takes a
* snapshot first. Their Runners' registrations come to this Mac (STATE/projects.json); `apply`
* then gives both Runners their settings.
*/
async function projectsCommand(flag) {
if (flag !== '--replace') throw new Error('Usage: cluster-poc.mjs projects --replace');
await assertCluster();
if (!(await stat(FANYU_PROFILE).catch(() => undefined))) throw new Error('Run fanyu-access first');
const unfinished = (
await inPod('postgres', ['psql', '-U', 'wishforge', '-d', 'wishforge', '-At', '-c', UNFINISHED_SQL])
).stdout.trim();
if (unfinished) throw new Error(`The cluster is still working; wait until these finish:\n${unfinished}`);
const before = Date.now();
await inPod('backup', ['node', 'scripts/cluster-poc-backup.mjs', 'once'], { allowFailure: true });
const { last } = JSON.parse((await inPod('backup', ['node', 'scripts/cluster-poc-backup.mjs', 'latest'])).stdout);
if (!(last?.ok && Date.parse(last.at) >= before))
throw new Error(`The snapshot before replacing the projects failed${last?.error ? `: ${last.error}` : ''}`);
console.log(`Snapshot ${last.name} taken on the node.`);
const output = '/var/lib/wishforge/platform-state/projects-setup.json';
await inPod('platform', ['rm', '-f', output]);
// kubectl can lose an exec's output and status: the file it writes says whether it finished.
const replaced = await inPod(
'platform',
[
'node',
'--env-file=/etc/wishforge/env/platform.env',
'--import',
'tsx',
'scripts/cluster-poc-projects.ts',
'--replace',
],
{ allowFailure: true },
);
const size = await inPod('platform', ['/bin/sh', '-c', `wc -c < ${output}`], { allowFailure: true });
if (size.code !== 0 || !/^\d+$/.test(size.stdout.trim()))
throw new Error(`Replacing the projects did not finish: ${replaced.stderr.trim().slice(-1500)}`);
if (replaced.stdout.trim()) console.log(replaced.stdout.trim());
await privateState();
const partial = `${PROJECTS_FILE}.partial`;
await download(output, partial, Number(size.stdout.trim()), 'platform');
JSON.parse(await readFile(partial, 'utf8'));
await rename(partial, PROJECTS_FILE);
await inPod('platform', ['rm', '-f', output]);
// Workspaces of the runs that are gone, and the Runner's records of their tasks.
await inPod('executor', [
'/bin/sh',
'-c',
'find /var/lib/wishforge/exec/runs -mindepth 1 -maxdepth 1 -exec rm -rf {} +',
]);
await inPod('runner', [
'/bin/sh',
'-c',
'find /var/lib/wishforge/runner-state -mindepth 1 -maxdepth 1 -exec rm -rf {} +',
]);
const disabled = (
await kubectl([
'-n',
NAMESPACE,
'get',
'statefulset',
'wishforge',
'-o',
'jsonpath={.spec.template.spec.containers[?(@.name=="runner")].env[?(@.name=="WF_RUNNER_DISABLED")].value}',
])
).stdout.trim();
await apply(...(disabled === 'true' ? [] : ['--runner']));
console.log(`The cluster now has the POC and Fanyu projects; their Runners' registrations are in ${PROJECTS_FILE}.`);
}
const COMMANDS = {
image,
public: publicEntry,
bootstrap,
'deploy-key': deployKey,
'gateway-key': gatewayKey,
secrets,
apply,
restore,
'fanyu-access': fanyuAccess,
projects: projectsCommand,
status,
};
export async function main(argv = process.argv.slice(2)) {
const command = COMMANDS[argv[0]];
if (!command) throw new Error(`Usage: cluster-poc.mjs ${Object.keys(COMMANDS).join('|')}`);
await command(...argv.slice(1));
}
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url)
main().catch(error => {
console.error(error.message);
process.exitCode = 1;
});