Brings the branch up to the current base (executor settings, project skills, Chinese run messages, cloud IDE removal, wish attachments) and resolves the ten conflicted files, keeping both sides: - wish-refiner: the tenant's executor settings pick the models first, then the Copilot model setting; the tenant credential still goes before the operator's login - worker providers: executor settings apply, and a run with its own credential still gets an empty home - main.ts: Copilot screenshot uploads and wish attachments each keep their own body limit - contracts, http, web types: default budget and skill auto-activation, Copilot and attachment entity kinds, ./text and ./attachments exports - globals.css: Copilot styles and attachment styles, one after the other - tests: the canceled-release reason is Chinese; the refiner's default models have no [1m] suffix, as this branch names them Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
1566 lines
66 KiB
JavaScript
1566 lines
66 KiB
JavaScript
#!/usr/bin/env node
|
|
// Operator CLI of WishForge's deployment in the POC cluster (docs/cluster-poc-deployment.md).
|
|
// Every command names its cluster explicitly and prints no secret. Importing it starts nothing.
|
|
// WishForge runs in the cluster only; this machine keeps the operator's inputs (STATE) and nothing else.
|
|
//
|
|
// node scripts/cluster-poc.mjs image build and push the image in the cluster (Kaniko)
|
|
// node scripts/cluster-poc.mjs bootstrap namespaces, storage, access rules
|
|
// node scripts/cluster-poc.mjs deploy-key create the Gitea deploy key; prints its public half
|
|
// node scripts/cluster-poc.mjs gateway-key [--stdin] the model gateway key, from the clipboard or stdin
|
|
// node scripts/cluster-poc.mjs secrets Secrets and ConfigMaps from this Mac's settings
|
|
// node scripts/cluster-poc.mjs apply [--maintenance] [--runner] the WishForge Pod
|
|
// node scripts/cluster-poc.mjs public [--production] the public HTTPS entry (billed load balancer)
|
|
// node scripts/cluster-poc.mjs fanyu-access --kubeconfig <Fanyu admin kubeconfig> WishForge's identity in Fanyu's cluster
|
|
// node scripts/cluster-poc.mjs projects --replace delete every project; create the POC and Fanyu projects
|
|
// node scripts/cluster-poc.mjs restore <snapshot> [--runner] replace the cluster's data with a node snapshot
|
|
// node scripts/cluster-poc.mjs status show the deployment's state
|
|
import { spawn } from 'node:child_process';
|
|
import { createHash, randomBytes } from 'node:crypto';
|
|
import { createReadStream, createWriteStream } from 'node:fs';
|
|
import { chmod, mkdir, mkdtemp, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
import { Resolver, getServers, resolve4 } from 'node:dns/promises';
|
|
import { isIP } from 'node:net';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { ARCHIVES, COUNTS_SQL } from './cluster-poc-backup.mjs';
|
|
|
|
export const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
export const STATE = path.join(ROOT, '.wishforge/cluster-poc');
|
|
export const KUBECONFIG = path.join(os.homedir(), '.config/droboticflow/kubeconfigs/poc.config');
|
|
/**
|
|
* The operator's inputs to the cluster's settings, kept private here: the keys and tokens the data
|
|
* depends on (carried.env, see CARRIED), the Runners' signing key, the plan catalog, the POC's
|
|
* profile and Gitea's pinned host keys.
|
|
*/
|
|
export const INPUTS = {
|
|
carried: path.join(STATE, 'carried.env'),
|
|
signingKey: path.join(STATE, 'runner-ed25519.pem'),
|
|
plans: path.join(STATE, 'plans.json'),
|
|
knownHosts: path.join(STATE, 'known_hosts'),
|
|
};
|
|
export const POC_PROFILE = path.join(STATE, 'poc-profile.json');
|
|
export const CLUSTER = 'https://120.48.181.105:6443';
|
|
export const NAMESPACE = 'wishforge';
|
|
export const CHECK_NAMESPACE = 'wishforge-checks';
|
|
/** Fanyu's cluster and namespace, which its project delivers to (fanyu-access.yaml). */
|
|
export const FANYU = { cluster: 'https://180.76.138.88:6443', namespace: 'droboticflow-fanyu' };
|
|
/** Pushes use the registry's public address; the cluster pulls through its VPC address. */
|
|
export const PULL_REGISTRY = 'ccr-29eug8s3-vpc.cnc.bj.baidubce.com/dataflow-poc';
|
|
export const IMAGE_REPOSITORY = 'wishforge-cluster-poc';
|
|
|
|
const POSTGRES_IMAGE =
|
|
'docker.io/library/postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3';
|
|
/** Every input of the image, pinned: base images by digest, sources by commit, binaries by checksum. */
|
|
export const IMAGE_INPUTS = {
|
|
NODE_IMAGE:
|
|
'docker.io/library/node:26-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9',
|
|
GO_IMAGE:
|
|
'docker.io/library/golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac',
|
|
CADDY_IMAGE:
|
|
'docker.io/library/caddy:2.11.7-alpine@sha256:eb4d35b7452d1ab6171fbdbe8d9275b2301dad6dad92115d91e19fb9b24d510f',
|
|
// The image's pg_dump and pg_restore are the database's own (the backup).
|
|
POSTGRES_IMAGE,
|
|
MINIO_RELEASE: 'RELEASE.2025-07-23T15-54-02Z',
|
|
MINIO_COMMIT: '7ced9663e6a791fef9dc6be798ff24cda9c730ac',
|
|
KUBECTL_VERSION: 'v1.34.2',
|
|
KUBECTL_SHA256: '9591f3d75e1581f3f7392e6ad119aab2f28ae7d6c6e083dc5d22469667f27253',
|
|
CLAUDE_CODE_VERSION: '2.1.292',
|
|
};
|
|
/** Third-party service images, pinned by digest; the cluster pulls them from Docker Hub. */
|
|
export const SERVICE_IMAGES = {
|
|
postgres: POSTGRES_IMAGE,
|
|
temporal:
|
|
'docker.io/temporalio/temporal:1.4.1@sha256:9844298258519ebece82b3253f1ee1fc4eb8d87b7abcc264cfb676be342c74a5',
|
|
};
|
|
/** Paths never sent to the build, whatever Git says: local agent settings and private state. */
|
|
const EXCLUDED = [/^\.claude\//, /^\.wishforge\//, /(^|\/)\.env(\..*)?$/];
|
|
|
|
function capture(executable, args, options = {}) {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn(executable, args, { cwd: ROOT, ...options, stdio: ['pipe', 'pipe', 'pipe'] });
|
|
const out = [];
|
|
const err = [];
|
|
child.stdout.on('data', chunk => out.push(chunk));
|
|
child.stderr.on('data', chunk => err.push(chunk));
|
|
child.once('error', reject);
|
|
child.once('close', code =>
|
|
resolve({ code, stdout: Buffer.concat(out).toString('utf8'), stderr: Buffer.concat(err).toString('utf8') }),
|
|
);
|
|
child.stdin.end(options.input);
|
|
});
|
|
}
|
|
export async function kubectl(args, { input, timeoutSeconds = 60, allowFailure = false } = {}) {
|
|
const result = await capture(
|
|
'kubectl',
|
|
['--kubeconfig', KUBECONFIG, `--request-timeout=${timeoutSeconds}s`, ...args],
|
|
{ input, env: { PATH: process.env.PATH, HOME: os.homedir() } },
|
|
);
|
|
if (result.code !== 0 && !allowFailure)
|
|
throw new Error(`kubectl ${args.slice(0, 3).join(' ')} failed: ${result.stderr.trim().slice(0, 800)}`);
|
|
return result;
|
|
}
|
|
/** Refuses to act unless the kubeconfig names the POC cluster. */
|
|
export async function assertCluster() {
|
|
const view = JSON.parse((await kubectl(['config', 'view', '--minify', '-o', 'json'])).stdout);
|
|
if (view.clusters?.length !== 1 || view.clusters[0].cluster.server !== CLUSTER)
|
|
throw new Error(`The kubeconfig does not name the POC cluster ${CLUSTER}`);
|
|
}
|
|
|
|
const sha256File = file =>
|
|
new Promise((resolve, reject) => {
|
|
const hash = createHash('sha256');
|
|
createReadStream(file)
|
|
.on('data', chunk => hash.update(chunk))
|
|
.on('error', reject)
|
|
.on('end', () => resolve(hash.digest('hex')));
|
|
});
|
|
|
|
/** kubectl for the image, downloaded once and kept only if it matches the published checksum. */
|
|
async function kubectlBinary() {
|
|
const file = path.join(STATE, `kubectl-${IMAGE_INPUTS.KUBECTL_VERSION}-linux-amd64`);
|
|
if (
|
|
await stat(file).then(
|
|
() => true,
|
|
() => false,
|
|
)
|
|
) {
|
|
if ((await sha256File(file)) === IMAGE_INPUTS.KUBECTL_SHA256) return file;
|
|
await rm(file, { force: true });
|
|
}
|
|
const url = `https://dl.k8s.io/release/${IMAGE_INPUTS.KUBECTL_VERSION}/bin/linux/amd64/kubectl`;
|
|
const response = await fetch(url);
|
|
if (!response.ok) throw new Error(`Cannot download kubectl (${response.status}); set HTTPS_PROXY if needed`);
|
|
const temporary = `${file}.${randomBytes(4).toString('hex')}.tmp`;
|
|
await writeFile(temporary, Buffer.from(await response.arrayBuffer()), { mode: 0o600 });
|
|
if ((await sha256File(temporary)) !== IMAGE_INPUTS.KUBECTL_SHA256) {
|
|
await rm(temporary, { force: true });
|
|
throw new Error('The downloaded kubectl does not match its published SHA-256');
|
|
}
|
|
await rename(temporary, file);
|
|
return file;
|
|
}
|
|
|
|
/** The files of the build: what Git tracks or would track, minus {@link EXCLUDED}. */
|
|
export function buildFiles(listing) {
|
|
return listing
|
|
.split('\0')
|
|
.filter(Boolean)
|
|
.filter(file => !EXCLUDED.some(pattern => pattern.test(file)));
|
|
}
|
|
|
|
async function sourceArchive(directory) {
|
|
const listing = await capture('git', ['ls-files', '-z', '--cached', '--others', '--exclude-standard']);
|
|
if (listing.code !== 0) throw new Error(`git ls-files failed: ${listing.stderr}`);
|
|
const files = [];
|
|
for (const file of buildFiles(listing.stdout))
|
|
if (
|
|
await stat(path.join(ROOT, file)).then(
|
|
info => info.isFile(),
|
|
() => false,
|
|
)
|
|
)
|
|
files.push(file);
|
|
const list = path.join(directory, 'files');
|
|
await writeFile(list, `${files.join('\0')}\0`);
|
|
const extra = path.join(directory, 'extra');
|
|
await mkdir(path.join(extra, '.cluster-poc-build'), { recursive: true });
|
|
await writeFile(path.join(extra, '.cluster-poc-build/kubectl'), await readFile(await kubectlBinary()));
|
|
const tarball = path.join(directory, 'source.tar');
|
|
const env = { PATH: process.env.PATH, COPYFILE_DISABLE: '1' };
|
|
for (const args of [
|
|
['--no-mac-metadata', '--null', '-cf', tarball, '-T', list],
|
|
['--no-mac-metadata', '-rf', tarball, '-C', extra, '.cluster-poc-build/kubectl'],
|
|
]) {
|
|
const result = await capture('tar', args, { env });
|
|
if (result.code !== 0) throw new Error(`tar failed: ${result.stderr}`);
|
|
}
|
|
const gzip = await capture('gzip', ['-n', '-9', tarball], { env });
|
|
if (gzip.code !== 0) throw new Error(`gzip failed: ${gzip.stderr}`);
|
|
return { file: `${tarball}.gz`, count: files.length };
|
|
}
|
|
|
|
const UNPACK = [
|
|
'set -e',
|
|
'sum=$(sha256sum /context/source.tar.gz)',
|
|
'[ "${sum%% *}" = "$1" ] || exit 76',
|
|
'mkdir -p /context/source && tar -xzf /context/source.tar.gz -C /context/source && touch /context/ready',
|
|
].join('\n');
|
|
|
|
/** The Kaniko Pod of one image build, in the POC namespace where the push credential lives. */
|
|
export function imageBuildPod(profile, input) {
|
|
return {
|
|
apiVersion: 'v1',
|
|
kind: 'Pod',
|
|
metadata: {
|
|
name: input.name,
|
|
namespace: profile.namespace,
|
|
labels: { 'app.kubernetes.io/managed-by': 'wishforge-cluster-poc', 'app.kubernetes.io/name': 'wishforge-image' },
|
|
},
|
|
spec: {
|
|
automountServiceAccountToken: false,
|
|
restartPolicy: 'Never',
|
|
activeDeadlineSeconds: 5400,
|
|
nodeSelector: { 'kubernetes.io/arch': 'amd64', 'kubernetes.io/os': 'linux' },
|
|
containers: [
|
|
{
|
|
name: 'kaniko',
|
|
image: profile.kanikoImage,
|
|
command: ['/busybox/sh', '-c'],
|
|
args: [
|
|
'until [ -f /context/ready ]; do /busybox/sleep 1; done; exec /kaniko/executor "$@"',
|
|
'--',
|
|
'--context=dir:///context/source',
|
|
'--dockerfile=/context/source/infra/images/cluster-poc.Dockerfile',
|
|
`--destination=${profile.registry}/${IMAGE_REPOSITORY}:${input.tag}`,
|
|
'--digest-file=/dev/termination-log',
|
|
...Object.entries(IMAGE_INPUTS).map(([key, value]) => `--build-arg=${key}=${value}`),
|
|
...profile.registryMirrors.map(mirror => `--registry-mirror=${mirror}`),
|
|
...(profile.registryMirrors.length ? ['--skip-default-registry-fallback'] : []),
|
|
'--snapshot-mode=redo',
|
|
'--verbosity=info',
|
|
],
|
|
resources: {
|
|
requests: { cpu: '1', memory: '2Gi' },
|
|
limits: { cpu: '6', memory: '12Gi', 'ephemeral-storage': '30Gi' },
|
|
},
|
|
volumeMounts: [
|
|
{ name: 'context', mountPath: '/context' },
|
|
{ name: 'registry', mountPath: '/kaniko/.docker', readOnly: true },
|
|
],
|
|
terminationMessagePolicy: 'File',
|
|
},
|
|
],
|
|
volumes: [
|
|
{ name: 'context', emptyDir: { sizeLimit: '4Gi' } },
|
|
{
|
|
name: 'registry',
|
|
secret: { secretName: profile.registrySecret, items: [{ key: '.dockerconfigjson', path: 'config.json' }] },
|
|
},
|
|
],
|
|
},
|
|
};
|
|
}
|
|
|
|
async function image() {
|
|
await assertCluster();
|
|
const profile = JSON.parse(await readFile(POC_PROFILE, 'utf8'));
|
|
await mkdir(STATE, { recursive: true, mode: 0o700 });
|
|
await chmod(STATE, 0o700);
|
|
const head = (await capture('git', ['rev-parse', 'HEAD'])).stdout.trim();
|
|
const dirty = (await capture('git', ['status', '--porcelain'])).stdout.trim().length > 0;
|
|
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..*$/, '').replace('T', '');
|
|
const tag = `${head.slice(0, 12)}${dirty ? '-dirty' : ''}-${stamp}`;
|
|
const directory = await mkdtemp(path.join(os.tmpdir(), 'wishforge-image-'));
|
|
const name = `wishforge-image-${randomBytes(4).toString('hex')}`;
|
|
try {
|
|
const archive = await sourceArchive(directory);
|
|
const sha = await sha256File(archive.file);
|
|
console.log(
|
|
`Source: ${archive.count} files, ${((await stat(archive.file)).size / 1e6).toFixed(1)} MB, sha256 ${sha.slice(0, 16)}…`,
|
|
);
|
|
await kubectl(['create', '-f', '-'], { input: JSON.stringify(imageBuildPod(profile, { name, tag })) });
|
|
await kubectl(['-n', profile.namespace, 'wait', '--for=condition=Ready', `pod/${name}`, '--timeout=600s'], {
|
|
timeoutSeconds: 620,
|
|
});
|
|
await kubectl(['-n', profile.namespace, 'cp', archive.file, `${name}:/context/source.tar.gz`, '-c', 'kaniko'], {
|
|
timeoutSeconds: 600,
|
|
});
|
|
await kubectl([
|
|
'-n',
|
|
profile.namespace,
|
|
'exec',
|
|
name,
|
|
'-c',
|
|
'kaniko',
|
|
'--',
|
|
'/busybox/sh',
|
|
'-c',
|
|
UNPACK,
|
|
'unpack',
|
|
sha,
|
|
]);
|
|
console.log(`Building ${profile.registry}/${IMAGE_REPOSITORY}:${tag} in pod ${name} …`);
|
|
let pod;
|
|
for (const until = Date.now() + 90 * 60_000; Date.now() < until; await delay(15_000)) {
|
|
const result = await kubectl(['-n', profile.namespace, 'get', 'pod', name, '-o', 'json'], { allowFailure: true });
|
|
if (result.code !== 0) continue;
|
|
pod = JSON.parse(result.stdout);
|
|
if (['Succeeded', 'Failed'].includes(pod.status?.phase)) break;
|
|
}
|
|
const terminated = pod?.status?.containerStatuses?.[0]?.state?.terminated;
|
|
if (pod?.status?.phase !== 'Succeeded' || !/^sha256:[a-f0-9]{64}$/.test(terminated?.message?.trim() ?? '')) {
|
|
const logs = await kubectl(['-n', profile.namespace, 'logs', name, '-c', 'kaniko', '--tail=80'], {
|
|
allowFailure: true,
|
|
});
|
|
throw new Error(`The image build failed (${pod?.status?.phase ?? 'unknown'}):\n${logs.stdout.slice(-6000)}`);
|
|
}
|
|
const digest = terminated.message.trim();
|
|
const record = {
|
|
repository: `${PULL_REGISTRY}/${IMAGE_REPOSITORY}`,
|
|
pushedAs: `${profile.registry}/${IMAGE_REPOSITORY}:${tag}`,
|
|
digest,
|
|
reference: `${PULL_REGISTRY}/${IMAGE_REPOSITORY}@${digest}`,
|
|
sourceCommit: head,
|
|
dirty,
|
|
sourceSha256: sha,
|
|
inputs: IMAGE_INPUTS,
|
|
builtAt: new Date().toISOString(),
|
|
};
|
|
await writeFile(path.join(STATE, 'image.json'), `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
|
|
console.log(`Image: ${record.reference}`);
|
|
} finally {
|
|
await kubectl(['-n', profile.namespace, 'delete', 'pod', name, '--ignore-not-found', '--wait=false'], {
|
|
allowFailure: true,
|
|
});
|
|
await rm(directory, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------------ configuration of the Pod
|
|
|
|
/** Where each container keeps its files: the paths infra/k8s/cluster-poc/platform.yaml mounts. */
|
|
export const PATHS = {
|
|
gitRoot: '/var/lib/wishforge/git',
|
|
localAccess: '/var/lib/wishforge/platform-state/local-access.json',
|
|
pocState: '/var/lib/wishforge/poc-state',
|
|
runnerState: '/var/lib/wishforge/runner-state',
|
|
execRoot: '/var/lib/wishforge/exec',
|
|
claudeHome: '/var/lib/wishforge/claude-home',
|
|
claudeHomeApi: '/var/lib/wishforge/claude-home-api',
|
|
// OpenSSH finds ~/.ssh through the user database, not $HOME: this is the image user's home.
|
|
gitHome: '/home/wishforge',
|
|
config: '/etc/wishforge/config',
|
|
signingKey: '/etc/wishforge/keys/runner-ed25519.pem',
|
|
kubeToken: '/var/run/wishforge/kube/token',
|
|
fanyuKubeconfig: '/etc/wishforge/fanyu/kubeconfig',
|
|
};
|
|
/**
|
|
* The Mac's settings the cluster keeps as they are: the keys and tokens that persisted data, issued
|
|
* grants and the Runner's registration rely on. Everything else is the cluster's own.
|
|
*/
|
|
export const CARRIED = [
|
|
'WF_TENANT_ID',
|
|
'WF_REGION',
|
|
'WF_DISPATCH_TENANTS',
|
|
'WF_DEV_AUTH_TOKEN',
|
|
'WF_WORKER_TOKEN',
|
|
'WF_VERIFIER_TOKEN',
|
|
'WF_DEPLOYER_TOKEN',
|
|
'WF_EVALUATOR_TOKEN',
|
|
'WF_VAULT_KEY',
|
|
'WF_SESSION_SECRET',
|
|
'WF_INTERNAL_TOKEN_KEY',
|
|
'WF_PROVISIONER_TOKEN',
|
|
'WF_RESOURCE_GRANT_SECRET',
|
|
'WF_PROVIDER_PROXY_SIGNING_KEY',
|
|
'WF_METRICS_TOKEN',
|
|
'WF_GIT_ADMIN_TOKEN',
|
|
'WF_GIT_TOKEN_SECRET',
|
|
'S3_BUCKET',
|
|
'S3_REGION',
|
|
'TEMPORAL_NAMESPACE',
|
|
];
|
|
const RUNNER_CARRIED = ['WF_RUNNER_TOKEN', 'WF_TENANT_ID', 'WF_ENVIRONMENT_ID', 'WF_RUNNER_PUBLIC_KEYS'];
|
|
|
|
const DOMAIN = /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/;
|
|
const PUBLIC_ORIGIN = new RegExp(`^https?://(?:\\d{1,3}(?:\\.\\d{1,3}){3}|${DOMAIN.source.slice(1, -1)})$`);
|
|
/** Public resolvers that show what the internet, Let's Encrypt included, sees of a domain. */
|
|
const PUBLIC_RESOLVERS = ['223.5.5.5', '119.29.29.29'];
|
|
/** A publicly routable IPv4 address: never a URL, private, shared or documentation address. */
|
|
export function publicIPv4(value) {
|
|
if (isIP(value) !== 4) throw new Error('A public IPv4 address is required');
|
|
const [a, b, c] = value.split('.').map(Number);
|
|
if (
|
|
a === 0 ||
|
|
a === 10 ||
|
|
a === 127 ||
|
|
a >= 224 ||
|
|
(a === 100 && b >= 64 && b <= 127) ||
|
|
(a === 169 && b === 254) ||
|
|
(a === 172 && b >= 16 && b <= 31) ||
|
|
(a === 192 && (b === 168 || b === 0 || (b === 88 && c === 99))) ||
|
|
(a === 198 && (b === 18 || b === 19 || (b === 51 && c === 100))) ||
|
|
(a === 203 && b === 0 && c === 113)
|
|
)
|
|
throw new Error('The entry must use a publicly routable IPv4 address');
|
|
return value;
|
|
}
|
|
/** A domain name for the public entry, as the operator's DNS points it at the load balancer. */
|
|
export function publicDomain(name) {
|
|
const domain = String(name ?? '')
|
|
.trim()
|
|
.toLowerCase();
|
|
if (!DOMAIN.test(domain)) throw new Error(`Not a domain name: ${name}`);
|
|
return domain;
|
|
}
|
|
/** The console's address through a port-forward to the Pod: the Mac's own console keeps port 3000. */
|
|
export const PORT_FORWARD_ORIGIN = 'http://127.0.0.1:13000';
|
|
/** The settings files of the platform, executor and Runner containers; see {@link CARRIED}. */
|
|
export function clusterEnvironments({ local, runner, runnerFanyu, image, generated, origins = [] }) {
|
|
const missing = [
|
|
...CARRIED.filter(key => !local[key]).map(key => `carried.env ${key}`),
|
|
...RUNNER_CARRIED.filter(key => !runner[key]).map(key => `runner.env ${key}`),
|
|
...(runnerFanyu ? RUNNER_CARRIED.filter(key => !runnerFanyu[key]).map(key => `Fanyu's Runner ${key}`) : []),
|
|
];
|
|
if (missing.length) throw new Error(`Missing settings: ${missing.join(', ')}`);
|
|
for (const origin of origins)
|
|
if (!PUBLIC_ORIGIN.test(origin)) throw new Error(`A public origin is http(s)://<IPv4 or domain>: ${origin}`);
|
|
const poc = {
|
|
WF_LOCAL_POC_BINDING_FILE: `${PATHS.pocState}/project.json`,
|
|
WF_POC_PROFILE_FILE: `${PATHS.pocState}/poc-profile.json`,
|
|
};
|
|
const platform = {
|
|
WF_DEPLOYMENT_MODE: 'development',
|
|
...Object.fromEntries(CARRIED.map(key => [key, local[key]])),
|
|
DATABASE_ADMIN_URL: `postgresql://wishforge:${encodeURIComponent(generated.postgresPassword)}@127.0.0.1:5432/wishforge`,
|
|
DATABASE_URL: `postgresql://wishforge_app:${encodeURIComponent(generated.appPassword)}@127.0.0.1:5432/wishforge`,
|
|
WF_DATABASE_APP_PASSWORD: generated.appPassword,
|
|
S3_ENDPOINT: 'http://127.0.0.1:9000',
|
|
S3_ACCESS_KEY_ID: generated.objectStorageUser,
|
|
S3_SECRET_ACCESS_KEY: generated.objectStoragePassword,
|
|
PORT: '4000',
|
|
HOST: '127.0.0.1',
|
|
WF_API_URL: 'http://127.0.0.1:4000',
|
|
WF_CONTROL_PLANE_URL: 'http://127.0.0.1:4000',
|
|
WF_PROVISIONER_URL: 'http://127.0.0.1:4450',
|
|
WF_PROVISIONER_BACKEND: 'local',
|
|
TEMPORAL_ADDRESS: '127.0.0.1:7233',
|
|
WF_GIT_PORT: '4504',
|
|
WF_GIT_SERVICE_URL: 'http://127.0.0.1:4504',
|
|
WF_GIT_PUBLIC_BASE_URL: 'http://127.0.0.1:4504',
|
|
WF_GIT_ALLOW_LOCAL_HTTP: 'true',
|
|
WF_GIT_ROOT: PATHS.gitRoot,
|
|
WF_LOCAL_ACCESS: 'true',
|
|
WF_LOCAL_ACCESS_FILE: PATHS.localAccess,
|
|
// The web launcher passes only this token name, not WF_DEV_AUTH_TOKEN, to the console.
|
|
WF_LOCAL_API_TOKEN: local.WF_DEV_AUTH_TOKEN,
|
|
// In the Pod, and through `kubectl port-forward pod/wishforge-0 13000:3000` from an operator's machine.
|
|
WF_LOCAL_ALLOWED_ORIGINS: ['http://127.0.0.1:3000', 'http://localhost:3000', PORT_FORWARD_ORIGIN, ...origins].join(
|
|
',',
|
|
),
|
|
WF_PUBLIC_URL: origins[0] ?? 'http://127.0.0.1:3000',
|
|
WF_WEB_SERVE_MODE: 'production',
|
|
WF_WEB_HOST: '127.0.0.1',
|
|
WF_WEB_PORT: '3000',
|
|
WF_RUNNER_SIGNING_KEY_FILE: PATHS.signingKey,
|
|
WF_PLAN_CATALOG_FILE: `${PATHS.config}/plans.json`,
|
|
WF_CLAUDE_BIN: '/usr/local/bin/claude',
|
|
WF_CLAUDE_LOCAL_AUTH_HOME: PATHS.claudeHomeApi,
|
|
WF_CLAUDE_ASK_MODEL: 'deepseek-flash@latest',
|
|
WF_CLAUDE_ASK_FALLBACK_MODEL: 'deepseek-pro@latest',
|
|
...poc,
|
|
};
|
|
const executor = {
|
|
WF_DEPLOYMENT_MODE: 'development',
|
|
WF_TENANT_ID: local.WF_TENANT_ID,
|
|
WF_RESOURCE_GRANT_SECRET: local.WF_RESOURCE_GRANT_SECRET,
|
|
WF_PROVISIONER_TOKEN: local.WF_PROVISIONER_TOKEN,
|
|
WF_PROVISIONER_BACKEND: 'local',
|
|
WF_CONTROL_PLANE_URL: 'http://127.0.0.1:4000',
|
|
TEMPORAL_ADDRESS: '127.0.0.1:7233',
|
|
TEMPORAL_NAMESPACE: local.TEMPORAL_NAMESPACE,
|
|
WF_LOCAL_RUN_ROOT: `${PATHS.execRoot}/runs`,
|
|
WF_GIT_ALLOW_LOCAL_HTTP: 'true',
|
|
WF_CLAUDE_BIN: '/usr/local/bin/claude',
|
|
WF_CLAUDE_LOCAL_AUTH_HOME: PATHS.claudeHome,
|
|
WF_HOST_CHECK_POD_NAMESPACE: CHECK_NAMESPACE,
|
|
WF_HOST_CHECK_POD_IMAGE: image,
|
|
WF_HOST_CHECK_POD_CLAIM: 'wishforge-exec',
|
|
WF_HOST_CHECK_POD_ROOT: PATHS.execRoot,
|
|
WF_HOST_CHECK_KUBECONFIG: `${PATHS.config}/kubeconfig`,
|
|
WF_HOST_CHECK_WRITE_PATHS: `${PATHS.execRoot}/check-cache`,
|
|
WF_HOST_CHECK_POD_CPU: '4',
|
|
WF_HOST_CHECK_POD_MEMORY: '8Gi',
|
|
...poc,
|
|
};
|
|
// Each Runner keeps its state in its own directory, mounted at the same path.
|
|
const runnerEnv = (values, keys, profile) => ({
|
|
WF_DEPLOYMENT_MODE: 'development',
|
|
WF_CONTROL_PLANE_URL: 'http://127.0.0.1:4000',
|
|
WF_RUNNER_ALLOW_LOCAL_HTTP: 'true',
|
|
...Object.fromEntries(RUNNER_CARRIED.map(key => [key, values[key]])),
|
|
// The Mac's runner.env names a file on the Mac; its content travels in the ConfigMap.
|
|
WF_RUNNER_PUBLIC_KEYS: `${PATHS.config}/${keys}`,
|
|
WF_RUNNER_EXECUTE: values.WF_RUNNER_EXECUTE === 'true' ? 'true' : 'false',
|
|
WF_RUNNER_STATE_DIR: PATHS.runnerState,
|
|
WF_POC_PROFILE_FILE: `${PATHS.pocState}/${profile}`,
|
|
});
|
|
return {
|
|
platform,
|
|
executor,
|
|
runner: runnerEnv(runner, 'runner-public.json', 'poc-profile.json'),
|
|
...(runnerFanyu ? { runnerFanyu: runnerEnv(runnerFanyu, 'runner-fanyu-public.json', 'fanyu-profile.json') } : {}),
|
|
};
|
|
}
|
|
|
|
/** A settings file Node's --env-file and the container entry both read; one value per line. */
|
|
export function envFile(values) {
|
|
const lines = Object.entries(values).map(([key, value]) => {
|
|
if (!/^[A-Z_][A-Z0-9_]*$/.test(key) || typeof value !== 'string' || /[\r\n\0']/.test(value))
|
|
throw new Error(`Cannot write ${key} as a single-line setting`);
|
|
return /^[A-Za-z0-9_@%:/.,+=?&[\]-]*$/.test(value) ? `${key}=${value}` : `${key}='${value}'`;
|
|
});
|
|
return `${lines.join('\n')}\n`;
|
|
}
|
|
|
|
/** WishForge's identity at the POC cluster's address, by its short-lived projected token. */
|
|
export function clusterKubeconfig(certificateAuthorityData) {
|
|
if (!/^[A-Za-z0-9+/=]+$/.test(certificateAuthorityData)) throw new Error('Invalid cluster CA data');
|
|
return JSON.stringify(
|
|
{
|
|
apiVersion: 'v1',
|
|
kind: 'Config',
|
|
clusters: [{ name: 'poc', cluster: { server: CLUSTER, 'certificate-authority-data': certificateAuthorityData } }],
|
|
users: [{ name: 'wishforge', user: { tokenFile: PATHS.kubeToken } }],
|
|
contexts: [{ name: 'poc', context: { cluster: 'poc', user: 'wishforge' } }],
|
|
'current-context': 'poc',
|
|
},
|
|
null,
|
|
2,
|
|
);
|
|
}
|
|
|
|
/** Until the public address exists: HTTP only, answering that the entry is not configured. */
|
|
export const PENDING_CADDYFILE = `{
|
|
admin off
|
|
auto_https off
|
|
}
|
|
:8080 {
|
|
respond "WishForge's public address is not configured yet." 503
|
|
}
|
|
`;
|
|
/**
|
|
* The public HTTPS entry at the load balancer's address: a Let's Encrypt certificate for the IP
|
|
* (short-lived profile, HTTP-01 only), HTTP GET/HEAD redirected to the standard HTTPS port, and the
|
|
* console behind it on loopback with forwarding headers it can trust.
|
|
*/
|
|
export function clusterCaddyfile(address, { staging = false, domain } = {}) {
|
|
const ip = publicIPv4(address);
|
|
const hosts = [...(domain ? [publicDomain(domain)] : []), ip];
|
|
const site = (scheme, port) => hosts.map(host => `${scheme}://${host}:${port}`).join(', ');
|
|
const ca = `https://acme${staging ? '-staging' : ''}-v02.api.letsencrypt.org/directory`;
|
|
return `{
|
|
admin off
|
|
skip_install_trust
|
|
http_port 8080
|
|
https_port 8443
|
|
default_sni ${ip}
|
|
auto_https disable_redirects
|
|
servers {
|
|
protocols h1 h2
|
|
}
|
|
}
|
|
${site('http', 8080)} {
|
|
@safe method GET HEAD
|
|
redir @safe https://{host}{uri} 308
|
|
respond 405
|
|
}
|
|
${site('https', 8443)} {
|
|
tls {
|
|
issuer acme {
|
|
dir ${ca}
|
|
test_dir ${ca}
|
|
profile shortlived
|
|
disable_tlsalpn_challenge
|
|
alt_http_port 8080
|
|
}
|
|
}
|
|
reverse_proxy 127.0.0.1:3000 {
|
|
header_up X-Forwarded-Proto https
|
|
header_up -Forwarded
|
|
}
|
|
}
|
|
`;
|
|
}
|
|
|
|
/**
|
|
* The public plain-HTTP entry at the load balancer's address, which the operator chose over HTTPS
|
|
* (`public --http`): the console behind it on loopback. Passwords and sessions travel unencrypted.
|
|
*/
|
|
export function clusterHttpCaddyfile(address, { domain } = {}) {
|
|
const ip = publicIPv4(address);
|
|
const site = [...(domain ? [publicDomain(domain)] : []), ip].map(host => `http://${host}:8080`).join(', ');
|
|
return `{
|
|
admin off
|
|
auto_https off
|
|
http_port 8080
|
|
servers {
|
|
protocols h1
|
|
}
|
|
}
|
|
${site} {
|
|
reverse_proxy 127.0.0.1:3000 {
|
|
header_up X-Forwarded-Proto http
|
|
header_up -Forwarded
|
|
}
|
|
}
|
|
`;
|
|
}
|
|
|
|
/** public-service.yaml as applied: a plain-HTTP entry has no HTTPS port. */
|
|
export function publicService(template, { http = false } = {}) {
|
|
const https = ' - { name: https, protocol: TCP, port: 443, targetPort: https }\n';
|
|
if (!template.includes(https)) throw new Error('public-service.yaml has no HTTPS port');
|
|
return http ? template.replace(https, '') : template;
|
|
}
|
|
|
|
const IMAGE_REFERENCE =
|
|
/^ccr-29eug8s3-vpc\.cnc\.bj\.baidubce\.com\/dataflow-poc\/wishforge-cluster-poc@sha256:[a-f0-9]{64}$/;
|
|
const PLACEHOLDER = name => `\${${name}}`;
|
|
/**
|
|
* platform.yaml with its image and switches filled in; nothing left unfilled. `settings` is the
|
|
* digest of the settings the containers read when they start, so a change to them rolls the Pod.
|
|
*/
|
|
export function renderPlatform(template, { image, runnerDisabled, maintenance, settings }) {
|
|
if (!IMAGE_REFERENCE.test(image)) throw new Error('The WishForge image must be the pinned cluster image');
|
|
if (!/^[a-f0-9]{32}$/.test(settings ?? '')) throw new Error('The settings digest is missing');
|
|
const rendered = template
|
|
.replaceAll(PLACEHOLDER('WISHFORGE_IMAGE'), image)
|
|
.replaceAll(PLACEHOLDER('SETTINGS_DIGEST'), settings)
|
|
.replaceAll(PLACEHOLDER('RUNNER_DISABLED'), String(Boolean(runnerDisabled)))
|
|
.replaceAll(PLACEHOLDER('MAINTENANCE'), String(Boolean(maintenance)));
|
|
if (rendered.includes('${')) throw new Error('platform.yaml has an unfilled placeholder');
|
|
return rendered;
|
|
}
|
|
|
|
// ------------------------------------------------------------------------------- commands
|
|
|
|
const MANIFESTS = path.join(ROOT, 'infra/k8s/cluster-poc');
|
|
const GENERATED = path.join(STATE, 'generated.json');
|
|
/** Written by fanyu-access: Fanyu's profile, naming WishForge's kubeconfig for Fanyu's cluster. */
|
|
const FANYU_PROFILE = path.join(STATE, 'fanyu-profile.json');
|
|
/** Written by projects --replace: the cluster's projects' bindings and their Runners' registrations. */
|
|
const PROJECTS_FILE = path.join(STATE, 'projects.json');
|
|
const secretValue = () => randomBytes(32).toString('base64url');
|
|
|
|
async function readEnv(file) {
|
|
const values = {};
|
|
for (const line of (await readFile(file, 'utf8')).split('\n')) {
|
|
const match = /^([A-Z_][A-Z0-9_]*)=(.*)$/.exec(line);
|
|
if (match) values[match[1]] = match[2].replace(/^(['"])(.*)\1$/, '$2');
|
|
}
|
|
return values;
|
|
}
|
|
async function privateState() {
|
|
await mkdir(STATE, { recursive: true, mode: 0o700 });
|
|
await chmod(STATE, 0o700);
|
|
}
|
|
async function writePrivate(file, content) {
|
|
await privateState();
|
|
const temporary = `${file}.${randomBytes(4).toString('hex')}.tmp`;
|
|
await writeFile(temporary, content, { mode: 0o600 });
|
|
await rename(temporary, file);
|
|
}
|
|
/** Passwords the cluster's own services use; made once, kept only in this Mac's private state. */
|
|
async function generatedSecrets() {
|
|
try {
|
|
return JSON.parse(await readFile(GENERATED, 'utf8'));
|
|
} catch (error) {
|
|
if (error.code !== 'ENOENT') throw error;
|
|
}
|
|
const generated = {
|
|
postgresPassword: secretValue(),
|
|
appPassword: secretValue(),
|
|
objectStorageUser: `wishforge-${randomBytes(6).toString('hex')}`,
|
|
objectStoragePassword: secretValue(),
|
|
};
|
|
await writePrivate(GENERATED, `${JSON.stringify(generated, null, 2)}\n`);
|
|
return generated;
|
|
}
|
|
async function imageRecord() {
|
|
try {
|
|
return JSON.parse(await readFile(path.join(STATE, 'image.json'), 'utf8'));
|
|
} catch {
|
|
throw new Error('No image yet; run: node scripts/cluster-poc.mjs image');
|
|
}
|
|
}
|
|
/** Applies Kubernetes objects from JSON on stdin, so no value appears in a command line. */
|
|
const applyObjects = objects =>
|
|
kubectl(['apply', '-f', '-'], { input: JSON.stringify({ apiVersion: 'v1', kind: 'List', items: objects }) });
|
|
const labels = { 'app.kubernetes.io/part-of': 'wishforge-cluster-poc' };
|
|
const secret = (name, stringData) => ({
|
|
apiVersion: 'v1',
|
|
kind: 'Secret',
|
|
type: 'Opaque',
|
|
metadata: { name, namespace: NAMESPACE, labels },
|
|
stringData,
|
|
});
|
|
const configMap = (name, data) => ({
|
|
apiVersion: 'v1',
|
|
kind: 'ConfigMap',
|
|
metadata: { name, namespace: NAMESPACE, labels },
|
|
data,
|
|
});
|
|
async function exists(kind, name, namespace) {
|
|
const scope = namespace ? ['-n', namespace] : [];
|
|
return (await kubectl([...scope, 'get', kind, name, '--ignore-not-found', '-o', 'name'])).stdout.trim() !== '';
|
|
}
|
|
|
|
async function bootstrap() {
|
|
await assertCluster();
|
|
await kubectl(['apply', '-f', path.join(MANIFESTS, 'namespaces.yaml')]);
|
|
if (!(await exists('pv', 'wishforge-data'))) {
|
|
const directories = path.join(MANIFESTS, 'bootstrap/directories.yaml');
|
|
await kubectl(['apply', '-f', directories]);
|
|
let phase = '';
|
|
for (const until = Date.now() + 5 * 60_000; Date.now() < until && !['Succeeded', 'Failed'].includes(phase);) {
|
|
await delay(3000);
|
|
phase = (
|
|
await kubectl(['-n', NAMESPACE, 'get', 'pod', 'wishforge-bootstrap', '-o', 'jsonpath={.status.phase}'], {
|
|
allowFailure: true,
|
|
})
|
|
).stdout.trim();
|
|
}
|
|
await kubectl(['delete', '-f', directories, '--wait=true'], { timeoutSeconds: 120 });
|
|
if (phase !== 'Succeeded') throw new Error(`Creating the data directory did not succeed (${phase || 'no phase'})`);
|
|
console.log('Data directory created on 192.168.80.249.');
|
|
}
|
|
for (const file of ['storage.yaml', 'rbac.yaml', 'admission.yaml'])
|
|
await kubectl(['apply', '-f', path.join(MANIFESTS, file)]);
|
|
console.log('Namespaces, storage, access rules and admission policies are in place.');
|
|
}
|
|
|
|
/** The Gitea deploy key: its private half goes to the cluster only; the public half is printed. */
|
|
async function deployKey(flag) {
|
|
await assertCluster();
|
|
const publicKey = path.join(STATE, 'deploy-key.pub');
|
|
if (flag !== '--rotate' && (await exists('secret', 'wishforge-git', NAMESPACE))) {
|
|
console.log((await readFile(publicKey, 'utf8')).trim());
|
|
return;
|
|
}
|
|
const directory = await mkdtemp(path.join(os.tmpdir(), 'wishforge-deploy-key-'));
|
|
try {
|
|
const key = path.join(directory, 'id_ed25519');
|
|
const made = await capture('ssh-keygen', [
|
|
'-q',
|
|
'-t',
|
|
'ed25519',
|
|
'-N',
|
|
'',
|
|
'-C',
|
|
'wishforge-cluster-poc',
|
|
'-f',
|
|
key,
|
|
]);
|
|
if (made.code !== 0) throw new Error(`ssh-keygen failed: ${made.stderr}`);
|
|
await applyObjects([secret('wishforge-git', { id_ed25519: await readFile(key, 'utf8') })]);
|
|
await writePrivate(publicKey, await readFile(`${key}.pub`, 'utf8'));
|
|
console.log((await readFile(`${key}.pub`, 'utf8')).trim());
|
|
} finally {
|
|
await rm(directory, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The Claude CLI's settings in the cluster: the company's Anthropic-protocol gateway, its models
|
|
* with their 1M context, and no traffic to services the gateway does not replace.
|
|
*/
|
|
export function claudeSettings(key) {
|
|
return {
|
|
env: {
|
|
ANTHROPIC_BASE_URL: 'https://ai-api.d-robotics.cc',
|
|
...(key ? { ANTHROPIC_API_KEY: key } : {}),
|
|
ANTHROPIC_MODEL: 'deepseek-pro@latest',
|
|
ANTHROPIC_DEFAULT_OPUS_MODEL: 'deepseek-pro@latest',
|
|
ANTHROPIC_DEFAULT_SONNET_MODEL: 'deepseek-pro@latest',
|
|
ANTHROPIC_DEFAULT_HAIKU_MODEL: 'deepseek-flash@latest',
|
|
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1',
|
|
DISABLE_AUTOUPDATER: '1',
|
|
DISABLE_TELEMETRY: '1',
|
|
DISABLE_ERROR_REPORTING: '1',
|
|
},
|
|
};
|
|
}
|
|
|
|
const readStdin = () =>
|
|
new Promise((resolve, reject) => {
|
|
const chunks = [];
|
|
process.stdin
|
|
.on('data', chunk => chunks.push(chunk))
|
|
.once('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
|
|
.once('error', reject);
|
|
});
|
|
/** The model gateway's key, from the clipboard or with --stdin from standard input, into the cluster only. */
|
|
async function gatewayKey(flag) {
|
|
await assertCluster();
|
|
if (flag !== undefined && flag !== '--stdin') throw new Error('Usage: cluster-poc.mjs gateway-key [--stdin]');
|
|
const source = flag === '--stdin' ? 'standard input' : 'clipboard';
|
|
const key = (flag === '--stdin' ? await readStdin() : (await capture('pbpaste', [])).stdout).trim();
|
|
if (!/^sk-[A-Za-z0-9_-]{16,200}$/.test(key))
|
|
throw new Error(
|
|
`The ${source} holds no gateway key (sk-…, ${key.length} characters read); provide the key and run this again`,
|
|
);
|
|
await applyObjects([
|
|
secret('wishforge-claude', { 'settings.json': `${JSON.stringify(claudeSettings(key), null, 2)}\n` }),
|
|
]);
|
|
console.log(`The gateway key is in the cluster (Secret wishforge-claude)${flag ? '.' : '; clear your clipboard.'}`);
|
|
}
|
|
|
|
/** The projects the cluster delivers and their Runners (projects --replace). */
|
|
async function clusterProjects() {
|
|
const pocProfile = { ...JSON.parse(await readFile(POC_PROFILE, 'utf8')), kubeconfig: `${PATHS.config}/kubeconfig` };
|
|
const fanyuProfile = await readFile(FANYU_PROFILE, 'utf8').then(JSON.parse, () => undefined);
|
|
const profiles = {
|
|
'poc-profile.json': pocProfile,
|
|
...(fanyuProfile ? { 'fanyu-profile.json': fanyuProfile } : {}),
|
|
};
|
|
const replaced = JSON.parse(await readFile(PROJECTS_FILE, 'utf8'));
|
|
for (const binding of replaced.bindings)
|
|
if (!profiles[path.basename(binding.profileFile)])
|
|
throw new Error(`No ${path.basename(binding.profileFile)} for project ${binding.projectId}; run fanyu-access`);
|
|
const values = registration => ({
|
|
WF_RUNNER_TOKEN: registration.token,
|
|
WF_TENANT_ID: replaced.tenantId,
|
|
WF_ENVIRONMENT_ID: registration.environmentId,
|
|
WF_RUNNER_PUBLIC_KEYS: 'in the ConfigMap',
|
|
WF_RUNNER_EXECUTE: 'true',
|
|
});
|
|
const { poc, fanyu } = replaced.runners;
|
|
return {
|
|
profiles,
|
|
bindings: replaced.bindings,
|
|
runner: values(poc),
|
|
runnerFanyu: fanyu ? values(fanyu) : undefined,
|
|
keys: {
|
|
'runner-public.json': `${JSON.stringify(poc.publicKeys, null, 2)}\n`,
|
|
...(fanyu ? { 'runner-fanyu-public.json': `${JSON.stringify(fanyu.publicKeys, null, 2)}\n` } : {}),
|
|
},
|
|
};
|
|
}
|
|
|
|
async function secrets() {
|
|
await assertCluster();
|
|
const local = await readEnv(INPUTS.carried);
|
|
const projects = await clusterProjects();
|
|
const { runner, runnerFanyu } = projects;
|
|
const image = (await imageRecord()).reference;
|
|
const generated = await generatedSecrets();
|
|
const http = generated.publicScheme === 'http';
|
|
const domain = generated.publicDomain;
|
|
const origins = generated.publicAddress
|
|
? [...(domain ? [domain] : []), generated.publicAddress].map(host => `${http ? 'http' : 'https'}://${host}`)
|
|
: [];
|
|
const environments = clusterEnvironments({ local, runner, runnerFanyu, image, generated, origins });
|
|
const ca = (
|
|
await kubectl([
|
|
'config',
|
|
'view',
|
|
'--raw',
|
|
'--minify',
|
|
'-o',
|
|
'jsonpath={.clusters[0].cluster.certificate-authority-data}',
|
|
])
|
|
).stdout.trim();
|
|
const hosts = (await readFile(INPUTS.knownHosts, 'utf8')).split('\n').filter(line => line && !line.startsWith('#'));
|
|
if (!hosts.length) throw new Error(`No pinned host key for gitea.d-robotics.cc in ${INPUTS.knownHosts}`);
|
|
// The Pod mounts the Claude settings; until gateway-key stores the key, they name the gateway only.
|
|
if (!(await exists('secret', 'wishforge-claude', NAMESPACE))) {
|
|
await applyObjects([
|
|
secret('wishforge-claude', { 'settings.json': `${JSON.stringify(claudeSettings(), null, 2)}\n` }),
|
|
]);
|
|
console.log('No gateway key yet: Claude runs and AI optimization stay unavailable until gateway-key.');
|
|
}
|
|
const objects = [
|
|
secret('wishforge-postgres', { password: generated.postgresPassword }),
|
|
secret('wishforge-minio', { user: generated.objectStorageUser, password: generated.objectStoragePassword }),
|
|
secret('wishforge-env', {
|
|
'platform.env': envFile(environments.platform),
|
|
'executor.env': envFile(environments.executor),
|
|
'runner.env': envFile(environments.runner),
|
|
...(environments.runnerFanyu ? { 'runner-fanyu.env': envFile(environments.runnerFanyu) } : {}),
|
|
}),
|
|
secret('wishforge-keys', { 'runner-ed25519.pem': await readFile(INPUTS.signingKey, 'utf8') }),
|
|
configMap('wishforge-config', {
|
|
'plans.json': await readFile(INPUTS.plans, 'utf8'),
|
|
...projects.keys,
|
|
...Object.fromEntries(
|
|
Object.entries(projects.profiles).map(([name, profile]) => [name, `${JSON.stringify(profile, null, 2)}\n`]),
|
|
),
|
|
'project.json': `${JSON.stringify(projects.bindings, null, 2)}\n`,
|
|
known_hosts: `${hosts.join('\n')}\n`,
|
|
kubeconfig: clusterKubeconfig(ca),
|
|
}),
|
|
configMap('wishforge-caddy', {
|
|
Caddyfile: !generated.publicAddress
|
|
? PENDING_CADDYFILE
|
|
: http
|
|
? clusterHttpCaddyfile(generated.publicAddress, { domain })
|
|
: clusterCaddyfile(generated.publicAddress, { staging: generated.acme !== 'production', domain }),
|
|
}),
|
|
];
|
|
await applyObjects(objects);
|
|
const digest = createHash('sha256').update(JSON.stringify(objects)).digest('hex').slice(0, 32);
|
|
console.log(`Secrets and ConfigMaps are in place: settings, keys, POC files and the Caddyfile (settings ${digest}).`);
|
|
return digest;
|
|
}
|
|
|
|
async function apply(...flags) {
|
|
await assertCluster();
|
|
// The containers read their settings, which name the check Pods' image too, only when they
|
|
// start: refresh them, and roll the Pod when they changed.
|
|
const settings = await secrets();
|
|
const rendered = renderPlatform(await readFile(path.join(MANIFESTS, 'platform.yaml'), 'utf8'), {
|
|
image: (await imageRecord()).reference,
|
|
settings,
|
|
runnerDisabled: !flags.includes('--runner'),
|
|
maintenance: flags.includes('--maintenance'),
|
|
});
|
|
await kubectl(['apply', '-f', '-'], { input: rendered });
|
|
// A StatefulSet rolls a Pod that is not Ready (one in maintenance, say) only once it is deleted.
|
|
let replaced = false;
|
|
await waitFor('the WishForge Pod to take the new revision', async () => {
|
|
const set = JSON.parse((await kubectl(['-n', NAMESPACE, 'get', 'statefulset', 'wishforge', '-o', 'json'])).stdout);
|
|
const pod = await kubectl(['-n', NAMESPACE, 'get', 'pod', 'wishforge-0', '-o', 'json'], { allowFailure: true });
|
|
if (pod.code !== 0 || set.metadata.generation !== set.status?.observedGeneration) return false;
|
|
if (JSON.parse(pod.stdout).metadata.labels?.['controller-revision-hash'] === set.status.updateRevision) return true;
|
|
if (!replaced) {
|
|
await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-0', '--wait=true'], { timeoutSeconds: 180 });
|
|
replaced = true;
|
|
}
|
|
return false;
|
|
});
|
|
console.log(`Applied the WishForge Pod${flags.length ? ` (${flags.join(' ')})` : ''}.`);
|
|
}
|
|
|
|
async function status() {
|
|
await assertCluster();
|
|
for (const [label, args] of [
|
|
['Pod', ['-n', NAMESPACE, 'get', 'pods', '-o', 'wide']],
|
|
['Storage', ['get', 'pv', 'wishforge-data', 'wishforge-checks-exec', '--ignore-not-found']],
|
|
['Check Pods', ['-n', CHECK_NAMESPACE, 'get', 'pods']],
|
|
['Public entry', ['-n', NAMESPACE, 'get', 'service', 'wishforge-public', '--ignore-not-found']],
|
|
]) {
|
|
const result = await kubectl(args, { allowFailure: true });
|
|
console.log(`== ${label} ==\n${result.stdout.trim() || result.stderr.trim() || '(none)'}`);
|
|
}
|
|
const pod = await kubectl(['-n', NAMESPACE, 'get', 'pod', 'wishforge-0', '-o', 'json'], { allowFailure: true });
|
|
if (pod.code === 0)
|
|
for (const container of JSON.parse(pod.stdout).status?.containerStatuses ?? [])
|
|
console.log(
|
|
`${container.name.padEnd(10)} ${container.ready ? 'ready' : 'not ready'}, restarts ${container.restartCount}${
|
|
container.state?.waiting ? `, ${container.state.waiting.reason}` : ''
|
|
}`,
|
|
);
|
|
const backups = await kubectl(backupExec(['node', 'scripts/cluster-poc-backup.mjs', 'latest']), {
|
|
allowFailure: true,
|
|
});
|
|
if (backups.code === 0) {
|
|
const { snapshots, manifest, last } = JSON.parse(backups.stdout);
|
|
console.log(
|
|
`== Backups ==\n${snapshots.length} on the node, newest ${manifest?.takenAt ?? 'none'}; last attempt ${
|
|
last ? `${last.ok ? 'succeeded' : `failed (${last.error})`} at ${last.at}` : 'none yet'
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------------------------- restore
|
|
|
|
/** What the cluster is still doing: projects are replaced and data restored only when nothing runs. */
|
|
export const UNFINISHED_SQL = `select kind || ' ' || id || ' ' || coalesce(data->>'status', '') from entities where
|
|
(kind = 'run' and data->>'status' in ('queued', 'running')) or
|
|
(kind = 'runner_task' and data->>'status' in ('pending', 'dispatching', 'unknown')) or
|
|
(kind = 'release' and data->>'status' in ('ready', 'deploying', 'unknown'))`;
|
|
async function waitFor(description, check, minutes = 10) {
|
|
for (const until = Date.now() + minutes * 60_000; Date.now() < until; await delay(5000)) if (await check()) return;
|
|
throw new Error(`Timed out waiting for ${description}`);
|
|
}
|
|
const podGone = async name =>
|
|
(
|
|
await kubectl(['-n', NAMESPACE, 'get', 'pod', name, '--ignore-not-found', '-o', 'name'], { allowFailure: true })
|
|
).stdout.trim() === '';
|
|
|
|
/** A restricted Pod with the data volume, to put a snapshot's files in place while the WishForge Pod is stopped. */
|
|
const restorePod = image => ({
|
|
apiVersion: 'v1',
|
|
kind: 'Pod',
|
|
metadata: { name: 'wishforge-restore', namespace: NAMESPACE, labels },
|
|
spec: {
|
|
restartPolicy: 'Never',
|
|
automountServiceAccountToken: false,
|
|
enableServiceLinks: false,
|
|
imagePullSecrets: [{ name: 'cce-image-credential-ccr-29eug8s3-vpc.cnc.bj.baidubce.com' }],
|
|
activeDeadlineSeconds: 3600,
|
|
securityContext: {
|
|
runAsNonRoot: true,
|
|
runAsUser: 10001,
|
|
runAsGroup: 10001,
|
|
fsGroup: 10001,
|
|
fsGroupChangePolicy: 'OnRootMismatch',
|
|
seccompProfile: { type: 'RuntimeDefault' },
|
|
},
|
|
containers: [
|
|
{
|
|
name: 'restore',
|
|
image,
|
|
command: ['/bin/sh', '-c', 'sleep 3600'],
|
|
securityContext: {
|
|
allowPrivilegeEscalation: false,
|
|
readOnlyRootFilesystem: true,
|
|
capabilities: { drop: ['ALL'] },
|
|
},
|
|
resources: { requests: { cpu: '100m', memory: '128Mi' }, limits: { cpu: '1', memory: '1Gi' } },
|
|
volumeMounts: [
|
|
{ name: 'data', mountPath: '/mnt/data' },
|
|
{ name: 'tmp', mountPath: '/tmp' },
|
|
],
|
|
},
|
|
],
|
|
volumes: [
|
|
{ name: 'data', persistentVolumeClaim: { claimName: 'wishforge-data' } },
|
|
{ name: 'tmp', emptyDir: { sizeLimit: '4Gi' } },
|
|
],
|
|
},
|
|
});
|
|
|
|
const SNAPSHOT = /^\d{8}T\d{6}Z$/;
|
|
/**
|
|
* Puts snapshot `name`'s files in place, in the restore Pod: each file is checked against the
|
|
* manifest first, and only then are the directories emptied and unpacked. `archives` are the
|
|
* directories the snapshot holds; the others are emptied. /tmp/verified and /tmp/restored mark how
|
|
* far it got, since kubectl can lose an exec's status.
|
|
*/
|
|
export function restoreScript(name, archives = ARCHIVES) {
|
|
if (!SNAPSHOT.test(name)) throw new Error(`Not a snapshot name: ${name}`);
|
|
if (archives.some(archive => !ARCHIVES.includes(archive))) throw new Error('Unknown archive');
|
|
const snapshot = `/mnt/data/backups/${name}`;
|
|
const reset = ['postgres', 'temporal', 'exec/runs', ...ARCHIVES];
|
|
return [
|
|
'set -e',
|
|
`node /opt/wishforge/scripts/cluster-poc-backup.mjs verify '${snapshot}'`,
|
|
'touch /tmp/verified',
|
|
'cd /mnt/data',
|
|
...reset.map(dir => `mkdir -p '${dir}' && find '${dir}' -mindepth 1 -delete && chmod 0700 '${dir}'`),
|
|
...archives.map(dir => `tar -xzf '${snapshot}/${dir}.tgz' -C '${dir}' --no-same-owner --no-same-permissions`),
|
|
`cp '${snapshot}/temporal.db' temporal/temporal.db && chmod 0600 temporal/temporal.db`,
|
|
'touch /tmp/restored',
|
|
'echo restored: $(find minio -type f | wc -l) object storage files, $(find git -type f | wc -l) Git files',
|
|
].join('\n');
|
|
}
|
|
|
|
/**
|
|
* Replaces the cluster's data with a snapshot on the node, with the WishForge Pod stopped. The
|
|
* database is then restored into a fresh PostgreSQL started with nothing else running, by the
|
|
* backup container, which reads the snapshot and reaches the database on loopback; the Pod starts
|
|
* only if every table's row count is `counts`.
|
|
*/
|
|
async function replaceData({ image, snapshot, archives, counts, runner }) {
|
|
const script = restoreScript(snapshot, archives);
|
|
if (await exists('statefulset', 'wishforge', NAMESPACE)) {
|
|
await kubectl(['-n', NAMESPACE, 'scale', 'statefulset', 'wishforge', '--replicas=0']);
|
|
await waitFor('the WishForge Pod to stop', () => podGone('wishforge-0'));
|
|
}
|
|
await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-restore', '--ignore-not-found', '--wait=true']);
|
|
await kubectl(['create', '-f', '-'], { input: JSON.stringify(restorePod(image)) });
|
|
const inRestorePod = (command, options) =>
|
|
kubectl(['-n', NAMESPACE, 'exec', 'wishforge-restore', '-c', 'restore', '--', '/bin/sh', '-c', command], options);
|
|
try {
|
|
await kubectl(['-n', NAMESPACE, 'wait', '--for=condition=Ready', 'pod/wishforge-restore', '--timeout=600s'], {
|
|
timeoutSeconds: 620,
|
|
});
|
|
const restored = await inRestorePod(script, { timeoutSeconds: 1200, allowFailure: true });
|
|
const marks = (await inRestorePod('ls /tmp', { allowFailure: true })).stdout;
|
|
if (!/\brestored\b/.test(marks)) {
|
|
if (/\bverified\b/.test(marks))
|
|
throw new Error(
|
|
`Unpacking snapshot ${snapshot} did not finish (${restored.stderr.trim().slice(0, 400)}); the Pod stays stopped, run this again`,
|
|
);
|
|
await kubectl(['-n', NAMESPACE, 'scale', 'statefulset', 'wishforge', '--replicas=1'], { allowFailure: true });
|
|
throw new Error(
|
|
`Snapshot ${snapshot} did not pass its check (${restored.stderr.trim().slice(0, 400)}); nothing was changed and the WishForge Pod is starting again`,
|
|
);
|
|
}
|
|
console.log(restored.stdout.trim() || 'Files restored.');
|
|
} finally {
|
|
await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-restore', '--ignore-not-found', '--wait=true'], {
|
|
allowFailure: true,
|
|
});
|
|
}
|
|
|
|
await apply('--maintenance');
|
|
await kubectl(['-n', NAMESPACE, 'scale', 'statefulset', 'wishforge', '--replicas=1']);
|
|
await waitFor('PostgreSQL in the cluster', async () => {
|
|
const ready = await kubectl(
|
|
[
|
|
'-n',
|
|
NAMESPACE,
|
|
'exec',
|
|
'wishforge-0',
|
|
'-c',
|
|
'postgres',
|
|
'--',
|
|
'pg_isready',
|
|
'-h',
|
|
'127.0.0.1',
|
|
'-U',
|
|
'wishforge',
|
|
'-d',
|
|
'wishforge',
|
|
],
|
|
{ allowFailure: true },
|
|
);
|
|
return ready.code === 0;
|
|
});
|
|
// kubectl can lose an exec's status: the row counts say whether the restore took.
|
|
const restored = await kubectl(
|
|
backupExec([
|
|
'pg_restore',
|
|
'-h',
|
|
'127.0.0.1',
|
|
'-U',
|
|
'wishforge',
|
|
'-d',
|
|
'wishforge',
|
|
'--no-owner',
|
|
'--no-privileges',
|
|
'--exit-on-error',
|
|
`/var/lib/wishforge/backups/${snapshot}/postgres.dump`,
|
|
]),
|
|
{ timeoutSeconds: 1800, allowFailure: true },
|
|
);
|
|
const clusterCounts = (
|
|
await kubectl([
|
|
'-n',
|
|
NAMESPACE,
|
|
'exec',
|
|
'wishforge-0',
|
|
'-c',
|
|
'postgres',
|
|
'--',
|
|
'psql',
|
|
'-U',
|
|
'wishforge',
|
|
'-d',
|
|
'wishforge',
|
|
'-At',
|
|
'-c',
|
|
COUNTS_SQL,
|
|
])
|
|
).stdout.trim();
|
|
if (clusterCounts !== counts)
|
|
throw new Error(
|
|
`Row counts differ after the restore (${restored.stderr.trim().slice(0, 800)}):\nSnapshot: ${counts}\nCluster: ${clusterCounts}`,
|
|
);
|
|
console.log(`Database restored; row counts match: ${counts}`);
|
|
await apply(...(runner ? ['--runner'] : []));
|
|
}
|
|
|
|
// ------------------------------------------------------------------------------ public entry
|
|
|
|
/** The load balancer's public IPv4, once CCE has assigned it; private addresses it also lists are skipped. */
|
|
export function publicAddressOf(service) {
|
|
for (const ingress of service?.status?.loadBalancer?.ingress ?? []) {
|
|
try {
|
|
return publicIPv4(ingress.ip);
|
|
} catch {
|
|
// A VPC address of the load balancer, not the public one.
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The public entry: WishForge's own load balancer and elastic IP (billed while they exist). HTTPS
|
|
* with a certificate for that IP from Let's Encrypt's staging CA first, and with --production the
|
|
* trusted one; or, with --http, plain HTTP on port 80 only. `--domain <name>` adds a domain the
|
|
* operator's DNS points at the load balancer, which becomes the console's address (the IP keeps
|
|
* working); later runs keep it. Each step restarts the WishForge Pod so the console accepts the
|
|
* new origin.
|
|
*/
|
|
async function publicEntry(...args) {
|
|
await assertCluster();
|
|
let flag, domain;
|
|
for (let i = 0; i < args.length; i++)
|
|
if (args[i] === '--domain' && args[i + 1] && !domain) domain = publicDomain(args[++i]);
|
|
else if (['--production', '--http'].includes(args[i]) && !flag) flag = args[i];
|
|
else throw new Error('Usage: cluster-poc.mjs public [--production|--http] [--domain <name>]');
|
|
if (!(await exists('statefulset', 'wishforge', NAMESPACE))) throw new Error('Deploy the WishForge Pod first');
|
|
const http = flag === '--http';
|
|
await kubectl(['apply', '-f', '-'], {
|
|
input: publicService(await readFile(path.join(MANIFESTS, 'public-service.yaml'), 'utf8'), { http }),
|
|
});
|
|
let address;
|
|
await waitFor(
|
|
'the load balancer address',
|
|
async () => {
|
|
const service = await kubectl(['-n', NAMESPACE, 'get', 'service', 'wishforge-public', '-o', 'json']);
|
|
address = publicAddressOf(JSON.parse(service.stdout));
|
|
return Boolean(address);
|
|
},
|
|
15,
|
|
);
|
|
const generated = { ...(await generatedSecrets()), publicAddress: address, publicScheme: http ? 'http' : 'https' };
|
|
if (http) delete generated.acme;
|
|
else generated.acme = flag === '--production' ? 'production' : 'staging';
|
|
if (domain) {
|
|
// Browsers and Let's Encrypt find the entry through public DNS: it must already point here.
|
|
const resolver = new Resolver({ timeout: 5000, tries: 2 });
|
|
resolver.setServers(PUBLIC_RESOLVERS);
|
|
const published = await resolver.resolve4(domain).catch(() => []);
|
|
if (!published.includes(address))
|
|
throw new Error(
|
|
`${domain} resolves publicly to ${published.join(', ') || 'nothing'}, not ${address}: point its A record here first`,
|
|
);
|
|
// A company DNS with its own copy of the zone answers its users from that copy instead.
|
|
if (!(await resolve4(domain).catch(() => [])).includes(address))
|
|
console.warn(
|
|
`This machine's DNS (${getServers().join(', ')}) does not resolve ${domain} to ${address}: people using ` +
|
|
'it cannot open the domain until that DNS has the record too.',
|
|
);
|
|
generated.publicDomain = domain;
|
|
}
|
|
const host = generated.publicDomain ?? address;
|
|
await writePrivate(GENERATED, `${JSON.stringify(generated, null, 2)}\n`);
|
|
await secrets();
|
|
await kubectl(['-n', NAMESPACE, 'delete', 'pod', 'wishforge-0', '--wait=true'], { timeoutSeconds: 180 });
|
|
console.log(
|
|
http
|
|
? `Public address ${address}: plain HTTP, port 80 only; the console will answer at http://${host} ` +
|
|
'once the Pod is ready. Passwords and sessions travel unencrypted.'
|
|
: `Public address ${address}: ${flag === '--production' ? 'trusted' : 'staging (untrusted)'} certificate ` +
|
|
`requested; the console will answer at https://${host} once the Pod is ready.`,
|
|
);
|
|
}
|
|
|
|
// ------------------------------------------------------------------------------- snapshots
|
|
|
|
const backupExec = args => ['-n', NAMESPACE, 'exec', 'wishforge-0', '-c', 'backup', '--', ...args];
|
|
/**
|
|
* Copies one file out of one of the WishForge Pod's containers. kubectl drops whatever of an exec's output is
|
|
* still in flight when the remote command exits, so the remote side sends the file and then waits
|
|
* for this end to close its input, which happens only once every byte has arrived.
|
|
*/
|
|
async function download(remote, local, bytes, container = 'backup') {
|
|
for (let attempt = 1; ; attempt++) {
|
|
const result = await new Promise((resolve, reject) => {
|
|
const child = spawn(
|
|
'kubectl',
|
|
[
|
|
'--kubeconfig',
|
|
KUBECONFIG,
|
|
'-n',
|
|
NAMESPACE,
|
|
'exec',
|
|
'-i',
|
|
'wishforge-0',
|
|
'-c',
|
|
container,
|
|
'--',
|
|
'/bin/sh',
|
|
'-c',
|
|
'cat "$0" || exit 1; read -r _',
|
|
remote,
|
|
],
|
|
{ cwd: ROOT, env: { PATH: process.env.PATH, HOME: os.homedir() }, stdio: ['pipe', 'pipe', 'pipe'] },
|
|
);
|
|
const err = [];
|
|
let received = 0;
|
|
// A link at a few dozen kB/s still finishes; a stalled one is given up.
|
|
const timer = setTimeout(() => child.kill(), 120_000 + bytes / 20);
|
|
const file = createWriteStream(local, { mode: 0o600 });
|
|
const written = new Promise((done, fail) => file.on('finish', done).on('error', fail));
|
|
if (bytes === 0) child.stdin.end('\n');
|
|
child.stdout.on('data', chunk => {
|
|
received += chunk.length;
|
|
if (received >= bytes) child.stdin.end('\n');
|
|
});
|
|
child.stdout.pipe(file);
|
|
child.stderr.on('data', chunk => err.push(chunk));
|
|
child.stdin.on('error', () => undefined);
|
|
child.once('error', reject);
|
|
child.once('close', () => {
|
|
clearTimeout(timer);
|
|
written.then(() => resolve({ received, stderr: Buffer.concat(err).toString('utf8') }), reject);
|
|
});
|
|
});
|
|
if (result.received === bytes) return;
|
|
if (attempt === 5)
|
|
throw new Error(
|
|
`Copying ${path.basename(remote)} failed after ${attempt} attempts (${result.received} of ${bytes} bytes): ${result.stderr.trim().slice(-400)}`,
|
|
);
|
|
await delay(3000 * attempt);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Replaces everything the cluster holds with one of the node's daily snapshots (`status` lists
|
|
* them): the database, Temporal, Git, object storage and the platform's, POC's and Runners'
|
|
* records. Starts the Runners only with --runner. The node keeps its snapshots.
|
|
*/
|
|
async function restore(name, ...flags) {
|
|
if (!name || name.startsWith('--') || flags.some(flag => flag !== '--runner'))
|
|
throw new Error('Usage: cluster-poc.mjs restore <snapshot, e.g. 20261102T192000Z> [--runner]');
|
|
restoreScript(name);
|
|
await assertCluster();
|
|
const image = (await imageRecord()).reference;
|
|
const { snapshots } = JSON.parse(
|
|
(await kubectl(backupExec(['node', 'scripts/cluster-poc-backup.mjs', 'latest']))).stdout,
|
|
);
|
|
if (!snapshots.includes(name))
|
|
throw new Error(`The node has no snapshot ${name}; it has ${snapshots.join(', ') || 'none'}`);
|
|
const remote = `/var/lib/wishforge/backups/${name}/manifest.json`;
|
|
const work = await mkdtemp(path.join(os.tmpdir(), 'wishforge-restore-'));
|
|
try {
|
|
const size = (await kubectl(backupExec(['/bin/sh', '-c', `wc -c < ${remote}`]))).stdout.trim();
|
|
await download(remote, path.join(work, 'manifest.json'), Number(size));
|
|
const manifest = JSON.parse(await readFile(path.join(work, 'manifest.json'), 'utf8'));
|
|
if (!manifest.database?.counts) throw new Error('The snapshot records no row counts to check the restore against');
|
|
for (const file of ['postgres.dump', 'temporal.db'])
|
|
if (!manifest.files[file]) throw new Error(`The snapshot has no ${file}`);
|
|
const archives = ARCHIVES.filter(archive => manifest.files[`${archive}.tgz`]);
|
|
console.log(`Restoring snapshot ${name} (taken ${manifest.takenAt}) …`);
|
|
await replaceData({
|
|
image,
|
|
snapshot: name,
|
|
archives,
|
|
counts: manifest.database.counts,
|
|
runner: flags.includes('--runner'),
|
|
});
|
|
console.log(`Restored snapshot ${name}${flags.includes('--runner') ? '' : '; the Runners stay idle'}.`);
|
|
} finally {
|
|
await rm(work, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------------------------ Fanyu's project
|
|
|
|
const FANYU_DEPLOYMENTS = {
|
|
api: 'droboticflow-api',
|
|
worker: 'droboticflow-worker',
|
|
coordinator: 'drosync-coordinator',
|
|
'annotation-processor': 'droboticflow-annotation-processor',
|
|
'annotation-validator': 'droboticflow-annotation-validator',
|
|
};
|
|
|
|
/** WishForge's kubeconfig for Fanyu's cluster: its Service Account's token, nothing else. */
|
|
export function fanyuKubeconfig({ certificateAuthorityData, token }) {
|
|
if (!/^[A-Za-z0-9+/=]+$/.test(certificateAuthorityData)) throw new Error('Invalid Fanyu cluster CA data');
|
|
if (!/^[A-Za-z0-9._-]+$/.test(token)) throw new Error('Invalid Fanyu Service Account token');
|
|
return JSON.stringify(
|
|
{
|
|
apiVersion: 'v1',
|
|
kind: 'Config',
|
|
clusters: [
|
|
{ name: 'fanyu', cluster: { server: FANYU.cluster, 'certificate-authority-data': certificateAuthorityData } },
|
|
],
|
|
users: [{ name: 'wishforge', user: { token } }],
|
|
contexts: [{ name: 'fanyu', context: { cluster: 'fanyu', user: 'wishforge', namespace: FANYU.namespace } }],
|
|
'current-context': 'fanyu',
|
|
},
|
|
null,
|
|
2,
|
|
);
|
|
}
|
|
|
|
/** Fanyu's profile, as WishForge's containers read it; the POC's pins the same Kaniko and mirrors. */
|
|
export function fanyuProfile({ poc, namespaceUid, deployments }) {
|
|
return {
|
|
schemaVersion: 1,
|
|
id: 'droboticflow-fanyu',
|
|
repository: poc.repository,
|
|
branch: 'dev',
|
|
kubeconfig: PATHS.fanyuKubeconfig,
|
|
clusterServer: FANYU.cluster,
|
|
namespace: FANYU.namespace,
|
|
namespaceUid,
|
|
deployments,
|
|
registry: 'ccr-29eug8s3-pub.cnc.bj.baidubce.com/fanyu',
|
|
registrySecret: 'droboticflow-registry',
|
|
applicationUrl: 'http://120.48.180.80',
|
|
kanikoImage: poc.kanikoImage,
|
|
registryMirrors: poc.registryMirrors,
|
|
rolloutTimeoutSeconds: poc.rolloutTimeoutSeconds ?? 600,
|
|
buildTimeoutSeconds: poc.buildTimeoutSeconds ?? 1800,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* WishForge's identity in Fanyu's cluster (fanyu-access.yaml), applied with Fanyu's administrator
|
|
* kubeconfig, which stays on this Mac: its token goes into the WishForge cluster's Secret
|
|
* wishforge-fanyu-kube only, after checking it can do what the delivery needs and no more.
|
|
*/
|
|
async function fanyuAccess(...args) {
|
|
if (args[0] !== '--kubeconfig' || !args[1] || args.length !== 2)
|
|
throw new Error('Usage: cluster-poc.mjs fanyu-access --kubeconfig <Fanyu administrator kubeconfig>');
|
|
await assertCluster();
|
|
const admin = path.resolve(args[1]);
|
|
const env = { PATH: process.env.PATH, HOME: os.homedir() };
|
|
const fanyu = async (kubeconfig, command, options = {}) => {
|
|
const result = await capture('kubectl', ['--kubeconfig', kubeconfig, '--request-timeout=60s', ...command], {
|
|
env,
|
|
input: options.input,
|
|
});
|
|
if (result.code !== 0 && !options.allowFailure)
|
|
throw new Error(`kubectl ${command.slice(0, 3).join(' ')} (Fanyu) failed: ${result.stderr.trim().slice(0, 600)}`);
|
|
return result;
|
|
};
|
|
const view = JSON.parse((await fanyu(admin, ['config', 'view', '--minify', '-o', 'json'])).stdout);
|
|
if (view.clusters?.length !== 1 || view.clusters[0].cluster.server !== FANYU.cluster)
|
|
throw new Error(`The kubeconfig does not name Fanyu's cluster ${FANYU.cluster}`);
|
|
await fanyu(admin, ['apply', '-f', path.join(MANIFESTS, 'fanyu-access.yaml')]);
|
|
let token;
|
|
await waitFor(
|
|
'the Fanyu Service Account token',
|
|
async () => {
|
|
token = JSON.parse(
|
|
(await fanyu(admin, ['-n', FANYU.namespace, 'get', 'secret', 'wishforge-token', '-o', 'json'])).stdout,
|
|
).data;
|
|
return Boolean(token?.token && token['ca.crt']);
|
|
},
|
|
2,
|
|
);
|
|
const kubeconfig = fanyuKubeconfig({
|
|
certificateAuthorityData: token['ca.crt'],
|
|
token: Buffer.from(token.token, 'base64').toString('utf8'),
|
|
});
|
|
const directory = await mkdtemp(path.join(os.tmpdir(), 'wishforge-fanyu-'));
|
|
try {
|
|
const file = path.join(directory, 'kubeconfig');
|
|
await writeFile(file, kubeconfig, { mode: 0o600 });
|
|
const may = async (...check) =>
|
|
(await fanyu(file, ['auth', 'can-i', ...check, '-n', FANYU.namespace], { allowFailure: true })).stdout.trim();
|
|
const expected = [
|
|
[['patch', 'deployment/droboticflow-annotation-validator'], 'yes'],
|
|
[['create', 'pods'], 'yes'],
|
|
[['get', 'secrets'], 'no'],
|
|
[['get', 'configmaps'], 'no'],
|
|
[['patch', 'deployment/droboticflow-headpose'], 'no'],
|
|
[['delete', 'deployments'], 'no'],
|
|
];
|
|
for (const [check, answer] of expected)
|
|
if ((await may(...check)) !== answer)
|
|
throw new Error(`WishForge's Fanyu identity: can-i ${check.join(' ')} is not ${answer}`);
|
|
} finally {
|
|
await rm(directory, { recursive: true, force: true });
|
|
}
|
|
const namespaceUid = (
|
|
await fanyu(admin, ['get', 'namespace', FANYU.namespace, '-o', 'jsonpath={.metadata.uid}'])
|
|
).stdout.trim();
|
|
const deployments = {};
|
|
for (const [name, deployment] of Object.entries(FANYU_DEPLOYMENTS))
|
|
deployments[name] = (
|
|
await fanyu(admin, ['-n', FANYU.namespace, 'get', 'deployment', deployment, '-o', 'jsonpath={.metadata.uid}'])
|
|
).stdout.trim();
|
|
const profile = fanyuProfile({ poc: JSON.parse(await readFile(POC_PROFILE, 'utf8')), namespaceUid, deployments });
|
|
const parsed = await capture(
|
|
process.execPath,
|
|
[
|
|
'--import',
|
|
'tsx',
|
|
'-e',
|
|
"import('./packages/runtime/src/poc-profile.ts').then(m => m.pocProfileSchema.parse(JSON.parse(process.argv[1])))",
|
|
JSON.stringify(profile),
|
|
],
|
|
{ env: process.env },
|
|
);
|
|
if (parsed.code !== 0) throw new Error(`Fanyu's profile is not valid: ${parsed.stderr.trim().slice(0, 800)}`);
|
|
await writePrivate(FANYU_PROFILE, `${JSON.stringify(profile, null, 2)}\n`);
|
|
await applyObjects([secret('wishforge-fanyu-kube', { kubeconfig })]);
|
|
console.log(
|
|
`WishForge's identity in Fanyu's cluster is in place (Secret wishforge-fanyu-kube); profile: ${FANYU_PROFILE}`,
|
|
);
|
|
}
|
|
|
|
const inPod = (container, command, options = {}) =>
|
|
kubectl(['-n', NAMESPACE, 'exec', 'wishforge-0', '-c', container, '--', ...command], {
|
|
timeoutSeconds: 600,
|
|
...options,
|
|
});
|
|
|
|
/**
|
|
* Deletes every project of the cluster's WishForge with all it holds, and creates the POC and
|
|
* Fanyu projects in their place (cluster-poc-projects.ts), with nothing running. The node takes a
|
|
* snapshot first. Their Runners' registrations come to this Mac (STATE/projects.json); `apply`
|
|
* then gives both Runners their settings.
|
|
*/
|
|
async function projectsCommand(flag) {
|
|
if (flag !== '--replace') throw new Error('Usage: cluster-poc.mjs projects --replace');
|
|
await assertCluster();
|
|
if (!(await stat(FANYU_PROFILE).catch(() => undefined))) throw new Error('Run fanyu-access first');
|
|
const unfinished = (
|
|
await inPod('postgres', ['psql', '-U', 'wishforge', '-d', 'wishforge', '-At', '-c', UNFINISHED_SQL])
|
|
).stdout.trim();
|
|
if (unfinished) throw new Error(`The cluster is still working; wait until these finish:\n${unfinished}`);
|
|
const before = Date.now();
|
|
await inPod('backup', ['node', 'scripts/cluster-poc-backup.mjs', 'once'], { allowFailure: true });
|
|
const { last } = JSON.parse((await inPod('backup', ['node', 'scripts/cluster-poc-backup.mjs', 'latest'])).stdout);
|
|
if (!(last?.ok && Date.parse(last.at) >= before))
|
|
throw new Error(`The snapshot before replacing the projects failed${last?.error ? `: ${last.error}` : ''}`);
|
|
console.log(`Snapshot ${last.name} taken on the node.`);
|
|
const output = '/var/lib/wishforge/platform-state/projects-setup.json';
|
|
await inPod('platform', ['rm', '-f', output]);
|
|
// kubectl can lose an exec's output and status: the file it writes says whether it finished.
|
|
const replaced = await inPod(
|
|
'platform',
|
|
[
|
|
'node',
|
|
'--env-file=/etc/wishforge/env/platform.env',
|
|
'--import',
|
|
'tsx',
|
|
'scripts/cluster-poc-projects.ts',
|
|
'--replace',
|
|
],
|
|
{ allowFailure: true },
|
|
);
|
|
const size = await inPod('platform', ['/bin/sh', '-c', `wc -c < ${output}`], { allowFailure: true });
|
|
if (size.code !== 0 || !/^\d+$/.test(size.stdout.trim()))
|
|
throw new Error(`Replacing the projects did not finish: ${replaced.stderr.trim().slice(-1500)}`);
|
|
if (replaced.stdout.trim()) console.log(replaced.stdout.trim());
|
|
await privateState();
|
|
const partial = `${PROJECTS_FILE}.partial`;
|
|
await download(output, partial, Number(size.stdout.trim()), 'platform');
|
|
JSON.parse(await readFile(partial, 'utf8'));
|
|
await rename(partial, PROJECTS_FILE);
|
|
await inPod('platform', ['rm', '-f', output]);
|
|
// Workspaces of the runs that are gone, and the Runner's records of their tasks.
|
|
await inPod('executor', [
|
|
'/bin/sh',
|
|
'-c',
|
|
'find /var/lib/wishforge/exec/runs -mindepth 1 -maxdepth 1 -exec rm -rf {} +',
|
|
]);
|
|
await inPod('runner', [
|
|
'/bin/sh',
|
|
'-c',
|
|
'find /var/lib/wishforge/runner-state -mindepth 1 -maxdepth 1 -exec rm -rf {} +',
|
|
]);
|
|
const disabled = (
|
|
await kubectl([
|
|
'-n',
|
|
NAMESPACE,
|
|
'get',
|
|
'statefulset',
|
|
'wishforge',
|
|
'-o',
|
|
'jsonpath={.spec.template.spec.containers[?(@.name=="runner")].env[?(@.name=="WF_RUNNER_DISABLED")].value}',
|
|
])
|
|
).stdout.trim();
|
|
await apply(...(disabled === 'true' ? [] : ['--runner']));
|
|
console.log(`The cluster now has the POC and Fanyu projects; their Runners' registrations are in ${PROJECTS_FILE}.`);
|
|
}
|
|
|
|
const COMMANDS = {
|
|
image,
|
|
public: publicEntry,
|
|
bootstrap,
|
|
'deploy-key': deployKey,
|
|
'gateway-key': gatewayKey,
|
|
secrets,
|
|
apply,
|
|
restore,
|
|
'fanyu-access': fanyuAccess,
|
|
projects: projectsCommand,
|
|
status,
|
|
};
|
|
export async function main(argv = process.argv.slice(2)) {
|
|
const command = COMMANDS[argv[0]];
|
|
if (!command) throw new Error(`Usage: cluster-poc.mjs ${Object.keys(COMMANDS).join('|')}`);
|
|
await command(...argv.slice(1));
|
|
}
|
|
|
|
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url)
|
|
main().catch(error => {
|
|
console.error(error.message);
|
|
process.exitCode = 1;
|
|
});
|