wishforge-ide/scripts/cluster-poc-backup.mjs
eust-w fb67484dd1 🔥 Run WishForge from the cluster only: drop the Mac deployment and its copies
- The cluster's settings come only from the operator inputs kept in
  .wishforge/cluster-poc (carried keys, signing key, plans, profiles,
  host keys); the Mac's .env.local and service files are no longer read
- restore puts back one of the node's snapshots inside the cluster: the
  files are checked against the manifest before anything is emptied, the
  backup container restores the database, row counts decide
- Snapshots also keep Fanyu's Runner records
- Remove migrate, the Mac backup copy and its launchd job, and the Mac
  public entry (frp gateway) code and manifests
- Mark the Mac operating guides as retired

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 02:31:17 +08:00

312 lines
13 KiB
JavaScript

#!/usr/bin/env node
// Daily snapshots of WishForge's data in the POC cluster (docs/cluster-poc-deployment.md), taken by
// the WishForge Pod's backup container onto the node's disk; `cluster-poc.mjs restore` puts one
// back. Importing it starts nothing.
//
// node scripts/cluster-poc-backup.mjs take one snapshot a day, until Kubernetes stops the container
// node scripts/cluster-poc-backup.mjs once take one snapshot now
// node scripts/cluster-poc-backup.mjs latest print the newest complete snapshot's manifest and the last result
// node scripts/cluster-poc-backup.mjs verify DIR check a snapshot's files against its manifest
import { spawn } from 'node:child_process';
import { createHash } from 'node:crypto';
import { createReadStream } from 'node:fs';
import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import { setTimeout as delay } from 'node:timers/promises';
/** Where the backup container finds what it saves (each mounted read-only) and keeps the snapshots. */
export const BACKUP_ROOT = '/var/lib/wishforge/backups';
export const DATA_ROOT = '/mnt/data';
/** The database beside it on the Pod's loopback; the password comes from PGPASSWORD. */
const DATABASE = { host: '127.0.0.1', port: 5432, user: 'wishforge', database: 'wishforge' };
/** The directories saved as archives, beside the database dump and Temporal's database. */
export const ARCHIVES = ['git', 'minio', 'platform-state', 'poc-state', 'runner-state', 'runner-state-fanyu'];
/** Not saved: the platform container's caches, rebuilt when it starts, and MinIO's scratch space and trash. */
const ARCHIVE_EXCLUDES = { 'platform-state': ['./home'], minio: ['./.minio.sys/tmp'] };
/** Snapshots kept on the node. */
export const KEEP = 7;
/** 03:20 in Beijing, when nobody is expected to be working. */
export const SCHEDULE = { hourUtc: 19, minute: 20 };
const NAME = /^\d{8}T\d{6}Z$/;
/** Row counts of every table, to compare a copy with its source. */
export const COUNTS_SQL = `select string_agg(table_name || '=' || (xpath('/row/c/text()', query_to_xml('select count(*) as c from public.' || quote_ident(table_name), false, true, '')))[1]::text, ',' order by table_name)
from information_schema.tables where table_schema = 'public' and table_type = 'BASE TABLE'`;
/** A snapshot's name: its UTC time, so names sort in the order they were taken. */
export const snapshotName = date =>
date
.toISOString()
.replace(/[-:]/g, '')
.replace(/\.\d+Z$/, 'Z');
/** The first scheduled time after `now`. */
export function nextRun(now, { hourUtc, minute } = SCHEDULE) {
const next = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), hourUtc, minute));
if (next <= now) next.setUTCDate(next.getUTCDate() + 1);
return next;
}
/** Which entries of the snapshot directory stay: the newest `keep` complete snapshots. Leftover partial ones go. */
export function retention(entries, keep = KEEP) {
const complete = entries.filter(entry => NAME.test(entry)).sort();
return {
keep: complete.slice(-keep),
remove: [
...complete.slice(0, Math.max(0, complete.length - keep)),
...entries.filter(e => e.startsWith('.partial-')),
],
};
}
/** Whether `files` (name → {bytes, sha256}) in `directory` are exactly what a manifest says. */
export async function verifyFiles(directory, files) {
for (const [file, expected] of Object.entries(files)) {
const location = path.join(directory, file);
const info = await stat(location).catch(() => undefined);
if (!info) throw new Error(`${file} is missing`);
if (info.size !== expected.bytes) throw new Error(`${file} has ${info.size} bytes, not ${expected.bytes}`);
if ((await sha256File(location)) !== expected.sha256) throw new Error(`${file} does not match its SHA-256`);
}
}
export const sha256File = file =>
new Promise((resolve, reject) => {
const hash = createHash('sha256');
createReadStream(file)
.on('data', chunk => hash.update(chunk))
.on('error', reject)
.on('end', () => resolve(hash.digest('hex')));
});
function run(executable, args, { signal, env = process.env } = {}) {
return new Promise((resolve, reject) => {
const child = spawn(executable, args, { env, signal, stdio: ['ignore', 'pipe', 'pipe'] });
const out = [];
const err = [];
child.stdout.on('data', chunk => out.push(chunk));
child.stderr.on('data', chunk => err.push(chunk));
child.once('error', reject);
child.once('close', code =>
resolve({ code, stdout: Buffer.concat(out).toString('utf8'), stderr: Buffer.concat(err).toString('utf8') }),
);
});
}
async function must(executable, args, options) {
const result = await run(executable, args, options);
if (result.code !== 0) throw new Error(`${path.basename(executable)} failed: ${result.stderr.trim().slice(0, 800)}`);
return result;
}
/**
* The database as of one moment: pg_dump reads the snapshot of a transaction that also counts every
* table's rows, so a restore can be checked against exactly what was dumped.
*/
async function dumpDatabase(file, { signal, database }) {
const { default: pg } = await import('pg');
const client = new pg.Client({
...database,
password: process.env.PGPASSWORD,
});
await client.connect();
try {
await client.query('begin isolation level repeatable read read only');
const snapshot = (await client.query('select pg_export_snapshot() as id')).rows[0].id;
const counts = (await client.query(`select (${COUNTS_SQL}) as counts`)).rows[0].counts;
await must(
'pg_dump',
[
'-h',
database.host,
'-p',
String(database.port),
'-U',
database.user,
'-d',
database.database,
'-Fc',
'--no-owner',
'--no-privileges',
`--snapshot=${snapshot}`,
'-f',
file,
],
{ signal },
);
await client.query('commit');
// The archive's table of contents reads back: the dump is complete.
await must('pg_restore', ['-l', file], { signal });
return counts;
} finally {
await client.end().catch(() => undefined);
}
}
/** Temporal's SQLite database through SQLite's online backup, which waits out its writers. */
async function copyTemporal(file, { signal, data }) {
const source = path.join(data, 'temporal/temporal.db');
await must('sqlite3', ['-readonly', '-cmd', '.timeout 60000', source, `.backup '${file}'`], { signal });
const check = await must('sqlite3', [file, 'pragma integrity_check'], { signal });
if (check.stdout.trim() !== 'ok')
throw new Error(`Temporal's copy fails its integrity check: ${check.stdout.trim()}`);
}
/** A directory as a gzip archive; tried again while files change under it (GNU tar's exit code 1). */
async function archive(name, file, { signal, data }) {
const args = [
'-czf',
file,
...(ARCHIVE_EXCLUDES[name] ?? []).map(entry => `--exclude=${entry}`),
'-C',
path.join(data, name),
'.',
];
for (let attempt = 1; ; attempt++) {
const result = await run('tar', args, { signal });
if (result.code === 0) return;
if (result.code !== 1 || attempt === 3)
throw new Error(`Archiving ${name} failed: ${result.stderr.trim().slice(0, 800)}`);
await delay(5000, undefined, { signal });
}
}
/** One snapshot, complete or not at all: built under a partial name, renamed once its manifest is written. */
export async function takeSnapshot({
root = BACKUP_ROOT,
data = DATA_ROOT,
database = DATABASE,
now = new Date(),
signal,
} = {}) {
await mkdir(root, { recursive: true, mode: 0o700 });
const lock = path.join(root, '.lock');
const held = await stat(lock).catch(() => undefined);
// A lock older than any snapshot takes is left by a container that was stopped mid-way.
if (held && Date.now() - held.mtimeMs < 2 * 3600_000) throw new Error('Another snapshot is being taken');
await rm(lock, { recursive: true, force: true });
await mkdir(lock);
const name = snapshotName(now);
const partial = path.join(root, `.partial-${name}`);
try {
for (const entry of retention(await readdir(root)).remove.filter(entry => entry.startsWith('.partial-')))
await rm(path.join(root, entry), { recursive: true, force: true });
await mkdir(partial, { mode: 0o700 });
const counts = await dumpDatabase(path.join(partial, 'postgres.dump'), { signal, database });
await copyTemporal(path.join(partial, 'temporal.db'), { signal, data });
for (const directory of ARCHIVES)
await archive(directory, path.join(partial, `${directory}.tgz`), { signal, data });
const files = {};
for (const file of (await readdir(partial)).sort()) {
const location = path.join(partial, file);
files[file] = { bytes: (await stat(location)).size, sha256: await sha256File(location) };
}
const manifest = {
name,
takenAt: now.toISOString(),
image: process.env.WF_IMAGE || undefined,
database: { counts },
files,
};
await writeFile(path.join(partial, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 });
await rename(partial, path.join(root, name));
for (const entry of retention(await readdir(root)).remove)
await rm(path.join(root, entry), { recursive: true, force: true });
return manifest;
} catch (error) {
await rm(partial, { recursive: true, force: true });
throw error;
} finally {
await rm(lock, { recursive: true, force: true });
}
}
async function record(root, result) {
await writeFile(path.join(root, 'last.json'), `${JSON.stringify(result, null, 2)}\n`, { mode: 0o600 });
}
/** The newest complete snapshot's manifest, and the last attempt's result. */
export async function latest(root = BACKUP_ROOT) {
const names = retention(await readdir(root).catch(() => [])).keep;
const newest = names.at(-1);
const read = async file => JSON.parse(await readFile(file, 'utf8'));
return {
snapshots: names,
manifest: newest ? await read(path.join(root, newest, 'manifest.json')) : undefined,
last: await read(path.join(root, 'last.json')).catch(() => undefined),
};
}
async function once(root, signal) {
const startedAt = new Date();
try {
const manifest = await takeSnapshot({ root, now: startedAt, signal });
const bytes = Object.values(manifest.files).reduce((sum, file) => sum + file.bytes, 0);
await record(root, { ok: true, name: manifest.name, at: new Date().toISOString(), bytes });
console.log(`[backup] snapshot ${manifest.name}: ${(bytes / 1e6).toFixed(1)} MB`);
return manifest;
} catch (error) {
if (signal?.aborted) throw error;
await record(root, { ok: false, at: new Date().toISOString(), error: error.message });
throw error;
}
}
/**
* Takes a snapshot every day at {@link SCHEDULE}, and at once when the newest is more than a day
* old (a Pod that was down at the scheduled time). A failure is retried an hour later.
*/
export async function backupLoop({ root = BACKUP_ROOT } = {}) {
const stop = new AbortController();
process.once('SIGTERM', () => stop.abort());
const sleep = ms => delay(ms, undefined, { signal: stop.signal });
try {
const newest = (await latest(root)).manifest;
let due = newest && Date.now() - Date.parse(newest.takenAt) < 24 * 3600_000 ? nextRun(new Date()) : new Date();
// The database starts beside this container: give it time before the first attempt.
if (due <= new Date()) await sleep(60_000);
for (;;) {
await sleep(Math.max(0, due - Date.now()));
try {
await once(root, stop.signal);
due = nextRun(new Date());
} catch (error) {
if (stop.signal.aborted) break;
console.error(`[backup] ${error.message}; trying again in an hour`);
due = new Date(Date.now() + 3600_000);
}
}
} catch (error) {
if (!stop.signal.aborted) throw error;
}
return 0;
}
export async function main(argv = process.argv.slice(2)) {
if (argv.length === 0) return backupLoop();
if (argv[0] === 'once' && argv.length === 1) {
await once(BACKUP_ROOT);
return 0;
}
if (argv[0] === 'verify' && argv.length === 2) {
const manifest = JSON.parse(await readFile(path.join(argv[1], 'manifest.json'), 'utf8'));
await verifyFiles(argv[1], manifest.files);
console.log(`[backup] snapshot ${manifest.name}: every file matches its manifest`);
return 0;
}
if (argv[0] === 'latest' && argv.length === 1) {
process.stdout.write(`${JSON.stringify(await latest())}\n`);
return 0;
}
throw new Error('Usage: cluster-poc-backup.mjs [once|latest|verify DIR]');
}
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url)
main().then(
code => process.exit(code),
error => {
console.error(`[backup] ${error.message}`);
process.exit(1);
},
);