- The cluster's settings come only from the operator inputs kept in .wishforge/cluster-poc (carried keys, signing key, plans, profiles, host keys); the Mac's .env.local and service files are no longer read - restore puts back one of the node's snapshots inside the cluster: the files are checked against the manifest before anything is emptied, the backup container restores the database, row counts decide - Snapshots also keep Fanyu's Runner records - Remove migrate, the Mac backup copy and its launchd job, and the Mac public entry (frp gateway) code and manifests - Mark the Mac operating guides as retired Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
312 lines
13 KiB
JavaScript
312 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
|
// Daily snapshots of WishForge's data in the POC cluster (docs/cluster-poc-deployment.md), taken by
|
|
// the WishForge Pod's backup container onto the node's disk; `cluster-poc.mjs restore` puts one
|
|
// back. Importing it starts nothing.
|
|
//
|
|
// node scripts/cluster-poc-backup.mjs take one snapshot a day, until Kubernetes stops the container
|
|
// node scripts/cluster-poc-backup.mjs once take one snapshot now
|
|
// node scripts/cluster-poc-backup.mjs latest print the newest complete snapshot's manifest and the last result
|
|
// node scripts/cluster-poc-backup.mjs verify DIR check a snapshot's files against its manifest
|
|
import { spawn } from 'node:child_process';
|
|
import { createHash } from 'node:crypto';
|
|
import { createReadStream } from 'node:fs';
|
|
import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import { pathToFileURL } from 'node:url';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
|
|
/** Where the backup container finds what it saves (each mounted read-only) and keeps the snapshots. */
|
|
export const BACKUP_ROOT = '/var/lib/wishforge/backups';
|
|
export const DATA_ROOT = '/mnt/data';
|
|
/** The database beside it on the Pod's loopback; the password comes from PGPASSWORD. */
|
|
const DATABASE = { host: '127.0.0.1', port: 5432, user: 'wishforge', database: 'wishforge' };
|
|
/** The directories saved as archives, beside the database dump and Temporal's database. */
|
|
export const ARCHIVES = ['git', 'minio', 'platform-state', 'poc-state', 'runner-state', 'runner-state-fanyu'];
|
|
/** Not saved: the platform container's caches, rebuilt when it starts, and MinIO's scratch space and trash. */
|
|
const ARCHIVE_EXCLUDES = { 'platform-state': ['./home'], minio: ['./.minio.sys/tmp'] };
|
|
/** Snapshots kept on the node. */
|
|
export const KEEP = 7;
|
|
/** 03:20 in Beijing, when nobody is expected to be working. */
|
|
export const SCHEDULE = { hourUtc: 19, minute: 20 };
|
|
const NAME = /^\d{8}T\d{6}Z$/;
|
|
|
|
/** Row counts of every table, to compare a copy with its source. */
|
|
export const COUNTS_SQL = `select string_agg(table_name || '=' || (xpath('/row/c/text()', query_to_xml('select count(*) as c from public.' || quote_ident(table_name), false, true, '')))[1]::text, ',' order by table_name)
|
|
from information_schema.tables where table_schema = 'public' and table_type = 'BASE TABLE'`;
|
|
|
|
/** A snapshot's name: its UTC time, so names sort in the order they were taken. */
|
|
export const snapshotName = date =>
|
|
date
|
|
.toISOString()
|
|
.replace(/[-:]/g, '')
|
|
.replace(/\.\d+Z$/, 'Z');
|
|
|
|
/** The first scheduled time after `now`. */
|
|
export function nextRun(now, { hourUtc, minute } = SCHEDULE) {
|
|
const next = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), hourUtc, minute));
|
|
if (next <= now) next.setUTCDate(next.getUTCDate() + 1);
|
|
return next;
|
|
}
|
|
|
|
/** Which entries of the snapshot directory stay: the newest `keep` complete snapshots. Leftover partial ones go. */
|
|
export function retention(entries, keep = KEEP) {
|
|
const complete = entries.filter(entry => NAME.test(entry)).sort();
|
|
return {
|
|
keep: complete.slice(-keep),
|
|
remove: [
|
|
...complete.slice(0, Math.max(0, complete.length - keep)),
|
|
...entries.filter(e => e.startsWith('.partial-')),
|
|
],
|
|
};
|
|
}
|
|
|
|
/** Whether `files` (name → {bytes, sha256}) in `directory` are exactly what a manifest says. */
|
|
export async function verifyFiles(directory, files) {
|
|
for (const [file, expected] of Object.entries(files)) {
|
|
const location = path.join(directory, file);
|
|
const info = await stat(location).catch(() => undefined);
|
|
if (!info) throw new Error(`${file} is missing`);
|
|
if (info.size !== expected.bytes) throw new Error(`${file} has ${info.size} bytes, not ${expected.bytes}`);
|
|
if ((await sha256File(location)) !== expected.sha256) throw new Error(`${file} does not match its SHA-256`);
|
|
}
|
|
}
|
|
|
|
export const sha256File = file =>
|
|
new Promise((resolve, reject) => {
|
|
const hash = createHash('sha256');
|
|
createReadStream(file)
|
|
.on('data', chunk => hash.update(chunk))
|
|
.on('error', reject)
|
|
.on('end', () => resolve(hash.digest('hex')));
|
|
});
|
|
|
|
function run(executable, args, { signal, env = process.env } = {}) {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn(executable, args, { env, signal, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
const out = [];
|
|
const err = [];
|
|
child.stdout.on('data', chunk => out.push(chunk));
|
|
child.stderr.on('data', chunk => err.push(chunk));
|
|
child.once('error', reject);
|
|
child.once('close', code =>
|
|
resolve({ code, stdout: Buffer.concat(out).toString('utf8'), stderr: Buffer.concat(err).toString('utf8') }),
|
|
);
|
|
});
|
|
}
|
|
async function must(executable, args, options) {
|
|
const result = await run(executable, args, options);
|
|
if (result.code !== 0) throw new Error(`${path.basename(executable)} failed: ${result.stderr.trim().slice(0, 800)}`);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* The database as of one moment: pg_dump reads the snapshot of a transaction that also counts every
|
|
* table's rows, so a restore can be checked against exactly what was dumped.
|
|
*/
|
|
async function dumpDatabase(file, { signal, database }) {
|
|
const { default: pg } = await import('pg');
|
|
const client = new pg.Client({
|
|
...database,
|
|
password: process.env.PGPASSWORD,
|
|
});
|
|
await client.connect();
|
|
try {
|
|
await client.query('begin isolation level repeatable read read only');
|
|
const snapshot = (await client.query('select pg_export_snapshot() as id')).rows[0].id;
|
|
const counts = (await client.query(`select (${COUNTS_SQL}) as counts`)).rows[0].counts;
|
|
await must(
|
|
'pg_dump',
|
|
[
|
|
'-h',
|
|
database.host,
|
|
'-p',
|
|
String(database.port),
|
|
'-U',
|
|
database.user,
|
|
'-d',
|
|
database.database,
|
|
'-Fc',
|
|
'--no-owner',
|
|
'--no-privileges',
|
|
`--snapshot=${snapshot}`,
|
|
'-f',
|
|
file,
|
|
],
|
|
{ signal },
|
|
);
|
|
await client.query('commit');
|
|
// The archive's table of contents reads back: the dump is complete.
|
|
await must('pg_restore', ['-l', file], { signal });
|
|
return counts;
|
|
} finally {
|
|
await client.end().catch(() => undefined);
|
|
}
|
|
}
|
|
|
|
/** Temporal's SQLite database through SQLite's online backup, which waits out its writers. */
|
|
async function copyTemporal(file, { signal, data }) {
|
|
const source = path.join(data, 'temporal/temporal.db');
|
|
await must('sqlite3', ['-readonly', '-cmd', '.timeout 60000', source, `.backup '${file}'`], { signal });
|
|
const check = await must('sqlite3', [file, 'pragma integrity_check'], { signal });
|
|
if (check.stdout.trim() !== 'ok')
|
|
throw new Error(`Temporal's copy fails its integrity check: ${check.stdout.trim()}`);
|
|
}
|
|
|
|
/** A directory as a gzip archive; tried again while files change under it (GNU tar's exit code 1). */
|
|
async function archive(name, file, { signal, data }) {
|
|
const args = [
|
|
'-czf',
|
|
file,
|
|
...(ARCHIVE_EXCLUDES[name] ?? []).map(entry => `--exclude=${entry}`),
|
|
'-C',
|
|
path.join(data, name),
|
|
'.',
|
|
];
|
|
for (let attempt = 1; ; attempt++) {
|
|
const result = await run('tar', args, { signal });
|
|
if (result.code === 0) return;
|
|
if (result.code !== 1 || attempt === 3)
|
|
throw new Error(`Archiving ${name} failed: ${result.stderr.trim().slice(0, 800)}`);
|
|
await delay(5000, undefined, { signal });
|
|
}
|
|
}
|
|
|
|
/** One snapshot, complete or not at all: built under a partial name, renamed once its manifest is written. */
|
|
export async function takeSnapshot({
|
|
root = BACKUP_ROOT,
|
|
data = DATA_ROOT,
|
|
database = DATABASE,
|
|
now = new Date(),
|
|
signal,
|
|
} = {}) {
|
|
await mkdir(root, { recursive: true, mode: 0o700 });
|
|
const lock = path.join(root, '.lock');
|
|
const held = await stat(lock).catch(() => undefined);
|
|
// A lock older than any snapshot takes is left by a container that was stopped mid-way.
|
|
if (held && Date.now() - held.mtimeMs < 2 * 3600_000) throw new Error('Another snapshot is being taken');
|
|
await rm(lock, { recursive: true, force: true });
|
|
await mkdir(lock);
|
|
const name = snapshotName(now);
|
|
const partial = path.join(root, `.partial-${name}`);
|
|
try {
|
|
for (const entry of retention(await readdir(root)).remove.filter(entry => entry.startsWith('.partial-')))
|
|
await rm(path.join(root, entry), { recursive: true, force: true });
|
|
await mkdir(partial, { mode: 0o700 });
|
|
const counts = await dumpDatabase(path.join(partial, 'postgres.dump'), { signal, database });
|
|
await copyTemporal(path.join(partial, 'temporal.db'), { signal, data });
|
|
for (const directory of ARCHIVES)
|
|
await archive(directory, path.join(partial, `${directory}.tgz`), { signal, data });
|
|
const files = {};
|
|
for (const file of (await readdir(partial)).sort()) {
|
|
const location = path.join(partial, file);
|
|
files[file] = { bytes: (await stat(location)).size, sha256: await sha256File(location) };
|
|
}
|
|
const manifest = {
|
|
name,
|
|
takenAt: now.toISOString(),
|
|
image: process.env.WF_IMAGE || undefined,
|
|
database: { counts },
|
|
files,
|
|
};
|
|
await writeFile(path.join(partial, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 });
|
|
await rename(partial, path.join(root, name));
|
|
for (const entry of retention(await readdir(root)).remove)
|
|
await rm(path.join(root, entry), { recursive: true, force: true });
|
|
return manifest;
|
|
} catch (error) {
|
|
await rm(partial, { recursive: true, force: true });
|
|
throw error;
|
|
} finally {
|
|
await rm(lock, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
async function record(root, result) {
|
|
await writeFile(path.join(root, 'last.json'), `${JSON.stringify(result, null, 2)}\n`, { mode: 0o600 });
|
|
}
|
|
|
|
/** The newest complete snapshot's manifest, and the last attempt's result. */
|
|
export async function latest(root = BACKUP_ROOT) {
|
|
const names = retention(await readdir(root).catch(() => [])).keep;
|
|
const newest = names.at(-1);
|
|
const read = async file => JSON.parse(await readFile(file, 'utf8'));
|
|
return {
|
|
snapshots: names,
|
|
manifest: newest ? await read(path.join(root, newest, 'manifest.json')) : undefined,
|
|
last: await read(path.join(root, 'last.json')).catch(() => undefined),
|
|
};
|
|
}
|
|
|
|
async function once(root, signal) {
|
|
const startedAt = new Date();
|
|
try {
|
|
const manifest = await takeSnapshot({ root, now: startedAt, signal });
|
|
const bytes = Object.values(manifest.files).reduce((sum, file) => sum + file.bytes, 0);
|
|
await record(root, { ok: true, name: manifest.name, at: new Date().toISOString(), bytes });
|
|
console.log(`[backup] snapshot ${manifest.name}: ${(bytes / 1e6).toFixed(1)} MB`);
|
|
return manifest;
|
|
} catch (error) {
|
|
if (signal?.aborted) throw error;
|
|
await record(root, { ok: false, at: new Date().toISOString(), error: error.message });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Takes a snapshot every day at {@link SCHEDULE}, and at once when the newest is more than a day
|
|
* old (a Pod that was down at the scheduled time). A failure is retried an hour later.
|
|
*/
|
|
export async function backupLoop({ root = BACKUP_ROOT } = {}) {
|
|
const stop = new AbortController();
|
|
process.once('SIGTERM', () => stop.abort());
|
|
const sleep = ms => delay(ms, undefined, { signal: stop.signal });
|
|
try {
|
|
const newest = (await latest(root)).manifest;
|
|
let due = newest && Date.now() - Date.parse(newest.takenAt) < 24 * 3600_000 ? nextRun(new Date()) : new Date();
|
|
// The database starts beside this container: give it time before the first attempt.
|
|
if (due <= new Date()) await sleep(60_000);
|
|
for (;;) {
|
|
await sleep(Math.max(0, due - Date.now()));
|
|
try {
|
|
await once(root, stop.signal);
|
|
due = nextRun(new Date());
|
|
} catch (error) {
|
|
if (stop.signal.aborted) break;
|
|
console.error(`[backup] ${error.message}; trying again in an hour`);
|
|
due = new Date(Date.now() + 3600_000);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
if (!stop.signal.aborted) throw error;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
export async function main(argv = process.argv.slice(2)) {
|
|
if (argv.length === 0) return backupLoop();
|
|
if (argv[0] === 'once' && argv.length === 1) {
|
|
await once(BACKUP_ROOT);
|
|
return 0;
|
|
}
|
|
if (argv[0] === 'verify' && argv.length === 2) {
|
|
const manifest = JSON.parse(await readFile(path.join(argv[1], 'manifest.json'), 'utf8'));
|
|
await verifyFiles(argv[1], manifest.files);
|
|
console.log(`[backup] snapshot ${manifest.name}: every file matches its manifest`);
|
|
return 0;
|
|
}
|
|
if (argv[0] === 'latest' && argv.length === 1) {
|
|
process.stdout.write(`${JSON.stringify(await latest())}\n`);
|
|
return 0;
|
|
}
|
|
throw new Error('Usage: cluster-poc-backup.mjs [once|latest|verify DIR]');
|
|
}
|
|
|
|
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url)
|
|
main().then(
|
|
code => process.exit(code),
|
|
error => {
|
|
console.error(`[backup] ${error.message}`);
|
|
process.exit(1);
|
|
},
|
|
);
|