#!/usr/bin/env node // Daily snapshots of WishForge's data in the POC cluster (docs/cluster-poc-deployment.md), taken by // the WishForge Pod's backup container onto the node's disk; `cluster-poc.mjs restore` puts one // back. Importing it starts nothing. // // node scripts/cluster-poc-backup.mjs take one snapshot a day, until Kubernetes stops the container // node scripts/cluster-poc-backup.mjs once take one snapshot now // node scripts/cluster-poc-backup.mjs latest print the newest complete snapshot's manifest and the last result // node scripts/cluster-poc-backup.mjs verify DIR check a snapshot's files against its manifest import { spawn } from 'node:child_process'; import { createHash } from 'node:crypto'; import { createReadStream } from 'node:fs'; import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { setTimeout as delay } from 'node:timers/promises'; /** Where the backup container finds what it saves (each mounted read-only) and keeps the snapshots. */ export const BACKUP_ROOT = '/var/lib/wishforge/backups'; export const DATA_ROOT = '/mnt/data'; /** The database beside it on the Pod's loopback; the password comes from PGPASSWORD. */ const DATABASE = { host: '127.0.0.1', port: 5432, user: 'wishforge', database: 'wishforge' }; /** The directories saved as archives, beside the database dump and Temporal's database. */ export const ARCHIVES = ['git', 'minio', 'platform-state', 'poc-state', 'runner-state', 'runner-state-fanyu']; /** Not saved: the platform container's caches, rebuilt when it starts, and MinIO's scratch space and trash. */ const ARCHIVE_EXCLUDES = { 'platform-state': ['./home'], minio: ['./.minio.sys/tmp'] }; /** Snapshots kept on the node. */ export const KEEP = 7; /** 03:20 in Beijing, when nobody is expected to be working. */ export const SCHEDULE = { hourUtc: 19, minute: 20 }; const NAME = /^\d{8}T\d{6}Z$/; /** Row counts of every table, to compare a copy with its source. */ export const COUNTS_SQL = `select string_agg(table_name || '=' || (xpath('/row/c/text()', query_to_xml('select count(*) as c from public.' || quote_ident(table_name), false, true, '')))[1]::text, ',' order by table_name) from information_schema.tables where table_schema = 'public' and table_type = 'BASE TABLE'`; /** A snapshot's name: its UTC time, so names sort in the order they were taken. */ export const snapshotName = date => date .toISOString() .replace(/[-:]/g, '') .replace(/\.\d+Z$/, 'Z'); /** The first scheduled time after `now`. */ export function nextRun(now, { hourUtc, minute } = SCHEDULE) { const next = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), hourUtc, minute)); if (next <= now) next.setUTCDate(next.getUTCDate() + 1); return next; } /** Which entries of the snapshot directory stay: the newest `keep` complete snapshots. Leftover partial ones go. */ export function retention(entries, keep = KEEP) { const complete = entries.filter(entry => NAME.test(entry)).sort(); return { keep: complete.slice(-keep), remove: [ ...complete.slice(0, Math.max(0, complete.length - keep)), ...entries.filter(e => e.startsWith('.partial-')), ], }; } /** Whether `files` (name → {bytes, sha256}) in `directory` are exactly what a manifest says. */ export async function verifyFiles(directory, files) { for (const [file, expected] of Object.entries(files)) { const location = path.join(directory, file); const info = await stat(location).catch(() => undefined); if (!info) throw new Error(`${file} is missing`); if (info.size !== expected.bytes) throw new Error(`${file} has ${info.size} bytes, not ${expected.bytes}`); if ((await sha256File(location)) !== expected.sha256) throw new Error(`${file} does not match its SHA-256`); } } export const sha256File = file => new Promise((resolve, reject) => { const hash = createHash('sha256'); createReadStream(file) .on('data', chunk => hash.update(chunk)) .on('error', reject) .on('end', () => resolve(hash.digest('hex'))); }); function run(executable, args, { signal, env = process.env } = {}) { return new Promise((resolve, reject) => { const child = spawn(executable, args, { env, signal, stdio: ['ignore', 'pipe', 'pipe'] }); const out = []; const err = []; child.stdout.on('data', chunk => out.push(chunk)); child.stderr.on('data', chunk => err.push(chunk)); child.once('error', reject); child.once('close', code => resolve({ code, stdout: Buffer.concat(out).toString('utf8'), stderr: Buffer.concat(err).toString('utf8') }), ); }); } async function must(executable, args, options) { const result = await run(executable, args, options); if (result.code !== 0) throw new Error(`${path.basename(executable)} failed: ${result.stderr.trim().slice(0, 800)}`); return result; } /** * The database as of one moment: pg_dump reads the snapshot of a transaction that also counts every * table's rows, so a restore can be checked against exactly what was dumped. */ async function dumpDatabase(file, { signal, database }) { const { default: pg } = await import('pg'); const client = new pg.Client({ ...database, password: process.env.PGPASSWORD, }); await client.connect(); try { await client.query('begin isolation level repeatable read read only'); const snapshot = (await client.query('select pg_export_snapshot() as id')).rows[0].id; const counts = (await client.query(`select (${COUNTS_SQL}) as counts`)).rows[0].counts; await must( 'pg_dump', [ '-h', database.host, '-p', String(database.port), '-U', database.user, '-d', database.database, '-Fc', '--no-owner', '--no-privileges', `--snapshot=${snapshot}`, '-f', file, ], { signal }, ); await client.query('commit'); // The archive's table of contents reads back: the dump is complete. await must('pg_restore', ['-l', file], { signal }); return counts; } finally { await client.end().catch(() => undefined); } } /** Temporal's SQLite database through SQLite's online backup, which waits out its writers. */ async function copyTemporal(file, { signal, data }) { const source = path.join(data, 'temporal/temporal.db'); await must('sqlite3', ['-readonly', '-cmd', '.timeout 60000', source, `.backup '${file}'`], { signal }); const check = await must('sqlite3', [file, 'pragma integrity_check'], { signal }); if (check.stdout.trim() !== 'ok') throw new Error(`Temporal's copy fails its integrity check: ${check.stdout.trim()}`); } /** A directory as a gzip archive; tried again while files change under it (GNU tar's exit code 1). */ async function archive(name, file, { signal, data }) { const args = [ '-czf', file, ...(ARCHIVE_EXCLUDES[name] ?? []).map(entry => `--exclude=${entry}`), '-C', path.join(data, name), '.', ]; for (let attempt = 1; ; attempt++) { const result = await run('tar', args, { signal }); if (result.code === 0) return; if (result.code !== 1 || attempt === 3) throw new Error(`Archiving ${name} failed: ${result.stderr.trim().slice(0, 800)}`); await delay(5000, undefined, { signal }); } } /** One snapshot, complete or not at all: built under a partial name, renamed once its manifest is written. */ export async function takeSnapshot({ root = BACKUP_ROOT, data = DATA_ROOT, database = DATABASE, now = new Date(), signal, } = {}) { await mkdir(root, { recursive: true, mode: 0o700 }); const lock = path.join(root, '.lock'); const held = await stat(lock).catch(() => undefined); // A lock older than any snapshot takes is left by a container that was stopped mid-way. if (held && Date.now() - held.mtimeMs < 2 * 3600_000) throw new Error('Another snapshot is being taken'); await rm(lock, { recursive: true, force: true }); await mkdir(lock); const name = snapshotName(now); const partial = path.join(root, `.partial-${name}`); try { for (const entry of retention(await readdir(root)).remove.filter(entry => entry.startsWith('.partial-'))) await rm(path.join(root, entry), { recursive: true, force: true }); await mkdir(partial, { mode: 0o700 }); const counts = await dumpDatabase(path.join(partial, 'postgres.dump'), { signal, database }); await copyTemporal(path.join(partial, 'temporal.db'), { signal, data }); for (const directory of ARCHIVES) await archive(directory, path.join(partial, `${directory}.tgz`), { signal, data }); const files = {}; for (const file of (await readdir(partial)).sort()) { const location = path.join(partial, file); files[file] = { bytes: (await stat(location)).size, sha256: await sha256File(location) }; } const manifest = { name, takenAt: now.toISOString(), image: process.env.WF_IMAGE || undefined, database: { counts }, files, }; await writeFile(path.join(partial, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 }); await rename(partial, path.join(root, name)); for (const entry of retention(await readdir(root)).remove) await rm(path.join(root, entry), { recursive: true, force: true }); return manifest; } catch (error) { await rm(partial, { recursive: true, force: true }); throw error; } finally { await rm(lock, { recursive: true, force: true }); } } async function record(root, result) { await writeFile(path.join(root, 'last.json'), `${JSON.stringify(result, null, 2)}\n`, { mode: 0o600 }); } /** The newest complete snapshot's manifest, and the last attempt's result. */ export async function latest(root = BACKUP_ROOT) { const names = retention(await readdir(root).catch(() => [])).keep; const newest = names.at(-1); const read = async file => JSON.parse(await readFile(file, 'utf8')); return { snapshots: names, manifest: newest ? await read(path.join(root, newest, 'manifest.json')) : undefined, last: await read(path.join(root, 'last.json')).catch(() => undefined), }; } async function once(root, signal) { const startedAt = new Date(); try { const manifest = await takeSnapshot({ root, now: startedAt, signal }); const bytes = Object.values(manifest.files).reduce((sum, file) => sum + file.bytes, 0); await record(root, { ok: true, name: manifest.name, at: new Date().toISOString(), bytes }); console.log(`[backup] snapshot ${manifest.name}: ${(bytes / 1e6).toFixed(1)} MB`); return manifest; } catch (error) { if (signal?.aborted) throw error; await record(root, { ok: false, at: new Date().toISOString(), error: error.message }); throw error; } } /** * Takes a snapshot every day at {@link SCHEDULE}, and at once when the newest is more than a day * old (a Pod that was down at the scheduled time). A failure is retried an hour later. */ export async function backupLoop({ root = BACKUP_ROOT } = {}) { const stop = new AbortController(); process.once('SIGTERM', () => stop.abort()); const sleep = ms => delay(ms, undefined, { signal: stop.signal }); try { const newest = (await latest(root)).manifest; let due = newest && Date.now() - Date.parse(newest.takenAt) < 24 * 3600_000 ? nextRun(new Date()) : new Date(); // The database starts beside this container: give it time before the first attempt. if (due <= new Date()) await sleep(60_000); for (;;) { await sleep(Math.max(0, due - Date.now())); try { await once(root, stop.signal); due = nextRun(new Date()); } catch (error) { if (stop.signal.aborted) break; console.error(`[backup] ${error.message}; trying again in an hour`); due = new Date(Date.now() + 3600_000); } } } catch (error) { if (!stop.signal.aborted) throw error; } return 0; } export async function main(argv = process.argv.slice(2)) { if (argv.length === 0) return backupLoop(); if (argv[0] === 'once' && argv.length === 1) { await once(BACKUP_ROOT); return 0; } if (argv[0] === 'verify' && argv.length === 2) { const manifest = JSON.parse(await readFile(path.join(argv[1], 'manifest.json'), 'utf8')); await verifyFiles(argv[1], manifest.files); console.log(`[backup] snapshot ${manifest.name}: every file matches its manifest`); return 0; } if (argv[0] === 'latest' && argv.length === 1) { process.stdout.write(`${JSON.stringify(await latest())}\n`); return 0; } throw new Error('Usage: cluster-poc-backup.mjs [once|latest|verify DIR]'); } if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) main().then( code => process.exit(code), error => { console.error(`[backup] ${error.message}`); process.exit(1); }, );