"""Run the snapshot script with local kubectl/docker stand-ins; no network or DB. The remote shell, split, cat and local SHA-256 check remain real commands. Run: python3 -m unittest discover -s tests/backend -p fetch_mongo_snapshot_script_test.py -v """ import hashlib import json import os import pathlib import shutil import signal import subprocess import sys import tempfile import time import unittest ROOT = pathlib.Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts/local/fetch-mongo-snapshot.sh" CHUNK = 512000 # Only external boundaries are faked. kubectl runs the supplied remote command # in a temporary filesystem; Docker records the bytes that would be restored. SHIM = r''' import hashlib, json, os, pathlib, re, shutil, subprocess, sys, time root = pathlib.Path(os.environ["SNAPSHOT_TEST_ROOT"]) name = pathlib.Path(sys.argv[0]).name args = sys.argv[1:] fault = os.environ.get("SNAPSHOT_TEST_FAULT", "") if name == "sleep": sys.exit(0) if name == "shasum": if fault == "hash_failed": raise SystemExit("simulated SHA-256 command failure") os.execv(os.environ["SNAPSHOT_TEST_SHASUM"], ["shasum"] + args) if name == "stat": if args[:2] != ["-c", "%s"]: raise SystemExit("unexpected stat arguments") print(pathlib.Path(args[2]).stat().st_size) sys.exit(0) with (root / (name + ".jsonl")).open("a") as log: log.write(json.dumps(args) + "\n") if name == "docker": if args[0] == "cp": shutil.copyfile(args[1], root / "container-backup.gz") elif args[:3] == ["exec", "fanyu-mongo-restore", "mongorestore"]: if "--drop" not in args or "--gzip" not in args: raise SystemExit("unexpected restore arguments") data = (root / "container-backup.gz").read_bytes() (root / "restored.json").write_text(json.dumps({ "size": len(data), "sha256": hashlib.sha256(data).hexdigest() })) elif args[0] not in ("rm", "run", "exec"): raise SystemExit("unexpected docker command") sys.exit(0) if name != "kubectl": raise SystemExit("unexpected shim") if args[2:4] == ["get", "pod"]: print("local-fixture-pod", end="") sys.exit(0) if args[2] != "exec" or "--" not in args: raise SystemExit("unexpected kubectl command") command = args[args.index("--") + 1:] if command[:2] == ["sh", "-c"] and "mktemp -d " in command[2]: if fault in ("mktemp_failed", "mktemp_invalid"): print("/tmp") sys.exit(1 if fault == "mktemp_failed" else 0) # Keep failure-injection tests safe even if production issues an unsafe rm. # The call is already recorded, so the test still detects the regression. if command[0] == "rm" and not re.fullmatch( r"/tmp/fanyu-mongo-snapshot\.[a-zA-Z0-9]+", command[-1]): raise SystemExit("refusing unsafe cleanup target in test sandbox") # Reject the legacy shared directory before its local counterpart can be # touched. This also makes a pre-fix regression run safe on a developer laptop. if any("/tmp/fetch-chunks" in part for part in command): raise SystemExit("unsafe shared temporary directory") remote = root / "remote" mapped = [part.replace("/tmp/", str(remote / "tmp") + "/") .replace("/workspace/", str(remote / "workspace") + "/") for part in command] if command[0] == "cat" and re.search(r"/p[0-9]+$", command[-1]): chunk = pathlib.Path(mapped[-1]) if fault == "missing": chunk.unlink(missing_ok=True) elif fault == "download_failed": sys.stdout.buffer.write(chunk.read_bytes()[:7]) sys.exit(1) elif fault == "truncated": sys.stdout.buffer.write(chunk.read_bytes()[:-1]) sys.exit(0) elif fault == "interrupt": (root / "download-started").touch() time.sleep(30) result = subprocess.run(mapped, stdout=subprocess.PIPE, cwd=root) sys.stdout.buffer.write(result.stdout.replace(str(remote).encode(), b"")) sys.exit(result.returncode) ''' class SnapshotScriptTest(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory(prefix="snapshot-script-test-") self.addCleanup(self.temp.cleanup) self.root = pathlib.Path(self.temp.name) self.bin = self.root / "bin" self.bin.mkdir() shim = self.bin / "shim" shim.write_text(f"#!{sys.executable}\n" + SHIM) shim.chmod(0o755) for name in ("kubectl", "docker", "sleep", "stat", "shasum"): (self.bin / name).symlink_to(shim) self.local_tmp = self.root / "local-tmp" self.remote_tmp = self.root / "remote/tmp" self.local_tmp.mkdir() self.remote_tmp.mkdir(parents=True) self.backup = self.root / "remote/workspace/system/backups/fixture" self.backup.mkdir(parents=True) self.env = dict(os.environ, PATH=f"{self.bin}:{os.environ['PATH']}", TMPDIR=str(self.local_tmp), KUBECONFIG="unused-test-config", SNAPSHOT_TEST_ROOT=str(self.root), SNAPSHOT_TEST_SHASUM=shutil.which("shasum")) # Another invocation's files must survive success, failure and signals. for directory in (self.local_tmp, self.remote_tmp): other = directory / "fanyu-mongo-snapshot.other-run" other.mkdir() (other / "keep").write_text("another invocation") def fixture(self, count=2, checksum=None): archive = self.backup / "mongo.archive.gz" digest = hashlib.sha256() # Distinct chunk contents expose incorrect ordering as well as omission. with archive.open("wb") as output: for index in range(count): block = bytes([index % 251]) * (CHUNK if index < count - 1 else 137) output.write(block) digest.update(block) self.expected = {"size": archive.stat().st_size, "sha256": digest.hexdigest()} (self.backup / "mongo.archive.gz.sha256").write_text( f"{checksum or digest.hexdigest()} mongo.archive.gz\n") def run_script(self, fault="", backup="fixture"): result = subprocess.run(["bash", str(SCRIPT), backup], env=dict( self.env, SNAPSHOT_TEST_FAULT=fault), capture_output=True, text=True, timeout=30) self.output = result.stdout + result.stderr return result def calls(self, command): log = self.root / f"{command}.jsonl" return [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] def assert_cleanup(self, success=False): for directory in (self.local_tmp, self.remote_tmp): self.assertEqual((directory / "fanyu-mongo-snapshot.other-run/keep").read_text(), "another invocation") owned = [p for p in directory.iterdir() if p.name != "fanyu-mongo-snapshot.other-run"] if success and directory == self.local_tmp: self.assertEqual(len(owned), 1, self.output) self.assertEqual([p.name for p in owned[0].iterdir()], ["mongo.archive.gz"]) else: self.assertEqual(owned, [], self.output) def test_restores_identical_archive_for_chunk_count_boundaries(self): for count in (1, 2, 9, 10, 99, 100): with self.subTest(chunks=count): self.fixture(count) result = self.run_script() self.assertEqual(result.returncode, 0, self.output) self.assertEqual(json.loads((self.root / "restored.json").read_text()), self.expected) self.assert_cleanup(success=True) for p in self.local_tmp.iterdir(): if p.name != "fanyu-mongo-snapshot.other-run": shutil.rmtree(p) def test_incomplete_downloads_cannot_touch_local_database(self): for fault in ("missing", "download_failed", "truncated", "hash_failed"): with self.subTest(fault=fault): self.fixture() result = self.run_script(fault) self.assertNotEqual(result.returncode, 0, self.output) self.assertEqual(self.calls("docker"), [], self.output) self.assert_cleanup() def test_hash_mismatch_cannot_touch_local_database(self): self.fixture(checksum="0" * 64) result = self.run_script() self.assertNotEqual(result.returncode, 0, self.output) self.assertEqual(self.calls("docker"), [], self.output) self.assert_cleanup() def test_missing_or_invalid_checksum_cannot_touch_local_database(self): for checksum in (None, "", "not-a-sha256"): with self.subTest(checksum=checksum): self.fixture() checksum_path = self.backup / "mongo.archive.gz.sha256" if checksum is None: checksum_path.unlink() else: checksum_path.write_text(checksum) result = self.run_script() self.assertNotEqual(result.returncode, 0, self.output) self.assertEqual(self.calls("docker"), [], self.output) self.assert_cleanup() def test_invalid_backup_names_are_rejected_before_remote_execution(self): for backup in ("../fixture", "fixture/other", "fixture;touch injected", "$(touch injected)", "-x", ""): with self.subTest(backup=backup): result = self.run_script(backup=backup) self.assertNotEqual(result.returncode, 0, self.output) self.assertEqual(self.calls("kubectl"), [], self.output) self.assertEqual(self.calls("docker"), [], self.output) def test_remote_mktemp_failure_or_invalid_output_never_becomes_cleanup_target(self): self.fixture() for fault in ("mktemp_failed", "mktemp_invalid"): with self.subTest(fault=fault): previous_calls = len(self.calls("kubectl")) result = self.run_script(fault) self.assertNotEqual(result.returncode, 0, self.output) remote_commands = [args[args.index("--") + 1:] for args in self.calls("kubectl")[previous_calls:] if "--" in args] self.assertEqual([command for command in remote_commands if command[0] == "rm"], [], self.output) self.assertEqual(self.calls("docker"), [], self.output) self.assert_cleanup() def test_interrupted_download_cleans_own_files_and_retry_succeeds(self): self.fixture() process = subprocess.Popen(["bash", str(SCRIPT), "fixture"], env=dict( self.env, SNAPSHOT_TEST_FAULT="interrupt"), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) try: deadline = time.monotonic() + 10 while not (self.root / "download-started").exists() and process.poll() is None: if time.monotonic() > deadline: self.fail("script never started downloading") time.sleep(0.02) if process.poll() is None: os.killpg(process.pid, signal.SIGTERM) stdout, stderr = process.communicate(timeout=10) self.output = stdout + stderr self.assertTrue((self.root / "download-started").exists(), self.output) self.assertNotEqual(process.returncode, 0, self.output) self.assertEqual(self.calls("docker"), [], self.output) self.assert_cleanup() finally: if process.poll() is None: os.killpg(process.pid, signal.SIGKILL) process.communicate() result = self.run_script() self.assertEqual(result.returncode, 0, self.output) self.assertEqual(json.loads((self.root / "restored.json").read_text()), self.expected) self.assert_cleanup(success=True) if __name__ == "__main__": unittest.main()