droboticflow/tests/backend/fetch_mongo_snapshot_script_test.py

259 lines
12 KiB
Python

"""Run the snapshot script with local kubectl/docker stand-ins; no network or DB.
The remote shell, split, cat and local SHA-256 check remain real commands.
Run: python3 -m unittest discover -s tests/backend -p fetch_mongo_snapshot_script_test.py -v
"""
import hashlib
import json
import os
import pathlib
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import unittest
ROOT = pathlib.Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts/local/fetch-mongo-snapshot.sh"
CHUNK = 512000
# Only external boundaries are faked. kubectl runs the supplied remote command
# in a temporary filesystem; Docker records the bytes that would be restored.
SHIM = r'''
import hashlib, json, os, pathlib, re, shutil, subprocess, sys, time
root = pathlib.Path(os.environ["SNAPSHOT_TEST_ROOT"])
name = pathlib.Path(sys.argv[0]).name
args = sys.argv[1:]
fault = os.environ.get("SNAPSHOT_TEST_FAULT", "")
if name == "sleep":
sys.exit(0)
if name == "shasum":
if fault == "hash_failed":
raise SystemExit("simulated SHA-256 command failure")
os.execv(os.environ["SNAPSHOT_TEST_SHASUM"], ["shasum"] + args)
if name == "stat":
if args[:2] != ["-c", "%s"]:
raise SystemExit("unexpected stat arguments")
print(pathlib.Path(args[2]).stat().st_size)
sys.exit(0)
with (root / (name + ".jsonl")).open("a") as log:
log.write(json.dumps(args) + "\n")
if name == "docker":
if args[0] == "cp":
shutil.copyfile(args[1], root / "container-backup.gz")
elif args[:3] == ["exec", "fanyu-mongo-restore", "mongorestore"]:
if "--drop" not in args or "--gzip" not in args:
raise SystemExit("unexpected restore arguments")
data = (root / "container-backup.gz").read_bytes()
(root / "restored.json").write_text(json.dumps({
"size": len(data), "sha256": hashlib.sha256(data).hexdigest()
}))
elif args[0] not in ("rm", "run", "exec"):
raise SystemExit("unexpected docker command")
sys.exit(0)
if name != "kubectl":
raise SystemExit("unexpected shim")
if args[2:4] == ["get", "pod"]:
print("local-fixture-pod", end="")
sys.exit(0)
if args[2] != "exec" or "--" not in args:
raise SystemExit("unexpected kubectl command")
command = args[args.index("--") + 1:]
if command[:2] == ["sh", "-c"] and "mktemp -d " in command[2]:
if fault in ("mktemp_failed", "mktemp_invalid"):
print("/tmp")
sys.exit(1 if fault == "mktemp_failed" else 0)
# Keep failure-injection tests safe even if production issues an unsafe rm.
# The call is already recorded, so the test still detects the regression.
if command[0] == "rm" and not re.fullmatch(
r"/tmp/fanyu-mongo-snapshot\.[a-zA-Z0-9]+", command[-1]):
raise SystemExit("refusing unsafe cleanup target in test sandbox")
# Reject the legacy shared directory before its local counterpart can be
# touched. This also makes a pre-fix regression run safe on a developer laptop.
if any("/tmp/fetch-chunks" in part for part in command):
raise SystemExit("unsafe shared temporary directory")
remote = root / "remote"
mapped = [part.replace("/tmp/", str(remote / "tmp") + "/")
.replace("/workspace/", str(remote / "workspace") + "/")
for part in command]
if command[0] == "cat" and re.search(r"/p[0-9]+$", command[-1]):
chunk = pathlib.Path(mapped[-1])
if fault == "missing":
chunk.unlink(missing_ok=True)
elif fault == "download_failed":
sys.stdout.buffer.write(chunk.read_bytes()[:7])
sys.exit(1)
elif fault == "truncated":
sys.stdout.buffer.write(chunk.read_bytes()[:-1])
sys.exit(0)
elif fault == "interrupt":
(root / "download-started").touch()
time.sleep(30)
result = subprocess.run(mapped, stdout=subprocess.PIPE, cwd=root)
sys.stdout.buffer.write(result.stdout.replace(str(remote).encode(), b""))
sys.exit(result.returncode)
'''
class SnapshotScriptTest(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix="snapshot-script-test-")
self.addCleanup(self.temp.cleanup)
self.root = pathlib.Path(self.temp.name)
self.bin = self.root / "bin"
self.bin.mkdir()
shim = self.bin / "shim"
shim.write_text(f"#!{sys.executable}\n" + SHIM)
shim.chmod(0o755)
for name in ("kubectl", "docker", "sleep", "stat", "shasum"):
(self.bin / name).symlink_to(shim)
self.local_tmp = self.root / "local-tmp"
self.remote_tmp = self.root / "remote/tmp"
self.local_tmp.mkdir()
self.remote_tmp.mkdir(parents=True)
self.backup = self.root / "remote/workspace/system/backups/fixture"
self.backup.mkdir(parents=True)
self.env = dict(os.environ, PATH=f"{self.bin}:{os.environ['PATH']}",
TMPDIR=str(self.local_tmp), KUBECONFIG="unused-test-config",
SNAPSHOT_TEST_ROOT=str(self.root),
SNAPSHOT_TEST_SHASUM=shutil.which("shasum"))
# Another invocation's files must survive success, failure and signals.
for directory in (self.local_tmp, self.remote_tmp):
other = directory / "fanyu-mongo-snapshot.other-run"
other.mkdir()
(other / "keep").write_text("another invocation")
def fixture(self, count=2, checksum=None):
archive = self.backup / "mongo.archive.gz"
digest = hashlib.sha256()
# Distinct chunk contents expose incorrect ordering as well as omission.
with archive.open("wb") as output:
for index in range(count):
block = bytes([index % 251]) * (CHUNK if index < count - 1 else 137)
output.write(block)
digest.update(block)
self.expected = {"size": archive.stat().st_size, "sha256": digest.hexdigest()}
(self.backup / "mongo.archive.gz.sha256").write_text(
f"{checksum or digest.hexdigest()} mongo.archive.gz\n")
def run_script(self, fault="", backup="fixture"):
result = subprocess.run(["bash", str(SCRIPT), backup], env=dict(
self.env, SNAPSHOT_TEST_FAULT=fault), capture_output=True, text=True, timeout=30)
self.output = result.stdout + result.stderr
return result
def calls(self, command):
log = self.root / f"{command}.jsonl"
return [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []
def assert_cleanup(self, success=False):
for directory in (self.local_tmp, self.remote_tmp):
self.assertEqual((directory / "fanyu-mongo-snapshot.other-run/keep").read_text(),
"another invocation")
owned = [p for p in directory.iterdir() if p.name != "fanyu-mongo-snapshot.other-run"]
if success and directory == self.local_tmp:
self.assertEqual(len(owned), 1, self.output)
self.assertEqual([p.name for p in owned[0].iterdir()], ["mongo.archive.gz"])
else:
self.assertEqual(owned, [], self.output)
def test_restores_identical_archive_for_chunk_count_boundaries(self):
for count in (1, 2, 9, 10, 99, 100):
with self.subTest(chunks=count):
self.fixture(count)
result = self.run_script()
self.assertEqual(result.returncode, 0, self.output)
self.assertEqual(json.loads((self.root / "restored.json").read_text()), self.expected)
self.assert_cleanup(success=True)
for p in self.local_tmp.iterdir():
if p.name != "fanyu-mongo-snapshot.other-run":
shutil.rmtree(p)
def test_incomplete_downloads_cannot_touch_local_database(self):
for fault in ("missing", "download_failed", "truncated", "hash_failed"):
with self.subTest(fault=fault):
self.fixture()
result = self.run_script(fault)
self.assertNotEqual(result.returncode, 0, self.output)
self.assertEqual(self.calls("docker"), [], self.output)
self.assert_cleanup()
def test_hash_mismatch_cannot_touch_local_database(self):
self.fixture(checksum="0" * 64)
result = self.run_script()
self.assertNotEqual(result.returncode, 0, self.output)
self.assertEqual(self.calls("docker"), [], self.output)
self.assert_cleanup()
def test_missing_or_invalid_checksum_cannot_touch_local_database(self):
for checksum in (None, "", "not-a-sha256"):
with self.subTest(checksum=checksum):
self.fixture()
checksum_path = self.backup / "mongo.archive.gz.sha256"
if checksum is None:
checksum_path.unlink()
else:
checksum_path.write_text(checksum)
result = self.run_script()
self.assertNotEqual(result.returncode, 0, self.output)
self.assertEqual(self.calls("docker"), [], self.output)
self.assert_cleanup()
def test_invalid_backup_names_are_rejected_before_remote_execution(self):
for backup in ("../fixture", "fixture/other", "fixture;touch injected", "$(touch injected)", "-x", ""):
with self.subTest(backup=backup):
result = self.run_script(backup=backup)
self.assertNotEqual(result.returncode, 0, self.output)
self.assertEqual(self.calls("kubectl"), [], self.output)
self.assertEqual(self.calls("docker"), [], self.output)
def test_remote_mktemp_failure_or_invalid_output_never_becomes_cleanup_target(self):
self.fixture()
for fault in ("mktemp_failed", "mktemp_invalid"):
with self.subTest(fault=fault):
previous_calls = len(self.calls("kubectl"))
result = self.run_script(fault)
self.assertNotEqual(result.returncode, 0, self.output)
remote_commands = [args[args.index("--") + 1:]
for args in self.calls("kubectl")[previous_calls:] if "--" in args]
self.assertEqual([command for command in remote_commands if command[0] == "rm"],
[], self.output)
self.assertEqual(self.calls("docker"), [], self.output)
self.assert_cleanup()
def test_interrupted_download_cleans_own_files_and_retry_succeeds(self):
self.fixture()
process = subprocess.Popen(["bash", str(SCRIPT), "fixture"], env=dict(
self.env, SNAPSHOT_TEST_FAULT="interrupt"), stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, start_new_session=True)
try:
deadline = time.monotonic() + 10
while not (self.root / "download-started").exists() and process.poll() is None:
if time.monotonic() > deadline:
self.fail("script never started downloading")
time.sleep(0.02)
if process.poll() is None:
os.killpg(process.pid, signal.SIGTERM)
stdout, stderr = process.communicate(timeout=10)
self.output = stdout + stderr
self.assertTrue((self.root / "download-started").exists(), self.output)
self.assertNotEqual(process.returncode, 0, self.output)
self.assertEqual(self.calls("docker"), [], self.output)
self.assert_cleanup()
finally:
if process.poll() is None:
os.killpg(process.pid, signal.SIGKILL)
process.communicate()
result = self.run_script()
self.assertEqual(result.returncode, 0, self.output)
self.assertEqual(json.loads((self.root / "restored.json").read_text()), self.expected)
self.assert_cleanup(success=True)
if __name__ == "__main__":
unittest.main()