forked from longtao.wu/droboticflow
259 lines
12 KiB
Python
259 lines
12 KiB
Python
"""Run the snapshot script with local kubectl/docker stand-ins; no network or DB.
|
|
|
|
The remote shell, split, cat and local SHA-256 check remain real commands.
|
|
Run: python3 -m unittest discover -s tests/backend -p fetch_mongo_snapshot_script_test.py -v
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts/local/fetch-mongo-snapshot.sh"
|
|
CHUNK = 512000
|
|
|
|
# Only external boundaries are faked. kubectl runs the supplied remote command
|
|
# in a temporary filesystem; Docker records the bytes that would be restored.
|
|
SHIM = r'''
|
|
import hashlib, json, os, pathlib, re, shutil, subprocess, sys, time
|
|
root = pathlib.Path(os.environ["SNAPSHOT_TEST_ROOT"])
|
|
name = pathlib.Path(sys.argv[0]).name
|
|
args = sys.argv[1:]
|
|
fault = os.environ.get("SNAPSHOT_TEST_FAULT", "")
|
|
if name == "sleep":
|
|
sys.exit(0)
|
|
if name == "shasum":
|
|
if fault == "hash_failed":
|
|
raise SystemExit("simulated SHA-256 command failure")
|
|
os.execv(os.environ["SNAPSHOT_TEST_SHASUM"], ["shasum"] + args)
|
|
if name == "stat":
|
|
if args[:2] != ["-c", "%s"]:
|
|
raise SystemExit("unexpected stat arguments")
|
|
print(pathlib.Path(args[2]).stat().st_size)
|
|
sys.exit(0)
|
|
with (root / (name + ".jsonl")).open("a") as log:
|
|
log.write(json.dumps(args) + "\n")
|
|
if name == "docker":
|
|
if args[0] == "cp":
|
|
shutil.copyfile(args[1], root / "container-backup.gz")
|
|
elif args[:3] == ["exec", "fanyu-mongo-restore", "mongorestore"]:
|
|
if "--drop" not in args or "--gzip" not in args:
|
|
raise SystemExit("unexpected restore arguments")
|
|
data = (root / "container-backup.gz").read_bytes()
|
|
(root / "restored.json").write_text(json.dumps({
|
|
"size": len(data), "sha256": hashlib.sha256(data).hexdigest()
|
|
}))
|
|
elif args[0] not in ("rm", "run", "exec"):
|
|
raise SystemExit("unexpected docker command")
|
|
sys.exit(0)
|
|
if name != "kubectl":
|
|
raise SystemExit("unexpected shim")
|
|
if args[2:4] == ["get", "pod"]:
|
|
print("local-fixture-pod", end="")
|
|
sys.exit(0)
|
|
if args[2] != "exec" or "--" not in args:
|
|
raise SystemExit("unexpected kubectl command")
|
|
command = args[args.index("--") + 1:]
|
|
if command[:2] == ["sh", "-c"] and "mktemp -d " in command[2]:
|
|
if fault in ("mktemp_failed", "mktemp_invalid"):
|
|
print("/tmp")
|
|
sys.exit(1 if fault == "mktemp_failed" else 0)
|
|
# Keep failure-injection tests safe even if production issues an unsafe rm.
|
|
# The call is already recorded, so the test still detects the regression.
|
|
if command[0] == "rm" and not re.fullmatch(
|
|
r"/tmp/fanyu-mongo-snapshot\.[a-zA-Z0-9]+", command[-1]):
|
|
raise SystemExit("refusing unsafe cleanup target in test sandbox")
|
|
# Reject the legacy shared directory before its local counterpart can be
|
|
# touched. This also makes a pre-fix regression run safe on a developer laptop.
|
|
if any("/tmp/fetch-chunks" in part for part in command):
|
|
raise SystemExit("unsafe shared temporary directory")
|
|
remote = root / "remote"
|
|
mapped = [part.replace("/tmp/", str(remote / "tmp") + "/")
|
|
.replace("/workspace/", str(remote / "workspace") + "/")
|
|
for part in command]
|
|
if command[0] == "cat" and re.search(r"/p[0-9]+$", command[-1]):
|
|
chunk = pathlib.Path(mapped[-1])
|
|
if fault == "missing":
|
|
chunk.unlink(missing_ok=True)
|
|
elif fault == "download_failed":
|
|
sys.stdout.buffer.write(chunk.read_bytes()[:7])
|
|
sys.exit(1)
|
|
elif fault == "truncated":
|
|
sys.stdout.buffer.write(chunk.read_bytes()[:-1])
|
|
sys.exit(0)
|
|
elif fault == "interrupt":
|
|
(root / "download-started").touch()
|
|
time.sleep(30)
|
|
result = subprocess.run(mapped, stdout=subprocess.PIPE, cwd=root)
|
|
sys.stdout.buffer.write(result.stdout.replace(str(remote).encode(), b""))
|
|
sys.exit(result.returncode)
|
|
'''
|
|
|
|
|
|
class SnapshotScriptTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory(prefix="snapshot-script-test-")
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = pathlib.Path(self.temp.name)
|
|
self.bin = self.root / "bin"
|
|
self.bin.mkdir()
|
|
shim = self.bin / "shim"
|
|
shim.write_text(f"#!{sys.executable}\n" + SHIM)
|
|
shim.chmod(0o755)
|
|
for name in ("kubectl", "docker", "sleep", "stat", "shasum"):
|
|
(self.bin / name).symlink_to(shim)
|
|
self.local_tmp = self.root / "local-tmp"
|
|
self.remote_tmp = self.root / "remote/tmp"
|
|
self.local_tmp.mkdir()
|
|
self.remote_tmp.mkdir(parents=True)
|
|
self.backup = self.root / "remote/workspace/system/backups/fixture"
|
|
self.backup.mkdir(parents=True)
|
|
self.env = dict(os.environ, PATH=f"{self.bin}:{os.environ['PATH']}",
|
|
TMPDIR=str(self.local_tmp), KUBECONFIG="unused-test-config",
|
|
SNAPSHOT_TEST_ROOT=str(self.root),
|
|
SNAPSHOT_TEST_SHASUM=shutil.which("shasum"))
|
|
# Another invocation's files must survive success, failure and signals.
|
|
for directory in (self.local_tmp, self.remote_tmp):
|
|
other = directory / "fanyu-mongo-snapshot.other-run"
|
|
other.mkdir()
|
|
(other / "keep").write_text("another invocation")
|
|
|
|
def fixture(self, count=2, checksum=None):
|
|
archive = self.backup / "mongo.archive.gz"
|
|
digest = hashlib.sha256()
|
|
# Distinct chunk contents expose incorrect ordering as well as omission.
|
|
with archive.open("wb") as output:
|
|
for index in range(count):
|
|
block = bytes([index % 251]) * (CHUNK if index < count - 1 else 137)
|
|
output.write(block)
|
|
digest.update(block)
|
|
self.expected = {"size": archive.stat().st_size, "sha256": digest.hexdigest()}
|
|
(self.backup / "mongo.archive.gz.sha256").write_text(
|
|
f"{checksum or digest.hexdigest()} mongo.archive.gz\n")
|
|
|
|
def run_script(self, fault="", backup="fixture"):
|
|
result = subprocess.run(["bash", str(SCRIPT), backup], env=dict(
|
|
self.env, SNAPSHOT_TEST_FAULT=fault), capture_output=True, text=True, timeout=30)
|
|
self.output = result.stdout + result.stderr
|
|
return result
|
|
|
|
def calls(self, command):
|
|
log = self.root / f"{command}.jsonl"
|
|
return [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []
|
|
|
|
def assert_cleanup(self, success=False):
|
|
for directory in (self.local_tmp, self.remote_tmp):
|
|
self.assertEqual((directory / "fanyu-mongo-snapshot.other-run/keep").read_text(),
|
|
"another invocation")
|
|
owned = [p for p in directory.iterdir() if p.name != "fanyu-mongo-snapshot.other-run"]
|
|
if success and directory == self.local_tmp:
|
|
self.assertEqual(len(owned), 1, self.output)
|
|
self.assertEqual([p.name for p in owned[0].iterdir()], ["mongo.archive.gz"])
|
|
else:
|
|
self.assertEqual(owned, [], self.output)
|
|
|
|
def test_restores_identical_archive_for_chunk_count_boundaries(self):
|
|
for count in (1, 2, 9, 10, 99, 100):
|
|
with self.subTest(chunks=count):
|
|
self.fixture(count)
|
|
result = self.run_script()
|
|
self.assertEqual(result.returncode, 0, self.output)
|
|
self.assertEqual(json.loads((self.root / "restored.json").read_text()), self.expected)
|
|
self.assert_cleanup(success=True)
|
|
for p in self.local_tmp.iterdir():
|
|
if p.name != "fanyu-mongo-snapshot.other-run":
|
|
shutil.rmtree(p)
|
|
|
|
def test_incomplete_downloads_cannot_touch_local_database(self):
|
|
for fault in ("missing", "download_failed", "truncated", "hash_failed"):
|
|
with self.subTest(fault=fault):
|
|
self.fixture()
|
|
result = self.run_script(fault)
|
|
self.assertNotEqual(result.returncode, 0, self.output)
|
|
self.assertEqual(self.calls("docker"), [], self.output)
|
|
self.assert_cleanup()
|
|
|
|
def test_hash_mismatch_cannot_touch_local_database(self):
|
|
self.fixture(checksum="0" * 64)
|
|
result = self.run_script()
|
|
self.assertNotEqual(result.returncode, 0, self.output)
|
|
self.assertEqual(self.calls("docker"), [], self.output)
|
|
self.assert_cleanup()
|
|
|
|
def test_missing_or_invalid_checksum_cannot_touch_local_database(self):
|
|
for checksum in (None, "", "not-a-sha256"):
|
|
with self.subTest(checksum=checksum):
|
|
self.fixture()
|
|
checksum_path = self.backup / "mongo.archive.gz.sha256"
|
|
if checksum is None:
|
|
checksum_path.unlink()
|
|
else:
|
|
checksum_path.write_text(checksum)
|
|
result = self.run_script()
|
|
self.assertNotEqual(result.returncode, 0, self.output)
|
|
self.assertEqual(self.calls("docker"), [], self.output)
|
|
self.assert_cleanup()
|
|
|
|
def test_invalid_backup_names_are_rejected_before_remote_execution(self):
|
|
for backup in ("../fixture", "fixture/other", "fixture;touch injected", "$(touch injected)", "-x", ""):
|
|
with self.subTest(backup=backup):
|
|
result = self.run_script(backup=backup)
|
|
self.assertNotEqual(result.returncode, 0, self.output)
|
|
self.assertEqual(self.calls("kubectl"), [], self.output)
|
|
self.assertEqual(self.calls("docker"), [], self.output)
|
|
|
|
def test_remote_mktemp_failure_or_invalid_output_never_becomes_cleanup_target(self):
|
|
self.fixture()
|
|
for fault in ("mktemp_failed", "mktemp_invalid"):
|
|
with self.subTest(fault=fault):
|
|
previous_calls = len(self.calls("kubectl"))
|
|
result = self.run_script(fault)
|
|
self.assertNotEqual(result.returncode, 0, self.output)
|
|
remote_commands = [args[args.index("--") + 1:]
|
|
for args in self.calls("kubectl")[previous_calls:] if "--" in args]
|
|
self.assertEqual([command for command in remote_commands if command[0] == "rm"],
|
|
[], self.output)
|
|
self.assertEqual(self.calls("docker"), [], self.output)
|
|
self.assert_cleanup()
|
|
|
|
def test_interrupted_download_cleans_own_files_and_retry_succeeds(self):
|
|
self.fixture()
|
|
process = subprocess.Popen(["bash", str(SCRIPT), "fixture"], env=dict(
|
|
self.env, SNAPSHOT_TEST_FAULT="interrupt"), stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, text=True, start_new_session=True)
|
|
try:
|
|
deadline = time.monotonic() + 10
|
|
while not (self.root / "download-started").exists() and process.poll() is None:
|
|
if time.monotonic() > deadline:
|
|
self.fail("script never started downloading")
|
|
time.sleep(0.02)
|
|
if process.poll() is None:
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
stdout, stderr = process.communicate(timeout=10)
|
|
self.output = stdout + stderr
|
|
self.assertTrue((self.root / "download-started").exists(), self.output)
|
|
self.assertNotEqual(process.returncode, 0, self.output)
|
|
self.assertEqual(self.calls("docker"), [], self.output)
|
|
self.assert_cleanup()
|
|
finally:
|
|
if process.poll() is None:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
process.communicate()
|
|
result = self.run_script()
|
|
self.assertEqual(result.returncode, 0, self.output)
|
|
self.assertEqual(json.loads((self.root / "restored.json").read_text()), self.expected)
|
|
self.assert_cleanup(success=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|