🛡️ Verify snapshot chunks before restore and isolate temporary cleanup

This commit is contained in:
d-robotics 2026-09-18 10:49:30 +08:00
parent 7ed51d3227
commit 024dfd0133
3 changed files with 327 additions and 21 deletions

View File

@ -109,7 +109,7 @@ kill $(lsof -nP -iTCP:5173 -sTCP:LISTEN -t) 2>/dev/null; npx vite --port 5173 --
| `env-local.sh` / `env-local-snapshot.sh` | 空库/快照库环境变量(后者含 SESSION_AUTH 与 WORKSPACE_MOUNT_PATH,**日常用后者**) |
| `run-operator.sh <slug>` | 单算子直跑(worker 同款调用方式,秒级调试) |
| `download-capture.sh` / `wire-local-capture.sh` | 真实数据下载与接线 |
| `fetch-mongo-snapshot.sh` | 从集群 PVC 分块拉全库备份并恢复到 27018(sha256 校验) |
| `fetch-mongo-snapshot.sh` | 从集群 PVC 分块拉全库备份并恢复到 27018;本地和远端使用独立临时目录,完整 SHA-256 校验通过后才替换本地容器;成功保留的快照路径由脚本输出 |
配套文档:`docs/local-dev-guide.zh-CN.md`(环境搭建与踩坑)、`docs/annotation-local-deployment.zh-CN.md`(标注模块部署+测试用例 T1-T12)、`docs/release-checklist.zh-CN.md`(发布三道闸门+回滚)。

View File

@ -5,8 +5,9 @@
# scripts/local/fetch-mongo-snapshot.sh [备份目录名]
# 默认备份目录: fanyu-v58-ca15d60d(PVC /workspace/system/backups/ 下)
#
# 集群影响: 零写入。仅在 api pod 内读取备份文件(读 PVC + 临时 split 文件,结束自动清理)。
# 本地产物: /tmp/fanyu-mongo-snapshot.gz(sha256 与远端校验一致才继续)
# 集群影响: 只读备份 PVC;在 api pod /tmp 下创建独立分块目录,退出时清理。
# 本地产物: ${TMPDIR:-/tmp}/fanyu-mongo-snapshot.XXXXXX/mongo.archive.gz
# (sha256 与远端校验一致才恢复;成功后保留,失败时清理)
# 恢复目标: 容器 fanyu-mongo-restore,端口 127.0.0.1:27018,库 drobodata-fanyu(--drop 重建)
#
# 恢复后使用:
@ -17,36 +18,82 @@ set -euo pipefail
KUBECONF="${KUBECONFIG:-/Users/d-robotics/Desktop/fanyu/droboticflow/local-data/fanyu.conf}"
NS="droboticflow-fanyu"
BACKUP_DIR="${1:-fanyu-v58-ca15d60d}"
BACKUP_DIR="${1-fanyu-v58-ca15d60d}"
ARCHIVE="mongo.archive.gz"
POD_BASE="deploy/droboticflow-api"
OUT="/tmp/fanyu-mongo-snapshot.gz"
CHUNK=512000
if [ "$#" -gt 1 ] || [[ ! "$BACKUP_DIR" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
echo "备份目录必须是单个目录名,仅允许字母、数字、点、下划线和连字符,且以字母或数字开头" >&2
exit 1
fi
umask 077
LOCAL_TMP=$(mktemp -d "${TMPDIR:-/tmp}/fanyu-mongo-snapshot.XXXXXX")
REMOTE_TMP=""
KEEP_ARCHIVE=0
OUT="$LOCAL_TMP/$ARCHIVE"
cleanup() {
local status=$?
trap - EXIT HUP INT TERM
if [[ "$REMOTE_TMP" =~ ^/tmp/fanyu-mongo-snapshot\.[a-zA-Z0-9]+$ ]]; then
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- rm -rf -- "$REMOTE_TMP" \
|| echo "远端临时目录清理失败: $REMOTE_TMP" >&2
fi
if [ "$KEEP_ARCHIVE" -eq 1 ]; then
rm -rf -- "$LOCAL_TMP/chunks" "$LOCAL_TMP/chunks.list"
else
rm -rf -- "$LOCAL_TMP"
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
echo "== 1/5 定位备份 =="
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD_BASE" -- sh -c \
"test -f /workspace/system/backups/$BACKUP_DIR/$ARCHIVE && stat -c '%s' /workspace/system/backups/$BACKUP_DIR/$ARCHIVE" \
# 固定同一个 pod 读取校验和、分块和下载;参数始终通过 argv 传递给远端 shell。
POD=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" get pod -l app=droboticflow-api -o jsonpath='{.items[0].metadata.name}')
[ -n "$POD" ] || { echo "找不到 api pod" >&2; exit 1; }
REMOTE_ARCHIVE="/workspace/system/backups/$BACKUP_DIR/$ARCHIVE"
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- sh -c \
'test -f "$1" && stat -c "%s" "$1"' snapshot-stat "$REMOTE_ARCHIVE" \
|| { echo "备份不存在: $BACKUP_DIR"; exit 1; }
EXPECTED_SHA=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD_BASE" -- cat "/workspace/system/backups/$BACKUP_DIR/$ARCHIVE.sha256" | awk '{print $1}')
SHA_FILE=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- cat "$REMOTE_ARCHIVE.sha256")
read -r EXPECTED_SHA _ <<< "$SHA_FILE"
[[ "$EXPECTED_SHA" =~ ^[a-fA-F0-9]{64}$ ]] || { echo "备份 sha256 文件格式无效" >&2; exit 1; }
EXPECTED_SHA=$(printf '%s' "$EXPECTED_SHA" | tr 'A-F' 'a-f')
echo "备份: $BACKUP_DIR/$ARCHIVE 期望sha256: ${EXPECTED_SHA:0:16}..."
echo "== 2/5 取 api pod 名并分块 =="
POD=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" get pod -l app=droboticflow-api -o jsonpath='{.items[0].metadata.name}')
echo "== 2/5 在独立临时目录中分块 =="
echo "pod: $POD"
REMOTE_TMP_CANDIDATE=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- sh -c \
'umask 077; mktemp -d /tmp/fanyu-mongo-snapshot.XXXXXX')
# 调用失败也可能返回部分输出;只把成功且校验通过的路径交给 EXIT 清理。
if [[ ! "$REMOTE_TMP_CANDIDATE" =~ ^/tmp/fanyu-mongo-snapshot\.[a-zA-Z0-9]+$ ]]; then
echo "远端临时目录无效" >&2
exit 1
fi
REMOTE_TMP="$REMOTE_TMP_CANDIDATE"
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- sh -c \
"rm -rf /tmp/fetch-chunks && mkdir -p /tmp/fetch-chunks && split -b $CHUNK -d /workspace/system/backups/$BACKUP_DIR/$ARCHIVE /tmp/fetch-chunks/p"
N=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- sh -c 'ls /tmp/fetch-chunks | wc -l' | tr -d ' \r')
'set -eu
split -b "$3" -d -a 6 "$1" "$2/p"
cd "$2"
printf "%s\n" p* | LC_ALL=C sort' \
snapshot-split "$REMOTE_ARCHIVE" "$REMOTE_TMP" "$CHUNK" > "$LOCAL_TMP/chunks.list"
N=$(wc -l < "$LOCAL_TMP/chunks.list" | tr -d ' ')
[ "$N" -gt 0 ] || { echo "没有备份分块" >&2; exit 1; }
echo "分块数: $N"
echo "== 3/5 逐块拉取(绕过长连接被网关重置的问题)=="
rm -rf /tmp/fetch-chunks-local && mkdir -p /tmp/fetch-chunks-local
WIDTH=$(printf '%s' "$N" | wc -c | tr -d ' ')
for i in $(seq -f "%0${WIDTH}g" 0 $((N-1))); do
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- cat "/tmp/fetch-chunks/p$i" > "/tmp/fetch-chunks-local/c$i" 2>/dev/null
done
cat /tmp/fetch-chunks-local/c* > "$OUT"
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- rm -rf /tmp/fetch-chunks
rm -rf /tmp/fetch-chunks-local
mkdir "$LOCAL_TMP/chunks"
: > "$OUT"
while IFS= read -r chunk; do
[[ "$chunk" =~ ^p[0-9]{6}$ ]] || { echo "无效的备份分块名" >&2; exit 1; }
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- cat "$REMOTE_TMP/$chunk" > "$LOCAL_TMP/chunks/$chunk"
cat "$LOCAL_TMP/chunks/$chunk" >> "$OUT"
done < "$LOCAL_TMP/chunks.list"
ACTUAL=$(shasum -a 256 "$OUT" | awk '{print $1}')
echo "本地sha256: $ACTUAL"
[ "$ACTUAL" = "$EXPECTED_SHA" ] || { echo "sha256 不匹配!文件可能不完整"; exit 1; }
@ -74,3 +121,4 @@ echo
echo "完成。快照文件: $OUT"
echo "本地连接: mongodb://127.0.0.1:27018/drobodata-fanyu"
echo "下一步: source scripts/local/env-local-snapshot.sh && npm run dev"
KEEP_ARCHIVE=1

View File

@ -0,0 +1,258 @@
"""Run the snapshot script with local kubectl/docker stand-ins; no network or DB.
The remote shell, split, cat and local SHA-256 check remain real commands.
Run: python3 -m unittest discover -s tests/backend -p fetch_mongo_snapshot_script_test.py -v
"""
import hashlib
import json
import os
import pathlib
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import unittest
ROOT = pathlib.Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts/local/fetch-mongo-snapshot.sh"
CHUNK = 512000
# Only external boundaries are faked. kubectl runs the supplied remote command
# in a temporary filesystem; Docker records the bytes that would be restored.
SHIM = r'''
import hashlib, json, os, pathlib, re, shutil, subprocess, sys, time
root = pathlib.Path(os.environ["SNAPSHOT_TEST_ROOT"])
name = pathlib.Path(sys.argv[0]).name
args = sys.argv[1:]
fault = os.environ.get("SNAPSHOT_TEST_FAULT", "")
if name == "sleep":
sys.exit(0)
if name == "shasum":
if fault == "hash_failed":
raise SystemExit("simulated SHA-256 command failure")
os.execv(os.environ["SNAPSHOT_TEST_SHASUM"], ["shasum"] + args)
if name == "stat":
if args[:2] != ["-c", "%s"]:
raise SystemExit("unexpected stat arguments")
print(pathlib.Path(args[2]).stat().st_size)
sys.exit(0)
with (root / (name + ".jsonl")).open("a") as log:
log.write(json.dumps(args) + "\n")
if name == "docker":
if args[0] == "cp":
shutil.copyfile(args[1], root / "container-backup.gz")
elif args[:3] == ["exec", "fanyu-mongo-restore", "mongorestore"]:
if "--drop" not in args or "--gzip" not in args:
raise SystemExit("unexpected restore arguments")
data = (root / "container-backup.gz").read_bytes()
(root / "restored.json").write_text(json.dumps({
"size": len(data), "sha256": hashlib.sha256(data).hexdigest()
}))
elif args[0] not in ("rm", "run", "exec"):
raise SystemExit("unexpected docker command")
sys.exit(0)
if name != "kubectl":
raise SystemExit("unexpected shim")
if args[2:4] == ["get", "pod"]:
print("local-fixture-pod", end="")
sys.exit(0)
if args[2] != "exec" or "--" not in args:
raise SystemExit("unexpected kubectl command")
command = args[args.index("--") + 1:]
if command[:2] == ["sh", "-c"] and "mktemp -d " in command[2]:
if fault in ("mktemp_failed", "mktemp_invalid"):
print("/tmp")
sys.exit(1 if fault == "mktemp_failed" else 0)
# Keep failure-injection tests safe even if production issues an unsafe rm.
# The call is already recorded, so the test still detects the regression.
if command[0] == "rm" and not re.fullmatch(
r"/tmp/fanyu-mongo-snapshot\.[a-zA-Z0-9]+", command[-1]):
raise SystemExit("refusing unsafe cleanup target in test sandbox")
# Reject the legacy shared directory before its local counterpart can be
# touched. This also makes a pre-fix regression run safe on a developer laptop.
if any("/tmp/fetch-chunks" in part for part in command):
raise SystemExit("unsafe shared temporary directory")
remote = root / "remote"
mapped = [part.replace("/tmp/", str(remote / "tmp") + "/")
.replace("/workspace/", str(remote / "workspace") + "/")
for part in command]
if command[0] == "cat" and re.search(r"/p[0-9]+$", command[-1]):
chunk = pathlib.Path(mapped[-1])
if fault == "missing":
chunk.unlink(missing_ok=True)
elif fault == "download_failed":
sys.stdout.buffer.write(chunk.read_bytes()[:7])
sys.exit(1)
elif fault == "truncated":
sys.stdout.buffer.write(chunk.read_bytes()[:-1])
sys.exit(0)
elif fault == "interrupt":
(root / "download-started").touch()
time.sleep(30)
result = subprocess.run(mapped, stdout=subprocess.PIPE, cwd=root)
sys.stdout.buffer.write(result.stdout.replace(str(remote).encode(), b""))
sys.exit(result.returncode)
'''
class SnapshotScriptTest(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix="snapshot-script-test-")
self.addCleanup(self.temp.cleanup)
self.root = pathlib.Path(self.temp.name)
self.bin = self.root / "bin"
self.bin.mkdir()
shim = self.bin / "shim"
shim.write_text(f"#!{sys.executable}\n" + SHIM)
shim.chmod(0o755)
for name in ("kubectl", "docker", "sleep", "stat", "shasum"):
(self.bin / name).symlink_to(shim)
self.local_tmp = self.root / "local-tmp"
self.remote_tmp = self.root / "remote/tmp"
self.local_tmp.mkdir()
self.remote_tmp.mkdir(parents=True)
self.backup = self.root / "remote/workspace/system/backups/fixture"
self.backup.mkdir(parents=True)
self.env = dict(os.environ, PATH=f"{self.bin}:{os.environ['PATH']}",
TMPDIR=str(self.local_tmp), KUBECONFIG="unused-test-config",
SNAPSHOT_TEST_ROOT=str(self.root),
SNAPSHOT_TEST_SHASUM=shutil.which("shasum"))
# Another invocation's files must survive success, failure and signals.
for directory in (self.local_tmp, self.remote_tmp):
other = directory / "fanyu-mongo-snapshot.other-run"
other.mkdir()
(other / "keep").write_text("another invocation")
def fixture(self, count=2, checksum=None):
archive = self.backup / "mongo.archive.gz"
digest = hashlib.sha256()
# Distinct chunk contents expose incorrect ordering as well as omission.
with archive.open("wb") as output:
for index in range(count):
block = bytes([index % 251]) * (CHUNK if index < count - 1 else 137)
output.write(block)
digest.update(block)
self.expected = {"size": archive.stat().st_size, "sha256": digest.hexdigest()}
(self.backup / "mongo.archive.gz.sha256").write_text(
f"{checksum or digest.hexdigest()} mongo.archive.gz\n")
def run_script(self, fault="", backup="fixture"):
result = subprocess.run(["bash", str(SCRIPT), backup], env=dict(
self.env, SNAPSHOT_TEST_FAULT=fault), capture_output=True, text=True, timeout=30)
self.output = result.stdout + result.stderr
return result
def calls(self, command):
log = self.root / f"{command}.jsonl"
return [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []
def assert_cleanup(self, success=False):
for directory in (self.local_tmp, self.remote_tmp):
self.assertEqual((directory / "fanyu-mongo-snapshot.other-run/keep").read_text(),
"another invocation")
owned = [p for p in directory.iterdir() if p.name != "fanyu-mongo-snapshot.other-run"]
if success and directory == self.local_tmp:
self.assertEqual(len(owned), 1, self.output)
self.assertEqual([p.name for p in owned[0].iterdir()], ["mongo.archive.gz"])
else:
self.assertEqual(owned, [], self.output)
def test_restores_identical_archive_for_chunk_count_boundaries(self):
for count in (1, 2, 9, 10, 99, 100):
with self.subTest(chunks=count):
self.fixture(count)
result = self.run_script()
self.assertEqual(result.returncode, 0, self.output)
self.assertEqual(json.loads((self.root / "restored.json").read_text()), self.expected)
self.assert_cleanup(success=True)
for p in self.local_tmp.iterdir():
if p.name != "fanyu-mongo-snapshot.other-run":
shutil.rmtree(p)
def test_incomplete_downloads_cannot_touch_local_database(self):
for fault in ("missing", "download_failed", "truncated", "hash_failed"):
with self.subTest(fault=fault):
self.fixture()
result = self.run_script(fault)
self.assertNotEqual(result.returncode, 0, self.output)
self.assertEqual(self.calls("docker"), [], self.output)
self.assert_cleanup()
def test_hash_mismatch_cannot_touch_local_database(self):
self.fixture(checksum="0" * 64)
result = self.run_script()
self.assertNotEqual(result.returncode, 0, self.output)
self.assertEqual(self.calls("docker"), [], self.output)
self.assert_cleanup()
def test_missing_or_invalid_checksum_cannot_touch_local_database(self):
for checksum in (None, "", "not-a-sha256"):
with self.subTest(checksum=checksum):
self.fixture()
checksum_path = self.backup / "mongo.archive.gz.sha256"
if checksum is None:
checksum_path.unlink()
else:
checksum_path.write_text(checksum)
result = self.run_script()
self.assertNotEqual(result.returncode, 0, self.output)
self.assertEqual(self.calls("docker"), [], self.output)
self.assert_cleanup()
def test_invalid_backup_names_are_rejected_before_remote_execution(self):
for backup in ("../fixture", "fixture/other", "fixture;touch injected", "$(touch injected)", "-x", ""):
with self.subTest(backup=backup):
result = self.run_script(backup=backup)
self.assertNotEqual(result.returncode, 0, self.output)
self.assertEqual(self.calls("kubectl"), [], self.output)
self.assertEqual(self.calls("docker"), [], self.output)
def test_remote_mktemp_failure_or_invalid_output_never_becomes_cleanup_target(self):
self.fixture()
for fault in ("mktemp_failed", "mktemp_invalid"):
with self.subTest(fault=fault):
previous_calls = len(self.calls("kubectl"))
result = self.run_script(fault)
self.assertNotEqual(result.returncode, 0, self.output)
remote_commands = [args[args.index("--") + 1:]
for args in self.calls("kubectl")[previous_calls:] if "--" in args]
self.assertEqual([command for command in remote_commands if command[0] == "rm"],
[], self.output)
self.assertEqual(self.calls("docker"), [], self.output)
self.assert_cleanup()
def test_interrupted_download_cleans_own_files_and_retry_succeeds(self):
self.fixture()
process = subprocess.Popen(["bash", str(SCRIPT), "fixture"], env=dict(
self.env, SNAPSHOT_TEST_FAULT="interrupt"), stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, start_new_session=True)
try:
deadline = time.monotonic() + 10
while not (self.root / "download-started").exists() and process.poll() is None:
if time.monotonic() > deadline:
self.fail("script never started downloading")
time.sleep(0.02)
if process.poll() is None:
os.killpg(process.pid, signal.SIGTERM)
stdout, stderr = process.communicate(timeout=10)
self.output = stdout + stderr
self.assertTrue((self.root / "download-started").exists(), self.output)
self.assertNotEqual(process.returncode, 0, self.output)
self.assertEqual(self.calls("docker"), [], self.output)
self.assert_cleanup()
finally:
if process.poll() is None:
os.killpg(process.pid, signal.SIGKILL)
process.communicate()
result = self.run_script()
self.assertEqual(result.returncode, 0, self.output)
self.assertEqual(json.loads((self.root / "restored.json").read_text()), self.expected)
self.assert_cleanup(success=True)
if __name__ == "__main__":
unittest.main()