🛡️ Verify snapshot chunks before restore and isolate temporary cleanup
This commit is contained in:
parent
7ed51d3227
commit
024dfd0133
@ -109,7 +109,7 @@ kill $(lsof -nP -iTCP:5173 -sTCP:LISTEN -t) 2>/dev/null; npx vite --port 5173 --
|
||||
| `env-local.sh` / `env-local-snapshot.sh` | 空库/快照库环境变量(后者含 SESSION_AUTH 与 WORKSPACE_MOUNT_PATH,**日常用后者**) |
|
||||
| `run-operator.sh <slug>` | 单算子直跑(worker 同款调用方式,秒级调试) |
|
||||
| `download-capture.sh` / `wire-local-capture.sh` | 真实数据下载与接线 |
|
||||
| `fetch-mongo-snapshot.sh` | 从集群 PVC 分块拉全库备份并恢复到 27018(sha256 校验) |
|
||||
| `fetch-mongo-snapshot.sh` | 从集群 PVC 分块拉全库备份并恢复到 27018;本地和远端使用独立临时目录,完整 SHA-256 校验通过后才替换本地容器;成功保留的快照路径由脚本输出 |
|
||||
|
||||
配套文档:`docs/local-dev-guide.zh-CN.md`(环境搭建与踩坑)、`docs/annotation-local-deployment.zh-CN.md`(标注模块部署+测试用例 T1-T12)、`docs/release-checklist.zh-CN.md`(发布三道闸门+回滚)。
|
||||
|
||||
|
||||
@ -5,8 +5,9 @@
|
||||
# scripts/local/fetch-mongo-snapshot.sh [备份目录名]
|
||||
# 默认备份目录: fanyu-v58-ca15d60d(PVC /workspace/system/backups/ 下)
|
||||
#
|
||||
# 集群影响: 零写入。仅在 api pod 内读取备份文件(读 PVC + 临时 split 文件,结束自动清理)。
|
||||
# 本地产物: /tmp/fanyu-mongo-snapshot.gz(sha256 与远端校验一致才继续)
|
||||
# 集群影响: 只读备份 PVC;在 api pod /tmp 下创建独立分块目录,退出时清理。
|
||||
# 本地产物: ${TMPDIR:-/tmp}/fanyu-mongo-snapshot.XXXXXX/mongo.archive.gz
|
||||
# (sha256 与远端校验一致才恢复;成功后保留,失败时清理)
|
||||
# 恢复目标: 容器 fanyu-mongo-restore,端口 127.0.0.1:27018,库 drobodata-fanyu(--drop 重建)
|
||||
#
|
||||
# 恢复后使用:
|
||||
@ -17,36 +18,82 @@ set -euo pipefail
|
||||
|
||||
KUBECONF="${KUBECONFIG:-/Users/d-robotics/Desktop/fanyu/droboticflow/local-data/fanyu.conf}"
|
||||
NS="droboticflow-fanyu"
|
||||
BACKUP_DIR="${1:-fanyu-v58-ca15d60d}"
|
||||
BACKUP_DIR="${1-fanyu-v58-ca15d60d}"
|
||||
ARCHIVE="mongo.archive.gz"
|
||||
POD_BASE="deploy/droboticflow-api"
|
||||
OUT="/tmp/fanyu-mongo-snapshot.gz"
|
||||
CHUNK=512000
|
||||
|
||||
if [ "$#" -gt 1 ] || [[ ! "$BACKUP_DIR" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
|
||||
echo "备份目录必须是单个目录名,仅允许字母、数字、点、下划线和连字符,且以字母或数字开头" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 077
|
||||
LOCAL_TMP=$(mktemp -d "${TMPDIR:-/tmp}/fanyu-mongo-snapshot.XXXXXX")
|
||||
REMOTE_TMP=""
|
||||
KEEP_ARCHIVE=0
|
||||
OUT="$LOCAL_TMP/$ARCHIVE"
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT HUP INT TERM
|
||||
if [[ "$REMOTE_TMP" =~ ^/tmp/fanyu-mongo-snapshot\.[a-zA-Z0-9]+$ ]]; then
|
||||
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- rm -rf -- "$REMOTE_TMP" \
|
||||
|| echo "远端临时目录清理失败: $REMOTE_TMP" >&2
|
||||
fi
|
||||
if [ "$KEEP_ARCHIVE" -eq 1 ]; then
|
||||
rm -rf -- "$LOCAL_TMP/chunks" "$LOCAL_TMP/chunks.list"
|
||||
else
|
||||
rm -rf -- "$LOCAL_TMP"
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
echo "== 1/5 定位备份 =="
|
||||
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD_BASE" -- sh -c \
|
||||
"test -f /workspace/system/backups/$BACKUP_DIR/$ARCHIVE && stat -c '%s' /workspace/system/backups/$BACKUP_DIR/$ARCHIVE" \
|
||||
# 固定同一个 pod 读取校验和、分块和下载;参数始终通过 argv 传递给远端 shell。
|
||||
POD=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" get pod -l app=droboticflow-api -o jsonpath='{.items[0].metadata.name}')
|
||||
[ -n "$POD" ] || { echo "找不到 api pod" >&2; exit 1; }
|
||||
REMOTE_ARCHIVE="/workspace/system/backups/$BACKUP_DIR/$ARCHIVE"
|
||||
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- sh -c \
|
||||
'test -f "$1" && stat -c "%s" "$1"' snapshot-stat "$REMOTE_ARCHIVE" \
|
||||
|| { echo "备份不存在: $BACKUP_DIR"; exit 1; }
|
||||
EXPECTED_SHA=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD_BASE" -- cat "/workspace/system/backups/$BACKUP_DIR/$ARCHIVE.sha256" | awk '{print $1}')
|
||||
SHA_FILE=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- cat "$REMOTE_ARCHIVE.sha256")
|
||||
read -r EXPECTED_SHA _ <<< "$SHA_FILE"
|
||||
[[ "$EXPECTED_SHA" =~ ^[a-fA-F0-9]{64}$ ]] || { echo "备份 sha256 文件格式无效" >&2; exit 1; }
|
||||
EXPECTED_SHA=$(printf '%s' "$EXPECTED_SHA" | tr 'A-F' 'a-f')
|
||||
echo "备份: $BACKUP_DIR/$ARCHIVE 期望sha256: ${EXPECTED_SHA:0:16}..."
|
||||
|
||||
echo "== 2/5 取 api pod 名并分块 =="
|
||||
POD=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" get pod -l app=droboticflow-api -o jsonpath='{.items[0].metadata.name}')
|
||||
echo "== 2/5 在独立临时目录中分块 =="
|
||||
echo "pod: $POD"
|
||||
REMOTE_TMP_CANDIDATE=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- sh -c \
|
||||
'umask 077; mktemp -d /tmp/fanyu-mongo-snapshot.XXXXXX')
|
||||
# 调用失败也可能返回部分输出;只把成功且校验通过的路径交给 EXIT 清理。
|
||||
if [[ ! "$REMOTE_TMP_CANDIDATE" =~ ^/tmp/fanyu-mongo-snapshot\.[a-zA-Z0-9]+$ ]]; then
|
||||
echo "远端临时目录无效" >&2
|
||||
exit 1
|
||||
fi
|
||||
REMOTE_TMP="$REMOTE_TMP_CANDIDATE"
|
||||
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- sh -c \
|
||||
"rm -rf /tmp/fetch-chunks && mkdir -p /tmp/fetch-chunks && split -b $CHUNK -d /workspace/system/backups/$BACKUP_DIR/$ARCHIVE /tmp/fetch-chunks/p"
|
||||
N=$(KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- sh -c 'ls /tmp/fetch-chunks | wc -l' | tr -d ' \r')
|
||||
'set -eu
|
||||
split -b "$3" -d -a 6 "$1" "$2/p"
|
||||
cd "$2"
|
||||
printf "%s\n" p* | LC_ALL=C sort' \
|
||||
snapshot-split "$REMOTE_ARCHIVE" "$REMOTE_TMP" "$CHUNK" > "$LOCAL_TMP/chunks.list"
|
||||
N=$(wc -l < "$LOCAL_TMP/chunks.list" | tr -d ' ')
|
||||
[ "$N" -gt 0 ] || { echo "没有备份分块" >&2; exit 1; }
|
||||
echo "分块数: $N"
|
||||
|
||||
echo "== 3/5 逐块拉取(绕过长连接被网关重置的问题)=="
|
||||
rm -rf /tmp/fetch-chunks-local && mkdir -p /tmp/fetch-chunks-local
|
||||
WIDTH=$(printf '%s' "$N" | wc -c | tr -d ' ')
|
||||
for i in $(seq -f "%0${WIDTH}g" 0 $((N-1))); do
|
||||
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- cat "/tmp/fetch-chunks/p$i" > "/tmp/fetch-chunks-local/c$i" 2>/dev/null
|
||||
done
|
||||
cat /tmp/fetch-chunks-local/c* > "$OUT"
|
||||
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- rm -rf /tmp/fetch-chunks
|
||||
rm -rf /tmp/fetch-chunks-local
|
||||
mkdir "$LOCAL_TMP/chunks"
|
||||
: > "$OUT"
|
||||
while IFS= read -r chunk; do
|
||||
[[ "$chunk" =~ ^p[0-9]{6}$ ]] || { echo "无效的备份分块名" >&2; exit 1; }
|
||||
KUBECONFIG="$KUBECONF" kubectl -n "$NS" exec "$POD" -- cat "$REMOTE_TMP/$chunk" > "$LOCAL_TMP/chunks/$chunk"
|
||||
cat "$LOCAL_TMP/chunks/$chunk" >> "$OUT"
|
||||
done < "$LOCAL_TMP/chunks.list"
|
||||
ACTUAL=$(shasum -a 256 "$OUT" | awk '{print $1}')
|
||||
echo "本地sha256: $ACTUAL"
|
||||
[ "$ACTUAL" = "$EXPECTED_SHA" ] || { echo "sha256 不匹配!文件可能不完整"; exit 1; }
|
||||
@ -74,3 +121,4 @@ echo
|
||||
echo "完成。快照文件: $OUT"
|
||||
echo "本地连接: mongodb://127.0.0.1:27018/drobodata-fanyu"
|
||||
echo "下一步: source scripts/local/env-local-snapshot.sh && npm run dev"
|
||||
KEEP_ARCHIVE=1
|
||||
|
||||
258
tests/backend/fetch_mongo_snapshot_script_test.py
Normal file
258
tests/backend/fetch_mongo_snapshot_script_test.py
Normal file
@ -0,0 +1,258 @@
|
||||
"""Run the snapshot script with local kubectl/docker stand-ins; no network or DB.
|
||||
|
||||
The remote shell, split, cat and local SHA-256 check remain real commands.
|
||||
Run: python3 -m unittest discover -s tests/backend -p fetch_mongo_snapshot_script_test.py -v
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts/local/fetch-mongo-snapshot.sh"
|
||||
CHUNK = 512000
|
||||
|
||||
# Only external boundaries are faked. kubectl runs the supplied remote command
|
||||
# in a temporary filesystem; Docker records the bytes that would be restored.
|
||||
SHIM = r'''
|
||||
import hashlib, json, os, pathlib, re, shutil, subprocess, sys, time
|
||||
root = pathlib.Path(os.environ["SNAPSHOT_TEST_ROOT"])
|
||||
name = pathlib.Path(sys.argv[0]).name
|
||||
args = sys.argv[1:]
|
||||
fault = os.environ.get("SNAPSHOT_TEST_FAULT", "")
|
||||
if name == "sleep":
|
||||
sys.exit(0)
|
||||
if name == "shasum":
|
||||
if fault == "hash_failed":
|
||||
raise SystemExit("simulated SHA-256 command failure")
|
||||
os.execv(os.environ["SNAPSHOT_TEST_SHASUM"], ["shasum"] + args)
|
||||
if name == "stat":
|
||||
if args[:2] != ["-c", "%s"]:
|
||||
raise SystemExit("unexpected stat arguments")
|
||||
print(pathlib.Path(args[2]).stat().st_size)
|
||||
sys.exit(0)
|
||||
with (root / (name + ".jsonl")).open("a") as log:
|
||||
log.write(json.dumps(args) + "\n")
|
||||
if name == "docker":
|
||||
if args[0] == "cp":
|
||||
shutil.copyfile(args[1], root / "container-backup.gz")
|
||||
elif args[:3] == ["exec", "fanyu-mongo-restore", "mongorestore"]:
|
||||
if "--drop" not in args or "--gzip" not in args:
|
||||
raise SystemExit("unexpected restore arguments")
|
||||
data = (root / "container-backup.gz").read_bytes()
|
||||
(root / "restored.json").write_text(json.dumps({
|
||||
"size": len(data), "sha256": hashlib.sha256(data).hexdigest()
|
||||
}))
|
||||
elif args[0] not in ("rm", "run", "exec"):
|
||||
raise SystemExit("unexpected docker command")
|
||||
sys.exit(0)
|
||||
if name != "kubectl":
|
||||
raise SystemExit("unexpected shim")
|
||||
if args[2:4] == ["get", "pod"]:
|
||||
print("local-fixture-pod", end="")
|
||||
sys.exit(0)
|
||||
if args[2] != "exec" or "--" not in args:
|
||||
raise SystemExit("unexpected kubectl command")
|
||||
command = args[args.index("--") + 1:]
|
||||
if command[:2] == ["sh", "-c"] and "mktemp -d " in command[2]:
|
||||
if fault in ("mktemp_failed", "mktemp_invalid"):
|
||||
print("/tmp")
|
||||
sys.exit(1 if fault == "mktemp_failed" else 0)
|
||||
# Keep failure-injection tests safe even if production issues an unsafe rm.
|
||||
# The call is already recorded, so the test still detects the regression.
|
||||
if command[0] == "rm" and not re.fullmatch(
|
||||
r"/tmp/fanyu-mongo-snapshot\.[a-zA-Z0-9]+", command[-1]):
|
||||
raise SystemExit("refusing unsafe cleanup target in test sandbox")
|
||||
# Reject the legacy shared directory before its local counterpart can be
|
||||
# touched. This also makes a pre-fix regression run safe on a developer laptop.
|
||||
if any("/tmp/fetch-chunks" in part for part in command):
|
||||
raise SystemExit("unsafe shared temporary directory")
|
||||
remote = root / "remote"
|
||||
mapped = [part.replace("/tmp/", str(remote / "tmp") + "/")
|
||||
.replace("/workspace/", str(remote / "workspace") + "/")
|
||||
for part in command]
|
||||
if command[0] == "cat" and re.search(r"/p[0-9]+$", command[-1]):
|
||||
chunk = pathlib.Path(mapped[-1])
|
||||
if fault == "missing":
|
||||
chunk.unlink(missing_ok=True)
|
||||
elif fault == "download_failed":
|
||||
sys.stdout.buffer.write(chunk.read_bytes()[:7])
|
||||
sys.exit(1)
|
||||
elif fault == "truncated":
|
||||
sys.stdout.buffer.write(chunk.read_bytes()[:-1])
|
||||
sys.exit(0)
|
||||
elif fault == "interrupt":
|
||||
(root / "download-started").touch()
|
||||
time.sleep(30)
|
||||
result = subprocess.run(mapped, stdout=subprocess.PIPE, cwd=root)
|
||||
sys.stdout.buffer.write(result.stdout.replace(str(remote).encode(), b""))
|
||||
sys.exit(result.returncode)
|
||||
'''
|
||||
|
||||
|
||||
class SnapshotScriptTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(prefix="snapshot-script-test-")
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = pathlib.Path(self.temp.name)
|
||||
self.bin = self.root / "bin"
|
||||
self.bin.mkdir()
|
||||
shim = self.bin / "shim"
|
||||
shim.write_text(f"#!{sys.executable}\n" + SHIM)
|
||||
shim.chmod(0o755)
|
||||
for name in ("kubectl", "docker", "sleep", "stat", "shasum"):
|
||||
(self.bin / name).symlink_to(shim)
|
||||
self.local_tmp = self.root / "local-tmp"
|
||||
self.remote_tmp = self.root / "remote/tmp"
|
||||
self.local_tmp.mkdir()
|
||||
self.remote_tmp.mkdir(parents=True)
|
||||
self.backup = self.root / "remote/workspace/system/backups/fixture"
|
||||
self.backup.mkdir(parents=True)
|
||||
self.env = dict(os.environ, PATH=f"{self.bin}:{os.environ['PATH']}",
|
||||
TMPDIR=str(self.local_tmp), KUBECONFIG="unused-test-config",
|
||||
SNAPSHOT_TEST_ROOT=str(self.root),
|
||||
SNAPSHOT_TEST_SHASUM=shutil.which("shasum"))
|
||||
# Another invocation's files must survive success, failure and signals.
|
||||
for directory in (self.local_tmp, self.remote_tmp):
|
||||
other = directory / "fanyu-mongo-snapshot.other-run"
|
||||
other.mkdir()
|
||||
(other / "keep").write_text("another invocation")
|
||||
|
||||
def fixture(self, count=2, checksum=None):
|
||||
archive = self.backup / "mongo.archive.gz"
|
||||
digest = hashlib.sha256()
|
||||
# Distinct chunk contents expose incorrect ordering as well as omission.
|
||||
with archive.open("wb") as output:
|
||||
for index in range(count):
|
||||
block = bytes([index % 251]) * (CHUNK if index < count - 1 else 137)
|
||||
output.write(block)
|
||||
digest.update(block)
|
||||
self.expected = {"size": archive.stat().st_size, "sha256": digest.hexdigest()}
|
||||
(self.backup / "mongo.archive.gz.sha256").write_text(
|
||||
f"{checksum or digest.hexdigest()} mongo.archive.gz\n")
|
||||
|
||||
def run_script(self, fault="", backup="fixture"):
|
||||
result = subprocess.run(["bash", str(SCRIPT), backup], env=dict(
|
||||
self.env, SNAPSHOT_TEST_FAULT=fault), capture_output=True, text=True, timeout=30)
|
||||
self.output = result.stdout + result.stderr
|
||||
return result
|
||||
|
||||
def calls(self, command):
|
||||
log = self.root / f"{command}.jsonl"
|
||||
return [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []
|
||||
|
||||
def assert_cleanup(self, success=False):
|
||||
for directory in (self.local_tmp, self.remote_tmp):
|
||||
self.assertEqual((directory / "fanyu-mongo-snapshot.other-run/keep").read_text(),
|
||||
"another invocation")
|
||||
owned = [p for p in directory.iterdir() if p.name != "fanyu-mongo-snapshot.other-run"]
|
||||
if success and directory == self.local_tmp:
|
||||
self.assertEqual(len(owned), 1, self.output)
|
||||
self.assertEqual([p.name for p in owned[0].iterdir()], ["mongo.archive.gz"])
|
||||
else:
|
||||
self.assertEqual(owned, [], self.output)
|
||||
|
||||
def test_restores_identical_archive_for_chunk_count_boundaries(self):
|
||||
for count in (1, 2, 9, 10, 99, 100):
|
||||
with self.subTest(chunks=count):
|
||||
self.fixture(count)
|
||||
result = self.run_script()
|
||||
self.assertEqual(result.returncode, 0, self.output)
|
||||
self.assertEqual(json.loads((self.root / "restored.json").read_text()), self.expected)
|
||||
self.assert_cleanup(success=True)
|
||||
for p in self.local_tmp.iterdir():
|
||||
if p.name != "fanyu-mongo-snapshot.other-run":
|
||||
shutil.rmtree(p)
|
||||
|
||||
def test_incomplete_downloads_cannot_touch_local_database(self):
|
||||
for fault in ("missing", "download_failed", "truncated", "hash_failed"):
|
||||
with self.subTest(fault=fault):
|
||||
self.fixture()
|
||||
result = self.run_script(fault)
|
||||
self.assertNotEqual(result.returncode, 0, self.output)
|
||||
self.assertEqual(self.calls("docker"), [], self.output)
|
||||
self.assert_cleanup()
|
||||
|
||||
def test_hash_mismatch_cannot_touch_local_database(self):
|
||||
self.fixture(checksum="0" * 64)
|
||||
result = self.run_script()
|
||||
self.assertNotEqual(result.returncode, 0, self.output)
|
||||
self.assertEqual(self.calls("docker"), [], self.output)
|
||||
self.assert_cleanup()
|
||||
|
||||
def test_missing_or_invalid_checksum_cannot_touch_local_database(self):
|
||||
for checksum in (None, "", "not-a-sha256"):
|
||||
with self.subTest(checksum=checksum):
|
||||
self.fixture()
|
||||
checksum_path = self.backup / "mongo.archive.gz.sha256"
|
||||
if checksum is None:
|
||||
checksum_path.unlink()
|
||||
else:
|
||||
checksum_path.write_text(checksum)
|
||||
result = self.run_script()
|
||||
self.assertNotEqual(result.returncode, 0, self.output)
|
||||
self.assertEqual(self.calls("docker"), [], self.output)
|
||||
self.assert_cleanup()
|
||||
|
||||
def test_invalid_backup_names_are_rejected_before_remote_execution(self):
|
||||
for backup in ("../fixture", "fixture/other", "fixture;touch injected", "$(touch injected)", "-x", ""):
|
||||
with self.subTest(backup=backup):
|
||||
result = self.run_script(backup=backup)
|
||||
self.assertNotEqual(result.returncode, 0, self.output)
|
||||
self.assertEqual(self.calls("kubectl"), [], self.output)
|
||||
self.assertEqual(self.calls("docker"), [], self.output)
|
||||
|
||||
def test_remote_mktemp_failure_or_invalid_output_never_becomes_cleanup_target(self):
|
||||
self.fixture()
|
||||
for fault in ("mktemp_failed", "mktemp_invalid"):
|
||||
with self.subTest(fault=fault):
|
||||
previous_calls = len(self.calls("kubectl"))
|
||||
result = self.run_script(fault)
|
||||
self.assertNotEqual(result.returncode, 0, self.output)
|
||||
remote_commands = [args[args.index("--") + 1:]
|
||||
for args in self.calls("kubectl")[previous_calls:] if "--" in args]
|
||||
self.assertEqual([command for command in remote_commands if command[0] == "rm"],
|
||||
[], self.output)
|
||||
self.assertEqual(self.calls("docker"), [], self.output)
|
||||
self.assert_cleanup()
|
||||
|
||||
def test_interrupted_download_cleans_own_files_and_retry_succeeds(self):
|
||||
self.fixture()
|
||||
process = subprocess.Popen(["bash", str(SCRIPT), "fixture"], env=dict(
|
||||
self.env, SNAPSHOT_TEST_FAULT="interrupt"), stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, text=True, start_new_session=True)
|
||||
try:
|
||||
deadline = time.monotonic() + 10
|
||||
while not (self.root / "download-started").exists() and process.poll() is None:
|
||||
if time.monotonic() > deadline:
|
||||
self.fail("script never started downloading")
|
||||
time.sleep(0.02)
|
||||
if process.poll() is None:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
stdout, stderr = process.communicate(timeout=10)
|
||||
self.output = stdout + stderr
|
||||
self.assertTrue((self.root / "download-started").exists(), self.output)
|
||||
self.assertNotEqual(process.returncode, 0, self.output)
|
||||
self.assertEqual(self.calls("docker"), [], self.output)
|
||||
self.assert_cleanup()
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.communicate()
|
||||
result = self.run_script()
|
||||
self.assertEqual(result.returncode, 0, self.output)
|
||||
self.assertEqual(json.loads((self.root / "restored.json").read_text()), self.expected)
|
||||
self.assert_cleanup(success=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
x
Reference in New Issue
Block a user