forked from longtao.wu/droboticflow
Bind recovery to the original receipt and scope, verify the held file and MD5, persist strict restart checkpoints, and atomically refresh identity without reusing cloud authorization. Cover direct retry, replacement protection, stale journals, and transactional failures; record Linux validation limits. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
523 lines
28 KiB
Python
523 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""Isolated native CLI/daemon cleanup acceptance against real Go API routes.
|
|
|
|
Build internal/api's test binary first. Upload planning/storage are synthetic;
|
|
cleanup uses real routing, device-token authentication, receipts and key leases,
|
|
with MemoryStore and a fake object-storage HEAD. No production config is read.
|
|
"""
|
|
import argparse
|
|
import ast
|
|
import copy
|
|
import datetime
|
|
import hashlib
|
|
import http.server
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import socket
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import types
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
TOKEN_VALUE = "synthetic-local-fixture-only"
|
|
HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
BASE_SOURCE = ROOT / "scripts/drosync-local-daily-acceptance.py"
|
|
BASE_TREE = ast.parse(BASE_SOURCE.read_text())
|
|
BASE_CLASSES = ast.Module(body=[node for node in BASE_TREE.body if isinstance(node, ast.ClassDef) and node.name in {"Control", "Handler"}], type_ignores=[])
|
|
|
|
|
|
def request_json(url, data=None, method=None, headers=None):
|
|
body = json.dumps(data).encode() if data is not None else None
|
|
request = urllib.request.Request(url, body, {"Content-Type": "application/json", **(headers or {})}, method=method)
|
|
with HTTP.open(request, timeout=60) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def isolated_env(root):
|
|
env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LANG": "en_US.UTF-8"}
|
|
for key, name in [("HOME", "home"), ("XDG_CONFIG_HOME", "xdg-config"), ("XDG_DATA_HOME", "xdg-data"), ("TMPDIR", "tmp")]:
|
|
path = root / name
|
|
path.mkdir(mode=0o700)
|
|
env[key] = str(path)
|
|
return env
|
|
|
|
|
|
class GoAPI:
|
|
def __init__(self, binary, root):
|
|
self.root = root
|
|
root.mkdir(mode=0o700)
|
|
self.ready = root / "ready"
|
|
self.log = (root / "server.txt").open("w")
|
|
env = isolated_env(root)
|
|
env["DROSYNC_TEST_CLEANUP_READY_FILE"] = str(self.ready)
|
|
self.proc = subprocess.Popen([str(binary), "-test.run", "^TestDrosyncCleanupHTTPHarness$", "-test.v", "-test.timeout", "25m"], env=env, stdout=self.log, stderr=self.log)
|
|
deadline = time.monotonic() + 30
|
|
while not self.ready.exists():
|
|
if self.proc.poll() is not None or time.monotonic() >= deadline:
|
|
self.close()
|
|
raise RuntimeError(f"API fixture failed to start: {root / 'server.txt'}")
|
|
time.sleep(0.05)
|
|
self.url = self.ready.read_text()
|
|
assert self.url.startswith("http://127.0.0.1:")
|
|
|
|
def control(self, action, data=None):
|
|
return request_json(self.url + "/_fixture/" + action, data, headers={"X-Drosync-Test-Control": "isolated-cleanup-fixture"})
|
|
|
|
def close(self):
|
|
if hasattr(self, "url") and self.proc.poll() is None:
|
|
try:
|
|
self.control("stop", {})
|
|
except (OSError, urllib.error.URLError):
|
|
pass
|
|
try:
|
|
self.proc.wait(timeout=15)
|
|
except subprocess.TimeoutExpired:
|
|
self.proc.terminate()
|
|
self.proc.wait(timeout=10)
|
|
self.log.close()
|
|
return self.proc.returncode
|
|
|
|
|
|
def remote_base(item):
|
|
return {"exists": True, "eTag": "fixture-etag", "sizeBytes": item["sizeBytes"], "versionId": "fixture-v1", "platformVersion": 1}
|
|
|
|
|
|
class Fixture:
|
|
def __init__(self, root, binaries, api, names):
|
|
self.root, self.binaries, self.api = root, binaries, api
|
|
root.mkdir(mode=0o700)
|
|
self.source = root / "capture"
|
|
self.source.mkdir()
|
|
self.data = root / "state"
|
|
self.config = root / "environment"
|
|
self.token = root / "device-token"
|
|
self.token.write_text(TOKEN_VALUE)
|
|
self.token.chmod(0o600)
|
|
self.config.write_text(f"DROSYNC_DATA_DIR={self.data}\nDROSYNC_SECRET_FILE={self.token}\nDROSYNC_LOG_PATH={root}/daemon.log\nDROSYNC_LOG_FALLBACK_PATH={root}/fallback.log\nDROSYNC_FILE_CONCURRENCY=2\nDROSYNC_PUT_CONCURRENCY=1\nDROSYNC_HASH_CONCURRENCY=2\n")
|
|
self.env = isolated_env(root)
|
|
self.commands, self.proof_calls = [], []
|
|
self.expire_proof = False
|
|
self.mutate_writer = False
|
|
self.writer = None
|
|
self.active_cloud = False
|
|
self.bad_device = False
|
|
self.original = {}
|
|
for name in names:
|
|
payload = b"" if name == "empty.bin" else ("payload:" + name).encode()[:24]
|
|
path = self.source / name
|
|
path.write_bytes(payload)
|
|
os.utime(path, (time.time() - 120,) * 2)
|
|
self.original[name] = payload
|
|
self.make_control()
|
|
self.log = (root / "daemon-stderr.txt").open("w")
|
|
self.proc = subprocess.Popen([str(binaries / "drosync-daemon"), "-c", str(self.config)], env=self.env, stdout=self.log, stderr=self.log)
|
|
deadline = time.monotonic() + 20
|
|
while not (self.data / "drosync.sock").exists():
|
|
if self.proc.poll() is not None or time.monotonic() >= deadline:
|
|
raise RuntimeError(f"daemon failed to start: {root / 'daemon-stderr.txt'}")
|
|
time.sleep(0.05)
|
|
self.cli("pause", expected=0)
|
|
with self.db() as db:
|
|
values = {"server_url": self.control_url, "credential_storage_profile": "fixture", "credential_location": json.dumps({"kind": "file", "path": str(self.token)}), "auth_token_hash": hashlib.sha256(self.token.read_bytes()).hexdigest(), "auth_credential_id": "credential", "device_id": "device"}
|
|
db.executemany("INSERT INTO daemon_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", values.items())
|
|
_, mapping = self.cli("mapping", "add", str(self.source), "--space", "space", "--remote-prefix", "node-test", expected=0)
|
|
self.mid = mapping["id"]
|
|
self.sync()
|
|
|
|
def make_control(self):
|
|
namespace = dict(globals(), OUT=self.root, args=types.SimpleNamespace(scenario="upload-clean"))
|
|
exec(compile(BASE_CLASSES, str(BASE_SOURCE), "exec"), namespace)
|
|
parent = namespace["Handler"]
|
|
owner = self
|
|
|
|
class Handler(parent):
|
|
def reply(self, data, status=200):
|
|
if self.path.endswith("/capabilities") and owner.bad_device:
|
|
data["deviceId"] = "foreign-device"
|
|
super().reply(data, status)
|
|
|
|
def handle_request(self):
|
|
control = self.server
|
|
path = urllib.parse.urlsplit(self.path).path
|
|
if path.endswith("/manifest-pages"):
|
|
control.calls.append((self.command, path))
|
|
body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0"))))
|
|
sid = path.split("/sessions/", 1)[1].split("/", 1)[0]
|
|
control.manifest_pages += 1
|
|
for manifest in body["items"]:
|
|
name = pathlib.PurePosixPath(manifest["relativePath"]).name
|
|
iid = sid + "-item-" + str(len(control.items)).zfill(5)
|
|
item = dict(manifest, id=iid, sessionId=sid, targetKey=manifest["relativePath"], status="planned", action="create")
|
|
if name in {"legacy.bin", "denied.bin"}:
|
|
item.update(status="denied", action="deny_archived_hash_change", lastError="已归档对象缺少 MD5 回执,暂停上传;旧 SHA256 回执退出内容比较和清理支持,保留云端原文件")
|
|
elif name == "failed.bin":
|
|
item.update(status="failed", lastError="synthetic failed item")
|
|
elif name == "same.bin" or control.archived.get(item["relativePath"]) == (item["sizeBytes"], item["md5sum"]):
|
|
item.update(status="unchanged", action="unchanged", expectedBase=remote_base(item))
|
|
control.archived[item["relativePath"]] = (item["sizeBytes"], item["md5sum"])
|
|
control.items[iid] = item
|
|
return self.reply({"accepted": len(body["items"])}, 202)
|
|
if path.endswith("/commit"):
|
|
control.calls.append((self.command, path))
|
|
self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
|
iid = path.split("/items/", 1)[1].split("/", 1)[0]
|
|
item = control.items[iid]
|
|
assert control.received[iid] == (item["sizeBytes"], item["md5sum"])
|
|
item.update(status="committed", committedVersion=remote_base(item))
|
|
control.archived[item["relativePath"]] = control.received[iid]
|
|
return self.reply({"operationId": "op"})
|
|
if path.endswith("/sessions") and self.command == "GET" and owner.active_cloud:
|
|
self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
|
return self.reply({"items": [{"id": "active", "mappingId": owner.mid, "syncSpaceId": "space", "status": "running"}], "nextCursor": ""})
|
|
if path.endswith("/cleanup-proof"):
|
|
raw = self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
|
headers = {"Content-Type": "application/json"}
|
|
for key in ("Authorization", "X-Drosync-Token", "Idempotency-Key"):
|
|
if self.headers.get(key):
|
|
headers[key] = self.headers[key]
|
|
request = urllib.request.Request(owner.api.url + path, raw, headers, method=self.command)
|
|
try:
|
|
with HTTP.open(request, timeout=60) as response:
|
|
status, body = response.status, json.load(response)
|
|
except urllib.error.HTTPError as error:
|
|
status, body = error.code, json.loads(error.read())
|
|
owner.proof_calls.append({"method": self.command, "path": path, "status": status})
|
|
if status == 200 and self.command == "POST":
|
|
if owner.expire_proof:
|
|
body["expiresAt"] = (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=1)).isoformat()
|
|
if owner.mutate_writer and body["relativePath"].endswith("/good.bin"):
|
|
owner.writer.seek(0)
|
|
owner.writer.write(b"X" * len(owner.original["good.bin"]))
|
|
owner.writer.flush()
|
|
os.fsync(owner.writer.fileno())
|
|
owner.mutate_writer = False
|
|
return self.reply(body, status)
|
|
return super().handle_request()
|
|
|
|
namespace["Handler"] = Handler
|
|
self.control = namespace["Control"]()
|
|
self.control.archived = {}
|
|
original_session = self.control.session
|
|
|
|
def session(control, sid):
|
|
result = original_session(sid)
|
|
rows = [item for item in control.items.values() if item["sessionId"] == sid]
|
|
for name, statuses in {"denied": {"denied"}, "failed": {"failed", "aborted", "expired"}, "unchanged": {"unchanged"}}.items():
|
|
result["counts"][name] = sum(item["status"] in statuses for item in rows)
|
|
return result
|
|
|
|
self.control.session = types.MethodType(session, self.control)
|
|
self.control_url = f"http://127.0.0.1:{self.control.server_address[1]}"
|
|
threading.Thread(target=self.control.serve_forever, daemon=True).start()
|
|
|
|
def restart(self):
|
|
self.proc.terminate()
|
|
self.proc.wait(timeout=10)
|
|
(self.data / "drosync.sock").unlink(missing_ok=True)
|
|
self.proc = subprocess.Popen([str(self.binaries / "drosync-daemon"), "-c", str(self.config)], env=self.env, stdout=self.log, stderr=self.log)
|
|
deadline = time.monotonic() + 20
|
|
while not (self.data / "drosync.sock").exists():
|
|
if self.proc.poll() is not None or time.monotonic() >= deadline:
|
|
raise RuntimeError("isolated daemon restart failed")
|
|
time.sleep(0.05)
|
|
self.cli("pause", expected=0)
|
|
|
|
def db(self):
|
|
return sqlite3.connect(self.data / "state.db", timeout=30)
|
|
|
|
def cli(self, *arguments, expected=None):
|
|
result = subprocess.run([str(self.binaries / "drosync"), "-c", str(self.config), "--json", *arguments], env=self.env, capture_output=True, text=True, timeout=300)
|
|
data = json.loads(result.stdout) if result.stdout else {"stderr": result.stderr}
|
|
self.commands.append({"args": arguments, "exitCode": result.returncode, "result": data, "stderr": result.stderr})
|
|
if expected is not None:
|
|
assert result.returncode == expected, (arguments, result.returncode, data)
|
|
return result.returncode, data
|
|
|
|
def sync(self):
|
|
with self.db() as db:
|
|
db.execute("UPDATE daemon_meta SET value='false' WHERE key='transfers_paused'")
|
|
code, result = self.cli("sync", self.mid)
|
|
self.cli("pause", expected=0)
|
|
assert code in (0, 50) and "session" in result, result
|
|
self.sid = result["session"]["id"]
|
|
assert result["session"]["status"] in ("completed", "partial"), result
|
|
self.upload_result = result
|
|
with self.db() as db:
|
|
db.execute("DELETE FROM daemon_meta WHERE key LIKE 'rescan:%'")
|
|
self.seed()
|
|
|
|
def seed(self, cloud_change=None):
|
|
items = [copy.deepcopy(item) for item in self.control.items.values() if item["sessionId"] == self.sid]
|
|
heads, objects = [], []
|
|
for item in items:
|
|
if item["status"] not in {"committed", "unchanged", "denied"}:
|
|
continue
|
|
digest = "" if item["status"] == "denied" else item["md5sum"]
|
|
heads.append({"key": item["targetKey"], "platformVersion": 1, "remoteETag": "fixture-etag", "versionId": "fixture-v1", "sizeBytes": item["sizeBytes"], "md5sum": digest, "sessionId": self.sid, "itemId": item["id"]})
|
|
metadata = {"drosync-md5sum": digest} if digest else {"drosync-sha256": "a" * 64}
|
|
obj = {"bucket": "bucket", "key": item["targetKey"], "eTag": "fixture-etag", "versionId": "fixture-v1", "sizeBytes": item["sizeBytes"], "metadata": metadata}
|
|
if cloud_change and item["status"] in {"committed", "unchanged"}:
|
|
cloud_change(obj)
|
|
objects.append(obj)
|
|
self.seed_model = {"session": self.control.session(self.sid), "items": items, "heads": heads, "objects": objects}
|
|
self.api.control("seed", self.seed_model)
|
|
|
|
def snapshot(self):
|
|
result = {}
|
|
for name in self.original:
|
|
path = self.source / name
|
|
if path.exists():
|
|
stat = path.stat()
|
|
result[name] = {"exists": True, "md5": hashlib.md5(path.read_bytes()).hexdigest(), "size": stat.st_size, "inode": stat.st_ino, "mtimeNs": stat.st_mtime_ns, "ctimeNs": stat.st_ctime_ns}
|
|
else:
|
|
result[name] = {"exists": False}
|
|
return result
|
|
|
|
def clean(self, execute=False):
|
|
return self.cli("clean", self.mid, "--session", self.sid, *(["--execute"] if execute else []))
|
|
|
|
def change_local_receipt(self, name, md5=None, unbound=False):
|
|
with self.db() as db:
|
|
row = db.execute("SELECT item_id,document FROM sync_items WHERE session_id=? AND json_extract(document,'$.relativePath')=?", (self.sid, "node-test/" + name)).fetchone()
|
|
item = json.loads(row[1])
|
|
if md5 is not None:
|
|
item["md5sum"] = md5
|
|
encoded = json.dumps(item)
|
|
db.execute("UPDATE sync_items SET document=? WHERE item_id=? AND session_id=?", (encoded, row[0], self.sid))
|
|
db.execute("UPDATE latest_sync_files SET document=? WHERE mapping_id=? AND relative_path=?", (encoded, self.mid, item["relativePath"]))
|
|
if unbound:
|
|
db.execute("UPDATE latest_sync_files SET source_identity=NULL WHERE mapping_id=? AND relative_path=?", (self.mid, item["relativePath"]))
|
|
|
|
def close(self):
|
|
if self.writer is not None:
|
|
self.writer.close()
|
|
if hasattr(self, "proc") and self.proc.poll() is None:
|
|
self.proc.terminate()
|
|
try:
|
|
self.proc.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
self.proc.kill()
|
|
self.proc.wait(timeout=5)
|
|
if hasattr(self, "log"):
|
|
self.log.close()
|
|
if hasattr(self, "control"):
|
|
self.control.shutdown()
|
|
self.control.server_close()
|
|
(self.root / "commands.json").write_text(json.dumps(self.commands, ensure_ascii=False, indent=2) + "\n")
|
|
(self.root / "proof-calls.json").write_text(json.dumps(self.proof_calls, indent=2) + "\n")
|
|
|
|
|
|
def assert_counts(result, candidates, deleted, blocked):
|
|
assert result["candidateFiles"] == candidates, result
|
|
assert result["deletedFiles"] == deleted, result
|
|
assert result["blockedFiles"] == blocked, result
|
|
assert result["recoveryNeeded"] == 0, result
|
|
|
|
|
|
def run_case(name, root, binaries, api, current_api):
|
|
names = ["good.bin", "same.bin", "legacy.bin"]
|
|
if name == "completed":
|
|
names.remove("legacy.bin")
|
|
elif name == "all-denied":
|
|
names = ["legacy.bin"]
|
|
elif name == "mixed":
|
|
names += ["failed.bin", "missing-md5.bin", "invalid-md5.bin", "unbound.bin", "changed.bin"]
|
|
elif name == "zero-unicode":
|
|
names = ["empty.bin", "中文 空格.bin", "legacy.bin"]
|
|
elif name == "pagination":
|
|
names = [f"file-{number:04d}.bin" for number in range(1002)] + ["legacy.bin"]
|
|
fixture = Fixture.__new__(Fixture)
|
|
try:
|
|
fixture.__init__(root, binaries, api, names)
|
|
result = {"case": name, "uploadStatus": fixture.upload_result["session"]["status"]}
|
|
original = fixture.snapshot()
|
|
if name == "old-client":
|
|
code, rejected = fixture.clean()
|
|
assert code != 0 and "clean requires a completed upload session from this mapping" in json.dumps(rejected), rejected
|
|
assert fixture.snapshot() == original and not fixture.proof_calls
|
|
result.update(expectedRejection=rejected, exitCode=code, sourcePreserved=True)
|
|
return result
|
|
if name == "mixed":
|
|
fixture.change_local_receipt("missing-md5.bin", md5="")
|
|
fixture.change_local_receipt("invalid-md5.bin", md5="z" * 32)
|
|
fixture.change_local_receipt("unbound.bin", unbound=True)
|
|
(fixture.source / "changed.bin").write_bytes(b"changed-local-content")
|
|
if name == "cloud-md5-missing":
|
|
fixture.seed(lambda obj: obj["metadata"].pop("drosync-md5sum", None))
|
|
if name == "gates":
|
|
with fixture.db() as db:
|
|
status, document, mapping = db.execute("SELECT status,document,mapping_id FROM sessions WHERE id=?", (fixture.sid,)).fetchone()
|
|
for local_status in ["running", "failed", "aborted", "expired", "conflicted"]:
|
|
with fixture.db() as db:
|
|
db.execute("UPDATE sessions SET status=? WHERE id=?", (local_status, fixture.sid))
|
|
code, response = fixture.clean(True)
|
|
assert code != 0 and "completed or partial" in json.dumps(response), response
|
|
with fixture.db() as db:
|
|
db.execute("UPDATE sessions SET status=?,document=? WHERE id=?", (status, json.dumps({**json.loads(document), "preview": True}), fixture.sid))
|
|
code, response = fixture.clean(True)
|
|
assert code != 0, response
|
|
with fixture.db() as db:
|
|
db.execute("UPDATE sessions SET document=?,mapping_id='foreign' WHERE id=?", (document, fixture.sid))
|
|
code, response = fixture.clean(True)
|
|
assert code != 0, response
|
|
with fixture.db() as db:
|
|
db.execute("UPDATE sessions SET mapping_id=? WHERE id=?", (mapping, fixture.sid))
|
|
for remote_status in ["running", "failed", "aborted", "expired", "conflicted"]:
|
|
fixture.control.sessions[fixture.sid]["status"] = remote_status
|
|
code, response = fixture.clean(True)
|
|
assert code != 0 and "cloud session" in json.dumps(response), response
|
|
fixture.control.sessions[fixture.sid]["status"] = status
|
|
fixture.active_cloud = True
|
|
code, response = fixture.clean(True)
|
|
assert code != 0 and "active cloud session" in json.dumps(response), response
|
|
fixture.active_cloud = False
|
|
fixture.bad_device = True
|
|
code, response = fixture.clean(True)
|
|
assert code != 0 and "identity differs" in json.dumps(response), response
|
|
fixture.bad_device = False
|
|
assert fixture.snapshot() == original and not fixture.proof_calls
|
|
result["blockedGateVariants"] = 14
|
|
if name == "old-api":
|
|
code, denied = fixture.clean(True)
|
|
assert code == 50
|
|
assert_counts(denied, 0, 0, 3)
|
|
assert all(item["exists"] for item in fixture.snapshot().values())
|
|
assert all(fixture.snapshot()[key]["md5"] == value["md5"] for key, value in original.items())
|
|
result["oldAPIRejection"] = denied
|
|
result["afterRejection"] = fixture.snapshot()
|
|
original_session = fixture.sid
|
|
fixture.restart()
|
|
fixture.api = current_api
|
|
fixture.seed()
|
|
retry_code, retry = fixture.clean(True)
|
|
result["retryAfterAPIUpgrade"] = retry
|
|
result["requiresResyncAfterRestore"] = any("source identity changed" in (row.get("reason") or "") for row in retry["files"])
|
|
assert not result["requiresResyncAfterRestore"], retry
|
|
assert retry_code == 50
|
|
assert_counts(retry, 2, 2, 1)
|
|
assert fixture.sid == original_session
|
|
assert sum(row["args"][0] == "sync" for row in fixture.commands) == 1
|
|
result["sameSessionDirectRetryAfterRestart"] = True
|
|
return result
|
|
if name == "cloud-md5-missing":
|
|
code, denied = fixture.clean(True)
|
|
assert code == 50
|
|
assert_counts(denied, 0, 0, 3)
|
|
assert all(fixture.snapshot()[key]["md5"] == value["md5"] for key, value in original.items())
|
|
result["retained"] = denied
|
|
return result
|
|
expected_good = 0 if name == "all-denied" else 1002 if name == "pagination" else 2
|
|
expected_blocked = 0 if name == "completed" else 6 if name == "mixed" else 1
|
|
before_preview = fixture.snapshot()
|
|
code, preview = fixture.clean()
|
|
assert code == (50 if expected_blocked else 0)
|
|
assert_counts(preview, expected_good, 0, expected_blocked)
|
|
assert fixture.snapshot() == before_preview, "preview changed sources"
|
|
result["preview"] = preview
|
|
if name == "writer":
|
|
fixture.writer = (fixture.source / "good.bin").open("r+b")
|
|
fixture.mutate_writer = True
|
|
expected_good, expected_blocked = 1, 2
|
|
if name == "expired-proof":
|
|
fixture.expire_proof = True
|
|
expected_good, expected_blocked = 0, 3
|
|
code, cleaned = fixture.clean(True)
|
|
assert code == (50 if expected_blocked else 0)
|
|
assert_counts(cleaned, expected_good, expected_good, expected_blocked)
|
|
after = fixture.snapshot()
|
|
for row in cleaned["files"]:
|
|
key = pathlib.PurePosixPath(row["relativePath"]).name
|
|
assert after[key]["exists"] == (row["status"] == "retained"), row
|
|
if name == "mixed":
|
|
for key in ["legacy.bin", "failed.bin", "missing-md5.bin", "invalid-md5.bin", "unbound.bin"]:
|
|
assert (fixture.source / key).read_bytes() == fixture.original[key]
|
|
assert (fixture.source / "changed.bin").read_bytes() == b"changed-local-content"
|
|
if name == "writer":
|
|
assert (fixture.source / "good.bin").read_bytes() == b"X" * len(fixture.original["good.bin"])
|
|
if name == "pagination":
|
|
assert cleaned["filesTruncated"] is True and len(cleaned["files"]) == 1000
|
|
assert sum(value["exists"] for value in after.values()) == 1
|
|
result["clean"] = cleaned
|
|
_, repeated = fixture.clean(True)
|
|
assert repeated["deletedFiles"] == 0
|
|
assert repeated["alreadyCleanedFiles"] == expected_good
|
|
result["repeat"] = repeated
|
|
if name == "expired-proof":
|
|
assert not any("source identity changed" in (row.get("reason") or "") for row in repeated["files"]), repeated
|
|
fixture.expire_proof = False
|
|
_, authorized = fixture.clean(True)
|
|
assert_counts(authorized, 2, 2, 1)
|
|
assert sum(row["args"][0] == "sync" for row in fixture.commands) == 1
|
|
result["retryWithFreshProofWithoutSync"] = authorized
|
|
if name == "recreated-source":
|
|
(fixture.source / "good.bin").write_bytes(b"new capture after clean")
|
|
_, recreated = fixture.clean(True)
|
|
assert recreated["deletedFiles"] == 0 and recreated["alreadyCleanedFiles"] == 1 and recreated["blockedFiles"] == 2
|
|
assert (fixture.source / "good.bin").read_bytes() == b"new capture after clean"
|
|
result["recreated"] = recreated
|
|
stats = fixture.api.control("stats")
|
|
assert stats["contentReads"] == 0
|
|
assert stats["objects"] == len(fixture.seed_model["objects"])
|
|
assert stats["sessions"][0]["status"] == fixture.upload_result["session"]["status"]
|
|
result["apiStats"] = stats
|
|
result["sourcesAfter"] = fixture.snapshot()
|
|
return result
|
|
finally:
|
|
if hasattr(fixture, "commands"):
|
|
fixture.close()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--bin-dir", type=pathlib.Path, required=True)
|
|
parser.add_argument("--api-test-bin", type=pathlib.Path, required=True)
|
|
parser.add_argument("--old-bin-dir", type=pathlib.Path)
|
|
parser.add_argument("--old-api-test-bin", type=pathlib.Path)
|
|
parser.add_argument("--output", type=pathlib.Path, required=True)
|
|
parser.add_argument("--cases", default="completed,partial,mixed,all-denied,gates,zero-unicode,writer,expired-proof,cloud-md5-missing,recreated-source,pagination,old-client,old-api")
|
|
args = parser.parse_args()
|
|
output = args.output.resolve()
|
|
output.mkdir(mode=0o700, parents=True, exist_ok=False)
|
|
current = GoAPI(args.api_test_bin.resolve(), output / "api-current")
|
|
legacy = None
|
|
results = []
|
|
try:
|
|
if args.old_api_test_bin:
|
|
legacy = GoAPI(args.old_api_test_bin.resolve(), output / "api-old")
|
|
for case in args.cases.split(","):
|
|
if case == "old-client" and not args.old_bin_dir or case == "old-api" and legacy is None:
|
|
continue
|
|
binaries = args.old_bin_dir.resolve() if case == "old-client" else args.bin_dir.resolve()
|
|
api = legacy if case == "old-api" else current
|
|
try:
|
|
result = run_case(case, output / case, binaries, api, current)
|
|
result["status"] = "PASS"
|
|
except Exception as error:
|
|
import traceback
|
|
result = {"case": case, "status": "FAIL", "error": str(error), "traceback": traceback.format_exc()}
|
|
(output / f"{case}.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
summary = {key: result[key] for key in ("case", "status", "uploadStatus", "error", "requiresResyncAfterRestore") if key in result}
|
|
results.append(summary)
|
|
print(json.dumps(summary, ensure_ascii=False), flush=True)
|
|
finally:
|
|
server_exits = {}
|
|
if legacy:
|
|
server_exits["old"] = legacy.close()
|
|
server_exits["current"] = current.close()
|
|
report = {"scope": "actual CLI/daemon and Go cleanup API with MemoryStore, seeded receipts and fake storage HEAD; upload control/storage synthetic", "baseHarnessSha256": hashlib.sha256(BASE_SOURCE.read_bytes()).hexdigest(), "apiExitCodes": server_exits, "results": results, "passed": sum(item["status"] == "PASS" for item in results), "failed": sum(item["status"] != "PASS" for item in results) + sum(code != 0 for code in server_exits.values())}
|
|
(output / "summary.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
return int(report["failed"] != 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|