#!/usr/bin/env python3 """Run an evidence-preserving drosync functional acceptance against a real POC/BOS.""" from __future__ import annotations import argparse import hashlib import http.cookiejar import json import os import pathlib import shutil import subprocess import sys import tempfile import time import urllib.error import urllib.parse import urllib.request import uuid from datetime import UTC, datetime from typing import Any REDACTED_KEYS = {"secret", "secretHash", "accessToken", "token", "tokenHash", "tokenHashes", "password"} def utc_now() -> str: return datetime.now(UTC).isoformat().replace("+00:00", "Z") def redact(value: Any) -> Any: if isinstance(value, dict): return { key: ("[REDACTED]" if key in REDACTED_KEYS or "password" in key.lower() else redact(item)) for key, item in value.items() } if isinstance(value, list): return [redact(item) for item in value] return value class AcceptanceError(RuntimeError): pass class ExternalBlock(AcceptanceError): pass class API: def __init__(self, base_url: str, api_token: str = "") -> None: self.base_url = base_url.rstrip("/") self.api_token = api_token self.cookies = http.cookiejar.CookieJar() handlers: list[Any] = [urllib.request.HTTPCookieProcessor(self.cookies)] hostname = (urllib.parse.urlparse(self.base_url).hostname or "").lower() if hostname in {"127.0.0.1", "::1", "localhost"}: handlers.insert(0, urllib.request.ProxyHandler({})) self.opener = urllib.request.build_opener(*handlers) self.calls: list[dict[str, Any]] = [] def csrf_token(self) -> str: for cookie in self.cookies: if cookie.name == "droboticflow_csrf": return cookie.value return "" def request( self, method: str, path: str, body: Any | None = None, *, idempotency_key: str = "", expected: tuple[int, ...] = (200,), stream_to: pathlib.Path | None = None, ) -> tuple[Any, dict[str, str]]: payload = None if body is None else json.dumps(body, ensure_ascii=False).encode() headers = {"Accept": "application/json"} if payload is not None: headers["Content-Type"] = "application/json" if self.api_token: headers["Authorization"] = f"Bearer {self.api_token}" csrf = self.csrf_token() if csrf and method not in {"GET", "HEAD", "OPTIONS"}: headers["X-Droboticflow-CSRF-Token"] = csrf if idempotency_key: headers["Idempotency-Key"] = idempotency_key request = urllib.request.Request(self.base_url + path, data=payload, headers=headers, method=method) started = time.monotonic() status = 0 response_headers: dict[str, str] = {} try: with self.opener.open(request, timeout=120) as response: status = response.status response_headers = dict(response.headers.items()) if stream_to is not None: with stream_to.open("wb") as output: shutil.copyfileobj(response, output, length=8 * 1024 * 1024) result: Any = {"downloadedTo": str(stream_to), "sizeBytes": stream_to.stat().st_size} else: raw = response.read() content_type = response.headers.get("Content-Type", "") result = json.loads(raw) if raw and "json" in content_type else raw.decode(errors="replace") except urllib.error.HTTPError as error: status = error.code response_headers = dict(error.headers.items()) raw = error.read() try: result = json.loads(raw) except (json.JSONDecodeError, UnicodeDecodeError): result = raw.decode(errors="replace") except urllib.error.URLError as error: raise ExternalBlock(f"POC endpoint is unreachable: {error}") from error call = { "at": utc_now(), "method": method, "path": path, "status": status, "elapsedMs": round((time.monotonic() - started) * 1000), "requestId": response_headers.get("X-Request-Id", response_headers.get("X-Request-ID", "")), } self.calls.append(call) if status not in expected: raise AcceptanceError(f"{method} {path} returned {status}, expected {expected}: {redact(result)}") return result, response_headers def run_json(command: list[str], *, stdin: str = "", timeout: int = 7_200) -> dict[str, Any]: completed = subprocess.run( command, input=stdin, text=True, capture_output=True, timeout=timeout, check=False, ) if completed.returncode != 0: raise AcceptanceError( f"command failed ({completed.returncode}): {' '.join(command)}\n" f"stdout={completed.stdout[-2000:]}\nstderr={completed.stderr[-2000:]}" ) try: return json.loads(completed.stdout) except json.JSONDecodeError as error: raise AcceptanceError(f"command did not emit one JSON value: {' '.join(command)}") from error def write_pattern(path: pathlib.Path, size_bytes: int) -> str: digest = hashlib.sha256() chunk = hashlib.sha256(str(path).encode()).digest() * (1024 * 32) remaining = size_bytes with path.open("wb") as output: while remaining: block = chunk[: min(len(chunk), remaining)] output.write(block) digest.update(block) remaining -= len(block) return digest.hexdigest() def sha256_file(path: pathlib.Path) -> str: digest = hashlib.sha256() with path.open("rb") as source: for block in iter(lambda: source.read(8 * 1024 * 1024), b""): digest.update(block) return digest.hexdigest() def object_path(connection_id: str, bucket: str, key: str) -> str: query = urllib.parse.urlencode({"connectionId": connection_id, "bucket": bucket, "key": key}) return f"/api/object-storage/content?{query}" def wait_for_session(api: API, mapping_id: str, after: str, timeout: int = 900) -> dict[str, Any]: deadline = time.monotonic() + timeout while time.monotonic() < deadline: page, _ = api.request("GET", "/api/object-sync/sessions?limit=200", expected=(200,)) candidates = [ item for item in page.get("items", []) if item.get("mappingId") == mapping_id and item.get("createdAt", "") >= after ] if candidates: current = sorted(candidates, key=lambda item: item.get("createdAt", ""), reverse=True)[0] if current.get("status") in {"completed", "partial", "conflicted", "failed", "aborted", "expired"}: return current time.sleep(2) raise AcceptanceError(f"session for mapping {mapping_id} did not reach a terminal state") def transport_security(base_url: str, require_https: bool) -> dict[str, Any]: parsed = urllib.parse.urlparse(base_url) if parsed.scheme not in {"http", "https"} or not parsed.netloc: raise ExternalBlock("control-plane base URL must be an absolute http:// or https:// URL") if require_https and parsed.scheme != "https": raise ExternalBlock("this acceptance run was configured to require HTTPS") encrypted = parsed.scheme == "https" return { "status": "PASS", "scheme": parsed.scheme, "encrypted": encrypted, "requireHttps": require_https, "acceptedRisk": ( None if encrypted else "control-plane credentials and metadata use an unencrypted HTTP connection" ), } def acceptance_status(public_scheme: str, audit_present: bool, require_https: bool = False) -> str: if not audit_present: return "FAIL" if require_https and public_scheme != "https": return "BLOCKED_EXTERNAL" return "PASS" def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--base-url", default=os.getenv("DROSYNC_ACCEPTANCE_BASE_URL", "")) parser.add_argument("--api-token", default=os.getenv("DROBOTICFLOW_API_TOKEN", "")) parser.add_argument("--email", default=os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", "")) parser.add_argument("--password", default=os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", "")) parser.add_argument("--connection-id", default=os.getenv("DROSYNC_ACCEPTANCE_CONNECTION_ID", "")) parser.add_argument("--bucket", default=os.getenv("DROSYNC_ACCEPTANCE_BUCKET", "")) parser.add_argument("--drosync-bin", default=os.getenv("DROSYNC_BIN", "drosync")) parser.add_argument("--profile", default="poc-acceptance") parser.add_argument("--stability-seconds", type=int, default=60) parser.add_argument("--multipart-bytes", type=int, default=300 * 1024 * 1024) parser.add_argument("--evidence-dir", default=".codex-artifacts/drosync-acceptance") parser.add_argument( "--require-https", action="store_true", default=os.getenv("DROSYNC_ACCEPTANCE_REQUIRE_HTTPS", "").lower() in {"1", "true", "yes"}, help="reject an HTTP control-plane URL; HTTP is allowed by default for the current POC", ) parser.add_argument( "--diagnostic-http", action="store_true", help=argparse.SUPPRESS, ) parser.add_argument("--keep-workdir", action="store_true") return parser.parse_args() def main() -> int: args = parse_args() run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8] evidence_dir = pathlib.Path(args.evidence_dir).resolve() / run_id evidence_dir.mkdir(parents=True, exist_ok=False) report: dict[str, Any] = { "schemaVersion": 1, "runId": run_id, "startedAt": utc_now(), "status": "FAIL", "layers": {}, } api: API | None = None daemon: subprocess.Popen[str] | None = None workdir: pathlib.Path | None = None credential_id = "" mapping_id = "" space_id = "" try: if not args.base_url or not args.connection_id or not args.bucket: raise ExternalBlock("--base-url, --connection-id and --bucket are required") transport = transport_security(args.base_url, args.require_https) parsed = urllib.parse.urlparse(args.base_url) if shutil.which(args.drosync_bin) is None and not pathlib.Path(args.drosync_bin).exists(): raise ExternalBlock(f"drosync executable was not found: {args.drosync_bin}") api = API(args.base_url, args.api_token) if args.email: api.request( "POST", "/api/auth/login", {"email": args.email, "password": args.password}, expected=(200,), ) api.request("GET", "/api/auth/me", expected=(200,)) prefix = f"drosync-acceptance/{run_id}" space_payload = { "name": f"drosync acceptance {run_id}", "description": "isolated automated POC acceptance", "connectionId": args.connection_id, "bucket": args.bucket, "rootPrefix": prefix, "operatorUserIds": [], } space, _ = api.request( "POST", "/api/object-sync/spaces", space_payload, idempotency_key=f"{run_id}-space", expected=(201,), ) space_id = space["id"] probed, _ = api.request( "POST", f"/api/object-sync/spaces/{space_id}/capability-probe", {}, idempotency_key=f"{run_id}-probe", expected=(200,), ) if probed.get("capability", {}).get("state") not in {"create_only", "update_enabled"}: raise ExternalBlock(f"BOS conditional create probe did not pass: {redact(probed.get('capability'))}") api.request( "PUT", f"/api/object-sync/spaces/{space_id}", {"enabled": True}, idempotency_key=f"{run_id}-enable", expected=(200,), ) issued, _ = api.request( "POST", "/api/object-sync/credentials", { "name": f"acceptance key {run_id}", "scopes": [ { "syncSpaceId": space_id, "allowedRelativePrefixes": [""], "actions": ["plan", "list_metadata", "create", "keep_both"], } ], }, idempotency_key=f"{run_id}-credential", expected=(201,), ) credential = issued["credential"] credential_id = credential["id"] secret = issued.pop("secret") workdir = pathlib.Path(tempfile.mkdtemp(prefix=f"drosync-{run_id}-")) local_root = workdir / "data" local_root.mkdir() (local_root / "nested 中文").mkdir() expected_hashes = { "empty.bin": write_pattern(local_root / "empty.bin", 0), "one-byte.bin": write_pattern(local_root / "one-byte.bin", 1), "nested 中文/space name.txt": write_pattern(local_root / "nested 中文" / "space name.txt", 337), "multipart-300m.bin": write_pattern(local_root / "multipart-300m.bin", args.multipart_bytes), } write_pattern(local_root / ".hidden.bin", 8) write_pattern(local_root / "ignored.tmp", 8) try: (local_root / "ignored-symlink").symlink_to(local_root / "one-byte.bin") except OSError: pass report["layers"]["localFixture"] = { "status": "PASS", "root": str(local_root), "expected": expected_hashes, "excluded": [".hidden.bin", "ignored.tmp", "ignored-symlink"], } status_command = [args.drosync_bin, "--json", "status"] status_probe = subprocess.run(status_command, text=True, capture_output=True, check=False) if status_probe.returncode != 0: daemon = subprocess.Popen( [args.drosync_bin, "--json", "daemon", "--foreground"], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) deadline = time.monotonic() + 20 while time.monotonic() < deadline: status_probe = subprocess.run(status_command, text=True, capture_output=True, check=False) if status_probe.returncode == 0: break time.sleep(0.5) if status_probe.returncode != 0: raise ExternalBlock("installed drosync daemon did not become ready") run_json( [ args.drosync_bin, "--json", "auth", "--server", args.base_url, "--profile", args.profile, "--access-key-id", credential["accessKeyId"], "--secret-stdin", ], stdin=secret, ) secret = "" mapping = run_json( [ args.drosync_bin, "--json", "mapping", "add", str(local_root), "--space", space_id, ] ) mapping_id = mapping.get("id") or mapping.get("mapping", {}).get("id") if not mapping_id: raise AcceptanceError(f"mapping add did not return an id: {redact(mapping)}") if args.stability_seconds > 0: time.sleep(args.stability_seconds) first_started = utc_now() first_client = run_json([args.drosync_bin, "--json", "sync", mapping_id], timeout=12 * 3600) first_session = wait_for_session(api, mapping_id, first_started) if first_session.get("status") not in {"completed", "conflicted"}: raise AcceptanceError(f"first sync did not complete: {redact(first_session)}") item_page, _ = api.request( "GET", f"/api/object-sync/sessions/{first_session['id']}/items?limit=1000", expected=(200,) ) items = item_page.get("items", []) paths = {item.get("relativePath") for item in items} excluded_present = paths.intersection({".hidden.bin", "ignored.tmp", "ignored-symlink"}) if excluded_present: raise AcceptanceError(f"excluded local entries reached the manifest: {sorted(excluded_present)}") remote_evidence: dict[str, Any] = {} for relative_path, expected_hash in expected_hashes.items(): key = f"{prefix}/{relative_path}" download = workdir / ("download-" + hashlib.sha256(relative_path.encode()).hexdigest()[:12]) _, headers = api.request( "GET", object_path(args.connection_id, args.bucket, key), expected=(200,), stream_to=download, ) actual_hash = sha256_file(download) if actual_hash != expected_hash: raise AcceptanceError(f"remote SHA-256 mismatch for {key}: {actual_hash} != {expected_hash}") remote_evidence[relative_path] = { "key": key, "sizeBytes": download.stat().st_size, "sha256": actual_hash, "etag": headers.get("ETag", ""), "versionId": headers.get("x-bce-version-id", ""), } download.unlink() (local_root / "one-byte.bin").unlink() second_started = utc_now() second_client = run_json([args.drosync_bin, "--json", "sync", mapping_id], timeout=12 * 3600) second_session = wait_for_session(api, mapping_id, second_started) if second_session.get("counts", {}).get("unchanged", 0) < len(expected_hashes) - 1: raise AcceptanceError(f"second sync did not report unchanged files: {redact(second_session)}") retained = workdir / "retained-one-byte" api.request( "GET", object_path(args.connection_id, args.bucket, f"{prefix}/one-byte.bin"), expected=(200,), stream_to=retained, ) if sha256_file(retained) != expected_hashes["one-byte.bin"]: raise AcceptanceError("remote no-delete invariant failed") retained.unlink() audit_page, _ = api.request("GET", "/api/object-sync/audit-events?limit=1000", expected=(200,)) related_audit = [ event for event in audit_page.get("items", []) if event.get("sessionId") in {first_session["id"], second_session["id"]} or event.get("target", {}).get("spaceId") == space_id ] report["layers"].update( { "pocApi": { "status": "PASS", "space": redact(probed), "credential": redact(credential), "sessions": [redact(first_session), redact(second_session)], "items": redact(items), }, "installedClient": { "status": "PASS", "binary": str(pathlib.Path(args.drosync_bin).resolve()), "firstSync": redact(first_client), "secondSync": redact(second_client), }, "realBos": {"status": "PASS", "objects": remote_evidence, "remoteDelete": "not_performed"}, "audit": {"status": "PASS" if related_audit else "FAIL", "events": redact(related_audit)}, "controlPlaneTransport": {**transport, "baseUrl": args.base_url}, "publicHttps": { "status": "PASS" if parsed.scheme == "https" else "NOT_RUN", "required": args.require_https, "baseUrl": args.base_url, }, } ) report["status"] = acceptance_status( parsed.scheme, bool(related_audit), args.require_https, ) except ExternalBlock as error: report["status"] = "BLOCKED_EXTERNAL" report["error"] = str(error) except Exception as error: # noqa: BLE001 - acceptance must always preserve a report report["status"] = "FAIL" report["error"] = str(error) finally: cleanup: dict[str, Any] = {} cleanup_failed = False if api is not None and credential_id: try: revoked, _ = api.request( "POST", f"/api/object-sync/credentials/{credential_id}/revoke", {}, idempotency_key=f"{run_id}-cleanup-credential", expected=(200,), ) cleanup["credential"] = { "status": "PASS", "id": credential_id, "enabled": revoked.get("credential", {}).get("enabled"), } except Exception as error: # noqa: BLE001 - preserve cleanup evidence cleanup_failed = True cleanup["credential"] = {"status": "FAIL", "error": str(error)} if api is not None and space_id: try: disabled, _ = api.request( "PUT", f"/api/object-sync/spaces/{space_id}", {"enabled": False}, idempotency_key=f"{run_id}-cleanup-space", expected=(200,), ) cleanup["space"] = { "status": "PASS", "id": space_id, "enabled": disabled.get("enabled"), } except Exception as error: # noqa: BLE001 - preserve cleanup evidence cleanup_failed = True cleanup["space"] = {"status": "FAIL", "error": str(error)} if mapping_id: try: cleanup["mapping"] = { "status": "PASS", "result": redact( run_json([args.drosync_bin, "--json", "mapping", "remove", mapping_id], timeout=60) ), } except Exception as error: # noqa: BLE001 - preserve cleanup evidence cleanup_failed = True cleanup["mapping"] = {"status": "FAIL", "error": str(error)} if cleanup: report["cleanup"] = cleanup if cleanup_failed: report["status"] = "FAIL" report["finishedAt"] = utc_now() if api is not None: report["apiCalls"] = redact(api.calls) if daemon is not None: daemon.terminate() try: daemon.wait(timeout=10) except subprocess.TimeoutExpired: daemon.kill() if workdir is not None: if args.keep_workdir: report["workdir"] = str(workdir) else: shutil.rmtree(workdir, ignore_errors=True) (evidence_dir / "report.json").write_text( json.dumps(redact(report), ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) print(json.dumps({"status": report["status"], "evidence": str(evidence_dir / "report.json")})) return 0 if report["status"] == "PASS" else 2 if __name__ == "__main__": sys.exit(main())