droboticflow/scripts/drosync-poc-acceptance.py

607 lines
25 KiB
Python

#!/usr/bin/env python3
"""Run an evidence-preserving drosync functional acceptance against a real POC/BOS."""
from __future__ import annotations
import argparse
import hashlib
import http.cookiejar
import json
import os
import pathlib
import shutil
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from datetime import UTC, datetime
from typing import Any
CLI_RUNTIME_CONFIG: pathlib.Path | None = None
def configure_cli_runtime(path: str | None) -> None:
"""Select the same explicit file used by the dedicated acceptance daemon."""
global CLI_RUNTIME_CONFIG
CLI_RUNTIME_CONFIG = pathlib.Path(path).resolve(strict=True) if path else None
if CLI_RUNTIME_CONFIG is not None and not CLI_RUNTIME_CONFIG.is_file():
raise ValueError("--drosync-config must be a readable regular file")
REDACTED_KEYS = {"secret", "secretHash", "accessToken", "token", "tokenHash", "tokenHashes", "password"}
def utc_now() -> str:
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
def redact(value: Any) -> Any:
if isinstance(value, dict):
return {
key: ("[REDACTED]" if key in REDACTED_KEYS or "password" in key.lower() else redact(item))
for key, item in value.items()
}
if isinstance(value, list):
return [redact(item) for item in value]
return value
class AcceptanceError(RuntimeError):
pass
class ExternalBlock(AcceptanceError):
pass
class API:
def __init__(self, base_url: str, api_token: str = "") -> None:
self.base_url = base_url.rstrip("/")
self.api_token = api_token
self.cookies = http.cookiejar.CookieJar()
handlers: list[Any] = [urllib.request.HTTPCookieProcessor(self.cookies)]
hostname = (urllib.parse.urlparse(self.base_url).hostname or "").lower()
if hostname in {"127.0.0.1", "::1", "localhost"}:
handlers.insert(0, urllib.request.ProxyHandler({}))
self.opener = urllib.request.build_opener(*handlers)
self.calls: list[dict[str, Any]] = []
def csrf_token(self) -> str:
for cookie in self.cookies:
if cookie.name == "droboticflow_csrf":
return cookie.value
return ""
def request(
self,
method: str,
path: str,
body: Any | None = None,
*,
idempotency_key: str = "",
expected: tuple[int, ...] = (200,),
stream_to: pathlib.Path | None = None,
) -> tuple[Any, dict[str, str]]:
payload = None if body is None else json.dumps(body, ensure_ascii=False).encode()
headers = {"Accept": "application/json"}
if payload is not None:
headers["Content-Type"] = "application/json"
if self.api_token:
headers["Authorization"] = f"Bearer {self.api_token}"
csrf = self.csrf_token()
if csrf and method not in {"GET", "HEAD", "OPTIONS"}:
headers["X-Droboticflow-CSRF-Token"] = csrf
if idempotency_key:
headers["Idempotency-Key"] = idempotency_key
request = urllib.request.Request(self.base_url + path, data=payload, headers=headers, method=method)
started = time.monotonic()
status = 0
response_headers: dict[str, str] = {}
try:
with self.opener.open(request, timeout=120) as response:
status = response.status
response_headers = dict(response.headers.items())
if stream_to is not None:
with stream_to.open("wb") as output:
shutil.copyfileobj(response, output, length=8 * 1024 * 1024)
result: Any = {"downloadedTo": str(stream_to), "sizeBytes": stream_to.stat().st_size}
else:
raw = response.read()
content_type = response.headers.get("Content-Type", "")
result = json.loads(raw) if raw and "json" in content_type else raw.decode(errors="replace")
except urllib.error.HTTPError as error:
status = error.code
response_headers = dict(error.headers.items())
raw = error.read()
try:
result = json.loads(raw)
except (json.JSONDecodeError, UnicodeDecodeError):
result = raw.decode(errors="replace")
except urllib.error.URLError as error:
raise ExternalBlock(f"POC endpoint is unreachable: {error}") from error
call = {
"at": utc_now(),
"method": method,
"path": path,
"status": status,
"elapsedMs": round((time.monotonic() - started) * 1000),
"requestId": response_headers.get("X-Request-Id", response_headers.get("X-Request-ID", "")),
}
self.calls.append(call)
if status not in expected:
raise AcceptanceError(f"{method} {path} returned {status}, expected {expected}: {redact(result)}")
return result, response_headers
def run_json(
command: list[str], *, stdin: str = "", timeout: int = 7_200,
expected_exit: int = 0,
) -> dict[str, Any]:
if CLI_RUNTIME_CONFIG is not None:
command = [command[0], "-c", str(CLI_RUNTIME_CONFIG), *command[1:]]
completed = subprocess.run(
command,
input=stdin,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
if completed.returncode != expected_exit:
raise AcceptanceError(
f"command failed ({completed.returncode}): {' '.join(command)}\n"
f"stdout={completed.stdout[-2000:]}\nstderr={completed.stderr[-2000:]}"
)
try:
result = json.loads(completed.stdout)
except json.JSONDecodeError as error:
raise AcceptanceError(f"command did not emit one JSON value: {' '.join(command)}") from error
if not isinstance(result, dict):
raise AcceptanceError(f"command did not emit a JSON object: {' '.join(command)}")
return result
def run_sync_json(command: list[str], *, expected_status: str = "completed", timeout: int = 7_200) -> dict[str, Any]:
"""Only a deliberately conflicting scenario may accept CLI exit 30."""
if expected_status not in {"completed", "conflicted"}:
raise ValueError("acceptance must expect completed or an explicit conflicted scenario")
result = run_json(command, timeout=timeout, expected_exit=30 if expected_status == "conflicted" else 0)
session = result.get("session")
if not isinstance(session, dict) or session.get("status") != expected_status or not session.get("id"):
raise AcceptanceError(f"sync result did not match expected {expected_status} session: {redact(result)}")
return result
def require_empty_daemon(binary: str) -> None:
try:
status = run_json([binary, "--json", "status"])
except AcceptanceError as error:
raise ExternalBlock("start the selected isolated drosync OS service before acceptance") from error
if status.get("mappingCount") != 0:
raise ExternalBlock("acceptance requires a dedicated empty daemon namespace; do not replace credentials on an existing mapping")
def require_server_protocol(binary: str) -> dict[str, Any]:
result = run_json([binary, "--json", "doctor"])
capabilities = result.get("capabilities", {})
if capabilities.get("features", {}).get("sessionHeartbeat") is not True:
raise ExternalBlock("server upgrade required: sessionHeartbeat capability must be enabled before acceptance")
return {"status": "PASS", "features": capabilities["features"]}
def cleanup_acceptance(
api: API | None, binary: str, run_id: str, mapping_id: str, credential_id: str, space_id: str,
) -> tuple[dict[str, Any], bool]:
"""Cancel/remove while the scoped credential is still usable; preserve failures."""
cleanup: dict[str, Any] = {}
failed = False
if mapping_id:
try:
result = run_json([binary, "--json", "mapping", "remove", mapping_id], timeout=900)
if result.get("removed") is not True:
raise AcceptanceError(f"mapping removal was not confirmed: {redact(result)}")
cleanup["mapping"] = {"status": "PASS", "result": redact(result)}
except Exception as error: # noqa: BLE001 - cleanup failures remain evidence
cleanup["mapping"] = {"status": "FAIL", "error": str(error)}
# Revocation/space disabling would prevent a subsequent remote abort.
for name, identity in [("credential", credential_id), ("space", space_id)]:
if identity:
cleanup[name] = {"status": "RETAINED_FOR_RETRY", "id": identity, "reason": "mapping cancellation was not confirmed"}
return cleanup, True
if api is not None and credential_id:
try:
revoked, _ = api.request("POST", f"/api/object-sync/credentials/{credential_id}/revoke", {},
idempotency_key=f"{run_id}-cleanup-credential", expected=(200,))
if revoked.get("credential", {}).get("enabled") is not False:
raise AcceptanceError("credential revocation was not confirmed")
cleanup["credential"] = {"status": "PASS", "id": credential_id, "enabled": False}
except Exception as error: # noqa: BLE001
failed = True
cleanup["credential"] = {"status": "FAIL", "error": str(error)}
if api is not None and space_id:
try:
disabled, _ = api.request("PUT", f"/api/object-sync/spaces/{space_id}", {"enabled": False},
idempotency_key=f"{run_id}-cleanup-space", expected=(200,))
if disabled.get("enabled") is not False:
raise AcceptanceError("space disabling was not confirmed")
cleanup["space"] = {"status": "PASS", "id": space_id, "enabled": False}
except Exception as error: # noqa: BLE001
failed = True
cleanup["space"] = {"status": "FAIL", "error": str(error)}
return cleanup, failed
def write_pattern(path: pathlib.Path, size_bytes: int) -> str:
digest = hashlib.md5()
chunk = hashlib.md5(str(path).encode()).digest() * (1024 * 32)
remaining = size_bytes
with path.open("wb") as output:
while remaining:
block = chunk[: min(len(chunk), remaining)]
output.write(block)
digest.update(block)
remaining -= len(block)
return digest.hexdigest()
def md5sum_file(path: pathlib.Path) -> str:
digest = hashlib.md5()
with path.open("rb") as source:
for block in iter(lambda: source.read(8 * 1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def object_path(connection_id: str, bucket: str, key: str) -> str:
query = urllib.parse.urlencode({"connectionId": connection_id, "bucket": bucket, "key": key})
return f"/api/object-storage/content?{query}"
def wait_for_session(api: API, session_id: str, timeout: int = 900) -> dict[str, Any]:
"""Verify the exact CLI receipt, not whichever mapping session is newest."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
current, _ = api.request("GET", f"/api/object-sync/sessions/{urllib.parse.quote(session_id, safe='')}", expected=(200,))
if current.get("id") != session_id:
raise AcceptanceError("session lookup returned a different session")
if current.get("status") in {"completed", "partial", "conflicted", "failed", "aborted", "expired"}:
return current
time.sleep(2)
raise AcceptanceError(f"session {session_id} did not reach a terminal state")
def transport_security(base_url: str, require_https: bool) -> dict[str, Any]:
parsed = urllib.parse.urlparse(base_url)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ExternalBlock("control-plane base URL must be an absolute http:// or https:// URL")
if require_https and parsed.scheme != "https":
raise ExternalBlock("this acceptance run was configured to require HTTPS")
encrypted = parsed.scheme == "https"
return {
"status": "PASS",
"scheme": parsed.scheme,
"encrypted": encrypted,
"requireHttps": require_https,
"acceptedRisk": (
None
if encrypted
else "control-plane credentials and metadata use an unencrypted HTTP connection"
),
}
def acceptance_status(public_scheme: str, audit_present: bool, require_https: bool = False) -> str:
if not audit_present:
return "FAIL"
if require_https and public_scheme != "https":
return "BLOCKED_EXTERNAL"
return "PASS"
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default=os.getenv("DROSYNC_ACCEPTANCE_BASE_URL", ""))
parser.add_argument("--api-token", default=os.getenv("DROBOTICFLOW_API_TOKEN", ""))
parser.add_argument("--email", default=os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", ""))
parser.add_argument("--password", default=os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", ""))
parser.add_argument("--connection-id", default=os.getenv("DROSYNC_ACCEPTANCE_CONNECTION_ID", ""))
parser.add_argument("--bucket", default=os.getenv("DROSYNC_ACCEPTANCE_BUCKET", ""))
parser.add_argument("--root-prefix", default="", help="Authorized connection root, e.g. ubt/")
parser.add_argument("--drosync-bin", default=os.getenv("DROSYNC_BIN", "drosync"))
parser.add_argument("--drosync-config", help="Runtime file used by the acceptance daemon (-c FILE)")
parser.add_argument("--profile", default="poc-acceptance")
parser.add_argument("--stability-seconds", type=int, default=60)
parser.add_argument("--multipart-bytes", type=int, default=300 * 1024 * 1024)
parser.add_argument("--evidence-dir", default=".codex-artifacts/drosync-acceptance")
parser.add_argument(
"--require-https",
action="store_true",
default=os.getenv("DROSYNC_ACCEPTANCE_REQUIRE_HTTPS", "").lower() in {"1", "true", "yes"},
help="reject an HTTP control-plane URL; HTTP is allowed by default for the current POC",
)
parser.add_argument(
"--diagnostic-http",
action="store_true",
help=argparse.SUPPRESS,
)
parser.add_argument("--keep-workdir", action="store_true")
return parser.parse_args()
def main() -> int:
args = parse_args()
configure_cli_runtime(args.drosync_config)
run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8]
evidence_dir = pathlib.Path(args.evidence_dir).resolve() / run_id
evidence_dir.mkdir(parents=True, exist_ok=False)
report: dict[str, Any] = {
"schemaVersion": 1,
"runId": run_id,
"startedAt": utc_now(),
"status": "FAIL",
"layers": {},
}
api: API | None = None
workdir: pathlib.Path | None = None
credential_id = ""
mapping_id = ""
space_id = ""
try:
if not args.base_url or not args.connection_id or not args.bucket:
raise ExternalBlock("--base-url, --connection-id and --bucket are required")
transport = transport_security(args.base_url, args.require_https)
parsed = urllib.parse.urlparse(args.base_url)
if shutil.which(args.drosync_bin) is None and not pathlib.Path(args.drosync_bin).exists():
raise ExternalBlock(f"drosync executable was not found: {args.drosync_bin}")
api = API(args.base_url, args.api_token)
if args.email:
api.request(
"POST",
"/api/auth/login",
{"email": args.email, "password": args.password},
expected=(200,),
)
api.request("GET", "/api/auth/me", expected=(200,))
root = args.root_prefix.strip("/")
if root and any(part in {"", ".", ".."} for part in root.split("/")):
raise AcceptanceError("--root-prefix must be a literal directory")
prefix = "/".join(part for part in [root, f"drosync-acceptance/{run_id}"] if part)
space_payload = {
"name": f"drosync acceptance {run_id}",
"description": "isolated automated POC acceptance",
"connectionId": args.connection_id,
"bucket": args.bucket,
"rootPrefix": prefix,
"operatorUserIds": [],
}
space, _ = api.request(
"POST",
"/api/object-sync/spaces",
space_payload,
idempotency_key=f"{run_id}-space",
expected=(201,),
)
space_id = space["id"]
probed, _ = api.request(
"POST",
f"/api/object-sync/spaces/{space_id}/capability-probe",
{},
idempotency_key=f"{run_id}-probe",
expected=(200,),
)
if probed.get("capability", {}).get("state") not in {"create_only", "update_enabled"}:
raise ExternalBlock(f"BOS conditional create probe did not pass: {redact(probed.get('capability'))}")
api.request(
"PUT",
f"/api/object-sync/spaces/{space_id}",
{"enabled": True},
idempotency_key=f"{run_id}-enable",
expected=(200,),
)
issued, _ = api.request(
"POST",
"/api/object-sync/credentials",
{
"name": f"acceptance key {run_id}",
"scopes": [
{
"syncSpaceId": space_id,
"allowedRelativePrefixes": [""],
"actions": ["plan", "list_metadata", "create", "keep_both"],
}
],
},
idempotency_key=f"{run_id}-credential",
expected=(201,),
)
credential = issued["credential"]
credential_id = credential["id"]
secret = issued.pop("secret")
workdir = pathlib.Path(tempfile.mkdtemp(prefix=f"drosync-{run_id}-"))
local_root = workdir / "data"
local_root.mkdir()
(local_root / "nested 中文").mkdir()
expected_hashes = {
"empty.bin": write_pattern(local_root / "empty.bin", 0),
"one-byte.bin": write_pattern(local_root / "one-byte.bin", 1),
"nested 中文/space name.txt": write_pattern(local_root / "nested 中文" / "space name.txt", 337),
"multipart-300m.bin": write_pattern(local_root / "multipart-300m.bin", args.multipart_bytes),
}
write_pattern(local_root / ".hidden.bin", 8)
write_pattern(local_root / "ignored.tmp", 8)
try:
(local_root / "ignored-symlink").symlink_to(local_root / "one-byte.bin")
except OSError:
pass
report["layers"]["localFixture"] = {
"status": "PASS",
"root": str(local_root),
"expected": expected_hashes,
"excluded": [".hidden.bin", "ignored.tmp", "ignored-symlink"],
}
require_empty_daemon(args.drosync_bin)
run_json(
[
args.drosync_bin,
"--json",
"auth",
"--server",
args.base_url,
"--profile",
args.profile,
"--access-key-id",
credential["accessKeyId"],
"--secret-stdin",
],
stdin=secret,
)
secret = ""
report["layers"]["serverProtocol"] = require_server_protocol(args.drosync_bin)
# Stabilize before registering, so the automatic scheduler cannot race
# the first explicit sync while this runner sleeps.
if args.stability_seconds > 0:
time.sleep(args.stability_seconds)
mapping = run_json(
[
args.drosync_bin,
"--json",
"mapping",
"add",
str(local_root),
"--space",
space_id,
]
)
mapping_id = mapping.get("id")
if not mapping_id:
raise AcceptanceError(f"mapping add did not return an id: {redact(mapping)}")
first_client = run_sync_json([args.drosync_bin, "--json", "sync", mapping_id], timeout=12 * 3600)
first_session = wait_for_session(api, first_client["session"]["id"])
if first_session.get("status") != "completed":
raise AcceptanceError(f"first sync did not complete: {redact(first_session)}")
item_page, _ = api.request(
"GET", f"/api/object-sync/sessions/{first_session['id']}/items?limit=1000", expected=(200,)
)
items = item_page.get("items", [])
paths = {item.get("relativePath") for item in items}
excluded_present = paths.intersection({".hidden.bin", "ignored.tmp", "ignored-symlink"})
if excluded_present:
raise AcceptanceError(f"excluded local entries reached the manifest: {sorted(excluded_present)}")
remote_evidence: dict[str, Any] = {}
for relative_path, expected_hash in expected_hashes.items():
key = f"{prefix}/{relative_path}"
download = workdir / ("download-" + hashlib.md5(relative_path.encode()).hexdigest()[:12])
_, headers = api.request(
"GET",
object_path(args.connection_id, args.bucket, key),
expected=(200,),
stream_to=download,
)
actual_hash = md5sum_file(download)
if actual_hash != expected_hash:
raise AcceptanceError(f"remote MD5 mismatch for {key}: {actual_hash} != {expected_hash}")
remote_evidence[relative_path] = {
"key": key,
"sizeBytes": download.stat().st_size,
"md5sum": actual_hash,
"etag": headers.get("ETag", ""),
"versionId": headers.get("x-bce-version-id", ""),
}
download.unlink()
(local_root / "one-byte.bin").unlink()
second_client = run_sync_json([args.drosync_bin, "--json", "sync", mapping_id], timeout=12 * 3600)
second_session = wait_for_session(api, second_client["session"]["id"])
if second_session.get("status") != "completed" or second_session.get("counts", {}).get("unchanged", 0) < len(expected_hashes) - 1:
raise AcceptanceError(f"second sync did not report unchanged files: {redact(second_session)}")
retained = workdir / "retained-one-byte"
api.request(
"GET",
object_path(args.connection_id, args.bucket, f"{prefix}/one-byte.bin"),
expected=(200,),
stream_to=retained,
)
if md5sum_file(retained) != expected_hashes["one-byte.bin"]:
raise AcceptanceError("remote no-delete invariant failed")
retained.unlink()
audit_page, _ = api.request("GET", "/api/object-sync/audit-events?limit=1000", expected=(200,))
related_audit = [
event
for event in audit_page.get("items", [])
if event.get("sessionId") in {first_session["id"], second_session["id"]}
or event.get("target", {}).get("spaceId") == space_id
]
report["layers"].update(
{
"pocApi": {
"status": "PASS",
"space": redact(probed),
"credential": redact(credential),
"sessions": [redact(first_session), redact(second_session)],
"items": redact(items),
},
"installedClient": {
"status": "PASS",
"binary": str(pathlib.Path(args.drosync_bin).resolve()),
"firstSync": redact(first_client),
"secondSync": redact(second_client),
},
"realBos": {"status": "PASS", "objects": remote_evidence, "remoteDelete": "not_performed"},
"audit": {"status": "PASS" if related_audit else "FAIL", "events": redact(related_audit)},
"controlPlaneTransport": {**transport, "baseUrl": args.base_url},
"publicHttps": {
"status": "PASS" if parsed.scheme == "https" else "NOT_RUN",
"required": args.require_https,
"baseUrl": args.base_url,
},
}
)
report["status"] = acceptance_status(
parsed.scheme,
bool(related_audit),
args.require_https,
)
except ExternalBlock as error:
report["status"] = "BLOCKED_EXTERNAL"
report["error"] = str(error)
except Exception as error: # noqa: BLE001 - acceptance must always preserve a report
report["status"] = "FAIL"
report["error"] = str(error)
finally:
cleanup, cleanup_failed = cleanup_acceptance(
api, args.drosync_bin, run_id, mapping_id, credential_id, space_id,
)
if cleanup:
report["cleanup"] = cleanup
if cleanup_failed:
report["status"] = "FAIL"
report["finishedAt"] = utc_now()
if api is not None:
report["apiCalls"] = redact(api.calls)
if workdir is not None:
if args.keep_workdir or cleanup_failed:
report["workdir"] = str(workdir)
else:
shutil.rmtree(workdir, ignore_errors=True)
(evidence_dir / "report.json").write_text(
json.dumps(redact(report), ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
print(json.dumps({"status": report["status"], "evidence": str(evidence_dir / "report.json")}))
return 0 if report["status"] == "PASS" else 2
if __name__ == "__main__":
sys.exit(main())