#!/usr/bin/env python3 """Destructive-cost-gated real BOS capacity certification for drosync. This runner intentionally has no small/default shortcut. Reduced values produce a diagnostic report, never a formal PASS. """ from __future__ import annotations import argparse import gzip import hashlib import json import os import pathlib import runpy import shutil import stat import subprocess import sys import time import urllib.parse import urllib.request import uuid from datetime import UTC, datetime from typing import Any FORMAL_OBJECT_COUNT = 1_000_000 FORMAL_TIB_FILES = 20 FORMAL_TIB_SIZE = 1 << 40 MIN_FREE_BYTES = 100 * (1 << 30) CONFIRMATION = "I_ACCEPT_REAL_20_TIB_BOS_COST" COMMON = runpy.run_path(str(pathlib.Path(__file__).with_name("drosync-poc-acceptance.py"))) API = COMMON["API"] AcceptanceError = COMMON["AcceptanceError"] ExternalBlock = COMMON["ExternalBlock"] object_path = COMMON["object_path"] redact = COMMON["redact"] run_json = COMMON["run_json"] transport_security = COMMON["transport_security"] utc_now = COMMON["utc_now"] wait_for_session = COMMON["wait_for_session"] def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--base-url", default=os.getenv("DROSYNC_ACCEPTANCE_BASE_URL", "")) parser.add_argument("--api-token", default=os.getenv("DROBOTICFLOW_API_TOKEN", "")) parser.add_argument("--email", default=os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", "")) parser.add_argument("--password", default=os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", "")) parser.add_argument("--connection-id", default=os.getenv("DROSYNC_ACCEPTANCE_CONNECTION_ID", "")) parser.add_argument("--bucket", default=os.getenv("DROSYNC_ACCEPTANCE_BUCKET", "")) parser.add_argument("--space-id", default=os.getenv("DROSYNC_CAPACITY_SPACE_ID", "")) parser.add_argument("--access-key-id", default=os.getenv("DROSYNC_CAPACITY_ACCESS_KEY_ID", "")) parser.add_argument("--secret-file", required=True) parser.add_argument("--volume", required=True) parser.add_argument("--drosync-bin", default=os.getenv("DROSYNC_BIN", "drosync")) parser.add_argument("--profile", default="capacity-certification") parser.add_argument("--object-count", type=int, default=FORMAL_OBJECT_COUNT) parser.add_argument("--tib-files", type=int, default=FORMAL_TIB_FILES) parser.add_argument("--tib-size", type=int, default=FORMAL_TIB_SIZE) parser.add_argument("--stability-seconds", type=int, default=60) parser.add_argument("--confirmation", required=True) parser.add_argument("--bos-cost-record", required=True) parser.add_argument("--bos-quota-record", required=True) parser.add_argument("--evidence-dir", default=".codex-artifacts/drosync-capacity") parser.add_argument("--retain-local", action="store_true") parser.add_argument("--resume-fixtures", action="store_true") parser.add_argument( "--require-https", action="store_true", default=os.getenv("DROSYNC_ACCEPTANCE_REQUIRE_HTTPS", "").lower() in {"1", "true", "yes"}, help="reject an HTTP control-plane URL; HTTP is allowed by default", ) return parser.parse_args() def private_secret(path: pathlib.Path) -> None: mode = stat.S_IMODE(path.stat().st_mode) if mode & 0o077: raise ExternalBlock(f"capacity Secret file must be 0600 or stricter: {path} mode={mode:o}") def deterministic_kib(index: int) -> bytes: seed = hashlib.sha256(f"drosync-capacity-{index}".encode()).digest() return seed * (1024 // len(seed)) def generate_million(root: pathlib.Path, count: int, manifest_path: pathlib.Path) -> str: root.mkdir(parents=True, exist_ok=True) aggregate = hashlib.sha256() with gzip.open(manifest_path, "wt", encoding="utf-8") as manifest: for index in range(count): relative = pathlib.Path(f"{index // 10_000:04d}") / f"{index:07d}.bin" target = root / relative target.parent.mkdir(parents=True, exist_ok=True) payload = deterministic_kib(index) digest = hashlib.sha256(payload).hexdigest() if not target.exists(): target.write_bytes(payload) record = {"relativePath": relative.as_posix(), "sizeBytes": 1024, "sha256": digest} line = json.dumps(record, sort_keys=True, separators=(",", ":")) manifest.write(line + "\n") aggregate.update(line.encode()) aggregate.update(b"\n") return aggregate.hexdigest() def generate_sparse_tib(root: pathlib.Path, count: int, size_bytes: int) -> list[dict[str, Any]]: root.mkdir(parents=True, exist_ok=True) records = [] for index in range(count): target = root / f"zero-{index:02d}.bin" if not target.exists() or target.stat().st_size != size_bytes: with target.open("wb") as output: output.truncate(size_bytes) records.append( { "relativePath": target.name, "sizeBytes": size_bytes, "contentGenerator": "sparse zero bytes", } ) return records def api_stream_sha256(api: Any, path: str) -> tuple[str, int]: headers = {"Accept": "application/octet-stream"} if api.api_token: headers["Authorization"] = f"Bearer {api.api_token}" request = urllib.request.Request(api.base_url + path, headers=headers, method="GET") digest = hashlib.sha256() total = 0 with api.opener.open(request, timeout=600) as response: while True: block = response.read(16 * 1024 * 1024) if not block: break digest.update(block) total += len(block) return digest.hexdigest(), total def session_items(api: Any, session_id: str) -> list[dict[str, Any]]: items: list[dict[str, Any]] = [] cursor = "" while True: query = urllib.parse.urlencode({"limit": 1000, **({"cursor": cursor} if cursor else {})}) page, _ = api.request( "GET", f"/api/object-sync/sessions/{session_id}/items?{query}", expected=(200,) ) items.extend(page.get("items", [])) cursor = page.get("nextCursor", "") if not cursor: return items def add_and_sync( binary: str, root: pathlib.Path, space_id: str, remote_prefix: str, stability_seconds: int, api: Any, ) -> tuple[dict[str, Any], dict[str, Any], float]: mapping = run_json( [binary, "--json", "mapping", "add", str(root), "--space", space_id, "--remote-prefix", remote_prefix] ) mapping_id = mapping.get("id") or mapping.get("mapping", {}).get("id") if not mapping_id: raise AcceptanceError(f"mapping add returned no id: {redact(mapping)}") if stability_seconds: time.sleep(stability_seconds) started_at = utc_now() started = time.monotonic() client = run_json([binary, "--json", "sync", mapping_id], timeout=30 * 24 * 3600) elapsed = time.monotonic() - started session = wait_for_session(api, mapping_id, started_at, timeout=30 * 24 * 3600) if session.get("status") not in {"completed", "conflicted"}: raise AcceptanceError(f"capacity session failed: {redact(session)}") return client, session, elapsed def main() -> int: args = parse_args() run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8] evidence_dir = pathlib.Path(args.evidence_dir).resolve() / run_id evidence_dir.mkdir(parents=True, exist_ok=False) report: dict[str, Any] = { "schemaVersion": 1, "runId": run_id, "startedAt": utc_now(), "status": "FAIL", "formalTargets": { "objectCount": FORMAL_OBJECT_COUNT, "tibFiles": FORMAL_TIB_FILES, "objectSizeBytes": FORMAL_TIB_SIZE, "sessionBytes": FORMAL_TIB_FILES * FORMAL_TIB_SIZE, }, } work_root: pathlib.Path | None = None try: if args.confirmation != CONFIRMATION: raise ExternalBlock(f"--confirmation must exactly equal {CONFIRMATION}") transport = transport_security(args.base_url, args.require_https) required = { "--base-url": args.base_url, "--connection-id": args.connection_id, "--bucket": args.bucket, "--space-id": args.space_id, "--access-key-id": args.access_key_id, } missing = [name for name, value in required.items() if not value] if missing: raise ExternalBlock("missing required arguments: " + ", ".join(missing)) if shutil.which(args.drosync_bin) is None and not pathlib.Path(args.drosync_bin).exists(): raise ExternalBlock(f"drosync executable was not found: {args.drosync_bin}") for record in [pathlib.Path(args.bos_cost_record), pathlib.Path(args.bos_quota_record)]: if not record.is_file() or record.stat().st_size == 0: raise ExternalBlock(f"required BOS price/quota evidence is missing: {record}") secret_file = pathlib.Path(args.secret_file).resolve() private_secret(secret_file) volume = pathlib.Path(args.volume).resolve() if not volume.is_dir(): raise ExternalBlock(f"capacity volume is unavailable: {volume}") usage = shutil.disk_usage(volume) if usage.free < MIN_FREE_BYTES: raise ExternalBlock( f"capacity volume has {usage.free} free bytes; at least {MIN_FREE_BYTES} are required" ) formal = ( args.object_count == FORMAL_OBJECT_COUNT and args.tib_files == FORMAL_TIB_FILES and args.tib_size == FORMAL_TIB_SIZE ) report["requestedTargets"] = { "objectCount": args.object_count, "tibFiles": args.tib_files, "objectSizeBytes": args.tib_size, "formal": formal, } report["preflight"] = { "status": "PASS", "baseUrl": args.base_url, "transportSecurity": transport, "volume": str(volume), "volumeFreeBytes": usage.free, "costRecord": str(pathlib.Path(args.bos_cost_record).resolve()), "quotaRecord": str(pathlib.Path(args.bos_quota_record).resolve()), } api = API(args.base_url, args.api_token) if args.email: api.request( "POST", "/api/auth/login", {"email": args.email, "password": args.password}, expected=(200,), ) spaces, _ = api.request("GET", "/api/object-sync/spaces", expected=(200,)) space = next((item for item in spaces.get("items", []) if item.get("id") == args.space_id), None) if not space or not space.get("enabled"): raise ExternalBlock("capacity Space is missing or disabled") run_json( [ args.drosync_bin, "--json", "auth", "--server", args.base_url, "--profile", args.profile, "--access-key-id", args.access_key_id, "--secret-file", str(secret_file), ] ) work_root = volume / f"drosync-capacity-{run_id}" work_root.mkdir(exist_ok=args.resume_fixtures) million_root = work_root / "million" tib_root = work_root / "tib" local_manifest = evidence_dir / "million-local-manifest.ndjson.gz" million_digest = generate_million(million_root, args.object_count, local_manifest) tib_records = generate_sparse_tib(tib_root, args.tib_files, args.tib_size) remote_base = f"capacity/{run_id}" million_client, million_session, million_elapsed = add_and_sync( args.drosync_bin, million_root, args.space_id, f"{remote_base}/million", args.stability_seconds, api, ) tib_client, tib_session, tib_elapsed = add_and_sync( args.drosync_bin, tib_root, args.space_id, f"{remote_base}/tib", args.stability_seconds, api, ) million_items = session_items(api, million_session["id"]) tib_items = session_items(api, tib_session["id"]) if len(million_items) != args.object_count or len(tib_items) != args.tib_files: raise AcceptanceError( f"manifest counts mismatch: million={len(million_items)} tib={len(tib_items)}" ) root_prefix = space.get("rootPrefix", "").strip("/") verified_manifest = evidence_dir / "bos-verification.ndjson.gz" verified_count = 0 verified_bytes = 0 with gzip.open(verified_manifest, "wt", encoding="utf-8") as output: for item in [*million_items, *tib_items]: key = "/".join( part for part in [ root_prefix, item.get("targetKey", "").removeprefix(root_prefix).lstrip("/"), ] if part ) path = object_path(args.connection_id, args.bucket, key) _, head = api.request("HEAD", path, expected=(200,)) digest, size = api_stream_sha256(api, path) if digest != item.get("sha256") or size != item.get("sizeBytes"): raise AcceptanceError( f"BOS verification mismatch for {key}: size={size}, sha256={digest}" ) record = { "key": key, "sizeBytes": size, "sha256": digest, "etag": head.get("ETag", ""), "versionId": head.get("x-bce-version-id", ""), "sessionId": item.get("sessionId"), "itemId": item.get("id"), } output.write(json.dumps(record, sort_keys=True) + "\n") verified_count += 1 verified_bytes += size report.update( { "status": "PASS" if formal else "DIAGNOSTIC_PASS", "fixtures": { "millionManifest": str(local_manifest), "millionManifestSha256": million_digest, "tib": tib_records, }, "sessions": { "million": redact(million_session), "tib": redact(tib_session), }, "client": { "million": redact(million_client), "tib": redact(tib_client), }, "elapsedSeconds": { "million": round(million_elapsed, 3), "tib": round(tib_elapsed, 3), }, "bosVerification": { "status": "PASS", "objectCount": verified_count, "bytes": verified_bytes, "manifest": str(verified_manifest), }, } ) except ExternalBlock as error: report["status"] = "BLOCKED_EXTERNAL" report["error"] = str(error) except Exception as error: # noqa: BLE001 report["status"] = "FAIL" report["error"] = str(error) finally: report["finishedAt"] = utc_now() if work_root is not None: if args.retain_local: report["localFixtures"] = str(work_root) else: shutil.rmtree(work_root, ignore_errors=True) report_path = evidence_dir / "report.json" report_path.write_text(json.dumps(redact(report), ensure_ascii=False, indent=2) + "\n") print(json.dumps({"status": report["status"], "evidence": str(report_path)})) return 0 if report["status"] == "PASS" else 2 if __name__ == "__main__": sys.exit(main())