droboticflow/scripts/drosync-capacity-acceptance.py

423 lines
17 KiB
Python

#!/usr/bin/env python3
"""Destructive-cost-gated real BOS capacity certification for drosync.
This runner intentionally has no small/default shortcut. Reduced values produce a
diagnostic report, never a formal PASS.
"""
from __future__ import annotations
import argparse
import gzip
import hashlib
import json
import os
import pathlib
import runpy
import shutil
import stat
import subprocess
import sys
import time
import urllib.parse
import urllib.request
import uuid
from datetime import UTC, datetime
from typing import Any
FORMAL_OBJECT_COUNT = 1_000_000
FORMAL_TIB_FILES = 20
FORMAL_TIB_SIZE = 1 << 40
MIN_FREE_BYTES = 100 * (1 << 30)
CONFIRMATION = "I_ACCEPT_REAL_20_TIB_BOS_COST"
COMMON = runpy.run_path(str(pathlib.Path(__file__).with_name("drosync-poc-acceptance.py")))
API = COMMON["API"]
AcceptanceError = COMMON["AcceptanceError"]
ExternalBlock = COMMON["ExternalBlock"]
object_path = COMMON["object_path"]
redact = COMMON["redact"]
run_json = COMMON["run_json"]
run_sync_json = COMMON["run_sync_json"]
require_empty_daemon = COMMON["require_empty_daemon"]
require_server_protocol = COMMON["require_server_protocol"]
transport_security = COMMON["transport_security"]
utc_now = COMMON["utc_now"]
wait_for_session = COMMON["wait_for_session"]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default=os.getenv("DROSYNC_ACCEPTANCE_BASE_URL", ""))
parser.add_argument("--api-token", default=os.getenv("DROBOTICFLOW_API_TOKEN", ""))
parser.add_argument("--email", default=os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", ""))
parser.add_argument("--password", default=os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", ""))
parser.add_argument("--connection-id", default=os.getenv("DROSYNC_ACCEPTANCE_CONNECTION_ID", ""))
parser.add_argument("--bucket", default=os.getenv("DROSYNC_ACCEPTANCE_BUCKET", ""))
parser.add_argument("--space-id", default=os.getenv("DROSYNC_CAPACITY_SPACE_ID", ""))
parser.add_argument("--access-key-id", default=os.getenv("DROSYNC_CAPACITY_ACCESS_KEY_ID", ""))
parser.add_argument("--secret-file", required=True)
parser.add_argument("--volume", required=True)
parser.add_argument("--drosync-bin", default=os.getenv("DROSYNC_BIN", "drosync"))
parser.add_argument("--drosync-config", help="Runtime file used by the acceptance daemon (-c FILE)")
parser.add_argument("--profile", default="capacity-certification")
parser.add_argument("--object-count", type=int, default=FORMAL_OBJECT_COUNT)
parser.add_argument("--tib-files", type=int, default=FORMAL_TIB_FILES)
parser.add_argument("--tib-size", type=int, default=FORMAL_TIB_SIZE)
parser.add_argument("--stability-seconds", type=int, default=60)
parser.add_argument("--confirmation", required=True)
parser.add_argument("--bos-cost-record", required=True)
parser.add_argument("--bos-quota-record", required=True)
parser.add_argument("--evidence-dir", default=".codex-artifacts/drosync-capacity")
parser.add_argument("--retain-local", action="store_true")
parser.add_argument("--resume-fixtures", action="store_true")
parser.add_argument(
"--require-https",
action="store_true",
default=os.getenv("DROSYNC_ACCEPTANCE_REQUIRE_HTTPS", "").lower() in {"1", "true", "yes"},
help="reject an HTTP control-plane URL; HTTP is allowed by default",
)
return parser.parse_args()
def private_secret(path: pathlib.Path) -> None:
mode = stat.S_IMODE(path.stat().st_mode)
if mode & 0o077:
raise ExternalBlock(f"capacity Secret file must be 0600 or stricter: {path} mode={mode:o}")
def deterministic_kib(index: int) -> bytes:
seed = hashlib.md5(f"drosync-capacity-{index}".encode()).digest()
return seed * (1024 // len(seed))
def generate_million(root: pathlib.Path, count: int, manifest_path: pathlib.Path) -> str:
root.mkdir(parents=True, exist_ok=True)
aggregate = hashlib.md5()
with gzip.open(manifest_path, "wt", encoding="utf-8") as manifest:
for index in range(count):
relative = pathlib.Path(f"{index // 10_000:04d}") / f"{index:07d}.bin"
target = root / relative
target.parent.mkdir(parents=True, exist_ok=True)
payload = deterministic_kib(index)
digest = hashlib.md5(payload).hexdigest()
if not target.exists():
target.write_bytes(payload)
record = {"relativePath": relative.as_posix(), "sizeBytes": 1024, "md5sum": digest}
line = json.dumps(record, sort_keys=True, separators=(",", ":"))
manifest.write(line + "\n")
aggregate.update(line.encode())
aggregate.update(b"\n")
return aggregate.hexdigest()
def generate_sparse_tib(root: pathlib.Path, count: int, size_bytes: int) -> list[dict[str, Any]]:
root.mkdir(parents=True, exist_ok=True)
records = []
for index in range(count):
target = root / f"zero-{index:02d}.bin"
if not target.exists() or target.stat().st_size != size_bytes:
with target.open("wb") as output:
output.truncate(size_bytes)
records.append(
{
"relativePath": target.name,
"sizeBytes": size_bytes,
"contentGenerator": "sparse zero bytes",
}
)
return records
def api_stream_md5sum(api: Any, path: str) -> tuple[str, int]:
headers = {"Accept": "application/octet-stream"}
if api.api_token:
headers["Authorization"] = f"Bearer {api.api_token}"
request = urllib.request.Request(api.base_url + path, headers=headers, method="GET")
digest = hashlib.md5()
total = 0
with api.opener.open(request, timeout=600) as response:
while True:
block = response.read(16 * 1024 * 1024)
if not block:
break
digest.update(block)
total += len(block)
return digest.hexdigest(), total
def session_items(api: Any, session_id: str) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
cursor = ""
while True:
query = urllib.parse.urlencode({"limit": 1000, **({"cursor": cursor} if cursor else {})})
page, _ = api.request(
"GET", f"/api/object-sync/sessions/{session_id}/items?{query}", expected=(200,)
)
items.extend(page.get("items", []))
cursor = page.get("nextCursor", "")
if not cursor:
return items
def add_and_sync(
binary: str,
root: pathlib.Path,
space_id: str,
remote_prefix: str,
stability_seconds: int,
api: Any,
created_mappings: list[str],
) -> tuple[dict[str, Any], dict[str, Any], float]:
if stability_seconds:
time.sleep(stability_seconds)
mapping = run_json(
[binary, "--json", "mapping", "add", str(root), "--space", space_id, "--remote-prefix", remote_prefix]
)
mapping_id = mapping.get("id")
if not mapping_id:
raise AcceptanceError(f"mapping add returned no id: {redact(mapping)}")
created_mappings.append(mapping_id)
started = time.monotonic()
client = run_sync_json([binary, "--json", "sync", mapping_id], timeout=30 * 24 * 3600)
elapsed = time.monotonic() - started
session = wait_for_session(api, client["session"]["id"], timeout=30 * 24 * 3600)
if session.get("status") != "completed":
raise AcceptanceError(f"capacity session failed: {redact(session)}")
return client, session, elapsed
def main() -> int:
args = parse_args()
COMMON["configure_cli_runtime"](args.drosync_config)
run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8]
evidence_dir = pathlib.Path(args.evidence_dir).resolve() / run_id
evidence_dir.mkdir(parents=True, exist_ok=False)
report: dict[str, Any] = {
"schemaVersion": 1,
"runId": run_id,
"startedAt": utc_now(),
"status": "FAIL",
"formalTargets": {
"objectCount": FORMAL_OBJECT_COUNT,
"tibFiles": FORMAL_TIB_FILES,
"objectSizeBytes": FORMAL_TIB_SIZE,
"sessionBytes": FORMAL_TIB_FILES * FORMAL_TIB_SIZE,
},
}
work_root: pathlib.Path | None = None
created_mappings: list[str] = []
try:
if args.confirmation != CONFIRMATION:
raise ExternalBlock(f"--confirmation must exactly equal {CONFIRMATION}")
transport = transport_security(args.base_url, args.require_https)
required = {
"--base-url": args.base_url,
"--connection-id": args.connection_id,
"--bucket": args.bucket,
"--space-id": args.space_id,
"--access-key-id": args.access_key_id,
}
missing = [name for name, value in required.items() if not value]
if missing:
raise ExternalBlock("missing required arguments: " + ", ".join(missing))
if shutil.which(args.drosync_bin) is None and not pathlib.Path(args.drosync_bin).exists():
raise ExternalBlock(f"drosync executable was not found: {args.drosync_bin}")
for record in [pathlib.Path(args.bos_cost_record), pathlib.Path(args.bos_quota_record)]:
if not record.is_file() or record.stat().st_size == 0:
raise ExternalBlock(f"required BOS price/quota evidence is missing: {record}")
secret_file = pathlib.Path(args.secret_file).resolve()
private_secret(secret_file)
volume = pathlib.Path(args.volume).resolve()
if not volume.is_dir():
raise ExternalBlock(f"capacity volume is unavailable: {volume}")
usage = shutil.disk_usage(volume)
if usage.free < MIN_FREE_BYTES:
raise ExternalBlock(
f"capacity volume has {usage.free} free bytes; at least {MIN_FREE_BYTES} are required"
)
formal = (
args.object_count == FORMAL_OBJECT_COUNT
and args.tib_files == FORMAL_TIB_FILES
and args.tib_size == FORMAL_TIB_SIZE
)
report["requestedTargets"] = {
"objectCount": args.object_count,
"tibFiles": args.tib_files,
"objectSizeBytes": args.tib_size,
"formal": formal,
}
report["preflight"] = {
"status": "PASS",
"baseUrl": args.base_url,
"transportSecurity": transport,
"volume": str(volume),
"volumeFreeBytes": usage.free,
"costRecord": str(pathlib.Path(args.bos_cost_record).resolve()),
"quotaRecord": str(pathlib.Path(args.bos_quota_record).resolve()),
}
api = API(args.base_url, args.api_token)
if args.email:
api.request(
"POST",
"/api/auth/login",
{"email": args.email, "password": args.password},
expected=(200,),
)
spaces, _ = api.request("GET", "/api/object-sync/spaces", expected=(200,))
space = next((item for item in spaces.get("items", []) if item.get("id") == args.space_id), None)
if not space or not space.get("enabled"):
raise ExternalBlock("capacity Space is missing or disabled")
require_empty_daemon(args.drosync_bin)
run_json(
[
args.drosync_bin,
"--json",
"auth",
"--server",
args.base_url,
"--profile",
args.profile,
"--access-key-id",
args.access_key_id,
"--secret-file",
str(secret_file),
]
)
report["preflight"]["serverProtocol"] = require_server_protocol(args.drosync_bin)
work_root = volume / f"drosync-capacity-{run_id}"
work_root.mkdir(exist_ok=args.resume_fixtures)
million_root = work_root / "million"
tib_root = work_root / "tib"
local_manifest = evidence_dir / "million-local-manifest.ndjson.gz"
million_digest = generate_million(million_root, args.object_count, local_manifest)
tib_records = generate_sparse_tib(tib_root, args.tib_files, args.tib_size)
remote_base = f"capacity/{run_id}"
million_client, million_session, million_elapsed = add_and_sync(
args.drosync_bin,
million_root,
args.space_id,
f"{remote_base}/million",
args.stability_seconds,
api,
created_mappings,
)
tib_client, tib_session, tib_elapsed = add_and_sync(
args.drosync_bin,
tib_root,
args.space_id,
f"{remote_base}/tib",
args.stability_seconds,
api,
created_mappings,
)
million_items = session_items(api, million_session["id"])
tib_items = session_items(api, tib_session["id"])
if len(million_items) != args.object_count or len(tib_items) != args.tib_files:
raise AcceptanceError(
f"manifest counts mismatch: million={len(million_items)} tib={len(tib_items)}"
)
root_prefix = space.get("rootPrefix", "").strip("/")
verified_manifest = evidence_dir / "bos-verification.ndjson.gz"
verified_count = 0
verified_bytes = 0
with gzip.open(verified_manifest, "wt", encoding="utf-8") as output:
for item in [*million_items, *tib_items]:
key = "/".join(
part
for part in [
root_prefix,
item.get("targetKey", "").removeprefix(root_prefix).lstrip("/"),
]
if part
)
path = object_path(args.connection_id, args.bucket, key)
_, head = api.request("HEAD", path, expected=(200,))
digest, size = api_stream_md5sum(api, path)
if digest != item.get("md5sum") or size != item.get("sizeBytes"):
raise AcceptanceError(
f"BOS verification mismatch for {key}: size={size}, md5sum={digest}"
)
record = {
"key": key,
"sizeBytes": size,
"md5sum": digest,
"etag": head.get("ETag", ""),
"versionId": head.get("x-bce-version-id", ""),
"sessionId": item.get("sessionId"),
"itemId": item.get("id"),
}
output.write(json.dumps(record, sort_keys=True) + "\n")
verified_count += 1
verified_bytes += size
report.update(
{
"status": "PASS" if formal else "DIAGNOSTIC_PASS",
"fixtures": {
"millionManifest": str(local_manifest),
"millionManifestSha256": million_digest,
"tib": tib_records,
},
"sessions": {
"million": redact(million_session),
"tib": redact(tib_session),
},
"client": {
"million": redact(million_client),
"tib": redact(tib_client),
},
"elapsedSeconds": {
"million": round(million_elapsed, 3),
"tib": round(tib_elapsed, 3),
},
"bosVerification": {
"status": "PASS",
"objectCount": verified_count,
"bytes": verified_bytes,
"manifest": str(verified_manifest),
},
}
)
except ExternalBlock as error:
report["status"] = "BLOCKED_EXTERNAL"
report["error"] = str(error)
except Exception as error: # noqa: BLE001
report["status"] = "FAIL"
report["error"] = str(error)
finally:
cleanup_failed = False
mapping_cleanup = []
for mapping_id in created_mappings:
try:
removed = run_json([args.drosync_bin, "--json", "mapping", "remove", mapping_id], timeout=900)
if removed.get("removed") is not True:
raise AcceptanceError(f"mapping removal was not confirmed: {redact(removed)}")
mapping_cleanup.append({"id": mapping_id, "status": "PASS", "result": redact(removed)})
except Exception as error: # noqa: BLE001 - preserve cancellation failure
cleanup_failed = True
mapping_cleanup.append({"id": mapping_id, "status": "FAIL", "error": str(error)})
if mapping_cleanup:
report["cleanup"] = {"mappings": mapping_cleanup, "credential": "user-supplied, not revoked"}
if cleanup_failed:
report["status"] = "FAIL"
report["finishedAt"] = utc_now()
if work_root is not None:
if args.retain_local or cleanup_failed:
report["localFixtures"] = str(work_root)
else:
shutil.rmtree(work_root, ignore_errors=True)
report_path = evidence_dir / "report.json"
report_path.write_text(json.dumps(redact(report), ensure_ascii=False, indent=2) + "\n")
print(json.dumps({"status": report["status"], "evidence": str(report_path)}))
return 0 if report["status"] == "PASS" else 2
if __name__ == "__main__":
sys.exit(main())