#!/usr/bin/env python3 from __future__ import annotations import argparse import concurrent.futures import hashlib import json import mimetypes import os from pathlib import Path import sys import threading import time import urllib.error import urllib.parse import urllib.request from typing import Any SCRIPT_DIR = Path(__file__).resolve().parent if str(SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(SCRIPT_DIR)) from lib.droboticflow_acceptance import DroboticflowClient PART_BYTES = 16 * 1024 * 1024 DIRECT_UPLOAD_THRESHOLD_BYTES = 100 * 1024 * 1024 PRESIGN_EXPIRES_SECONDS = 3600 class UploadError(RuntimeError): pass def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(4 * 1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Upload and verify UBT MCAP archives and URDF through Drobodata." ) parser.add_argument("--api-base-url", required=True) parser.add_argument("--connection-id", required=True) parser.add_argument("--bucket", required=True) parser.add_argument("--prefix", required=True) parser.add_argument("--archive-dir", type=Path, required=True) parser.add_argument("--urdf", type=Path, required=True) parser.add_argument("--receipt", type=Path, required=True) parser.add_argument("--workers", type=int, default=4) parser.add_argument( "--file-workers", type=int, default=1, help="Number of distinct files uploaded concurrently.", ) return parser.parse_args() def api_path(connection_id: str, suffix: str) -> str: query = urllib.parse.urlencode({"connectionId": connection_id}) return f"/api/object-storage/uploads/{suffix}?{query}" def put_presigned(url: str, headers: dict[str, str], body: bytes) -> dict[str, str]: request = urllib.request.Request(url, data=body, headers=headers, method="PUT") last_error: BaseException | None = None for attempt in range(1, 4): try: with urllib.request.urlopen(request, timeout=300) as response: if response.status < 200 or response.status >= 300: raise UploadError(f"presigned PUT returned {response.status}") return dict(response.headers.items()) except (urllib.error.URLError, TimeoutError) as exc: last_error = exc if attempt == 3: break time.sleep(attempt) raise UploadError(f"presigned PUT failed after retries: {last_error}") def object_listing( client: DroboticflowClient, connection_id: str, bucket: str, prefix: str, ) -> dict[str, dict[str, Any]]: marker = "" found: dict[str, dict[str, Any]] = {} while True: query = urllib.parse.urlencode( { "connectionId": connection_id, "bucket": bucket, "prefix": prefix, "marker": marker, "limit": 1000, } ) payload = client.request_json("GET", f"/api/object-storage/objects?{query}") for item in payload.get("items", []): if item.get("kind") != "directory": found[str(item.get("key", ""))] = item next_marker = str(payload.get("nextMarker", "") or "") if not next_marker or next_marker == marker: return found marker = next_marker class MultipartUploader: def __init__( self, client: DroboticflowClient, connection_id: str, bucket: str, workers: int, ) -> None: self.client = client self.connection_id = connection_id self.bucket = bucket self.workers = max(1, workers) self.api_lock = threading.Lock() def api(self, suffix: str, payload: dict[str, Any]) -> dict[str, Any]: with self.api_lock: return self.client.request_json( "POST", api_path(self.connection_id, suffix), payload, retries=3, retry_delay_seconds=1.0, ) def upload_direct(self, path: Path, key: str, content_type: str) -> None: presigned = self.api( "presign-put", { "bucket": self.bucket, "key": key, "contentType": content_type, "expiresInSeconds": PRESIGN_EXPIRES_SECONDS, }, ) put_presigned( str(presigned["url"]), {str(k): str(v) for k, v in (presigned.get("headers") or {}).items()}, path.read_bytes(), ) def upload_multipart(self, path: Path, key: str, content_type: str) -> None: initiated = self.api( "multipart/initiate", { "bucket": self.bucket, "key": key, "contentType": content_type, "expiresInSeconds": PRESIGN_EXPIRES_SECONDS, }, ) upload_id = str(initiated["uploadId"]) size = path.stat().st_size parts = [ (part_number, start, min(start + PART_BYTES, size)) for part_number, start in enumerate(range(0, size, PART_BYTES), start=1) ] completed_bytes = 0 completed_lock = threading.Lock() last_report = 0.0 def upload_part(part: tuple[int, int, int]) -> dict[str, Any]: nonlocal completed_bytes, last_report part_number, start, end = part signed = self.api( "multipart/part-url", { "bucket": self.bucket, "key": key, "uploadId": upload_id, "partNumber": part_number, "expiresInSeconds": PRESIGN_EXPIRES_SECONDS, }, ) with path.open("rb") as handle: handle.seek(start) body = handle.read(end - start) response_headers = put_presigned( str(signed["url"]), {str(k): str(v) for k, v in (signed.get("headers") or {}).items()}, body, ) etag = next( ( value.strip().strip('"') for name, value in response_headers.items() if name.lower() == "etag" ), "", ) if not etag: raise UploadError(f"part {part_number} did not return ETag") with completed_lock: completed_bytes += end - start now = time.monotonic() if now - last_report >= 10 or completed_bytes == size: percent = completed_bytes * 100 / size print( f"PROGRESS {path.name} {completed_bytes}/{size} " f"({percent:.1f}%)", flush=True, ) last_report = now return {"partNumber": part_number, "eTag": etag} try: with concurrent.futures.ThreadPoolExecutor( max_workers=min(self.workers, len(parts)) ) as executor: uploaded = list(executor.map(upload_part, parts)) uploaded.sort(key=lambda item: int(item["partNumber"])) self.api( "multipart/complete", { "bucket": self.bucket, "key": key, "uploadId": upload_id, "parts": uploaded, }, ) except BaseException: try: self.api( "multipart/abort", { "bucket": self.bucket, "key": key, "uploadId": upload_id, }, ) except BaseException: pass raise def upload(self, path: Path, key: str) -> None: content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream" if path.stat().st_size >= DIRECT_UPLOAD_THRESHOLD_BYTES: self.upload_multipart(path, key, content_type) else: self.upload_direct(path, key, content_type) def main() -> int: args = parse_args() token = os.getenv("DROBOTICFLOW_API_TOKEN", "").strip() email = os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", "").strip() password = os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", "") if not token: raise UploadError("DROBOTICFLOW_API_TOKEN is required") if not email or not password: raise UploadError( "DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD are required" ) if args.workers < 1: raise UploadError("--workers must be positive") if args.file_workers < 1: raise UploadError("--file-workers must be positive") archives = sorted(args.archive_dir.glob("*.zip")) if len(archives) != 11: raise UploadError(f"expected 11 zip archives, found {len(archives)}") if not args.urdf.is_file(): raise UploadError(f"URDF not found: {args.urdf}") files = archives + [args.urdf] prefix = args.prefix.strip("/") client = DroboticflowClient( args.api_base_url, api_token=token, error_factory=UploadError, ) client.login_if_configured(email, password, label="Drobodata") existing = object_listing( client, args.connection_id, args.bucket, prefix ) uploader = MultipartUploader( client, args.connection_id, args.bucket, args.workers ) def process_file(index_and_path: tuple[int, Path]) -> dict[str, Any]: index, path = index_and_path key = f"{prefix}/{path.name}" size = path.stat().st_size digest = sha256_file(path) current = existing.get(key) if current: remote_size = int(current.get("sizeBytes", current.get("size", 0)) or 0) if remote_size != size: raise UploadError( f"existing object has wrong size: {key}: {remote_size} != {size}" ) action = "verified-existing" else: print( f"UPLOAD {index}/{len(files)} {path.name} {size} bytes", flush=True, ) uploader.upload(path, key) action = "uploaded" print(f"DONE {index}/{len(files)} {path.name} {action}", flush=True) return { "localPath": str(path.resolve()), "key": key, "sizeBytes": size, "sha256": digest, "action": action, } indexed_files = list(enumerate(files, start=1)) if args.file_workers == 1: receipt_items = [process_file(item) for item in indexed_files] else: with concurrent.futures.ThreadPoolExecutor( max_workers=min(args.file_workers, len(files)) ) as executor: receipt_items = list(executor.map(process_file, indexed_files)) remote = object_listing(client, args.connection_id, args.bucket, prefix) for item in receipt_items: remote_item = remote.get(item["key"]) if not remote_item: raise UploadError(f"uploaded object is not listed: {item['key']}") remote_size = int( remote_item.get("sizeBytes", remote_item.get("size", 0)) or 0 ) if remote_size != item["sizeBytes"]: raise UploadError( f"remote size mismatch: {item['key']}: " f"{remote_size} != {item['sizeBytes']}" ) item["remote"] = { "sizeBytes": remote_size, "etag": str(remote_item.get("etag", remote_item.get("eTag", ""))), "lastModified": remote_item.get("lastModified"), } receipt = { "apiBaseUrl": args.api_base_url, "connectionId": args.connection_id, "bucket": args.bucket, "prefix": prefix, "fileCount": len(receipt_items), "totalBytes": sum(item["sizeBytes"] for item in receipt_items), "items": receipt_items, } args.receipt.parent.mkdir(parents=True, exist_ok=True) args.receipt.write_text( json.dumps(receipt, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) print( f"VERIFIED {receipt['fileCount']} files {receipt['totalBytes']} bytes " f"receipt={args.receipt}", flush=True, ) return 0 if __name__ == "__main__": raise SystemExit(main())