droboticflow/scripts/ubt_mcap_assets_upload.py
2026-07-24 17:04:09 +08:00

370 lines
12 KiB
Python

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import concurrent.futures
import hashlib
import json
import mimetypes
import os
from pathlib import Path
import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from lib.droboticflow_acceptance import DroboticflowClient
PART_BYTES = 16 * 1024 * 1024
DIRECT_UPLOAD_THRESHOLD_BYTES = 100 * 1024 * 1024
PRESIGN_EXPIRES_SECONDS = 3600
class UploadError(RuntimeError):
pass
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(4 * 1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Upload and verify UBT MCAP archives and URDF through Drobodata."
)
parser.add_argument("--api-base-url", required=True)
parser.add_argument("--connection-id", required=True)
parser.add_argument("--bucket", required=True)
parser.add_argument("--prefix", required=True)
parser.add_argument("--archive-dir", type=Path, required=True)
parser.add_argument("--urdf", type=Path, required=True)
parser.add_argument("--receipt", type=Path, required=True)
parser.add_argument("--workers", type=int, default=4)
parser.add_argument(
"--file-workers",
type=int,
default=1,
help="Number of distinct files uploaded concurrently.",
)
return parser.parse_args()
def api_path(connection_id: str, suffix: str) -> str:
query = urllib.parse.urlencode({"connectionId": connection_id})
return f"/api/object-storage/uploads/{suffix}?{query}"
def put_presigned(url: str, headers: dict[str, str], body: bytes) -> dict[str, str]:
request = urllib.request.Request(url, data=body, headers=headers, method="PUT")
last_error: BaseException | None = None
for attempt in range(1, 4):
try:
with urllib.request.urlopen(request, timeout=300) as response:
if response.status < 200 or response.status >= 300:
raise UploadError(f"presigned PUT returned {response.status}")
return dict(response.headers.items())
except (urllib.error.URLError, TimeoutError) as exc:
last_error = exc
if attempt == 3:
break
time.sleep(attempt)
raise UploadError(f"presigned PUT failed after retries: {last_error}")
def object_listing(
client: DroboticflowClient,
connection_id: str,
bucket: str,
prefix: str,
) -> dict[str, dict[str, Any]]:
marker = ""
found: dict[str, dict[str, Any]] = {}
while True:
query = urllib.parse.urlencode(
{
"connectionId": connection_id,
"bucket": bucket,
"prefix": prefix,
"marker": marker,
"limit": 1000,
}
)
payload = client.request_json("GET", f"/api/object-storage/objects?{query}")
for item in payload.get("items", []):
if item.get("kind") != "directory":
found[str(item.get("key", ""))] = item
next_marker = str(payload.get("nextMarker", "") or "")
if not next_marker or next_marker == marker:
return found
marker = next_marker
class MultipartUploader:
def __init__(
self,
client: DroboticflowClient,
connection_id: str,
bucket: str,
workers: int,
) -> None:
self.client = client
self.connection_id = connection_id
self.bucket = bucket
self.workers = max(1, workers)
self.api_lock = threading.Lock()
def api(self, suffix: str, payload: dict[str, Any]) -> dict[str, Any]:
with self.api_lock:
return self.client.request_json(
"POST",
api_path(self.connection_id, suffix),
payload,
retries=3,
retry_delay_seconds=1.0,
)
def upload_direct(self, path: Path, key: str, content_type: str) -> None:
presigned = self.api(
"presign-put",
{
"bucket": self.bucket,
"key": key,
"contentType": content_type,
"expiresInSeconds": PRESIGN_EXPIRES_SECONDS,
},
)
put_presigned(
str(presigned["url"]),
{str(k): str(v) for k, v in (presigned.get("headers") or {}).items()},
path.read_bytes(),
)
def upload_multipart(self, path: Path, key: str, content_type: str) -> None:
initiated = self.api(
"multipart/initiate",
{
"bucket": self.bucket,
"key": key,
"contentType": content_type,
"expiresInSeconds": PRESIGN_EXPIRES_SECONDS,
},
)
upload_id = str(initiated["uploadId"])
size = path.stat().st_size
parts = [
(part_number, start, min(start + PART_BYTES, size))
for part_number, start in enumerate(range(0, size, PART_BYTES), start=1)
]
completed_bytes = 0
completed_lock = threading.Lock()
last_report = 0.0
def upload_part(part: tuple[int, int, int]) -> dict[str, Any]:
nonlocal completed_bytes, last_report
part_number, start, end = part
signed = self.api(
"multipart/part-url",
{
"bucket": self.bucket,
"key": key,
"uploadId": upload_id,
"partNumber": part_number,
"expiresInSeconds": PRESIGN_EXPIRES_SECONDS,
},
)
with path.open("rb") as handle:
handle.seek(start)
body = handle.read(end - start)
response_headers = put_presigned(
str(signed["url"]),
{str(k): str(v) for k, v in (signed.get("headers") or {}).items()},
body,
)
etag = next(
(
value.strip().strip('"')
for name, value in response_headers.items()
if name.lower() == "etag"
),
"",
)
if not etag:
raise UploadError(f"part {part_number} did not return ETag")
with completed_lock:
completed_bytes += end - start
now = time.monotonic()
if now - last_report >= 10 or completed_bytes == size:
percent = completed_bytes * 100 / size
print(
f"PROGRESS {path.name} {completed_bytes}/{size} "
f"({percent:.1f}%)",
flush=True,
)
last_report = now
return {"partNumber": part_number, "eTag": etag}
try:
with concurrent.futures.ThreadPoolExecutor(
max_workers=min(self.workers, len(parts))
) as executor:
uploaded = list(executor.map(upload_part, parts))
uploaded.sort(key=lambda item: int(item["partNumber"]))
self.api(
"multipart/complete",
{
"bucket": self.bucket,
"key": key,
"uploadId": upload_id,
"parts": uploaded,
},
)
except BaseException:
try:
self.api(
"multipart/abort",
{
"bucket": self.bucket,
"key": key,
"uploadId": upload_id,
},
)
except BaseException:
pass
raise
def upload(self, path: Path, key: str) -> None:
content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
if path.stat().st_size >= DIRECT_UPLOAD_THRESHOLD_BYTES:
self.upload_multipart(path, key, content_type)
else:
self.upload_direct(path, key, content_type)
def main() -> int:
args = parse_args()
token = os.getenv("DROBOTICFLOW_API_TOKEN", "").strip()
email = os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", "").strip()
password = os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", "")
if not token:
raise UploadError("DROBOTICFLOW_API_TOKEN is required")
if not email or not password:
raise UploadError(
"DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD are required"
)
if args.workers < 1:
raise UploadError("--workers must be positive")
if args.file_workers < 1:
raise UploadError("--file-workers must be positive")
archives = sorted(args.archive_dir.glob("*.zip"))
if len(archives) != 11:
raise UploadError(f"expected 11 zip archives, found {len(archives)}")
if not args.urdf.is_file():
raise UploadError(f"URDF not found: {args.urdf}")
files = archives + [args.urdf]
prefix = args.prefix.strip("/")
client = DroboticflowClient(
args.api_base_url,
api_token=token,
error_factory=UploadError,
)
client.login_if_configured(email, password, label="Drobodata")
existing = object_listing(
client, args.connection_id, args.bucket, prefix
)
uploader = MultipartUploader(
client, args.connection_id, args.bucket, args.workers
)
def process_file(index_and_path: tuple[int, Path]) -> dict[str, Any]:
index, path = index_and_path
key = f"{prefix}/{path.name}"
size = path.stat().st_size
digest = sha256_file(path)
current = existing.get(key)
if current:
remote_size = int(current.get("sizeBytes", current.get("size", 0)) or 0)
if remote_size != size:
raise UploadError(
f"existing object has wrong size: {key}: {remote_size} != {size}"
)
action = "verified-existing"
else:
print(
f"UPLOAD {index}/{len(files)} {path.name} {size} bytes",
flush=True,
)
uploader.upload(path, key)
action = "uploaded"
print(f"DONE {index}/{len(files)} {path.name} {action}", flush=True)
return {
"localPath": str(path.resolve()),
"key": key,
"sizeBytes": size,
"sha256": digest,
"action": action,
}
indexed_files = list(enumerate(files, start=1))
if args.file_workers == 1:
receipt_items = [process_file(item) for item in indexed_files]
else:
with concurrent.futures.ThreadPoolExecutor(
max_workers=min(args.file_workers, len(files))
) as executor:
receipt_items = list(executor.map(process_file, indexed_files))
remote = object_listing(client, args.connection_id, args.bucket, prefix)
for item in receipt_items:
remote_item = remote.get(item["key"])
if not remote_item:
raise UploadError(f"uploaded object is not listed: {item['key']}")
remote_size = int(
remote_item.get("sizeBytes", remote_item.get("size", 0)) or 0
)
if remote_size != item["sizeBytes"]:
raise UploadError(
f"remote size mismatch: {item['key']}: "
f"{remote_size} != {item['sizeBytes']}"
)
item["remote"] = {
"sizeBytes": remote_size,
"etag": str(remote_item.get("etag", remote_item.get("eTag", ""))),
"lastModified": remote_item.get("lastModified"),
}
receipt = {
"apiBaseUrl": args.api_base_url,
"connectionId": args.connection_id,
"bucket": args.bucket,
"prefix": prefix,
"fileCount": len(receipt_items),
"totalBytes": sum(item["sizeBytes"] for item in receipt_items),
"items": receipt_items,
}
args.receipt.parent.mkdir(parents=True, exist_ok=True)
args.receipt.write_text(
json.dumps(receipt, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
print(
f"VERIFIED {receipt['fileCount']} files {receipt['totalBytes']} bytes "
f"receipt={args.receipt}",
flush=True,
)
return 0
if __name__ == "__main__":
raise SystemExit(main())