forked from longtao.wu/droboticflow
370 lines
12 KiB
Python
370 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import hashlib
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from typing import Any
|
|
|
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
if str(SCRIPT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPT_DIR))
|
|
|
|
from lib.droboticflow_acceptance import DroboticflowClient
|
|
|
|
|
|
PART_BYTES = 16 * 1024 * 1024
|
|
DIRECT_UPLOAD_THRESHOLD_BYTES = 100 * 1024 * 1024
|
|
PRESIGN_EXPIRES_SECONDS = 3600
|
|
|
|
|
|
class UploadError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(4 * 1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Upload and verify UBT MCAP archives and URDF through Drobodata."
|
|
)
|
|
parser.add_argument("--api-base-url", required=True)
|
|
parser.add_argument("--connection-id", required=True)
|
|
parser.add_argument("--bucket", required=True)
|
|
parser.add_argument("--prefix", required=True)
|
|
parser.add_argument("--archive-dir", type=Path, required=True)
|
|
parser.add_argument("--urdf", type=Path, required=True)
|
|
parser.add_argument("--receipt", type=Path, required=True)
|
|
parser.add_argument("--workers", type=int, default=4)
|
|
parser.add_argument(
|
|
"--file-workers",
|
|
type=int,
|
|
default=1,
|
|
help="Number of distinct files uploaded concurrently.",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def api_path(connection_id: str, suffix: str) -> str:
|
|
query = urllib.parse.urlencode({"connectionId": connection_id})
|
|
return f"/api/object-storage/uploads/{suffix}?{query}"
|
|
|
|
|
|
def put_presigned(url: str, headers: dict[str, str], body: bytes) -> dict[str, str]:
|
|
request = urllib.request.Request(url, data=body, headers=headers, method="PUT")
|
|
last_error: BaseException | None = None
|
|
for attempt in range(1, 4):
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=300) as response:
|
|
if response.status < 200 or response.status >= 300:
|
|
raise UploadError(f"presigned PUT returned {response.status}")
|
|
return dict(response.headers.items())
|
|
except (urllib.error.URLError, TimeoutError) as exc:
|
|
last_error = exc
|
|
if attempt == 3:
|
|
break
|
|
time.sleep(attempt)
|
|
raise UploadError(f"presigned PUT failed after retries: {last_error}")
|
|
|
|
|
|
def object_listing(
|
|
client: DroboticflowClient,
|
|
connection_id: str,
|
|
bucket: str,
|
|
prefix: str,
|
|
) -> dict[str, dict[str, Any]]:
|
|
marker = ""
|
|
found: dict[str, dict[str, Any]] = {}
|
|
while True:
|
|
query = urllib.parse.urlencode(
|
|
{
|
|
"connectionId": connection_id,
|
|
"bucket": bucket,
|
|
"prefix": prefix,
|
|
"marker": marker,
|
|
"limit": 1000,
|
|
}
|
|
)
|
|
payload = client.request_json("GET", f"/api/object-storage/objects?{query}")
|
|
for item in payload.get("items", []):
|
|
if item.get("kind") != "directory":
|
|
found[str(item.get("key", ""))] = item
|
|
next_marker = str(payload.get("nextMarker", "") or "")
|
|
if not next_marker or next_marker == marker:
|
|
return found
|
|
marker = next_marker
|
|
|
|
|
|
class MultipartUploader:
|
|
def __init__(
|
|
self,
|
|
client: DroboticflowClient,
|
|
connection_id: str,
|
|
bucket: str,
|
|
workers: int,
|
|
) -> None:
|
|
self.client = client
|
|
self.connection_id = connection_id
|
|
self.bucket = bucket
|
|
self.workers = max(1, workers)
|
|
self.api_lock = threading.Lock()
|
|
|
|
def api(self, suffix: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
with self.api_lock:
|
|
return self.client.request_json(
|
|
"POST",
|
|
api_path(self.connection_id, suffix),
|
|
payload,
|
|
retries=3,
|
|
retry_delay_seconds=1.0,
|
|
)
|
|
|
|
def upload_direct(self, path: Path, key: str, content_type: str) -> None:
|
|
presigned = self.api(
|
|
"presign-put",
|
|
{
|
|
"bucket": self.bucket,
|
|
"key": key,
|
|
"contentType": content_type,
|
|
"expiresInSeconds": PRESIGN_EXPIRES_SECONDS,
|
|
},
|
|
)
|
|
put_presigned(
|
|
str(presigned["url"]),
|
|
{str(k): str(v) for k, v in (presigned.get("headers") or {}).items()},
|
|
path.read_bytes(),
|
|
)
|
|
|
|
def upload_multipart(self, path: Path, key: str, content_type: str) -> None:
|
|
initiated = self.api(
|
|
"multipart/initiate",
|
|
{
|
|
"bucket": self.bucket,
|
|
"key": key,
|
|
"contentType": content_type,
|
|
"expiresInSeconds": PRESIGN_EXPIRES_SECONDS,
|
|
},
|
|
)
|
|
upload_id = str(initiated["uploadId"])
|
|
size = path.stat().st_size
|
|
parts = [
|
|
(part_number, start, min(start + PART_BYTES, size))
|
|
for part_number, start in enumerate(range(0, size, PART_BYTES), start=1)
|
|
]
|
|
completed_bytes = 0
|
|
completed_lock = threading.Lock()
|
|
last_report = 0.0
|
|
|
|
def upload_part(part: tuple[int, int, int]) -> dict[str, Any]:
|
|
nonlocal completed_bytes, last_report
|
|
part_number, start, end = part
|
|
signed = self.api(
|
|
"multipart/part-url",
|
|
{
|
|
"bucket": self.bucket,
|
|
"key": key,
|
|
"uploadId": upload_id,
|
|
"partNumber": part_number,
|
|
"expiresInSeconds": PRESIGN_EXPIRES_SECONDS,
|
|
},
|
|
)
|
|
with path.open("rb") as handle:
|
|
handle.seek(start)
|
|
body = handle.read(end - start)
|
|
response_headers = put_presigned(
|
|
str(signed["url"]),
|
|
{str(k): str(v) for k, v in (signed.get("headers") or {}).items()},
|
|
body,
|
|
)
|
|
etag = next(
|
|
(
|
|
value.strip().strip('"')
|
|
for name, value in response_headers.items()
|
|
if name.lower() == "etag"
|
|
),
|
|
"",
|
|
)
|
|
if not etag:
|
|
raise UploadError(f"part {part_number} did not return ETag")
|
|
with completed_lock:
|
|
completed_bytes += end - start
|
|
now = time.monotonic()
|
|
if now - last_report >= 10 or completed_bytes == size:
|
|
percent = completed_bytes * 100 / size
|
|
print(
|
|
f"PROGRESS {path.name} {completed_bytes}/{size} "
|
|
f"({percent:.1f}%)",
|
|
flush=True,
|
|
)
|
|
last_report = now
|
|
return {"partNumber": part_number, "eTag": etag}
|
|
|
|
try:
|
|
with concurrent.futures.ThreadPoolExecutor(
|
|
max_workers=min(self.workers, len(parts))
|
|
) as executor:
|
|
uploaded = list(executor.map(upload_part, parts))
|
|
uploaded.sort(key=lambda item: int(item["partNumber"]))
|
|
self.api(
|
|
"multipart/complete",
|
|
{
|
|
"bucket": self.bucket,
|
|
"key": key,
|
|
"uploadId": upload_id,
|
|
"parts": uploaded,
|
|
},
|
|
)
|
|
except BaseException:
|
|
try:
|
|
self.api(
|
|
"multipart/abort",
|
|
{
|
|
"bucket": self.bucket,
|
|
"key": key,
|
|
"uploadId": upload_id,
|
|
},
|
|
)
|
|
except BaseException:
|
|
pass
|
|
raise
|
|
|
|
def upload(self, path: Path, key: str) -> None:
|
|
content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
|
|
if path.stat().st_size >= DIRECT_UPLOAD_THRESHOLD_BYTES:
|
|
self.upload_multipart(path, key, content_type)
|
|
else:
|
|
self.upload_direct(path, key, content_type)
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
token = os.getenv("DROBOTICFLOW_API_TOKEN", "").strip()
|
|
email = os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", "").strip()
|
|
password = os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", "")
|
|
if not token:
|
|
raise UploadError("DROBOTICFLOW_API_TOKEN is required")
|
|
if not email or not password:
|
|
raise UploadError(
|
|
"DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD are required"
|
|
)
|
|
if args.workers < 1:
|
|
raise UploadError("--workers must be positive")
|
|
if args.file_workers < 1:
|
|
raise UploadError("--file-workers must be positive")
|
|
archives = sorted(args.archive_dir.glob("*.zip"))
|
|
if len(archives) != 11:
|
|
raise UploadError(f"expected 11 zip archives, found {len(archives)}")
|
|
if not args.urdf.is_file():
|
|
raise UploadError(f"URDF not found: {args.urdf}")
|
|
files = archives + [args.urdf]
|
|
prefix = args.prefix.strip("/")
|
|
|
|
client = DroboticflowClient(
|
|
args.api_base_url,
|
|
api_token=token,
|
|
error_factory=UploadError,
|
|
)
|
|
client.login_if_configured(email, password, label="Drobodata")
|
|
existing = object_listing(
|
|
client, args.connection_id, args.bucket, prefix
|
|
)
|
|
uploader = MultipartUploader(
|
|
client, args.connection_id, args.bucket, args.workers
|
|
)
|
|
def process_file(index_and_path: tuple[int, Path]) -> dict[str, Any]:
|
|
index, path = index_and_path
|
|
key = f"{prefix}/{path.name}"
|
|
size = path.stat().st_size
|
|
digest = sha256_file(path)
|
|
current = existing.get(key)
|
|
if current:
|
|
remote_size = int(current.get("sizeBytes", current.get("size", 0)) or 0)
|
|
if remote_size != size:
|
|
raise UploadError(
|
|
f"existing object has wrong size: {key}: {remote_size} != {size}"
|
|
)
|
|
action = "verified-existing"
|
|
else:
|
|
print(
|
|
f"UPLOAD {index}/{len(files)} {path.name} {size} bytes",
|
|
flush=True,
|
|
)
|
|
uploader.upload(path, key)
|
|
action = "uploaded"
|
|
print(f"DONE {index}/{len(files)} {path.name} {action}", flush=True)
|
|
return {
|
|
"localPath": str(path.resolve()),
|
|
"key": key,
|
|
"sizeBytes": size,
|
|
"sha256": digest,
|
|
"action": action,
|
|
}
|
|
|
|
indexed_files = list(enumerate(files, start=1))
|
|
if args.file_workers == 1:
|
|
receipt_items = [process_file(item) for item in indexed_files]
|
|
else:
|
|
with concurrent.futures.ThreadPoolExecutor(
|
|
max_workers=min(args.file_workers, len(files))
|
|
) as executor:
|
|
receipt_items = list(executor.map(process_file, indexed_files))
|
|
|
|
remote = object_listing(client, args.connection_id, args.bucket, prefix)
|
|
for item in receipt_items:
|
|
remote_item = remote.get(item["key"])
|
|
if not remote_item:
|
|
raise UploadError(f"uploaded object is not listed: {item['key']}")
|
|
remote_size = int(
|
|
remote_item.get("sizeBytes", remote_item.get("size", 0)) or 0
|
|
)
|
|
if remote_size != item["sizeBytes"]:
|
|
raise UploadError(
|
|
f"remote size mismatch: {item['key']}: "
|
|
f"{remote_size} != {item['sizeBytes']}"
|
|
)
|
|
item["remote"] = {
|
|
"sizeBytes": remote_size,
|
|
"etag": str(remote_item.get("etag", remote_item.get("eTag", ""))),
|
|
"lastModified": remote_item.get("lastModified"),
|
|
}
|
|
receipt = {
|
|
"apiBaseUrl": args.api_base_url,
|
|
"connectionId": args.connection_id,
|
|
"bucket": args.bucket,
|
|
"prefix": prefix,
|
|
"fileCount": len(receipt_items),
|
|
"totalBytes": sum(item["sizeBytes"] for item in receipt_items),
|
|
"items": receipt_items,
|
|
}
|
|
args.receipt.parent.mkdir(parents=True, exist_ok=True)
|
|
args.receipt.write_text(
|
|
json.dumps(receipt, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
print(
|
|
f"VERIFIED {receipt['fileCount']} files {receipt['totalBytes']} bytes "
|
|
f"receipt={args.receipt}",
|
|
flush=True,
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|