⚡ Parallelize UBT asset uploads

This commit is contained in:
eust-w 2026-07-24 17:04:09 +08:00
parent 76fece8744
commit e53fc69a4e

View File

@ -52,6 +52,12 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--urdf", type=Path, required=True)
parser.add_argument("--receipt", type=Path, required=True)
parser.add_argument("--workers", type=int, default=4)
parser.add_argument(
"--file-workers",
type=int,
default=1,
help="Number of distinct files uploaded concurrently.",
)
return parser.parse_args()
@ -259,6 +265,8 @@ def main() -> int:
)
if args.workers < 1:
raise UploadError("--workers must be positive")
if args.file_workers < 1:
raise UploadError("--file-workers must be positive")
archives = sorted(args.archive_dir.glob("*.zip"))
if len(archives) != 11:
raise UploadError(f"expected 11 zip archives, found {len(archives)}")
@ -279,8 +287,8 @@ def main() -> int:
uploader = MultipartUploader(
client, args.connection_id, args.bucket, args.workers
)
receipt_items: list[dict[str, Any]] = []
for index, path in enumerate(files, start=1):
def process_file(index_and_path: tuple[int, Path]) -> dict[str, Any]:
index, path = index_and_path
key = f"{prefix}/{path.name}"
size = path.stat().st_size
digest = sha256_file(path)
@ -299,16 +307,23 @@ def main() -> int:
)
uploader.upload(path, key)
action = "uploaded"
receipt_items.append(
{
"localPath": str(path.resolve()),
"key": key,
"sizeBytes": size,
"sha256": digest,
"action": action,
}
)
print(f"DONE {index}/{len(files)} {path.name} {action}", flush=True)
return {
"localPath": str(path.resolve()),
"key": key,
"sizeBytes": size,
"sha256": digest,
"action": action,
}
indexed_files = list(enumerate(files, start=1))
if args.file_workers == 1:
receipt_items = [process_file(item) for item in indexed_files]
else:
with concurrent.futures.ThreadPoolExecutor(
max_workers=min(args.file_workers, len(files))
) as executor:
receipt_items = list(executor.map(process_file, indexed_files))
remote = object_listing(client, args.connection_id, args.bucket, prefix)
for item in receipt_items: