diff --git a/scripts/ubt_mcap_assets_upload.py b/scripts/ubt_mcap_assets_upload.py index 09f378cf..97b590fd 100644 --- a/scripts/ubt_mcap_assets_upload.py +++ b/scripts/ubt_mcap_assets_upload.py @@ -52,6 +52,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--urdf", type=Path, required=True) parser.add_argument("--receipt", type=Path, required=True) parser.add_argument("--workers", type=int, default=4) + parser.add_argument( + "--file-workers", + type=int, + default=1, + help="Number of distinct files uploaded concurrently.", + ) return parser.parse_args() @@ -259,6 +265,8 @@ def main() -> int: ) if args.workers < 1: raise UploadError("--workers must be positive") + if args.file_workers < 1: + raise UploadError("--file-workers must be positive") archives = sorted(args.archive_dir.glob("*.zip")) if len(archives) != 11: raise UploadError(f"expected 11 zip archives, found {len(archives)}") @@ -279,8 +287,8 @@ def main() -> int: uploader = MultipartUploader( client, args.connection_id, args.bucket, args.workers ) - receipt_items: list[dict[str, Any]] = [] - for index, path in enumerate(files, start=1): + def process_file(index_and_path: tuple[int, Path]) -> dict[str, Any]: + index, path = index_and_path key = f"{prefix}/{path.name}" size = path.stat().st_size digest = sha256_file(path) @@ -299,16 +307,23 @@ def main() -> int: ) uploader.upload(path, key) action = "uploaded" - receipt_items.append( - { - "localPath": str(path.resolve()), - "key": key, - "sizeBytes": size, - "sha256": digest, - "action": action, - } - ) print(f"DONE {index}/{len(files)} {path.name} {action}", flush=True) + return { + "localPath": str(path.resolve()), + "key": key, + "sizeBytes": size, + "sha256": digest, + "action": action, + } + + indexed_files = list(enumerate(files, start=1)) + if args.file_workers == 1: + receipt_items = [process_file(item) for item in indexed_files] + else: + with concurrent.futures.ThreadPoolExecutor( + max_workers=min(args.file_workers, len(files)) + ) as executor: + receipt_items = list(executor.map(process_file, indexed_files)) remote = object_listing(client, args.connection_id, args.bucket, prefix) for item in receipt_items: