forked from longtao.wu/droboticflow
⚡ Parallelize UBT asset uploads
This commit is contained in:
parent
76fece8744
commit
e53fc69a4e
@ -52,6 +52,12 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--urdf", type=Path, required=True)
|
||||
parser.add_argument("--receipt", type=Path, required=True)
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
parser.add_argument(
|
||||
"--file-workers",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of distinct files uploaded concurrently.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@ -259,6 +265,8 @@ def main() -> int:
|
||||
)
|
||||
if args.workers < 1:
|
||||
raise UploadError("--workers must be positive")
|
||||
if args.file_workers < 1:
|
||||
raise UploadError("--file-workers must be positive")
|
||||
archives = sorted(args.archive_dir.glob("*.zip"))
|
||||
if len(archives) != 11:
|
||||
raise UploadError(f"expected 11 zip archives, found {len(archives)}")
|
||||
@ -279,8 +287,8 @@ def main() -> int:
|
||||
uploader = MultipartUploader(
|
||||
client, args.connection_id, args.bucket, args.workers
|
||||
)
|
||||
receipt_items: list[dict[str, Any]] = []
|
||||
for index, path in enumerate(files, start=1):
|
||||
def process_file(index_and_path: tuple[int, Path]) -> dict[str, Any]:
|
||||
index, path = index_and_path
|
||||
key = f"{prefix}/{path.name}"
|
||||
size = path.stat().st_size
|
||||
digest = sha256_file(path)
|
||||
@ -299,16 +307,23 @@ def main() -> int:
|
||||
)
|
||||
uploader.upload(path, key)
|
||||
action = "uploaded"
|
||||
receipt_items.append(
|
||||
{
|
||||
"localPath": str(path.resolve()),
|
||||
"key": key,
|
||||
"sizeBytes": size,
|
||||
"sha256": digest,
|
||||
"action": action,
|
||||
}
|
||||
)
|
||||
print(f"DONE {index}/{len(files)} {path.name} {action}", flush=True)
|
||||
return {
|
||||
"localPath": str(path.resolve()),
|
||||
"key": key,
|
||||
"sizeBytes": size,
|
||||
"sha256": digest,
|
||||
"action": action,
|
||||
}
|
||||
|
||||
indexed_files = list(enumerate(files, start=1))
|
||||
if args.file_workers == 1:
|
||||
receipt_items = [process_file(item) for item in indexed_files]
|
||||
else:
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=min(args.file_workers, len(files))
|
||||
) as executor:
|
||||
receipt_items = list(executor.map(process_file, indexed_files))
|
||||
|
||||
remote = object_listing(client, args.connection_id, args.bucket, prefix)
|
||||
for item in receipt_items:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user