From fda8cfd4b1ceaec507d49135c3fdff96dc515aeb Mon Sep 17 00:00:00 2001 From: eust-w Date: Fri, 24 Jul 2026 17:17:07 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Parallelize=20UBT=20archive=20prepa?= =?UTF-8?q?ration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ubt-mcap-pipeline/ubt_pipeline/prepare.py | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/lightweight-operators/ubt-mcap-pipeline/ubt_pipeline/prepare.py b/lightweight-operators/ubt-mcap-pipeline/ubt_pipeline/prepare.py index 64d563a8..1f9cc8d3 100644 --- a/lightweight-operators/ubt-mcap-pipeline/ubt_pipeline/prepare.py +++ b/lightweight-operators/ubt-mcap-pipeline/ubt_pipeline/prepare.py @@ -1,5 +1,6 @@ from __future__ import annotations +import concurrent.futures import contextlib import fnmatch import os @@ -256,6 +257,22 @@ def _select_objects( return sorted(archives, key=lambda item: item["key"]), urdf_matches[0] +def _stage_archive( + index: int, + archive_object: dict[str, Any], + scratch: pathlib.Path, +) -> tuple[int, dict[str, Any], pathlib.Path]: + local_archive = ( + scratch / f"{index:02d}-{pathlib.PurePosixPath(archive_object['key']).name}" + ) + extract_dir = scratch / f"extract-{index:02d}" + download_object(build_bos_client(), archive_object, local_archive) + safe_extract_zip(local_archive, extract_dir) + with contextlib.suppress(OSError): + local_archive.unlink() + return index, archive_object, extract_dir + + def run_prepare() -> None: dataset_id = safe_identifier(env_text("DATASET_ID"), "DATASET_ID") user_name = safe_identifier(env_text("USER_NAME", "ubt"), "USER_NAME") @@ -335,16 +352,17 @@ def run_prepare() -> None: episodes: list[dict[str, Any]] = [] seen: set[str] = set() - for index, archive_object in enumerate(archives): - local_archive = ( - scratch - / f"{index:02d}-{pathlib.PurePosixPath(archive_object['key']).name}" + prepare_workers = min(3, len(archives)) + with concurrent.futures.ThreadPoolExecutor( + max_workers=prepare_workers + ) as executor: + staged_archives = list( + executor.map( + lambda item: _stage_archive(item[0], item[1], scratch), + enumerate(archives), + ) ) - extract_dir = scratch / f"extract-{index:02d}" - download_object(client, archive_object, local_archive) - safe_extract_zip(local_archive, extract_dir) - with contextlib.suppress(FileNotFoundError): - local_archive.unlink() + for _, archive_object, extract_dir in sorted(staged_archives): discovered = discover_episode_dirs(extract_dir) if not discovered: raise RuntimeError(