⚡ Parallelize UBT archive preparation

This commit is contained in:
eust-w 2026-07-24 17:17:07 +08:00
parent fcac815629
commit fda8cfd4b1

View File

@ -1,5 +1,6 @@
from __future__ import annotations
import concurrent.futures
import contextlib
import fnmatch
import os
@ -256,6 +257,22 @@ def _select_objects(
return sorted(archives, key=lambda item: item["key"]), urdf_matches[0]
def _stage_archive(
index: int,
archive_object: dict[str, Any],
scratch: pathlib.Path,
) -> tuple[int, dict[str, Any], pathlib.Path]:
local_archive = (
scratch / f"{index:02d}-{pathlib.PurePosixPath(archive_object['key']).name}"
)
extract_dir = scratch / f"extract-{index:02d}"
download_object(build_bos_client(), archive_object, local_archive)
safe_extract_zip(local_archive, extract_dir)
with contextlib.suppress(OSError):
local_archive.unlink()
return index, archive_object, extract_dir
def run_prepare() -> None:
dataset_id = safe_identifier(env_text("DATASET_ID"), "DATASET_ID")
user_name = safe_identifier(env_text("USER_NAME", "ubt"), "USER_NAME")
@ -335,16 +352,17 @@ def run_prepare() -> None:
episodes: list[dict[str, Any]] = []
seen: set[str] = set()
for index, archive_object in enumerate(archives):
local_archive = (
scratch
/ f"{index:02d}-{pathlib.PurePosixPath(archive_object['key']).name}"
prepare_workers = min(3, len(archives))
with concurrent.futures.ThreadPoolExecutor(
max_workers=prepare_workers
) as executor:
staged_archives = list(
executor.map(
lambda item: _stage_archive(item[0], item[1], scratch),
enumerate(archives),
)
)
extract_dir = scratch / f"extract-{index:02d}"
download_object(client, archive_object, local_archive)
safe_extract_zip(local_archive, extract_dir)
with contextlib.suppress(FileNotFoundError):
local_archive.unlink()
for _, archive_object, extract_dir in sorted(staged_archives):
discovered = discover_episode_dirs(extract_dir)
if not discovered:
raise RuntimeError(