forked from longtao.wu/droboticflow
⚡ Parallelize UBT archive preparation
This commit is contained in:
parent
fcac815629
commit
fda8cfd4b1
@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import fnmatch
|
||||
import os
|
||||
@ -256,6 +257,22 @@ def _select_objects(
|
||||
return sorted(archives, key=lambda item: item["key"]), urdf_matches[0]
|
||||
|
||||
|
||||
def _stage_archive(
|
||||
index: int,
|
||||
archive_object: dict[str, Any],
|
||||
scratch: pathlib.Path,
|
||||
) -> tuple[int, dict[str, Any], pathlib.Path]:
|
||||
local_archive = (
|
||||
scratch / f"{index:02d}-{pathlib.PurePosixPath(archive_object['key']).name}"
|
||||
)
|
||||
extract_dir = scratch / f"extract-{index:02d}"
|
||||
download_object(build_bos_client(), archive_object, local_archive)
|
||||
safe_extract_zip(local_archive, extract_dir)
|
||||
with contextlib.suppress(OSError):
|
||||
local_archive.unlink()
|
||||
return index, archive_object, extract_dir
|
||||
|
||||
|
||||
def run_prepare() -> None:
|
||||
dataset_id = safe_identifier(env_text("DATASET_ID"), "DATASET_ID")
|
||||
user_name = safe_identifier(env_text("USER_NAME", "ubt"), "USER_NAME")
|
||||
@ -335,16 +352,17 @@ def run_prepare() -> None:
|
||||
|
||||
episodes: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for index, archive_object in enumerate(archives):
|
||||
local_archive = (
|
||||
scratch
|
||||
/ f"{index:02d}-{pathlib.PurePosixPath(archive_object['key']).name}"
|
||||
prepare_workers = min(3, len(archives))
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=prepare_workers
|
||||
) as executor:
|
||||
staged_archives = list(
|
||||
executor.map(
|
||||
lambda item: _stage_archive(item[0], item[1], scratch),
|
||||
enumerate(archives),
|
||||
)
|
||||
)
|
||||
extract_dir = scratch / f"extract-{index:02d}"
|
||||
download_object(client, archive_object, local_archive)
|
||||
safe_extract_zip(local_archive, extract_dir)
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
local_archive.unlink()
|
||||
for _, archive_object, extract_dir in sorted(staged_archives):
|
||||
discovered = discover_episode_dirs(extract_dir)
|
||||
if not discovered:
|
||||
raise RuntimeError(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user