2026-10-09 16:41:13 +08:00

237 lines
5.9 KiB
Bash
Executable File

#!/usr/bin/env bash
# Download the RoboDojo dataset from Hugging Face to XPolicyLab/data/
#
# Usage:
# bash scripts/RoboDojo/download_robodojo_data.sh <type>
#
# Example:
# bash scripts/RoboDojo/download_robodojo_data.sh hdf5
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" # XPolicyLab root directory
DATA_ROOT="${PROJECT_ROOT}/../data"
DATA_TYPE="${1:-}" # Data format: lerobot_v3.0 / lerobot_v2.1 / hdf5 / demo / real
HF_REPO_ID="${HF_REPO_ID:-RoboDojo-Benchmark/Robodojo}"
HF_REVISION="${HF_REVISION:-main}"
HF_MAX_WORKERS="${HF_MAX_WORKERS:-16}"
HF_RETRY_WAIT="${HF_RETRY_WAIT:-60}"
HF_MAX_RETRIES="${HF_MAX_RETRIES:-10}"
HF_DOWNLOAD_TIMEOUT="${HF_DOWNLOAD_TIMEOUT:-300}"
usage() {
cat <<'EOF'
Usage: bash scripts/RoboDojo/download_robodojo_data.sh <type>
Types:
lerobot_v3.0 -> RoboDojo_lerobot_v30_video
lerobot_v2.1 -> RoboDojo_lerobot_v21_video
hdf5 -> RoboDojo
demo -> demo
real -> RoboDojo_real
Example:
bash scripts/RoboDojo/download_robodojo_data.sh hdf5
bash scripts/RoboDojo/download_robodojo_data.sh real
Environment:
HF_REPO_ID default: RoboDojo-Benchmark/Robodojo
HF_REVISION default: main
HF_MAX_WORKERS parallel file downloads (default: 16)
HF_RETRY_WAIT seconds to wait before retry (default: 60)
HF_MAX_RETRIES max retry attempts (default: 10)
HF_DOWNLOAD_TIMEOUT per-file timeout in seconds (default: 300)
HF_FORCE_DOWNLOAD set to 1 to re-download even if complete
EOF
}
if [[ -z "${DATA_TYPE}" ]]; then
usage
exit 1
fi
mkdir -p "${DATA_ROOT}"
resolve_data_paths() {
case "${DATA_TYPE}" in
lerobot_v3.0)
DATA_DIR_NAME="RoboDojo_lerobot_v30_video"
;;
lerobot_v2.1)
DATA_DIR_NAME="RoboDojo_lerobot_v21_video"
;;
hdf5)
DATA_DIR_NAME="RoboDojo"
;;
demo)
DATA_DIR_NAME="demo"
;;
real|RoboDojo_real)
DATA_DIR_NAME="RoboDojo_real"
;;
*)
echo "Invalid type: ${DATA_TYPE}" >&2
return 1
;;
esac
REMOTE_DIR="data/${DATA_DIR_NAME}"
TARGET_DIR="${DATA_ROOT}/${DATA_DIR_NAME}"
}
ensure_hf_deps() {
if ! command -v python3 >/dev/null 2>&1; then
echo "python3 not found" >&2
exit 1
fi
python3 - <<'PY'
import importlib.util
import subprocess
import sys
missing = [
name for name in ("huggingface_hub", "hf_transfer")
if importlib.util.find_spec(name) is None
]
if missing:
subprocess.check_call([
sys.executable,
"-m",
"pip",
"install",
"-U",
"huggingface_hub",
"hf_transfer",
])
PY
}
# HuggingFace: snapshot_download + allow_patterns + hf_transfer parallel batch download for large files
download_hf_folder() {
local repo_id="$1"
local remote_dir="$2"
local target_dir="$3"
local complete_marker="${target_dir}/.download_complete"
if [[ -f "${complete_marker}" && "${HF_FORCE_DOWNLOAD:-0}" != "1" ]]; then
echo "==> Target already exists, skip: ${target_dir}"
return 0
fi
if [[ -d "${target_dir}" && "${HF_FORCE_DOWNLOAD:-0}" != "1" ]]; then
echo "==> Resuming partial download: ${target_dir}"
elif [[ -d "${target_dir}" ]]; then
echo "==> Force re-download: ${target_dir}"
rm -rf "${target_dir}"
fi
ensure_hf_deps
# hf_transfer must be enabled before importing huggingface_hub
export HF_HUB_ENABLE_HF_TRANSFER=1
export HF_HUB_DOWNLOAD_TIMEOUT="${HF_DOWNLOAD_TIMEOUT}"
export HF_TRANSFER_MAX_CONCURRENT_DOWNLOADS="${HF_MAX_WORKERS}"
echo "==> Downloading ${remote_dir} from HuggingFace ${repo_id}"
echo "==> Parallel workers: ${HF_MAX_WORKERS}, timeout: ${HF_DOWNLOAD_TIMEOUT}s"
HF_REPO_ID="${repo_id}" \
HF_REVISION="${HF_REVISION}" \
REMOTE_DIR="${remote_dir}" \
DATA_ROOT="${DATA_ROOT}" \
TARGET_DIR="${target_dir}" \
HF_MAX_WORKERS="${HF_MAX_WORKERS}" \
HF_RETRY_WAIT="${HF_RETRY_WAIT}" \
HF_MAX_RETRIES="${HF_MAX_RETRIES}" \
python3 - <<'PY'
import os
import sys
import time
from pathlib import Path
from huggingface_hub import snapshot_download
from huggingface_hub.utils import HfHubHTTPError
repo_id = os.environ["HF_REPO_ID"]
revision = os.environ["HF_REVISION"]
remote_dir = os.environ["REMOTE_DIR"]
data_root = Path(os.environ["DATA_ROOT"])
target_dir = Path(os.environ["TARGET_DIR"])
complete_marker = target_dir / ".download_complete"
max_workers = int(os.environ["HF_MAX_WORKERS"])
retry_wait = int(os.environ["HF_RETRY_WAIT"])
max_retries = int(os.environ["HF_MAX_RETRIES"])
data_root.mkdir(parents=True, exist_ok=True)
download_root = data_root.parent
def is_retryable(exc: BaseException) -> bool:
err = str(exc).lower()
if isinstance(exc, HfHubHTTPError) and exc.response is not None:
if exc.response.status_code in (408, 429, 500, 502, 503, 504):
return True
return any(
token in err
for token in (
"429",
"too many requests",
"rate limit",
"timeout",
"timed out",
"connection reset",
"connection aborted",
"temporary failure",
"503",
"502",
)
)
attempt = 0
while True:
attempt += 1
try:
print(f"==> Download attempt {attempt}/{max_retries} (max_workers={max_workers})")
snapshot_download(
repo_id=repo_id,
repo_type="dataset",
revision=revision,
local_dir=str(download_root),
allow_patterns=[f"{remote_dir}/**"],
max_workers=max_workers,
)
break
except Exception as exc:
if attempt >= max_retries or not is_retryable(exc):
raise
print(
f"==> Transient error, retry in {retry_wait}s: {exc}",
file=sys.stderr,
)
time.sleep(retry_wait)
if not target_dir.is_dir():
print(f"Remote folder not found: {remote_dir}", file=sys.stderr)
sys.exit(1)
complete_marker.write_text(f"repo_id={repo_id}\nrevision={revision}\n")
print(f"==> Saved to {target_dir}")
PY
if [[ ! -d "${target_dir}" ]]; then
echo "Remote folder not found: ${remote_dir}" >&2
exit 1
fi
echo "==> Download complete: ${target_dir}"
}
if ! resolve_data_paths; then
usage
exit 1
fi
download_hf_folder "${HF_REPO_ID}" "${REMOTE_DIR}" "${TARGET_DIR}"