#!/usr/bin/env python3 """匿名拉取 GCS 公开桶对象到本地 (无 gsutil 依赖)。 用法: fetch_gcs_public.py [--required] 走 storage.googleapis.com 公开 JSON API (列表 + 逐对象下载), 经 http_proxy/HTTPS_PROXY (若设置)。openpi-assets 与 big_vision 桶均公开可匿名读。 """ from __future__ import annotations import argparse import json import os import sys import time import urllib.error import urllib.request from pathlib import Path from urllib.parse import quote def _opener() -> urllib.request.OpenerDirector: proxy = os.environ.get("https_proxy") or os.environ.get("HTTPS_PROXY") or os.environ.get("http_proxy") or os.environ.get("HTTP_PROXY") handlers = [] if proxy: # no_proxy 语义按主机粒度; storage.googleapis.com 不在本地网段, 走代理 handlers.append(urllib.request.ProxyHandler({"http": proxy, "https": proxy})) return urllib.request.build_opener(*handlers) def list_objects(opener, bucket: str, prefix: str) -> list[str]: names: list[str] = [] page_token = None while True: url = ( f"https://storage.googleapis.com/storage/v1/b/{quote(bucket)}/o" f"?prefix={quote(prefix, safe='')}&maxResults=1000" + (f"&pageToken={page_token}" if page_token else "") ) with opener.open(url, timeout=60) as resp: data = json.load(resp) names.extend(item["name"] for item in data.get("items", [])) page_token = data.get("nextPageToken") if not page_token: break return names def download(opener, bucket: str, name: str, dest: Path) -> None: dest.parent.mkdir(parents=True, exist_ok=True) if dest.exists() and dest.stat().st_size > 0: return url = f"https://storage.googleapis.com/storage/v1/b/{quote(bucket)}/o/{quote(name, safe='')}?alt=media" tmp = dest.with_suffix(dest.suffix + ".partial") last_err: Exception | None = None for attempt in range(5): try: with opener.open(url, timeout=300) as resp, open(tmp, "wb") as f: while chunk := resp.read(1 << 20): f.write(chunk) tmp.rename(dest) return except Exception as exc: # noqa: BLE001 last_err = exc time.sleep(3 * (attempt + 1)) raise RuntimeError(f"download failed: {name}: {last_err}") def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("bucket") parser.add_argument("prefix") parser.add_argument("output_dir") parser.add_argument("--required", action="store_true", help="对象列表为空时非 0 退出") args = parser.parse_args() opener = _opener() names: list[str] | None = None try: names = list_objects(opener, args.bucket, args.prefix) except Exception as exc: # noqa: BLE001 某些桶 (big_vision) 禁匿名 list 但对象公开 print(f"[fetch] list failed ({type(exc).__name__}: {exc}); trying direct object GET", flush=True) out = Path(args.output_dir) / args.prefix try: download(opener, args.bucket, args.prefix, out) except Exception: # 直读也失败: 若 list 失败是可重试网络错误, 重新抛原始异常语义 if isinstance(exc, urllib.error.HTTPError) and exc.code not in (401, 403): raise raise print(f"[fetch] done: {out}", flush=True) return assert names is not None if not names: msg = f"no objects under gs://{args.bucket}/{args.prefix}" if args.required: sys.exit(f"[fetch] {msg}") print(f"[fetch] {msg} (skipped)") return out_root = Path(args.output_dir) total = sum(1 for n in names if not n.endswith("/")) print(f"[fetch] gs://{args.bucket}/{args.prefix}: {total} objects -> {out_root}") for i, name in enumerate(names, 1): if name.endswith("/"): continue rel = name[len(args.prefix):].lstrip("/") download(opener, args.bucket, name, out_root / rel) if i % 50 == 0 or i == total: print(f"[fetch] {i}/{total}", flush=True) print(f"[fetch] done: {out_root}") if __name__ == "__main__": main()