OpenPi_POC_Dev/scripts/fetch_gcs_public.py
2026-10-09 16:41:13 +08:00

117 lines
4.2 KiB
Python

#!/usr/bin/env python3
"""匿名拉取 GCS 公开桶对象到本地 (无 gsutil 依赖)。
用法: fetch_gcs_public.py <bucket> <prefix> <output_dir> [--required]
走 storage.googleapis.com 公开 JSON API (列表 + 逐对象下载), 经 http_proxy/HTTPS_PROXY
(若设置)。openpi-assets 与 big_vision 桶均公开可匿名读。
"""
from __future__ import annotations
import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
from urllib.parse import quote
def _opener() -> urllib.request.OpenerDirector:
proxy = os.environ.get("https_proxy") or os.environ.get("HTTPS_PROXY") or os.environ.get("http_proxy") or os.environ.get("HTTP_PROXY")
handlers = []
if proxy:
# no_proxy 语义按主机粒度; storage.googleapis.com 不在本地网段, 走代理
handlers.append(urllib.request.ProxyHandler({"http": proxy, "https": proxy}))
return urllib.request.build_opener(*handlers)
def list_objects(opener, bucket: str, prefix: str) -> list[str]:
names: list[str] = []
page_token = None
while True:
url = (
f"https://storage.googleapis.com/storage/v1/b/{quote(bucket)}/o"
f"?prefix={quote(prefix, safe='')}&maxResults=1000"
+ (f"&pageToken={page_token}" if page_token else "")
)
with opener.open(url, timeout=60) as resp:
data = json.load(resp)
names.extend(item["name"] for item in data.get("items", []))
page_token = data.get("nextPageToken")
if not page_token:
break
return names
def download(opener, bucket: str, name: str, dest: Path) -> None:
dest.parent.mkdir(parents=True, exist_ok=True)
if dest.exists() and dest.stat().st_size > 0:
return
url = f"https://storage.googleapis.com/storage/v1/b/{quote(bucket)}/o/{quote(name, safe='')}?alt=media"
tmp = dest.with_suffix(dest.suffix + ".partial")
last_err: Exception | None = None
for attempt in range(5):
try:
with opener.open(url, timeout=300) as resp, open(tmp, "wb") as f:
while chunk := resp.read(1 << 20):
f.write(chunk)
tmp.rename(dest)
return
except Exception as exc: # noqa: BLE001
last_err = exc
time.sleep(3 * (attempt + 1))
raise RuntimeError(f"download failed: {name}: {last_err}")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("bucket")
parser.add_argument("prefix")
parser.add_argument("output_dir")
parser.add_argument("--required", action="store_true", help="对象列表为空时非 0 退出")
args = parser.parse_args()
opener = _opener()
names: list[str] | None = None
try:
names = list_objects(opener, args.bucket, args.prefix)
except Exception as exc: # noqa: BLE001 某些桶 (big_vision) 禁匿名 list 但对象公开
print(f"[fetch] list failed ({type(exc).__name__}: {exc}); trying direct object GET", flush=True)
out = Path(args.output_dir) / args.prefix
try:
download(opener, args.bucket, args.prefix, out)
except Exception:
# 直读也失败: 若 list 失败是可重试网络错误, 重新抛原始异常语义
if isinstance(exc, urllib.error.HTTPError) and exc.code not in (401, 403):
raise
raise
print(f"[fetch] done: {out}", flush=True)
return
assert names is not None
if not names:
msg = f"no objects under gs://{args.bucket}/{args.prefix}"
if args.required:
sys.exit(f"[fetch] {msg}")
print(f"[fetch] {msg} (skipped)")
return
out_root = Path(args.output_dir)
total = sum(1 for n in names if not n.endswith("/"))
print(f"[fetch] gs://{args.bucket}/{args.prefix}: {total} objects -> {out_root}")
for i, name in enumerate(names, 1):
if name.endswith("/"):
continue
rel = name[len(args.prefix):].lstrip("/")
download(opener, args.bucket, name, out_root / rel)
if i % 50 == 0 or i == total:
print(f"[fetch] {i}/{total}", flush=True)
print(f"[fetch] done: {out_root}")
if __name__ == "__main__":
main()