117 lines
4.2 KiB
Python
117 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""匿名拉取 GCS 公开桶对象到本地 (无 gsutil 依赖)。
|
|
|
|
用法: fetch_gcs_public.py <bucket> <prefix> <output_dir> [--required]
|
|
|
|
走 storage.googleapis.com 公开 JSON API (列表 + 逐对象下载), 经 http_proxy/HTTPS_PROXY
|
|
(若设置)。openpi-assets 与 big_vision 桶均公开可匿名读。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
|
|
def _opener() -> urllib.request.OpenerDirector:
|
|
proxy = os.environ.get("https_proxy") or os.environ.get("HTTPS_PROXY") or os.environ.get("http_proxy") or os.environ.get("HTTP_PROXY")
|
|
handlers = []
|
|
if proxy:
|
|
# no_proxy 语义按主机粒度; storage.googleapis.com 不在本地网段, 走代理
|
|
handlers.append(urllib.request.ProxyHandler({"http": proxy, "https": proxy}))
|
|
return urllib.request.build_opener(*handlers)
|
|
|
|
|
|
def list_objects(opener, bucket: str, prefix: str) -> list[str]:
|
|
names: list[str] = []
|
|
page_token = None
|
|
while True:
|
|
url = (
|
|
f"https://storage.googleapis.com/storage/v1/b/{quote(bucket)}/o"
|
|
f"?prefix={quote(prefix, safe='')}&maxResults=1000"
|
|
+ (f"&pageToken={page_token}" if page_token else "")
|
|
)
|
|
with opener.open(url, timeout=60) as resp:
|
|
data = json.load(resp)
|
|
names.extend(item["name"] for item in data.get("items", []))
|
|
page_token = data.get("nextPageToken")
|
|
if not page_token:
|
|
break
|
|
return names
|
|
|
|
|
|
def download(opener, bucket: str, name: str, dest: Path) -> None:
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
if dest.exists() and dest.stat().st_size > 0:
|
|
return
|
|
url = f"https://storage.googleapis.com/storage/v1/b/{quote(bucket)}/o/{quote(name, safe='')}?alt=media"
|
|
tmp = dest.with_suffix(dest.suffix + ".partial")
|
|
last_err: Exception | None = None
|
|
for attempt in range(5):
|
|
try:
|
|
with opener.open(url, timeout=300) as resp, open(tmp, "wb") as f:
|
|
while chunk := resp.read(1 << 20):
|
|
f.write(chunk)
|
|
tmp.rename(dest)
|
|
return
|
|
except Exception as exc: # noqa: BLE001
|
|
last_err = exc
|
|
time.sleep(3 * (attempt + 1))
|
|
raise RuntimeError(f"download failed: {name}: {last_err}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("bucket")
|
|
parser.add_argument("prefix")
|
|
parser.add_argument("output_dir")
|
|
parser.add_argument("--required", action="store_true", help="对象列表为空时非 0 退出")
|
|
args = parser.parse_args()
|
|
|
|
opener = _opener()
|
|
names: list[str] | None = None
|
|
try:
|
|
names = list_objects(opener, args.bucket, args.prefix)
|
|
except Exception as exc: # noqa: BLE001 某些桶 (big_vision) 禁匿名 list 但对象公开
|
|
print(f"[fetch] list failed ({type(exc).__name__}: {exc}); trying direct object GET", flush=True)
|
|
out = Path(args.output_dir) / args.prefix
|
|
try:
|
|
download(opener, args.bucket, args.prefix, out)
|
|
except Exception:
|
|
# 直读也失败: 若 list 失败是可重试网络错误, 重新抛原始异常语义
|
|
if isinstance(exc, urllib.error.HTTPError) and exc.code not in (401, 403):
|
|
raise
|
|
raise
|
|
print(f"[fetch] done: {out}", flush=True)
|
|
return
|
|
assert names is not None
|
|
if not names:
|
|
msg = f"no objects under gs://{args.bucket}/{args.prefix}"
|
|
if args.required:
|
|
sys.exit(f"[fetch] {msg}")
|
|
print(f"[fetch] {msg} (skipped)")
|
|
return
|
|
|
|
out_root = Path(args.output_dir)
|
|
total = sum(1 for n in names if not n.endswith("/"))
|
|
print(f"[fetch] gs://{args.bucket}/{args.prefix}: {total} objects -> {out_root}")
|
|
for i, name in enumerate(names, 1):
|
|
if name.endswith("/"):
|
|
continue
|
|
rel = name[len(args.prefix):].lstrip("/")
|
|
download(opener, args.bucket, name, out_root / rel)
|
|
if i % 50 == 0 or i == total:
|
|
print(f"[fetch] {i}/{total}", flush=True)
|
|
print(f"[fetch] done: {out_root}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|