48 lines
2.1 KiB
Python

#!/usr/bin/env python3
"""fetch_model_v3.py — mmBERT-base 从 ModelScope 下载 (绕开 hf-mirror 完全)"""
import os, sys, urllib.request
MS = "https://www.modelscope.cn/models/jhu-clsp/mmBERT-base/resolve/master"
UA = {"User-Agent": "Mozilla/5.0"}
dest = sys.argv[1] if len(sys.argv) > 1 else "/tmp/base"
os.makedirs(dest, exist_ok=True)
os.makedirs(os.path.join(dest, "tokenizer"), exist_ok=True)
os.makedirs(os.path.join(dest, "encoder"), exist_ok=True)
FILES = {
"config.json": os.path.join(dest, "config.json"),
"tokenizer.json": os.path.join(dest, "tokenizer", "tokenizer.json"),
"tokenizer_config.json": os.path.join(dest, "tokenizer", "tokenizer_config.json"),
"special_tokens_map.json": os.path.join(dest, "tokenizer", "special_tokens_map.json"),
"encoder/config.json": os.path.join(dest, "encoder", "config.json"),
"model.safetensors": os.path.join(dest, "model.safetensors"),
}
MIN = {"model.safetensors": 500_000_000}
for fn, path in FILES.items():
want = MIN.get(fn, 100)
if os.path.exists(path) and os.path.getsize(path) >= want:
print(f"[fetch] {fn} cached ({os.path.getsize(path)//1024//1024}MB)", flush=True)
continue
url = f"{MS}/{fn}"
for attempt in range(1, 5):
try:
t0 = time.time()
req = urllib.request.Request(url, headers=UA)
with urllib.request.urlopen(req, timeout=180) as r, open(path + ".tmp", "wb") as f:
total = 0
while True:
b = r.read(8 * 1024 * 1024)
if not b: break
f.write(b); total += len(b)
os.replace(path + ".tmp", path)
print(f"[fetch] {fn}: {total//1024//1024}MB {time.time()-t0:.0f}s", flush=True)
if total >= want: break
raise RuntimeError(f"size {total} < {want}")
except Exception as e:
print(f"[fetch] {fn} attempt {attempt}: {str(e)[:100]}", flush=True)
if attempt == 4: sys.exit(f"FAIL {fn}")
import time as _t; _t.sleep(10 * attempt)
import time
print("[fetch] ALL-OK", flush=True)