44 lines
1.7 KiB
Python
44 lines
1.7 KiB
Python
#!/usr/bin/env python3
|
|
"""fetch_model_v2.py — 从 hf-mirror 下载 mmBERT-base(现代Bert, Laya-multilingual 同源底座)
|
|
用法: python3 fetch_model_v2.py <目标目录>
|
|
"""
|
|
import os, sys, time, urllib.request
|
|
|
|
BASE = "https://hf-mirror.com/jhu-clsp/mmBERT-base/resolve/main"
|
|
FILES = ["config.json", "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "pytorch_model.bin"]
|
|
MIN_SIZE = {"pytorch_model.bin": 500_000_000}
|
|
|
|
dest = sys.argv[1]
|
|
os.makedirs(dest, exist_ok=True)
|
|
UA = {"User-Agent": "Mozilla/5.0"}
|
|
for fn in FILES:
|
|
path = os.path.join(dest, fn)
|
|
want = MIN_SIZE.get(fn, 1)
|
|
if os.path.exists(path) and os.path.getsize(path) >= want:
|
|
print(f"[fetch] {fn} 已存在({os.path.getsize(path)//1024//1024}MB) 跳过", flush=True)
|
|
continue
|
|
url = f"{BASE}/{fn}"
|
|
for attempt in range(1, 7):
|
|
try:
|
|
t0 = time.time()
|
|
req = urllib.request.Request(url, headers=UA)
|
|
with urllib.request.urlopen(req, timeout=120) as r, open(path + ".tmp", "wb") as f:
|
|
total = 0
|
|
while True:
|
|
b = r.read(8 * 1024 * 1024)
|
|
if not b:
|
|
break
|
|
f.write(b)
|
|
total += len(b)
|
|
os.replace(path + ".tmp", path)
|
|
print(f"[fetch] {fn}: {total//1024//1024}MB {time.time()-t0:.0f}s", flush=True)
|
|
if total >= want:
|
|
break
|
|
raise RuntimeError(f"size {total} < {want}")
|
|
except Exception as e:
|
|
print(f"[fetch] {fn} 第{attempt}次失败: {str(e)[:120]}", flush=True)
|
|
if attempt == 6:
|
|
sys.exit(f"FAIL-model {fn}")
|
|
time.sleep(5 * attempt)
|
|
print("[fetch] ALL-OK", flush=True)
|