add fetch_model.py (modelscope直链+重试)
This commit is contained in:
parent
01174c4377
commit
e015029ca2
42
fetch_model.py
Normal file
42
fetch_model.py
Normal file
@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
"""fetch_model.py — 从 ModelScope 直链下载 xlm-roberta-base(纯urllib, 带重试与校验)
|
||||
用法: python3 fetch_model.py <目标目录>
|
||||
"""
|
||||
import os, sys, time, urllib.request
|
||||
|
||||
BASE = "https://www.modelscope.cn/models/AI-ModelScope/xlm-roberta-base/resolve/master"
|
||||
FILES = ["config.json", "tokenizer.json", "sentencepiece.bpe.model", "model.safetensors"]
|
||||
MIN_SIZE = {"model.safetensors": 1_100_000_000}
|
||||
|
||||
dest = sys.argv[1]
|
||||
os.makedirs(dest, exist_ok=True)
|
||||
for fn in FILES:
|
||||
path = os.path.join(dest, fn)
|
||||
want = MIN_SIZE.get(fn, 1)
|
||||
if os.path.exists(path) and os.path.getsize(path) >= want:
|
||||
print(f"[fetch] {fn} 已存在({os.path.getsize(path)//1024//1024}MB) 跳过", flush=True)
|
||||
continue
|
||||
url = f"{BASE}/{fn}"
|
||||
for attempt in range(1, 7):
|
||||
try:
|
||||
t0 = time.time()
|
||||
req = urllib.request.Request(url)
|
||||
with urllib.request.urlopen(req, timeout=60) as r, open(path + ".tmp", "wb") as f:
|
||||
total = 0
|
||||
while True:
|
||||
b = r.read(4 * 1024 * 1024)
|
||||
if not b:
|
||||
break
|
||||
f.write(b)
|
||||
total += len(b)
|
||||
os.replace(path + ".tmp", path)
|
||||
print(f"[fetch] {fn}: {total//1024//1024}MB {time.time()-t0:.0f}s", flush=True)
|
||||
if total >= want:
|
||||
break
|
||||
raise RuntimeError(f"size {total} < {want}")
|
||||
except Exception as e:
|
||||
print(f"[fetch] {fn} 第{attempt}次失败: {str(e)[:120]}", flush=True)
|
||||
if attempt == 6:
|
||||
sys.exit(f"FAIL-model {fn}")
|
||||
time.sleep(3 * attempt)
|
||||
print("[fetch] ALL-OK", flush=True)
|
||||
Loading…
x
Reference in New Issue
Block a user