#!/usr/bin/env python3 """fetch_model.py — 从 ModelScope 直链下载 xlm-roberta-base(纯urllib, 带重试与校验) 用法: python3 fetch_model.py <目标目录> """ import os, sys, time, urllib.request BASE = "https://www.modelscope.cn/models/AI-ModelScope/xlm-roberta-base/resolve/master" FILES = ["config.json", "tokenizer.json", "sentencepiece.bpe.model", "model.safetensors"] MIN_SIZE = {"model.safetensors": 1_100_000_000} dest = sys.argv[1] os.makedirs(dest, exist_ok=True) for fn in FILES: path = os.path.join(dest, fn) want = MIN_SIZE.get(fn, 1) if os.path.exists(path) and os.path.getsize(path) >= want: print(f"[fetch] {fn} 已存在({os.path.getsize(path)//1024//1024}MB) 跳过", flush=True) continue url = f"{BASE}/{fn}" for attempt in range(1, 7): try: t0 = time.time() req = urllib.request.Request(url) with urllib.request.urlopen(req, timeout=60) as r, open(path + ".tmp", "wb") as f: total = 0 while True: b = r.read(4 * 1024 * 1024) if not b: break f.write(b) total += len(b) os.replace(path + ".tmp", path) print(f"[fetch] {fn}: {total//1024//1024}MB {time.time()-t0:.0f}s", flush=True) if total >= want: break raise RuntimeError(f"size {total} < {want}") except Exception as e: print(f"[fetch] {fn} 第{attempt}次失败: {str(e)[:120]}", flush=True) if attempt == 6: sys.exit(f"FAIL-model {fn}") time.sleep(3 * attempt) print("[fetch] ALL-OK", flush=True)