From e015029ca210b7b33f912288cc5bcf1d335e7641 Mon Sep 17 00:00:00 2001 From: "dingbin.wang" Date: Thu, 15 Oct 2026 20:51:03 +0800 Subject: [PATCH] =?UTF-8?q?add=20fetch=5Fmodel.py=20(modelscope=E7=9B=B4?= =?UTF-8?q?=E9=93=BE+=E9=87=8D=E8=AF=95)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fetch_model.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 fetch_model.py diff --git a/fetch_model.py b/fetch_model.py new file mode 100644 index 0000000..a4f479a --- /dev/null +++ b/fetch_model.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""fetch_model.py — 从 ModelScope 直链下载 xlm-roberta-base(纯urllib, 带重试与校验) +用法: python3 fetch_model.py <目标目录> +""" +import os, sys, time, urllib.request + +BASE = "https://www.modelscope.cn/models/AI-ModelScope/xlm-roberta-base/resolve/master" +FILES = ["config.json", "tokenizer.json", "sentencepiece.bpe.model", "model.safetensors"] +MIN_SIZE = {"model.safetensors": 1_100_000_000} + +dest = sys.argv[1] +os.makedirs(dest, exist_ok=True) +for fn in FILES: + path = os.path.join(dest, fn) + want = MIN_SIZE.get(fn, 1) + if os.path.exists(path) and os.path.getsize(path) >= want: + print(f"[fetch] {fn} 已存在({os.path.getsize(path)//1024//1024}MB) 跳过", flush=True) + continue + url = f"{BASE}/{fn}" + for attempt in range(1, 7): + try: + t0 = time.time() + req = urllib.request.Request(url) + with urllib.request.urlopen(req, timeout=60) as r, open(path + ".tmp", "wb") as f: + total = 0 + while True: + b = r.read(4 * 1024 * 1024) + if not b: + break + f.write(b) + total += len(b) + os.replace(path + ".tmp", path) + print(f"[fetch] {fn}: {total//1024//1024}MB {time.time()-t0:.0f}s", flush=True) + if total >= want: + break + raise RuntimeError(f"size {total} < {want}") + except Exception as e: + print(f"[fetch] {fn} 第{attempt}次失败: {str(e)[:120]}", flush=True) + if attempt == 6: + sys.exit(f"FAIL-model {fn}") + time.sleep(3 * attempt) +print("[fetch] ALL-OK", flush=True)