#!/usr/bin/env python3 """fetch_model_v3.py — mmBERT-base 从 ModelScope 下载 (绕开 hf-mirror 完全)""" import os, sys, urllib.request MS = "https://www.modelscope.cn/models/jhu-clsp/mmBERT-base/resolve/master" UA = {"User-Agent": "Mozilla/5.0"} dest = sys.argv[1] if len(sys.argv) > 1 else "/tmp/base" os.makedirs(dest, exist_ok=True) os.makedirs(os.path.join(dest, "tokenizer"), exist_ok=True) os.makedirs(os.path.join(dest, "encoder"), exist_ok=True) FILES = { "config.json": os.path.join(dest, "config.json"), "tokenizer.json": os.path.join(dest, "tokenizer", "tokenizer.json"), "tokenizer_config.json": os.path.join(dest, "tokenizer", "tokenizer_config.json"), "special_tokens_map.json": os.path.join(dest, "tokenizer", "special_tokens_map.json"), "encoder/config.json": os.path.join(dest, "encoder", "config.json"), "model.safetensors": os.path.join(dest, "model.safetensors"), } MIN = {"model.safetensors": 500_000_000} for fn, path in FILES.items(): want = MIN.get(fn, 100) if os.path.exists(path) and os.path.getsize(path) >= want: print(f"[fetch] {fn} cached ({os.path.getsize(path)//1024//1024}MB)", flush=True) continue url = f"{MS}/{fn}" for attempt in range(1, 5): try: t0 = time.time() req = urllib.request.Request(url, headers=UA) with urllib.request.urlopen(req, timeout=180) as r, open(path + ".tmp", "wb") as f: total = 0 while True: b = r.read(8 * 1024 * 1024) if not b: break f.write(b); total += len(b) os.replace(path + ".tmp", path) print(f"[fetch] {fn}: {total//1024//1024}MB {time.time()-t0:.0f}s", flush=True) if total >= want: break raise RuntimeError(f"size {total} < {want}") except Exception as e: print(f"[fetch] {fn} attempt {attempt}: {str(e)[:100]}", flush=True) if attempt == 4: sys.exit(f"FAIL {fn}") import time as _t; _t.sleep(10 * attempt) import time print("[fetch] ALL-OK", flush=True)