86 lines
2.9 KiB
Python
86 lines
2.9 KiB
Python
"""01-fetch 子进程:把 models.csv 的 weights URI 落成一个文件。
|
||
|
||
支持:
|
||
https://… 直接下载(带断点续传)
|
||
torchvision://<arch>/<WEIGHTS_ENUM> 例 torchvision://resnet18/IMAGENET1K_V1 → 保存 state_dict .pth
|
||
hf://<repo>/<file> huggingface hub
|
||
file:///abs/path 本机已有
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import shutil
|
||
import sys
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
|
||
def fetch_http(url: str, out: Path) -> Path:
|
||
tmp = out.with_suffix(out.suffix + ".part")
|
||
pos = tmp.stat().st_size if tmp.exists() else 0
|
||
req = urllib.request.Request(url, headers={"Range": f"bytes={pos}-"} if pos else {})
|
||
with urllib.request.urlopen(req, timeout=60) as r, open(tmp, "ab") as f:
|
||
shutil.copyfileobj(r, f, length=1 << 20)
|
||
tmp.rename(out)
|
||
return out
|
||
|
||
|
||
def fetch_torchvision(arch: str, enum: str, out: Path) -> Path:
|
||
import torch, torchvision.models as tvm
|
||
w = getattr(tvm.get_model_weights(arch), enum) # enum 可以是 DEFAULT
|
||
model = tvm.get_model(arch, weights=w)
|
||
torch.save(model.state_dict(), out)
|
||
return out
|
||
|
||
|
||
def fetch_hf(repo: str, filename: str, out: Path) -> Path:
|
||
from huggingface_hub import hf_hub_download
|
||
p = hf_hub_download(repo_id=repo, filename=filename)
|
||
shutil.copy2(p, out)
|
||
return out
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--model", required=True)
|
||
ap.add_argument("--uri", required=True)
|
||
ap.add_argument("--out-dir", required=True)
|
||
a = ap.parse_args()
|
||
out_dir = Path(a.out_dir)
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
uri = a.uri
|
||
if uri.startswith("ultralytics://"):
|
||
# 权重名自动下载(GitHub release assets);YOLO() 会把 .pt 放到 cwd
|
||
import os
|
||
wname = uri[len("ultralytics://"):]
|
||
os.chdir(out_dir)
|
||
from ultralytics import YOLO
|
||
y = YOLO(wname)
|
||
src = Path(getattr(y, "ckpt_path", None) or wname)
|
||
if not src.exists():
|
||
src = next(out_dir.glob("*.pt"))
|
||
p = out_dir / src.name
|
||
if src.resolve() != p.resolve():
|
||
shutil.copy2(src, p)
|
||
elif uri.startswith("torchvision://"):
|
||
arch, enum = uri[len("torchvision://"):].split("/", 1)
|
||
p = fetch_torchvision(arch, enum, out_dir / f"{a.model}.pth")
|
||
elif uri.startswith("hf://"):
|
||
repo, fn = uri[len("hf://"):].split("/", 2)[:2] if uri.count("/") == 3 else uri[len("hf://"):].rsplit("/", 1)
|
||
p = fetch_hf(repo, fn, out_dir / Path(fn).name)
|
||
elif uri.startswith("file://"):
|
||
src = Path(uri[len("file://"):])
|
||
p = out_dir / src.name
|
||
shutil.copy2(src, p)
|
||
elif uri.startswith(("http://", "https://")):
|
||
p = fetch_http(uri, out_dir / Path(uri.split("?")[0]).name)
|
||
else:
|
||
print(f"unknown weights uri: {uri}", file=sys.stderr)
|
||
return 2
|
||
print(p)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|