data-gen-bpu/harness/tools/fetch_weights.py

86 lines
2.9 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""01-fetch 子进程:把 models.csv 的 weights URI 落成一个文件。
支持:
https://… 直接下载(带断点续传)
torchvision://<arch>/<WEIGHTS_ENUM> 例 torchvision://resnet18/IMAGENET1K_V1 → 保存 state_dict .pth
hf://<repo>/<file> huggingface hub
file:///abs/path 本机已有
"""
from __future__ import annotations
import argparse
import shutil
import sys
import urllib.request
from pathlib import Path
def fetch_http(url: str, out: Path) -> Path:
tmp = out.with_suffix(out.suffix + ".part")
pos = tmp.stat().st_size if tmp.exists() else 0
req = urllib.request.Request(url, headers={"Range": f"bytes={pos}-"} if pos else {})
with urllib.request.urlopen(req, timeout=60) as r, open(tmp, "ab") as f:
shutil.copyfileobj(r, f, length=1 << 20)
tmp.rename(out)
return out
def fetch_torchvision(arch: str, enum: str, out: Path) -> Path:
import torch, torchvision.models as tvm
w = getattr(tvm.get_model_weights(arch), enum) # enum 可以是 DEFAULT
model = tvm.get_model(arch, weights=w)
torch.save(model.state_dict(), out)
return out
def fetch_hf(repo: str, filename: str, out: Path) -> Path:
from huggingface_hub import hf_hub_download
p = hf_hub_download(repo_id=repo, filename=filename)
shutil.copy2(p, out)
return out
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--model", required=True)
ap.add_argument("--uri", required=True)
ap.add_argument("--out-dir", required=True)
a = ap.parse_args()
out_dir = Path(a.out_dir)
out_dir.mkdir(parents=True, exist_ok=True)
uri = a.uri
if uri.startswith("ultralytics://"):
# 权重名自动下载(GitHub release assets);YOLO() 会把 .pt 放到 cwd
import os
wname = uri[len("ultralytics://"):]
os.chdir(out_dir)
from ultralytics import YOLO
y = YOLO(wname)
src = Path(getattr(y, "ckpt_path", None) or wname)
if not src.exists():
src = next(out_dir.glob("*.pt"))
p = out_dir / src.name
if src.resolve() != p.resolve():
shutil.copy2(src, p)
elif uri.startswith("torchvision://"):
arch, enum = uri[len("torchvision://"):].split("/", 1)
p = fetch_torchvision(arch, enum, out_dir / f"{a.model}.pth")
elif uri.startswith("hf://"):
repo, fn = uri[len("hf://"):].split("/", 2)[:2] if uri.count("/") == 3 else uri[len("hf://"):].rsplit("/", 1)
p = fetch_hf(repo, fn, out_dir / Path(fn).name)
elif uri.startswith("file://"):
src = Path(uri[len("file://"):])
p = out_dir / src.name
shutil.copy2(src, p)
elif uri.startswith(("http://", "https://")):
p = fetch_http(uri, out_dir / Path(uri.split("?")[0]).name)
else:
print(f"unknown weights uri: {uri}", file=sys.stderr)
return 2
print(p)
return 0
if __name__ == "__main__":
sys.exit(main())