241 lines
11 KiB
Python
241 lines
11 KiB
Python
"""《视觉ModelZoo_10类_不含VLM_338个候选模型.xlsx》→ registry/models.csv。
|
||
|
||
~/toolchains/bench-venv/bin/python scripts/excel_to_models.py registry/models_源清单.xlsx [--dry-run]
|
||
|
||
sheet「模型清单」列:ID | 任务分类 | 模型/有效变体 | 首选评测集 | 补充评测集 | 指标 | 官方仓库/模型库 | 权重下载位置 | 下载说明 | 端侧优先级 | 数据集入口
|
||
|
||
每行给一个 status,决定它现在能不能自动跑:
|
||
ready harness 现在就能端到端跑:torchvision 分类(权重 API 自动下)、ultralytics 检测(权重名自动下)
|
||
needs_eval 能取权重、能导出,但评测器还没写(ultralytics seg / pose;P4)
|
||
needs_adapter 需要模型仓库专用的取权重/导出适配器(mmdet / mmseg / mmpose / 各家 GitHub 仓库;P4–P5)
|
||
skip 不是神经网络(SIFT / ORB / AKAZE),不量化
|
||
matrix.yaml 默认只展开 status=ready。其余留在 registry 里,bench status 会报"未开始:needs_adapter N"。
|
||
|
||
torchvision 的 builder 名与默认权重的预处理尺寸从 torchvision 本身查(所以要在 bench venv 里跑)。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import re
|
||
import sys
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
|
||
TASK_MAP = [ # 顺序即优先级:先匹配更具体的
|
||
("iseg", ["实例分割", "提示分割"]),
|
||
("stereo", ["双目", "视差", "立体"]),
|
||
("occ", ["占据", "占用", "occupancy"]),
|
||
("flow", ["光流"]),
|
||
("feat", ["局部特征", "特征提取", "匹配"]),
|
||
("pose", ["关键点", "姿态"]),
|
||
("mde", ["单目深度", "深度"]),
|
||
("seg", ["语义分割", "分割"]),
|
||
("det", ["目标检测", "检测"]),
|
||
("cls", ["分类"]),
|
||
]
|
||
DATASET_MAP = [("imagenet1k", ["imagenet"]), ("coco2017", ["coco"]), ("ade20k", ["ade20k"]), ("cityscapes", ["cityscapes"]),
|
||
("lvis", ["lvis"]), ("kitti2015", ["kitti 2015", "kitti2015", "kitti"]), ("nyuv2", ["nyu"]), ("sintel", ["sintel"]),
|
||
("hpatches", ["hpatches"]), ("occ3d_nuscenes", ["occ3d", "nuscenes"]), ("eth3d", ["eth3d"]), ("semantickitti", ["semantickitti"])]
|
||
DEFAULT_INPUT = {"cls": "1x3x224x224", "det": "1x3x640x640", "iseg": "1x3x640x640", "pose": "1x3x640x640",
|
||
"seg": "1x3x512x1024", "mde": "1x3x518x518", "stereo": "1x3x384x1248", "flow": "1x3x384x1024",
|
||
"feat": "1x1x480x640", "occ": "1x3x256x704"}
|
||
CLASSICAL = {"sift", "orb", "akaze"}
|
||
|
||
|
||
def norm(s) -> str:
|
||
return re.sub(r"\s+", " ", str(s or "")).strip()
|
||
|
||
|
||
def slug(s: str) -> str:
|
||
s = s.lower().replace("+", "plus").replace("/", "-").replace(" ", "-").replace("_", "-")
|
||
s = re.sub(r"[^a-z0-9.\-]", "", s)
|
||
s = re.sub(r"-+", "-", s).strip("-.")
|
||
return s.replace("-", ".") # job_id 段内不允许 '-'
|
||
|
||
|
||
def map_task(v: str) -> str | None:
|
||
for t, keys in TASK_MAP:
|
||
if any(k in v for k in keys):
|
||
return t
|
||
return None
|
||
|
||
|
||
def map_dataset(v: str, task: str) -> str:
|
||
low = v.lower()
|
||
for ds, keys in DATASET_MAP:
|
||
if any(k in low for k in keys):
|
||
return ds
|
||
return {"cls": "imagenet1k", "det": "coco2017", "iseg": "coco2017", "pose": "coco2017", "seg": "ade20k",
|
||
"mde": "nyuv2", "stereo": "kitti2015", "flow": "sintel", "feat": "hpatches", "occ": "occ3d_nuscenes"}[task]
|
||
|
||
|
||
def map_priority(v: str) -> str:
|
||
return v.strip()[:1].upper() if v.strip()[:1].upper() in "ABC" else "C"
|
||
|
||
|
||
def source_of(repo: str) -> tuple[str, str]:
|
||
"""→ (source, family)"""
|
||
r = repo.lower()
|
||
if "pytorch.org/vision" in r or "torchvision" in r:
|
||
return "torchvision", "torchvision"
|
||
if "ultralytics" in r:
|
||
return "ultralytics", "ultralytics"
|
||
for fam in ("mmsegmentation", "mmdetection", "mmpose", "mmdepth", "mmflow"):
|
||
if fam in r:
|
||
return "openmmlab", fam
|
||
if "opencv" in r:
|
||
return "opencv", "opencv"
|
||
m = re.search(r"github\.com/([^/]+/[^/]+)", r)
|
||
if m:
|
||
return "custom", m.group(1)
|
||
m = re.search(r"huggingface\.co/([^/]+/[^/]+)", r)
|
||
if m:
|
||
return "hf", m.group(1)
|
||
return "custom", r[:40]
|
||
|
||
|
||
# ---------------------------------------------------------------- torchvision 解析
|
||
|
||
_TV_MODELS: list[str] | None = None
|
||
|
||
|
||
def tv_models() -> list[str]:
|
||
global _TV_MODELS
|
||
if _TV_MODELS is None:
|
||
try:
|
||
import torchvision.models as tvm
|
||
_TV_MODELS = sorted(tvm.list_models())
|
||
except Exception as e: # noqa: BLE001
|
||
print(f"WARN torchvision 不可用({e}),builder 名靠规则猜", file=sys.stderr)
|
||
_TV_MODELS = []
|
||
return _TV_MODELS
|
||
|
||
|
||
def tv_builder(display: str) -> str | None:
|
||
"""'ResNeXt-50-32x4d' → 'resnext50_32x4d';'ViT-B/16' → 'vit_b_16';'ShuffleNetV2-x1.0' → 'shufflenet_v2_x1_0'"""
|
||
key = re.sub(r"[^a-z0-9]", "", display.lower())
|
||
cands = {re.sub(r"[^a-z0-9]", "", m): m for m in tv_models()}
|
||
if key in cands:
|
||
return cands[key]
|
||
# 常见写法差异
|
||
alt = key.replace("mobilenetv2", "mobilenetv2").replace("wideresnet", "wideresnet")
|
||
for k, m in cands.items():
|
||
if k == alt:
|
||
return m
|
||
# 'regnety400mf' → 'regnet_y_400mf';'maxvitt' → 'maxvit_t'
|
||
for k, m in cands.items():
|
||
if k.replace("_", "") == key:
|
||
return m
|
||
return None
|
||
|
||
|
||
def tv_meta(builder: str) -> dict:
|
||
"""DEFAULT 权重的 resize / crop / 参数量。"""
|
||
try:
|
||
import torchvision.models as tvm
|
||
w = tvm.get_model_weights(builder).DEFAULT
|
||
t = w.transforms()
|
||
crop = getattr(t, "crop_size", [224])
|
||
resize = getattr(t, "resize_size", [256])
|
||
crop = crop[0] if isinstance(crop, (list, tuple)) else crop
|
||
resize = resize[0] if isinstance(resize, (list, tuple)) else resize
|
||
return {"weights": f"torchvision://{builder}/{w.name}", "crop": int(crop), "resize": int(resize),
|
||
"params_m": round(w.meta.get("num_params", 0) / 1e6, 2) if w.meta.get("num_params") else "",
|
||
"official": w.meta.get("_metrics", {}).get("ImageNet-1K", {}).get("acc@1", "")}
|
||
except Exception as e: # noqa: BLE001
|
||
return {"weights": f"torchvision://{builder}/DEFAULT", "crop": 224, "resize": 256, "params_m": "", "official": "",
|
||
"note": f"tv_meta failed: {e}"}
|
||
|
||
|
||
# ---------------------------------------------------------------- 主流程
|
||
|
||
def convert(xlsx: Path) -> tuple[list[dict], list[str]]:
|
||
import openpyxl
|
||
wb = openpyxl.load_workbook(xlsx, read_only=True, data_only=True)
|
||
ws = wb["模型清单"] if "模型清单" in wb.sheetnames else wb.worksheets[0]
|
||
rows = list(ws.iter_rows(values_only=True))
|
||
hdr = [norm(c) for c in rows[0]]
|
||
col = {name: hdr.index(name) for name in hdr}
|
||
need = ["任务分类", "模型/有效变体", "首选评测集", "官方仓库/模型库", "权重下载位置", "下载说明", "端侧优先级"]
|
||
for n in need:
|
||
if n not in col:
|
||
raise SystemExit(f"Excel 缺列 {n!r},现有:{hdr}")
|
||
out, warns, seen = [], [], Counter()
|
||
for r in rows[1:]:
|
||
if not any(c not in (None, "") for c in r):
|
||
continue
|
||
g = lambda name: norm(r[col[name]])
|
||
name = g("模型/有效变体")
|
||
task = map_task(g("任务分类"))
|
||
if not task:
|
||
warns.append(f"{name}: 任务 {g('任务分类')!r} 无法映射"); continue
|
||
source, family = source_of(g("官方仓库/模型库") + " " + g("权重下载位置"))
|
||
mid = slug(name)
|
||
seen[mid] += 1
|
||
if seen[mid] > 1:
|
||
mid = f"{mid}.{seen[mid]}"
|
||
rec = {"model_id": mid, "name": name, "task": task, "priority": map_priority(g("端侧优先级")),
|
||
"family": family, "source": source, "repo": g("官方仓库/模型库"), "weights": g("权重下载位置"),
|
||
"input_spec": DEFAULT_INPUT[task], "resize": "", "dataset": map_dataset(g("首选评测集"), task),
|
||
"params_m": "", "official_metric": "", "status": "needs_adapter",
|
||
"notes": g("下载说明"), "excel_id": norm(r[col["ID"]]) if "ID" in col else ""}
|
||
low = name.lower()
|
||
if source == "torchvision":
|
||
b = tv_builder(name)
|
||
if not b:
|
||
warns.append(f"{name}: torchvision 里找不到 builder"); rec["status"] = "needs_adapter"
|
||
else:
|
||
m = tv_meta(b)
|
||
rec.update({"model_id": b.replace("_", "."), "weights": m["weights"], "params_m": m["params_m"],
|
||
"official_metric": m["official"], "resize": m["resize"]})
|
||
if task == "cls":
|
||
rec["input_spec"] = f"1x3x{m['crop']}x{m['crop']}"
|
||
rec["status"] = "ready"
|
||
else:
|
||
rec["status"] = "needs_adapter" # RAFT/SpyNet 双输入
|
||
if m.get("note"):
|
||
warns.append(f"{name}: {m['note']}")
|
||
elif source == "ultralytics":
|
||
wname = low.replace("yolo11", "yolo11").replace(" ", "") + ".pt" # yolov8n / yolo11n-seg / fastsam-s
|
||
if wname.startswith("fastsam"):
|
||
wname = "FastSAM-" + wname.split("-", 1)[1]
|
||
rec["weights"] = f"ultralytics://{wname}"
|
||
rec["status"] = "ready" if task == "det" else "needs_eval"
|
||
elif source == "opencv" or any(k == re.sub(r"[^a-z]", "", low) for k in CLASSICAL):
|
||
rec["status"] = "skip"; rec["notes"] = "经典算子,非神经网络,不量化"
|
||
out.append(rec)
|
||
return out, warns
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("xlsx")
|
||
ap.add_argument("--out", default=str(Path(__file__).resolve().parent.parent / "registry" / "models.csv"))
|
||
ap.add_argument("--dry-run", action="store_true")
|
||
a = ap.parse_args()
|
||
rows, warns = convert(Path(a.xlsx))
|
||
for w in warns:
|
||
print("WARN", w, file=sys.stderr)
|
||
print("rows:", len(rows), file=sys.stderr)
|
||
print("task:", dict(Counter(r["task"] for r in rows)), file=sys.stderr)
|
||
print("priority:", dict(Counter(r["priority"] for r in rows)), file=sys.stderr)
|
||
print("status:", dict(Counter(r["status"] for r in rows)), file=sys.stderr)
|
||
print("family:", dict(Counter(r["family"] for r in rows).most_common(12)), file=sys.stderr)
|
||
ready = [r for r in rows if r["status"] == "ready"]
|
||
print(f"ready ({len(ready)}):", ", ".join(r["model_id"] for r in ready), file=sys.stderr)
|
||
dup = [k for k, v in Counter(r["model_id"] for r in rows).items() if v > 1]
|
||
if dup:
|
||
raise SystemExit(f"model_id 重复:{dup}")
|
||
if a.dry_run:
|
||
return 0
|
||
fields = list(rows[0].keys())
|
||
with open(a.out, "w", newline="", encoding="utf-8") as f:
|
||
w = csv.DictWriter(f, fieldnames=fields); w.writeheader(); w.writerows(rows)
|
||
print(f"wrote {len(rows)} → {a.out}", file=sys.stderr)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|