"""《视觉ModelZoo_10类_不含VLM_338个候选模型.xlsx》→ registry/models.csv。 ~/toolchains/bench-venv/bin/python scripts/excel_to_models.py registry/models_源清单.xlsx [--dry-run] sheet「模型清单」列:ID | 任务分类 | 模型/有效变体 | 首选评测集 | 补充评测集 | 指标 | 官方仓库/模型库 | 权重下载位置 | 下载说明 | 端侧优先级 | 数据集入口 每行给一个 status,决定它现在能不能自动跑: ready harness 现在就能端到端跑:torchvision 分类(权重 API 自动下)、ultralytics 检测(权重名自动下) needs_eval 能取权重、能导出,但评测器还没写(ultralytics seg / pose;P4) needs_adapter 需要模型仓库专用的取权重/导出适配器(mmdet / mmseg / mmpose / 各家 GitHub 仓库;P4–P5) skip 不是神经网络(SIFT / ORB / AKAZE),不量化 matrix.yaml 默认只展开 status=ready。其余留在 registry 里,bench status 会报"未开始:needs_adapter N"。 torchvision 的 builder 名与默认权重的预处理尺寸从 torchvision 本身查(所以要在 bench venv 里跑)。 """ from __future__ import annotations import argparse import csv import re import sys from collections import Counter from pathlib import Path TASK_MAP = [ # 顺序即优先级:先匹配更具体的 ("iseg", ["实例分割", "提示分割"]), ("stereo", ["双目", "视差", "立体"]), ("occ", ["占据", "占用", "occupancy"]), ("flow", ["光流"]), ("feat", ["局部特征", "特征提取", "匹配"]), ("pose", ["关键点", "姿态"]), ("mde", ["单目深度", "深度"]), ("seg", ["语义分割", "分割"]), ("det", ["目标检测", "检测"]), ("cls", ["分类"]), ] DATASET_MAP = [("imagenet1k", ["imagenet"]), ("coco2017", ["coco"]), ("ade20k", ["ade20k"]), ("cityscapes", ["cityscapes"]), ("lvis", ["lvis"]), ("kitti2015", ["kitti 2015", "kitti2015", "kitti"]), ("nyuv2", ["nyu"]), ("sintel", ["sintel"]), ("hpatches", ["hpatches"]), ("occ3d_nuscenes", ["occ3d", "nuscenes"]), ("eth3d", ["eth3d"]), ("semantickitti", ["semantickitti"])] DEFAULT_INPUT = {"cls": "1x3x224x224", "det": "1x3x640x640", "iseg": "1x3x640x640", "pose": "1x3x640x640", "seg": "1x3x512x1024", "mde": "1x3x518x518", "stereo": "1x3x384x1248", "flow": "1x3x384x1024", "feat": "1x1x480x640", "occ": "1x3x256x704"} CLASSICAL = {"sift", "orb", "akaze"} def norm(s) -> str: return re.sub(r"\s+", " ", str(s or "")).strip() def slug(s: str) -> str: s = s.lower().replace("+", "plus").replace("/", "-").replace(" ", "-").replace("_", "-") s = re.sub(r"[^a-z0-9.\-]", "", s) s = re.sub(r"-+", "-", s).strip("-.") return s.replace("-", ".") # job_id 段内不允许 '-' def map_task(v: str) -> str | None: for t, keys in TASK_MAP: if any(k in v for k in keys): return t return None def map_dataset(v: str, task: str) -> str: low = v.lower() for ds, keys in DATASET_MAP: if any(k in low for k in keys): return ds return {"cls": "imagenet1k", "det": "coco2017", "iseg": "coco2017", "pose": "coco2017", "seg": "ade20k", "mde": "nyuv2", "stereo": "kitti2015", "flow": "sintel", "feat": "hpatches", "occ": "occ3d_nuscenes"}[task] def map_priority(v: str) -> str: return v.strip()[:1].upper() if v.strip()[:1].upper() in "ABC" else "C" def source_of(repo: str) -> tuple[str, str]: """→ (source, family)""" r = repo.lower() if "pytorch.org/vision" in r or "torchvision" in r: return "torchvision", "torchvision" if "ultralytics" in r: return "ultralytics", "ultralytics" for fam in ("mmsegmentation", "mmdetection", "mmpose", "mmdepth", "mmflow"): if fam in r: return "openmmlab", fam if "opencv" in r: return "opencv", "opencv" m = re.search(r"github\.com/([^/]+/[^/]+)", r) if m: return "custom", m.group(1) m = re.search(r"huggingface\.co/([^/]+/[^/]+)", r) if m: return "hf", m.group(1) return "custom", r[:40] # ---------------------------------------------------------------- torchvision 解析 _TV_MODELS: list[str] | None = None def tv_models() -> list[str]: global _TV_MODELS if _TV_MODELS is None: try: import torchvision.models as tvm _TV_MODELS = sorted(tvm.list_models()) except Exception as e: # noqa: BLE001 print(f"WARN torchvision 不可用({e}),builder 名靠规则猜", file=sys.stderr) _TV_MODELS = [] return _TV_MODELS def tv_builder(display: str) -> str | None: """'ResNeXt-50-32x4d' → 'resnext50_32x4d';'ViT-B/16' → 'vit_b_16';'ShuffleNetV2-x1.0' → 'shufflenet_v2_x1_0'""" key = re.sub(r"[^a-z0-9]", "", display.lower()) cands = {re.sub(r"[^a-z0-9]", "", m): m for m in tv_models()} if key in cands: return cands[key] # 常见写法差异 alt = key.replace("mobilenetv2", "mobilenetv2").replace("wideresnet", "wideresnet") for k, m in cands.items(): if k == alt: return m # 'regnety400mf' → 'regnet_y_400mf';'maxvitt' → 'maxvit_t' for k, m in cands.items(): if k.replace("_", "") == key: return m return None def tv_meta(builder: str) -> dict: """DEFAULT 权重的 resize / crop / 参数量。""" try: import torchvision.models as tvm w = tvm.get_model_weights(builder).DEFAULT t = w.transforms() crop = getattr(t, "crop_size", [224]) resize = getattr(t, "resize_size", [256]) crop = crop[0] if isinstance(crop, (list, tuple)) else crop resize = resize[0] if isinstance(resize, (list, tuple)) else resize return {"weights": f"torchvision://{builder}/{w.name}", "crop": int(crop), "resize": int(resize), "params_m": round(w.meta.get("num_params", 0) / 1e6, 2) if w.meta.get("num_params") else "", "official": w.meta.get("_metrics", {}).get("ImageNet-1K", {}).get("acc@1", "")} except Exception as e: # noqa: BLE001 return {"weights": f"torchvision://{builder}/DEFAULT", "crop": 224, "resize": 256, "params_m": "", "official": "", "note": f"tv_meta failed: {e}"} # ---------------------------------------------------------------- 主流程 def convert(xlsx: Path) -> tuple[list[dict], list[str]]: import openpyxl wb = openpyxl.load_workbook(xlsx, read_only=True, data_only=True) ws = wb["模型清单"] if "模型清单" in wb.sheetnames else wb.worksheets[0] rows = list(ws.iter_rows(values_only=True)) hdr = [norm(c) for c in rows[0]] col = {name: hdr.index(name) for name in hdr} need = ["任务分类", "模型/有效变体", "首选评测集", "官方仓库/模型库", "权重下载位置", "下载说明", "端侧优先级"] for n in need: if n not in col: raise SystemExit(f"Excel 缺列 {n!r},现有:{hdr}") out, warns, seen = [], [], Counter() for r in rows[1:]: if not any(c not in (None, "") for c in r): continue g = lambda name: norm(r[col[name]]) name = g("模型/有效变体") task = map_task(g("任务分类")) if not task: warns.append(f"{name}: 任务 {g('任务分类')!r} 无法映射"); continue source, family = source_of(g("官方仓库/模型库") + " " + g("权重下载位置")) mid = slug(name) seen[mid] += 1 if seen[mid] > 1: mid = f"{mid}.{seen[mid]}" rec = {"model_id": mid, "name": name, "task": task, "priority": map_priority(g("端侧优先级")), "family": family, "source": source, "repo": g("官方仓库/模型库"), "weights": g("权重下载位置"), "input_spec": DEFAULT_INPUT[task], "resize": "", "dataset": map_dataset(g("首选评测集"), task), "params_m": "", "official_metric": "", "status": "needs_adapter", "notes": g("下载说明"), "excel_id": norm(r[col["ID"]]) if "ID" in col else ""} low = name.lower() if source == "torchvision": b = tv_builder(name) if not b: warns.append(f"{name}: torchvision 里找不到 builder"); rec["status"] = "needs_adapter" else: m = tv_meta(b) rec.update({"model_id": b.replace("_", "."), "weights": m["weights"], "params_m": m["params_m"], "official_metric": m["official"], "resize": m["resize"]}) if task == "cls": rec["input_spec"] = f"1x3x{m['crop']}x{m['crop']}" rec["status"] = "ready" else: rec["status"] = "needs_adapter" # RAFT/SpyNet 双输入 if m.get("note"): warns.append(f"{name}: {m['note']}") elif source == "ultralytics": wname = low.replace("yolo11", "yolo11").replace(" ", "") + ".pt" # yolov8n / yolo11n-seg / fastsam-s if wname.startswith("fastsam"): wname = "FastSAM-" + wname.split("-", 1)[1] rec["weights"] = f"ultralytics://{wname}" rec["status"] = "ready" if task == "det" else "needs_eval" elif source == "opencv" or any(k == re.sub(r"[^a-z]", "", low) for k in CLASSICAL): rec["status"] = "skip"; rec["notes"] = "经典算子,非神经网络,不量化" out.append(rec) return out, warns def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("xlsx") ap.add_argument("--out", default=str(Path(__file__).resolve().parent.parent / "registry" / "models.csv")) ap.add_argument("--dry-run", action="store_true") a = ap.parse_args() rows, warns = convert(Path(a.xlsx)) for w in warns: print("WARN", w, file=sys.stderr) print("rows:", len(rows), file=sys.stderr) print("task:", dict(Counter(r["task"] for r in rows)), file=sys.stderr) print("priority:", dict(Counter(r["priority"] for r in rows)), file=sys.stderr) print("status:", dict(Counter(r["status"] for r in rows)), file=sys.stderr) print("family:", dict(Counter(r["family"] for r in rows).most_common(12)), file=sys.stderr) ready = [r for r in rows if r["status"] == "ready"] print(f"ready ({len(ready)}):", ", ".join(r["model_id"] for r in ready), file=sys.stderr) dup = [k for k, v in Counter(r["model_id"] for r in rows).items() if v > 1] if dup: raise SystemExit(f"model_id 重复:{dup}") if a.dry_run: return 0 fields = list(rows[0].keys()) with open(a.out, "w", newline="", encoding="utf-8") as f: w = csv.DictWriter(f, fieldnames=fields); w.writeheader(); w.writerows(rows) print(f"wrote {len(rows)} → {a.out}", file=sys.stderr) return 0 if __name__ == "__main__": sys.exit(main())