data-gen-bpu/scripts/excel_to_models.py

241 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""《视觉ModelZoo_10类_不含VLM_338个候选模型.xlsx》→ registry/models.csv。
~/toolchains/bench-venv/bin/python scripts/excel_to_models.py registry/models_源清单.xlsx [--dry-run]
sheet「模型清单」列:ID | 任务分类 | 模型/有效变体 | 首选评测集 | 补充评测集 | 指标 | 官方仓库/模型库 | 权重下载位置 | 下载说明 | 端侧优先级 | 数据集入口
每行给一个 status,决定它现在能不能自动跑:
ready harness 现在就能端到端跑:torchvision 分类(权重 API 自动下)、ultralytics 检测(权重名自动下)
needs_eval 能取权重、能导出,但评测器还没写(ultralytics seg / pose;P4)
needs_adapter 需要模型仓库专用的取权重/导出适配器(mmdet / mmseg / mmpose / 各家 GitHub 仓库;P4–P5)
skip 不是神经网络(SIFT / ORB / AKAZE),不量化
matrix.yaml 默认只展开 status=ready。其余留在 registry 里,bench status 会报"未开始:needs_adapter N"。
torchvision 的 builder 名与默认权重的预处理尺寸从 torchvision 本身查(所以要在 bench venv 里跑)。
"""
from __future__ import annotations
import argparse
import csv
import re
import sys
from collections import Counter
from pathlib import Path
TASK_MAP = [ # 顺序即优先级:先匹配更具体的
("iseg", ["实例分割", "提示分割"]),
("stereo", ["双目", "视差", "立体"]),
("occ", ["占据", "占用", "occupancy"]),
("flow", ["光流"]),
("feat", ["局部特征", "特征提取", "匹配"]),
("pose", ["关键点", "姿态"]),
("mde", ["单目深度", "深度"]),
("seg", ["语义分割", "分割"]),
("det", ["目标检测", "检测"]),
("cls", ["分类"]),
]
DATASET_MAP = [("imagenet1k", ["imagenet"]), ("coco2017", ["coco"]), ("ade20k", ["ade20k"]), ("cityscapes", ["cityscapes"]),
("lvis", ["lvis"]), ("kitti2015", ["kitti 2015", "kitti2015", "kitti"]), ("nyuv2", ["nyu"]), ("sintel", ["sintel"]),
("hpatches", ["hpatches"]), ("occ3d_nuscenes", ["occ3d", "nuscenes"]), ("eth3d", ["eth3d"]), ("semantickitti", ["semantickitti"])]
DEFAULT_INPUT = {"cls": "1x3x224x224", "det": "1x3x640x640", "iseg": "1x3x640x640", "pose": "1x3x640x640",
"seg": "1x3x512x1024", "mde": "1x3x518x518", "stereo": "1x3x384x1248", "flow": "1x3x384x1024",
"feat": "1x1x480x640", "occ": "1x3x256x704"}
CLASSICAL = {"sift", "orb", "akaze"}
def norm(s) -> str:
return re.sub(r"\s+", " ", str(s or "")).strip()
def slug(s: str) -> str:
s = s.lower().replace("+", "plus").replace("/", "-").replace(" ", "-").replace("_", "-")
s = re.sub(r"[^a-z0-9.\-]", "", s)
s = re.sub(r"-+", "-", s).strip("-.")
return s.replace("-", ".") # job_id 段内不允许 '-'
def map_task(v: str) -> str | None:
for t, keys in TASK_MAP:
if any(k in v for k in keys):
return t
return None
def map_dataset(v: str, task: str) -> str:
low = v.lower()
for ds, keys in DATASET_MAP:
if any(k in low for k in keys):
return ds
return {"cls": "imagenet1k", "det": "coco2017", "iseg": "coco2017", "pose": "coco2017", "seg": "ade20k",
"mde": "nyuv2", "stereo": "kitti2015", "flow": "sintel", "feat": "hpatches", "occ": "occ3d_nuscenes"}[task]
def map_priority(v: str) -> str:
return v.strip()[:1].upper() if v.strip()[:1].upper() in "ABC" else "C"
def source_of(repo: str) -> tuple[str, str]:
"""→ (source, family)"""
r = repo.lower()
if "pytorch.org/vision" in r or "torchvision" in r:
return "torchvision", "torchvision"
if "ultralytics" in r:
return "ultralytics", "ultralytics"
for fam in ("mmsegmentation", "mmdetection", "mmpose", "mmdepth", "mmflow"):
if fam in r:
return "openmmlab", fam
if "opencv" in r:
return "opencv", "opencv"
m = re.search(r"github\.com/([^/]+/[^/]+)", r)
if m:
return "custom", m.group(1)
m = re.search(r"huggingface\.co/([^/]+/[^/]+)", r)
if m:
return "hf", m.group(1)
return "custom", r[:40]
# ---------------------------------------------------------------- torchvision 解析
_TV_MODELS: list[str] | None = None
def tv_models() -> list[str]:
global _TV_MODELS
if _TV_MODELS is None:
try:
import torchvision.models as tvm
_TV_MODELS = sorted(tvm.list_models())
except Exception as e: # noqa: BLE001
print(f"WARN torchvision 不可用({e}),builder 名靠规则猜", file=sys.stderr)
_TV_MODELS = []
return _TV_MODELS
def tv_builder(display: str) -> str | None:
"""'ResNeXt-50-32x4d' → 'resnext50_32x4d';'ViT-B/16' → 'vit_b_16';'ShuffleNetV2-x1.0' → 'shufflenet_v2_x1_0'"""
key = re.sub(r"[^a-z0-9]", "", display.lower())
cands = {re.sub(r"[^a-z0-9]", "", m): m for m in tv_models()}
if key in cands:
return cands[key]
# 常见写法差异
alt = key.replace("mobilenetv2", "mobilenetv2").replace("wideresnet", "wideresnet")
for k, m in cands.items():
if k == alt:
return m
# 'regnety400mf' → 'regnet_y_400mf';'maxvitt' → 'maxvit_t'
for k, m in cands.items():
if k.replace("_", "") == key:
return m
return None
def tv_meta(builder: str) -> dict:
"""DEFAULT 权重的 resize / crop / 参数量。"""
try:
import torchvision.models as tvm
w = tvm.get_model_weights(builder).DEFAULT
t = w.transforms()
crop = getattr(t, "crop_size", [224])
resize = getattr(t, "resize_size", [256])
crop = crop[0] if isinstance(crop, (list, tuple)) else crop
resize = resize[0] if isinstance(resize, (list, tuple)) else resize
return {"weights": f"torchvision://{builder}/{w.name}", "crop": int(crop), "resize": int(resize),
"params_m": round(w.meta.get("num_params", 0) / 1e6, 2) if w.meta.get("num_params") else "",
"official": w.meta.get("_metrics", {}).get("ImageNet-1K", {}).get("acc@1", "")}
except Exception as e: # noqa: BLE001
return {"weights": f"torchvision://{builder}/DEFAULT", "crop": 224, "resize": 256, "params_m": "", "official": "",
"note": f"tv_meta failed: {e}"}
# ---------------------------------------------------------------- 主流程
def convert(xlsx: Path) -> tuple[list[dict], list[str]]:
import openpyxl
wb = openpyxl.load_workbook(xlsx, read_only=True, data_only=True)
ws = wb["模型清单"] if "模型清单" in wb.sheetnames else wb.worksheets[0]
rows = list(ws.iter_rows(values_only=True))
hdr = [norm(c) for c in rows[0]]
col = {name: hdr.index(name) for name in hdr}
need = ["任务分类", "模型/有效变体", "首选评测集", "官方仓库/模型库", "权重下载位置", "下载说明", "端侧优先级"]
for n in need:
if n not in col:
raise SystemExit(f"Excel 缺列 {n!r},现有:{hdr}")
out, warns, seen = [], [], Counter()
for r in rows[1:]:
if not any(c not in (None, "") for c in r):
continue
g = lambda name: norm(r[col[name]])
name = g("模型/有效变体")
task = map_task(g("任务分类"))
if not task:
warns.append(f"{name}: 任务 {g('任务分类')!r} 无法映射"); continue
source, family = source_of(g("官方仓库/模型库") + " " + g("权重下载位置"))
mid = slug(name)
seen[mid] += 1
if seen[mid] > 1:
mid = f"{mid}.{seen[mid]}"
rec = {"model_id": mid, "name": name, "task": task, "priority": map_priority(g("端侧优先级")),
"family": family, "source": source, "repo": g("官方仓库/模型库"), "weights": g("权重下载位置"),
"input_spec": DEFAULT_INPUT[task], "resize": "", "dataset": map_dataset(g("首选评测集"), task),
"params_m": "", "official_metric": "", "status": "needs_adapter",
"notes": g("下载说明"), "excel_id": norm(r[col["ID"]]) if "ID" in col else ""}
low = name.lower()
if source == "torchvision":
b = tv_builder(name)
if not b:
warns.append(f"{name}: torchvision 里找不到 builder"); rec["status"] = "needs_adapter"
else:
m = tv_meta(b)
rec.update({"model_id": b.replace("_", "."), "weights": m["weights"], "params_m": m["params_m"],
"official_metric": m["official"], "resize": m["resize"]})
if task == "cls":
rec["input_spec"] = f"1x3x{m['crop']}x{m['crop']}"
rec["status"] = "ready"
else:
rec["status"] = "needs_adapter" # RAFT/SpyNet 双输入
if m.get("note"):
warns.append(f"{name}: {m['note']}")
elif source == "ultralytics":
wname = low.replace("yolo11", "yolo11").replace(" ", "") + ".pt" # yolov8n / yolo11n-seg / fastsam-s
if wname.startswith("fastsam"):
wname = "FastSAM-" + wname.split("-", 1)[1]
rec["weights"] = f"ultralytics://{wname}"
rec["status"] = "ready" if task == "det" else "needs_eval"
elif source == "opencv" or any(k == re.sub(r"[^a-z]", "", low) for k in CLASSICAL):
rec["status"] = "skip"; rec["notes"] = "经典算子,非神经网络,不量化"
out.append(rec)
return out, warns
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("xlsx")
ap.add_argument("--out", default=str(Path(__file__).resolve().parent.parent / "registry" / "models.csv"))
ap.add_argument("--dry-run", action="store_true")
a = ap.parse_args()
rows, warns = convert(Path(a.xlsx))
for w in warns:
print("WARN", w, file=sys.stderr)
print("rows:", len(rows), file=sys.stderr)
print("task:", dict(Counter(r["task"] for r in rows)), file=sys.stderr)
print("priority:", dict(Counter(r["priority"] for r in rows)), file=sys.stderr)
print("status:", dict(Counter(r["status"] for r in rows)), file=sys.stderr)
print("family:", dict(Counter(r["family"] for r in rows).most_common(12)), file=sys.stderr)
ready = [r for r in rows if r["status"] == "ready"]
print(f"ready ({len(ready)}):", ", ".join(r["model_id"] for r in ready), file=sys.stderr)
dup = [k for k, v in Counter(r["model_id"] for r in rows).items() if v > 1]
if dup:
raise SystemExit(f"model_id 重复:{dup}")
if a.dry_run:
return 0
fields = list(rows[0].keys())
with open(a.out, "w", newline="", encoding="utf-8") as f:
w = csv.DictWriter(f, fieldnames=fields); w.writeheader(); w.writerows(rows)
print(f"wrote {len(rows)} → {a.out}", file=sys.stderr)
return 0
if __name__ == "__main__":
sys.exit(main())