data-gen-bpu/scripts/make_calib.py

189 lines
9.2 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""T0 校准集:data/calib/<task>/<input_spec>/NNNNN.npy + manifest.json。同规格模型共用。
python scripts/make_calib.py --task cls --input-spec 1x3x224x224 [--n 128] [--from auto|train|val-holdout]
来源优先级(auto):
1. datasets.yaml calib.train_parquet(HF train-*.parquet,固定 seed 选片、选行)
2. datasets.yaml calib.train_dir(本机 train 图片目录)
3. pool 里带 "train" 的图片
4. val-holdout:从 val 里抽 *不在 T1 里* 的图 —— 与 T2 全量有交集,manifest 记 source,出交付数前必须换 train
预处理与评测器完全同一份代码(harness/eval/common.py),layout NCHW,含 batch 维。
"""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import random
import sys
import tarfile
from pathlib import Path
import yaml
BENCH = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(BENCH))
REG = yaml.safe_load((BENCH / "registry" / "datasets.yaml").read_text())
POOL = BENCH / REG.get("pool_root", "data/pool")
LOCAL = Path(REG.get("local_root", "/mnt/data"))
TASK_DATASET = {"cls": "imagenet1k", "det": "coco2017", "iseg": "coco2017", "pose": "coco2017", "seg": "cityscapes",
"mde": "nyuv2", "stereo": "kitti2015", "flow": "kitti2015", "feat": "hpatches", "occ": "nuscenes"}
EXTS = {".jpg", ".jpeg", ".png", ".JPEG", ".JPG"}
def list_images(root: Path, split_hint: str = "") -> list[Path]:
return sorted(p for p in root.rglob("*") if p.suffix in EXTS and split_hint in str(p.relative_to(root)).lower())
def t1_files(ds: str) -> set[str]:
d = REG["datasets"][ds]
tar = BENCH / "data" / "t1" / ds / f"{(d.get('t1') or {}).get('seed', 20260914)}.tar"
if not tar.exists():
return set()
with tarfile.open(tar) as t:
man = json.loads(t.extractfile("manifest.json").read())
return {it["src"] for it in man["items"]}
def from_train_parquet(cfg: dict, n: int, seed: int):
"""返回 (PIL 图列表, 描述列表)。固定 seed 选 4 片、每片均匀抽。"""
import pyarrow.parquet as pq
from PIL import Image
root = LOCAL / cfg["train_parquet"]
shards = sorted(root.glob(cfg.get("train_glob", "train-*.parquet")))
if not shards:
raise FileNotFoundError(f"no train parquet under {root}")
rng = random.Random(seed)
picked = sorted(rng.sample(shards, min(4, len(shards))))
per = -(-n // len(picked))
imgs, desc = [], []
for sh in picked:
pf = pq.ParquetFile(sh)
total = pf.metadata.num_rows
rows = sorted(rng.sample(range(total), min(per, total)))
names = pf.schema_arrow.names # 顶层列名(HF: image{bytes,path}, label)
img_col = "image" if "image" in names else names[0]
col = pf.read(columns=[img_col]).column(0)
for r in rows:
v = col[r].as_py()
data = v.get("bytes") if isinstance(v, dict) else v
imgs.append(Image.open(io.BytesIO(data)).convert("RGB")); desc.append(f"{sh.name}#{r}")
if len(imgs) >= n:
return imgs, desc
return imgs, desc
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--task", required=True)
ap.add_argument("--input-spec", required=True)
ap.add_argument("--model", help="Use this model's official input recipe in a separate calibration directory")
ap.add_argument("--n", type=int, default=128)
ap.add_argument("--from", dest="src", choices=["auto", "train", "val-holdout"], default="auto")
ap.add_argument("--seed", type=int, default=20260914)
a = ap.parse_args()
import numpy as np
from PIL import Image
from harness.eval.common import preprocess_cls, letterbox
from harness.adapters import calibration_dir
from harness.registry import model_row
row = model_row(a.model) if a.model else {"task": a.task, "input_spec": a.input_spec, "model_id": ""}
if a.model and (row["task"] != a.task or row["input_spec"] != a.input_spec):
raise SystemExit("--task/--input-spec must match the model recipe")
ds = row.get("dataset") or TASK_DATASET[a.task]
d = REG["datasets"][ds]
ccfg = d.get("calib") or {}
pool = POOL / ds / d.get("version", "v1")
shape = tuple(int(x) for x in a.input_spec.split("x"))
rng = random.Random(a.seed)
out = calibration_dir(row)
if a.model and out.exists():
manifest = json.loads((out / "manifest.json").read_text())
files = sorted((out / manifest["npy_dir"]).glob("*.npy"))
digest = hashlib.sha256()
for path in files:
digest.update(path.read_bytes())
expected_recipe = row.get("preprocess") or "harness.eval.common (same as evaluator)"
if (manifest.get("count") != a.n or len(files) != a.n or manifest.get("seed") != a.seed
or json.dumps(manifest.get("preprocess"), sort_keys=True) != json.dumps(expected_recipe, sort_keys=True)
or manifest.get("sha256") != digest.hexdigest()
or (a.src == "train" and not manifest.get("source", "").startswith("train"))):
raise ValueError(f"existing model calibration differs; refusing to overwrite {out}")
print(f"{out} n={a.n} sha={digest.hexdigest()[:12]} (reused)")
return 0
imgs, desc, source = [], [], None
if a.src in ("auto", "train"):
if ds == "cityscapes":
import zipfile
archive = LOCAL / d["sources"][0]["path"]
with zipfile.ZipFile(archive) as source_zip:
names = sorted(name for name in source_zip.namelist()
if name.startswith("leftImg8bit/train/") and name.endswith("_leftImg8bit.png"))
if len(names) < a.n:
raise ValueError(f"only {len(names)} Cityscapes training images")
rng.shuffle(names)
desc = [f"{archive}#{name}" for name in sorted(names[:a.n])]
imgs = [Image.open(io.BytesIO(source_zip.read(name))).convert("RGB") for name in sorted(names[:a.n])]
source = "train-zip"
if ccfg.get("train_parquet") and (LOCAL / ccfg["train_parquet"]).exists():
try:
imgs, desc = from_train_parquet(ccfg, a.n, a.seed); source = "train-parquet"
except Exception as e:
print(f"WARN train_parquet 失败:{e}", file=sys.stderr)
if not imgs and ccfg.get("train_dir") and (LOCAL / ccfg["train_dir"]).exists():
files = list_images(LOCAL / ccfg["train_dir"])
if len(files) >= a.n:
rng.shuffle(files); files = sorted(files[:a.n])
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p) for p in files]; source = "train-dir"
if not imgs:
files = list_images(pool, "train")
if len(files) >= a.n:
rng.shuffle(files); files = sorted(files[:a.n])
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p.relative_to(pool)) for p in files]; source = "train"
if not imgs:
if a.src == "train":
raise SystemExit("没有可用的 train 来源")
excl = t1_files(ds)
files = [p for p in list_images(pool) if str(p.relative_to(pool)) not in excl]
if len(files) < a.n:
raise SystemExit(f"val-holdout 可用图片 {len(files)} < {a.n}")
rng.shuffle(files); files = sorted(files[:a.n])
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p.relative_to(pool)) for p in files]; source = "val-holdout"
print(f"WARN: {ds} 没有 train 来源,用 val-holdout(排除 T1 的 {len(excl)} 张)。出 T2 交付数前须换 train。", file=sys.stderr)
npy_dir = out / "npy_f32" # hb_compile 读 cal_data_dir 下的全部文件 → manifest.json 不能放同一目录;_f32 后缀是 OE 约定
if out.exists():
import shutil; shutil.rmtree(out)
npy_dir.mkdir(parents=True, exist_ok=True)
h = hashlib.sha256()
for i, img in enumerate(imgs[:a.n]):
if a.task == "cls":
x = preprocess_cls(img, size=shape[-1], resize=round(shape[-1] * 256 / 224), model=a.model or "")
elif a.task in ("seg", "det") and a.model:
from harness.adapters import get
transform = getattr(get(a.model), "preprocess", None)
if a.task == "seg" and transform is None:
raise ValueError(f"no segmentation preprocessing recipe for {a.model}")
x = transform(img, a.input_spec) if transform else letterbox(img, shape[-1])[0]
else:
x = letterbox(img, shape[-1])[0]
assert x.shape == shape, (x.shape, shape)
fp = npy_dir / f"{i:05d}.npy"
np.save(fp, x.astype(np.float32))
h.update(fp.read_bytes())
(out / "manifest.json").write_text(json.dumps({
"task": a.task, "input_spec": a.input_spec, "count": len(imgs[:a.n]), "seed": a.seed, "source": source,
"dataset": ds, "sha256": h.hexdigest(), "files": desc[:a.n],
"preprocess": row.get("preprocess") or "harness.eval.common (same as evaluator)",
"layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"}, indent=2))
print(f"{out} n={len(imgs[:a.n])} source={source} sha={h.hexdigest()[:12]}")
return 0
if __name__ == "__main__":
sys.exit(main())