189 lines
9.2 KiB
Python
189 lines
9.2 KiB
Python
"""T0 校准集:data/calib/<task>/<input_spec>/NNNNN.npy + manifest.json。同规格模型共用。
|
||
|
||
python scripts/make_calib.py --task cls --input-spec 1x3x224x224 [--n 128] [--from auto|train|val-holdout]
|
||
|
||
来源优先级(auto):
|
||
1. datasets.yaml calib.train_parquet(HF train-*.parquet,固定 seed 选片、选行)
|
||
2. datasets.yaml calib.train_dir(本机 train 图片目录)
|
||
3. pool 里带 "train" 的图片
|
||
4. val-holdout:从 val 里抽 *不在 T1 里* 的图 —— 与 T2 全量有交集,manifest 记 source,出交付数前必须换 train
|
||
预处理与评测器完全同一份代码(harness/eval/common.py),layout NCHW,含 batch 维。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import random
|
||
import sys
|
||
import tarfile
|
||
from pathlib import Path
|
||
|
||
import yaml
|
||
|
||
BENCH = Path(__file__).resolve().parent.parent
|
||
sys.path.insert(0, str(BENCH))
|
||
REG = yaml.safe_load((BENCH / "registry" / "datasets.yaml").read_text())
|
||
POOL = BENCH / REG.get("pool_root", "data/pool")
|
||
LOCAL = Path(REG.get("local_root", "/mnt/data"))
|
||
|
||
TASK_DATASET = {"cls": "imagenet1k", "det": "coco2017", "iseg": "coco2017", "pose": "coco2017", "seg": "cityscapes",
|
||
"mde": "nyuv2", "stereo": "kitti2015", "flow": "kitti2015", "feat": "hpatches", "occ": "nuscenes"}
|
||
EXTS = {".jpg", ".jpeg", ".png", ".JPEG", ".JPG"}
|
||
|
||
|
||
def list_images(root: Path, split_hint: str = "") -> list[Path]:
|
||
return sorted(p for p in root.rglob("*") if p.suffix in EXTS and split_hint in str(p.relative_to(root)).lower())
|
||
|
||
|
||
def t1_files(ds: str) -> set[str]:
|
||
d = REG["datasets"][ds]
|
||
tar = BENCH / "data" / "t1" / ds / f"{(d.get('t1') or {}).get('seed', 20260914)}.tar"
|
||
if not tar.exists():
|
||
return set()
|
||
with tarfile.open(tar) as t:
|
||
man = json.loads(t.extractfile("manifest.json").read())
|
||
return {it["src"] for it in man["items"]}
|
||
|
||
|
||
def from_train_parquet(cfg: dict, n: int, seed: int):
|
||
"""返回 (PIL 图列表, 描述列表)。固定 seed 选 4 片、每片均匀抽。"""
|
||
import pyarrow.parquet as pq
|
||
from PIL import Image
|
||
root = LOCAL / cfg["train_parquet"]
|
||
shards = sorted(root.glob(cfg.get("train_glob", "train-*.parquet")))
|
||
if not shards:
|
||
raise FileNotFoundError(f"no train parquet under {root}")
|
||
rng = random.Random(seed)
|
||
picked = sorted(rng.sample(shards, min(4, len(shards))))
|
||
per = -(-n // len(picked))
|
||
imgs, desc = [], []
|
||
for sh in picked:
|
||
pf = pq.ParquetFile(sh)
|
||
total = pf.metadata.num_rows
|
||
rows = sorted(rng.sample(range(total), min(per, total)))
|
||
names = pf.schema_arrow.names # 顶层列名(HF: image{bytes,path}, label)
|
||
img_col = "image" if "image" in names else names[0]
|
||
col = pf.read(columns=[img_col]).column(0)
|
||
for r in rows:
|
||
v = col[r].as_py()
|
||
data = v.get("bytes") if isinstance(v, dict) else v
|
||
imgs.append(Image.open(io.BytesIO(data)).convert("RGB")); desc.append(f"{sh.name}#{r}")
|
||
if len(imgs) >= n:
|
||
return imgs, desc
|
||
return imgs, desc
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--task", required=True)
|
||
ap.add_argument("--input-spec", required=True)
|
||
ap.add_argument("--model", help="Use this model's official input recipe in a separate calibration directory")
|
||
ap.add_argument("--n", type=int, default=128)
|
||
ap.add_argument("--from", dest="src", choices=["auto", "train", "val-holdout"], default="auto")
|
||
ap.add_argument("--seed", type=int, default=20260914)
|
||
a = ap.parse_args()
|
||
import numpy as np
|
||
from PIL import Image
|
||
from harness.eval.common import preprocess_cls, letterbox
|
||
|
||
from harness.adapters import calibration_dir
|
||
from harness.registry import model_row
|
||
row = model_row(a.model) if a.model else {"task": a.task, "input_spec": a.input_spec, "model_id": ""}
|
||
if a.model and (row["task"] != a.task or row["input_spec"] != a.input_spec):
|
||
raise SystemExit("--task/--input-spec must match the model recipe")
|
||
ds = row.get("dataset") or TASK_DATASET[a.task]
|
||
d = REG["datasets"][ds]
|
||
ccfg = d.get("calib") or {}
|
||
pool = POOL / ds / d.get("version", "v1")
|
||
shape = tuple(int(x) for x in a.input_spec.split("x"))
|
||
rng = random.Random(a.seed)
|
||
out = calibration_dir(row)
|
||
if a.model and out.exists():
|
||
manifest = json.loads((out / "manifest.json").read_text())
|
||
files = sorted((out / manifest["npy_dir"]).glob("*.npy"))
|
||
digest = hashlib.sha256()
|
||
for path in files:
|
||
digest.update(path.read_bytes())
|
||
expected_recipe = row.get("preprocess") or "harness.eval.common (same as evaluator)"
|
||
if (manifest.get("count") != a.n or len(files) != a.n or manifest.get("seed") != a.seed
|
||
or json.dumps(manifest.get("preprocess"), sort_keys=True) != json.dumps(expected_recipe, sort_keys=True)
|
||
or manifest.get("sha256") != digest.hexdigest()
|
||
or (a.src == "train" and not manifest.get("source", "").startswith("train"))):
|
||
raise ValueError(f"existing model calibration differs; refusing to overwrite {out}")
|
||
print(f"{out} n={a.n} sha={digest.hexdigest()[:12]} (reused)")
|
||
return 0
|
||
|
||
imgs, desc, source = [], [], None
|
||
if a.src in ("auto", "train"):
|
||
if ds == "cityscapes":
|
||
import zipfile
|
||
archive = LOCAL / d["sources"][0]["path"]
|
||
with zipfile.ZipFile(archive) as source_zip:
|
||
names = sorted(name for name in source_zip.namelist()
|
||
if name.startswith("leftImg8bit/train/") and name.endswith("_leftImg8bit.png"))
|
||
if len(names) < a.n:
|
||
raise ValueError(f"only {len(names)} Cityscapes training images")
|
||
rng.shuffle(names)
|
||
desc = [f"{archive}#{name}" for name in sorted(names[:a.n])]
|
||
imgs = [Image.open(io.BytesIO(source_zip.read(name))).convert("RGB") for name in sorted(names[:a.n])]
|
||
source = "train-zip"
|
||
if ccfg.get("train_parquet") and (LOCAL / ccfg["train_parquet"]).exists():
|
||
try:
|
||
imgs, desc = from_train_parquet(ccfg, a.n, a.seed); source = "train-parquet"
|
||
except Exception as e:
|
||
print(f"WARN train_parquet 失败:{e}", file=sys.stderr)
|
||
if not imgs and ccfg.get("train_dir") and (LOCAL / ccfg["train_dir"]).exists():
|
||
files = list_images(LOCAL / ccfg["train_dir"])
|
||
if len(files) >= a.n:
|
||
rng.shuffle(files); files = sorted(files[:a.n])
|
||
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p) for p in files]; source = "train-dir"
|
||
if not imgs:
|
||
files = list_images(pool, "train")
|
||
if len(files) >= a.n:
|
||
rng.shuffle(files); files = sorted(files[:a.n])
|
||
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p.relative_to(pool)) for p in files]; source = "train"
|
||
if not imgs:
|
||
if a.src == "train":
|
||
raise SystemExit("没有可用的 train 来源")
|
||
excl = t1_files(ds)
|
||
files = [p for p in list_images(pool) if str(p.relative_to(pool)) not in excl]
|
||
if len(files) < a.n:
|
||
raise SystemExit(f"val-holdout 可用图片 {len(files)} < {a.n}")
|
||
rng.shuffle(files); files = sorted(files[:a.n])
|
||
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p.relative_to(pool)) for p in files]; source = "val-holdout"
|
||
print(f"WARN: {ds} 没有 train 来源,用 val-holdout(排除 T1 的 {len(excl)} 张)。出 T2 交付数前须换 train。", file=sys.stderr)
|
||
|
||
npy_dir = out / "npy_f32" # hb_compile 读 cal_data_dir 下的全部文件 → manifest.json 不能放同一目录;_f32 后缀是 OE 约定
|
||
if out.exists():
|
||
import shutil; shutil.rmtree(out)
|
||
npy_dir.mkdir(parents=True, exist_ok=True)
|
||
h = hashlib.sha256()
|
||
for i, img in enumerate(imgs[:a.n]):
|
||
if a.task == "cls":
|
||
x = preprocess_cls(img, size=shape[-1], resize=round(shape[-1] * 256 / 224), model=a.model or "")
|
||
elif a.task in ("seg", "det", "mde", "pose") and a.model:
|
||
from harness.adapters import get
|
||
transform = getattr(get(a.model), "preprocess", None)
|
||
if a.task in ("seg", "mde") and transform is None:
|
||
raise ValueError(f"no {a.task} preprocessing recipe for {a.model}")
|
||
x = transform(img, a.input_spec) if transform else letterbox(img, shape[-1])[0]
|
||
else:
|
||
x = letterbox(img, shape[-1])[0]
|
||
assert x.shape == shape, (x.shape, shape)
|
||
fp = npy_dir / f"{i:05d}.npy"
|
||
np.save(fp, x.astype(np.float32))
|
||
h.update(fp.read_bytes())
|
||
(out / "manifest.json").write_text(json.dumps({
|
||
"task": a.task, "input_spec": a.input_spec, "count": len(imgs[:a.n]), "seed": a.seed, "source": source,
|
||
"dataset": ds, "sha256": h.hexdigest(), "files": desc[:a.n],
|
||
"preprocess": row.get("preprocess") or "harness.eval.common (same as evaluator)",
|
||
"layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"}, indent=2))
|
||
print(f"{out} n={len(imgs[:a.n])} source={source} sha={h.hexdigest()[:12]}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|