data-gen-bpu/scripts/make_calib.py

150 lines
6.7 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""T0 校准集:data/calib/<task>/<input_spec>/NNNNN.npy + manifest.json。同规格模型共用。
python scripts/make_calib.py --task cls --input-spec 1x3x224x224 [--n 128] [--from auto|train|val-holdout]
来源优先级(auto):
1. datasets.yaml calib.train_parquet(HF train-*.parquet,固定 seed 选片、选行)
2. datasets.yaml calib.train_dir(本机 train 图片目录)
3. pool 里带 "train" 的图片
4. val-holdout:从 val 里抽 *不在 T1 里* 的图 —— 与 T2 全量有交集,manifest 记 source,出交付数前必须换 train
预处理与评测器完全同一份代码(harness/eval/common.py),layout NCHW,含 batch 维。
"""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import random
import sys
import tarfile
from pathlib import Path
import yaml
BENCH = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(BENCH))
REG = yaml.safe_load((BENCH / "registry" / "datasets.yaml").read_text())
POOL = BENCH / REG.get("pool_root", "data/pool")
LOCAL = Path(REG.get("local_root", "/mnt/data"))
TASK_DATASET = {"cls": "imagenet1k", "det": "coco2017", "iseg": "coco2017", "pose": "coco2017", "seg": "cityscapes",
"mde": "nyuv2", "stereo": "kitti2015", "flow": "kitti2015", "feat": "hpatches", "occ": "nuscenes"}
EXTS = {".jpg", ".jpeg", ".png", ".JPEG", ".JPG"}
def list_images(root: Path, split_hint: str = "") -> list[Path]:
return sorted(p for p in root.rglob("*") if p.suffix in EXTS and split_hint in str(p.relative_to(root)).lower())
def t1_files(ds: str) -> set[str]:
d = REG["datasets"][ds]
tar = BENCH / "data" / "t1" / ds / f"{(d.get('t1') or {}).get('seed', 20260914)}.tar"
if not tar.exists():
return set()
with tarfile.open(tar) as t:
man = json.loads(t.extractfile("manifest.json").read())
return {it["src"] for it in man["items"]}
def from_train_parquet(cfg: dict, n: int, seed: int):
"""返回 (PIL 图列表, 描述列表)。固定 seed 选 4 片、每片均匀抽。"""
import pyarrow.parquet as pq
from PIL import Image
root = LOCAL / cfg["train_parquet"]
shards = sorted(root.glob(cfg.get("train_glob", "train-*.parquet")))
if not shards:
raise FileNotFoundError(f"no train parquet under {root}")
rng = random.Random(seed)
picked = sorted(rng.sample(shards, min(4, len(shards))))
per = -(-n // len(picked))
imgs, desc = [], []
for sh in picked:
pf = pq.ParquetFile(sh)
total = pf.metadata.num_rows
rows = sorted(rng.sample(range(total), min(per, total)))
names = pf.schema_arrow.names # 顶层列名(HF: image{bytes,path}, label)
img_col = "image" if "image" in names else names[0]
col = pf.read(columns=[img_col]).column(0)
for r in rows:
v = col[r].as_py()
data = v.get("bytes") if isinstance(v, dict) else v
imgs.append(Image.open(io.BytesIO(data)).convert("RGB")); desc.append(f"{sh.name}#{r}")
if len(imgs) >= n:
return imgs, desc
return imgs, desc
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--task", required=True)
ap.add_argument("--input-spec", required=True)
ap.add_argument("--n", type=int, default=128)
ap.add_argument("--from", dest="src", choices=["auto", "train", "val-holdout"], default="auto")
ap.add_argument("--seed", type=int, default=20260914)
a = ap.parse_args()
import numpy as np
from PIL import Image
from harness.eval.common import preprocess_cls, letterbox
ds = TASK_DATASET[a.task]
d = REG["datasets"][ds]
ccfg = d.get("calib") or {}
pool = POOL / ds / d.get("version", "v1")
shape = tuple(int(x) for x in a.input_spec.split("x"))
rng = random.Random(a.seed)
imgs, desc, source = [], [], None
if a.src in ("auto", "train"):
if ccfg.get("train_parquet") and (LOCAL / ccfg["train_parquet"]).exists():
try:
imgs, desc = from_train_parquet(ccfg, a.n, a.seed); source = "train-parquet"
except Exception as e:
print(f"WARN train_parquet 失败:{e}", file=sys.stderr)
if not imgs and ccfg.get("train_dir") and (LOCAL / ccfg["train_dir"]).exists():
files = list_images(LOCAL / ccfg["train_dir"])
if len(files) >= a.n:
rng.shuffle(files); files = sorted(files[:a.n])
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p) for p in files]; source = "train-dir"
if not imgs:
files = list_images(pool, "train")
if len(files) >= a.n:
rng.shuffle(files); files = sorted(files[:a.n])
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p.relative_to(pool)) for p in files]; source = "train"
if not imgs:
if a.src == "train":
raise SystemExit("没有可用的 train 来源")
excl = t1_files(ds)
files = [p for p in list_images(pool) if str(p.relative_to(pool)) not in excl]
if len(files) < a.n:
raise SystemExit(f"val-holdout 可用图片 {len(files)} < {a.n}")
rng.shuffle(files); files = sorted(files[:a.n])
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p.relative_to(pool)) for p in files]; source = "val-holdout"
print(f"WARN: {ds} 没有 train 来源,用 val-holdout(排除 T1 的 {len(excl)} 张)。出 T2 交付数前须换 train。", file=sys.stderr)
out = BENCH / "data" / "calib" / a.task / a.input_spec
npy_dir = out / "npy_f32" # hb_compile 读 cal_data_dir 下的全部文件 → manifest.json 不能放同一目录;_f32 后缀是 OE 约定
if out.exists():
import shutil; shutil.rmtree(out)
npy_dir.mkdir(parents=True, exist_ok=True)
h = hashlib.sha256()
for i, img in enumerate(imgs[:a.n]):
if a.task == "cls":
x = preprocess_cls(img, size=shape[-1], resize=round(shape[-1] * 256 / 224))
else:
x = letterbox(img, shape[-1])[0]
assert x.shape == shape, (x.shape, shape)
fp = npy_dir / f"{i:05d}.npy"
np.save(fp, x.astype(np.float32))
h.update(fp.read_bytes())
(out / "manifest.json").write_text(json.dumps({
"task": a.task, "input_spec": a.input_spec, "count": len(imgs[:a.n]), "seed": a.seed, "source": source,
"dataset": ds, "sha256": h.hexdigest(), "files": desc[:a.n],
"preprocess": "harness.eval.common (same as evaluator)", "layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"}, indent=2))
print(f"{out} n={len(imgs[:a.n])} source={source} sha={h.hexdigest()[:12]}")
return 0
if __name__ == "__main__":
sys.exit(main())