data-gen-bpu/scripts/make_calib.py

267 lines
14 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""T0 校准集:data/calib/<task>/<input_spec>/NNNNN.npy + manifest.json。同规格模型共用。
python scripts/make_calib.py --task cls --input-spec 1x3x224x224 [--n 128] [--from auto|train|val-holdout]
来源优先级(auto):
1. datasets.yaml calib.train_parquet(HF train-*.parquet,固定 seed 选片、选行)
2. datasets.yaml calib.train_dir(本机 train 图片目录)
3. pool 里带 "train" 的图片
4. val-holdout:从 val 里抽 *不在 T1 里* 的图 —— 与 T2 全量有交集,manifest 记 source,出交付数前必须换 train
预处理与评测器完全同一份代码(harness/eval/common.py),layout NCHW,含 batch 维。
"""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import random
import sys
import tarfile
from pathlib import Path
import yaml
BENCH = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(BENCH))
REG = yaml.safe_load((BENCH / "registry" / "datasets.yaml").read_text())
POOL = BENCH / REG.get("pool_root", "data/pool")
LOCAL = Path(REG.get("local_root", "/mnt/data"))
TASK_DATASET = {"cls": "imagenet1k", "det": "coco2017", "iseg": "coco2017", "pose": "coco2017", "seg": "cityscapes",
"mde": "nyuv2", "stereo": "kitti2015", "flow": "kitti2015", "feat": "hpatches", "occ": "nuscenes"}
EXTS = {".jpg", ".jpeg", ".png", ".JPEG", ".JPG"}
def list_images(root: Path, split_hint: str = "") -> list[Path]:
return sorted(p for p in root.rglob("*") if p.suffix in EXTS and split_hint in str(p.relative_to(root)).lower())
def t1_files(ds: str) -> set[str]:
d = REG["datasets"][ds]
tar = BENCH / "data" / "t1" / ds / f"{(d.get('t1') or {}).get('seed', 20260914)}.tar"
if not tar.exists():
return set()
with tarfile.open(tar) as t:
man = json.loads(t.extractfile("manifest.json").read())
return {it["src"] for it in man["items"]}
def from_train_parquet(cfg: dict, n: int, seed: int, local: Path = LOCAL):
"""返回 (PIL 图列表, 描述列表)。固定 seed 选 4 片、每片均匀抽。"""
import pyarrow.parquet as pq
from PIL import Image
root = local / cfg["train_parquet"]
shards = sorted(root.glob(cfg.get("train_glob", "train-*.parquet")))
if not shards:
raise FileNotFoundError(f"no train parquet under {root}")
rng = random.Random(seed)
picked = sorted(rng.sample(shards, min(4, len(shards))))
per = -(-n // len(picked))
imgs, desc = [], []
for sh in picked:
pf = pq.ParquetFile(sh)
total = pf.metadata.num_rows
rows = sorted(rng.sample(range(total), min(per, total)))
names = pf.schema_arrow.names # 顶层列名(HF: image{bytes,path}, label)
img_col = "image" if "image" in names else names[0]
col = pf.read(columns=[img_col]).column(0)
for r in rows:
v = col[r].as_py()
data = v.get("bytes") if isinstance(v, dict) else v
imgs.append(Image.open(io.BytesIO(data)).convert("RGB")); desc.append(f"{sh.name}#{r}")
if len(imgs) >= n:
return imgs, desc
return imgs, desc
def select_images(ds: str, d: dict, pool: Path, n: int, seed: int, src: str, local: Path = LOCAL):
"""One deterministic source selector shared by the CLI and protected recipe stage."""
from PIL import Image
ccfg = d.get("calib") or {}
rng = random.Random(seed)
imgs, desc, source = [], [], None
if src in ("auto", "train"):
if ds == "cityscapes" and d.get("sources") and (local / d["sources"][0]["path"]).is_file():
import zipfile
archive = local / d["sources"][0]["path"]
with zipfile.ZipFile(archive) as source_zip:
names = sorted(name for name in source_zip.namelist()
if name.startswith("leftImg8bit/train/") and name.endswith("_leftImg8bit.png"))
if len(names) < n:
raise ValueError(f"only {len(names)} Cityscapes training images")
rng.shuffle(names)
desc = [f"{archive}#{name}" for name in sorted(names[:n])]
imgs = [Image.open(io.BytesIO(source_zip.read(name))).convert("RGB") for name in sorted(names[:n])]
source = "train-zip"
if ccfg.get("train_parquet") and (local / ccfg["train_parquet"]).exists():
try:
imgs, desc = from_train_parquet(ccfg, n, seed, local); source = "train-parquet"
except Exception as e:
print(f"WARN train_parquet 失败:{e}", file=sys.stderr)
if not imgs and ccfg.get("train_dir") and (local / ccfg["train_dir"]).exists():
files = list_images(local / ccfg["train_dir"])
if len(files) >= n:
rng.shuffle(files); files = sorted(files[:n])
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p) for p in files]; source = "train-dir"
if not imgs:
files = list_images(pool, "train")
if len(files) >= n:
rng.shuffle(files); files = sorted(files[:n])
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p.relative_to(pool)) for p in files]; source = "train"
if not imgs:
if src == "train":
raise ValueError(f"no training images: configure registry/datasets.yaml datasets.{ds}.calib.train_dir or a training archive, then create a new job")
excl = t1_files(ds)
files = [p for p in list_images(pool) if str(p.relative_to(pool)) not in excl]
if len(files) < n:
raise SystemExit(f"val-holdout 可用图片 {len(files)} < {n}")
rng.shuffle(files); files = sorted(files[:n])
imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p.relative_to(pool)) for p in files]; source = "val-holdout"
print(f"WARN: {ds} 没有 train 来源,用 val-holdout(排除 T1 的 {len(excl)} 张)。出 T2 交付数前须换 train。", file=sys.stderr)
return imgs, desc, source
def make_recipe_calibration(job: dict, out: Path, calls: Path, python: Path, runtime_identity: dict) -> dict:
"""Prepare current deployment inputs from platform-owned T0 images only.
Selection and T1 exclusion stay in trusted Python. The model receives RGB
arrays through the existing jail, never a dataset path, labels or writable data.
"""
import numpy as np
from harness.eval.common import RecipeRunner, T1
source = job["data"]["calib"].get("recipe_source")
if not source or source.get("dataset") != job["data"]["dataset"]:
raise ValueError("recipe calibration requires frozen dataset training sources; configure registry/datasets.yaml and create a new job")
if out.exists():
raise ValueError("recipe calibration output already exists; submit a new attempt")
n = job["data"]["calib"]["n"]
if type(n) is not int or not 0 < n <= 4096:
raise ValueError("invalid frozen calibration sample count")
imgs, descriptions, origin = select_images(source["dataset"], source["dataset_config"], Path(source["pool"]),
n, source["seed"], "train", Path(source["local_root"]))
if len(imgs) != n or len(descriptions) != n or not origin.startswith("train"):
raise ValueError("recipe calibration requires the complete platform training selection")
def image_hash(image):
rgb = np.asarray(image, dtype=np.uint8)
digest = hashlib.sha256(json.dumps(list(rgb.shape)).encode())
digest.update(rgb.tobytes())
return digest.hexdigest()
# Compare decoded image identity: the same RGB pixels in a different file
# encoding must not leak from the scoring split into calibration.
t1_config = job["data"]["t1"]
t1 = T1.open(t1_config["path"], t1_config.get("manifest_sha"))
try:
excluded = {image_hash(t1.image(item)) for item in t1.items}
finally:
t1.tar.close()
identities = [image_hash(image) for image in imgs]
if set(identities) & excluded:
raise ValueError("platform calibration training images overlap T1; fix the training source")
recipe = job["recipe"]
runner = RecipeRunner(recipe, str(python), calls, source["dataset"])
npy_dir = out / "npy_f32"
npy_dir.mkdir(parents=True)
digest = hashlib.sha256()
for start in range(0, n, runner.CHUNK):
batch = runner.prepare(imgs[start:start + runner.CHUNK])
for offset, value in enumerate(batch):
target = npy_dir / f"{start + offset:05d}.npy"
np.save(target, value, allow_pickle=False)
digest.update(target.read_bytes())
manifest = {"task": job["task"], "dataset": source["dataset"], "input_spec": recipe["input_spec"],
"count": n, "seed": source["seed"], "source": origin, "sha256": digest.hexdigest(),
"files": descriptions, "raw_rgb_sha256": identities, "t1": t1_config,
"preprocessing_identity": {"submission_sha256": recipe["submission_sha256"],
"reference_sha256": recipe["reference_sha256"], "runtime": runtime_identity,
"input_spec": recipe["input_spec"], "preprocess": recipe["preprocess"]},
"layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"}
(out / "manifest.json").write_text(json.dumps(manifest, indent=2))
return manifest
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--task")
ap.add_argument("--input-spec")
ap.add_argument("--recipe-job", help="protected Worker calibration request, executed in the configured model environment")
ap.add_argument("--out", help="new task-attempt calibration directory for --recipe-job")
ap.add_argument("--model", help="Use this model's official input recipe in a separate calibration directory")
ap.add_argument("--n", type=int, default=128)
ap.add_argument("--from", dest="src", choices=["auto", "train", "val-holdout"], default="auto")
ap.add_argument("--seed", type=int, default=20260914)
a = ap.parse_args()
if a.recipe_job:
if not a.out:
ap.error("--recipe-job requires --out")
request = json.loads(Path(a.recipe_job).read_text())
out = Path(a.out)
make_recipe_calibration(request["job"], out, out.parent / "calibration_calls", Path(sys.executable), request["runtime_identity"])
return 0
if not a.task or not a.input_spec:
ap.error("--task and --input-spec are required")
import numpy as np
from harness.eval.common import preprocess_cls, letterbox
from harness.adapters import calibration_dir
from harness.registry import model_row
row = model_row(a.model) if a.model else {"task": a.task, "input_spec": a.input_spec, "model_id": ""}
if a.model and (row["task"] != a.task or row["input_spec"] != a.input_spec):
raise SystemExit("--task/--input-spec must match the model recipe")
ds = row.get("dataset") or TASK_DATASET[a.task]
d = REG["datasets"][ds]
pool = POOL / ds / d.get("version", "v1")
shape = tuple(int(x) for x in a.input_spec.split("x"))
out = calibration_dir(row)
if a.model and out.exists():
manifest = json.loads((out / "manifest.json").read_text())
files = sorted((out / manifest["npy_dir"]).glob("*.npy"))
digest = hashlib.sha256()
for path in files:
digest.update(path.read_bytes())
expected_recipe = row.get("preprocess") or "harness.eval.common (same as evaluator)"
if (manifest.get("count") != a.n or len(files) != a.n or manifest.get("seed") != a.seed
or json.dumps(manifest.get("preprocess"), sort_keys=True) != json.dumps(expected_recipe, sort_keys=True)
or manifest.get("sha256") != digest.hexdigest()
or (a.src == "train" and not manifest.get("source", "").startswith("train"))):
raise ValueError(f"existing model calibration differs; refusing to overwrite {out}")
print(f"{out} n={a.n} sha={digest.hexdigest()[:12]} (reused)")
return 0
imgs, desc, source = select_images(ds, d, pool, a.n, a.seed, a.src)
npy_dir = out / "npy_f32" # hb_compile 读 cal_data_dir 下的全部文件 → manifest.json 不能放同一目录;_f32 后缀是 OE 约定
if out.exists():
import shutil; shutil.rmtree(out)
npy_dir.mkdir(parents=True, exist_ok=True)
h = hashlib.sha256()
for i, img in enumerate(imgs[:a.n]):
if a.task == "cls":
x = preprocess_cls(img, size=shape[-1], resize=round(shape[-1] * 256 / 224), model=a.model or "")
elif a.task in ("seg", "det", "mde", "pose") and a.model:
from harness.adapters import get
transform = getattr(get(a.model), "preprocess", None)
if a.task in ("seg", "mde") and transform is None:
raise ValueError(f"no {a.task} preprocessing recipe for {a.model}")
x = transform(img, a.input_spec) if transform else letterbox(img, shape[-1])[0]
else:
x = letterbox(img, shape[-1])[0]
assert x.shape == shape, (x.shape, shape)
fp = npy_dir / f"{i:05d}.npy"
np.save(fp, x.astype(np.float32))
h.update(fp.read_bytes())
(out / "manifest.json").write_text(json.dumps({
"task": a.task, "input_spec": a.input_spec, "count": len(imgs[:a.n]), "seed": a.seed, "source": source,
"dataset": ds, "sha256": h.hexdigest(), "files": desc[:a.n],
"preprocess": row.get("preprocess") or "harness.eval.common (same as evaluator)",
"layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"}, indent=2))
print(f"{out} n={len(imgs[:a.n])} source={source} sha={h.hexdigest()[:12]}")
return 0
if __name__ == "__main__":
sys.exit(main())