"""T0 校准集:data/calib///NNNNN.npy + manifest.json。同规格模型共用。 python scripts/make_calib.py --task cls --input-spec 1x3x224x224 [--n 128] [--from auto|train|val-holdout] 来源优先级(auto): 1. datasets.yaml calib.train_parquet(HF train-*.parquet,固定 seed 选片、选行) 2. datasets.yaml calib.train_dir(本机 train 图片目录) 3. pool 里带 "train" 的图片 4. val-holdout:从 val 里抽 *不在 T1 里* 的图 —— 与 T2 全量有交集,manifest 记 source,出交付数前必须换 train 预处理与评测器完全同一份代码(harness/eval/common.py),layout NCHW,含 batch 维。 """ from __future__ import annotations import argparse import hashlib import io import json import random import sys import tarfile from pathlib import Path import yaml BENCH = Path(__file__).resolve().parent.parent sys.path.insert(0, str(BENCH)) REG = yaml.safe_load((BENCH / "registry" / "datasets.yaml").read_text()) POOL = BENCH / REG.get("pool_root", "data/pool") LOCAL = Path(REG.get("local_root", "/mnt/data")) TASK_DATASET = {"cls": "imagenet1k", "det": "coco2017", "iseg": "coco2017", "pose": "coco2017", "seg": "cityscapes", "mde": "nyuv2", "stereo": "kitti2015", "flow": "kitti2015", "feat": "hpatches", "occ": "nuscenes"} EXTS = {".jpg", ".jpeg", ".png", ".JPEG", ".JPG"} def list_images(root: Path, split_hint: str = "") -> list[Path]: return sorted(p for p in root.rglob("*") if p.suffix in EXTS and split_hint in str(p.relative_to(root)).lower()) def t1_files(ds: str) -> set[str]: d = REG["datasets"][ds] tar = BENCH / "data" / "t1" / ds / f"{(d.get('t1') or {}).get('seed', 20260914)}.tar" if not tar.exists(): return set() with tarfile.open(tar) as t: man = json.loads(t.extractfile("manifest.json").read()) return {it["src"] for it in man["items"]} def from_train_parquet(cfg: dict, n: int, seed: int, local: Path = LOCAL): """返回 (PIL 图列表, 描述列表)。固定 seed 选 4 片、每片均匀抽。""" import pyarrow.parquet as pq from PIL import Image root = local / cfg["train_parquet"] shards = sorted(root.glob(cfg.get("train_glob", "train-*.parquet"))) if not shards: raise FileNotFoundError(f"no train parquet under {root}") rng = random.Random(seed) picked = sorted(rng.sample(shards, min(4, len(shards)))) per = -(-n // len(picked)) imgs, desc = [], [] for sh in picked: pf = pq.ParquetFile(sh) total = pf.metadata.num_rows rows = sorted(rng.sample(range(total), min(per, total))) names = pf.schema_arrow.names # 顶层列名(HF: image{bytes,path}, label) img_col = "image" if "image" in names else names[0] col = pf.read(columns=[img_col]).column(0) for r in rows: v = col[r].as_py() data = v.get("bytes") if isinstance(v, dict) else v imgs.append(Image.open(io.BytesIO(data)).convert("RGB")); desc.append(f"{sh.name}#{r}") if len(imgs) >= n: return imgs, desc return imgs, desc def select_images(ds: str, d: dict, pool: Path, n: int, seed: int, src: str, local: Path = LOCAL): """One deterministic source selector shared by the CLI and protected recipe stage.""" from PIL import Image ccfg = d.get("calib") or {} rng = random.Random(seed) imgs, desc, source = [], [], None if src in ("auto", "train"): if ds == "cityscapes" and d.get("sources") and (local / d["sources"][0]["path"]).is_file(): import zipfile archive = local / d["sources"][0]["path"] with zipfile.ZipFile(archive) as source_zip: names = sorted(name for name in source_zip.namelist() if name.startswith("leftImg8bit/train/") and name.endswith("_leftImg8bit.png")) if len(names) < n: raise ValueError(f"only {len(names)} Cityscapes training images") rng.shuffle(names) desc = [f"{archive}#{name}" for name in sorted(names[:n])] imgs = [Image.open(io.BytesIO(source_zip.read(name))).convert("RGB") for name in sorted(names[:n])] source = "train-zip" if ccfg.get("train_parquet") and (local / ccfg["train_parquet"]).exists(): try: imgs, desc = from_train_parquet(ccfg, n, seed, local); source = "train-parquet" except Exception as e: print(f"WARN train_parquet 失败:{e}", file=sys.stderr) if not imgs and ccfg.get("train_dir") and (local / ccfg["train_dir"]).exists(): files = list_images(local / ccfg["train_dir"]) if len(files) >= n: rng.shuffle(files); files = sorted(files[:n]) imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p) for p in files]; source = "train-dir" if not imgs: files = list_images(pool, "train") if len(files) >= n: rng.shuffle(files); files = sorted(files[:n]) imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p.relative_to(pool)) for p in files]; source = "train" if not imgs: if src == "train": raise ValueError(f"no training images: configure registry/datasets.yaml datasets.{ds}.calib.train_dir or a training archive, then create a new job") excl = t1_files(ds) files = [p for p in list_images(pool) if str(p.relative_to(pool)) not in excl] if len(files) < n: raise SystemExit(f"val-holdout 可用图片 {len(files)} < {n}") rng.shuffle(files); files = sorted(files[:n]) imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p.relative_to(pool)) for p in files]; source = "val-holdout" print(f"WARN: {ds} 没有 train 来源,用 val-holdout(排除 T1 的 {len(excl)} 张)。出 T2 交付数前须换 train。", file=sys.stderr) return imgs, desc, source def make_recipe_calibration(job: dict, out: Path, calls: Path, python: Path, runtime_identity: dict) -> dict: """Prepare current deployment inputs from platform-owned T0 images only. Selection and T1 exclusion stay in trusted Python. The model receives RGB arrays through the existing jail, never a dataset path, labels or writable data. """ import numpy as np from harness.eval.common import RecipeRunner, T1 source = job["data"]["calib"].get("recipe_source") if not source or source.get("dataset") != job["data"]["dataset"]: raise ValueError("recipe calibration requires frozen dataset training sources; configure registry/datasets.yaml and create a new job") if out.exists(): raise ValueError("recipe calibration output already exists; submit a new attempt") n = job["data"]["calib"]["n"] if type(n) is not int or not 0 < n <= 4096: raise ValueError("invalid frozen calibration sample count") imgs, descriptions, origin = select_images(source["dataset"], source["dataset_config"], Path(source["pool"]), n, source["seed"], "train", Path(source["local_root"])) if len(imgs) != n or len(descriptions) != n or not origin.startswith("train"): raise ValueError("recipe calibration requires the complete platform training selection") def image_hash(image): rgb = np.asarray(image, dtype=np.uint8) digest = hashlib.sha256(json.dumps(list(rgb.shape)).encode()) digest.update(rgb.tobytes()) return digest.hexdigest() # Compare decoded image identity: the same RGB pixels in a different file # encoding must not leak from the scoring split into calibration. t1_config = job["data"]["t1"] t1 = T1.open(t1_config["path"], t1_config.get("manifest_sha")) try: excluded = {image_hash(t1.image(item)) for item in t1.items} finally: t1.tar.close() identities = [image_hash(image) for image in imgs] if set(identities) & excluded: raise ValueError("platform calibration training images overlap T1; fix the training source") recipe = job["recipe"] runner = RecipeRunner(recipe, str(python), calls, source["dataset"]) npy_dir = out / "npy_f32" npy_dir.mkdir(parents=True) digest = hashlib.sha256() for start in range(0, n, runner.CHUNK): batch = runner.prepare(imgs[start:start + runner.CHUNK]) for offset, value in enumerate(batch): target = npy_dir / f"{start + offset:05d}.npy" np.save(target, value, allow_pickle=False) digest.update(target.read_bytes()) manifest = {"task": job["task"], "dataset": source["dataset"], "input_spec": recipe["input_spec"], "count": n, "seed": source["seed"], "source": origin, "sha256": digest.hexdigest(), "files": descriptions, "raw_rgb_sha256": identities, "t1": t1_config, "preprocessing_identity": {"submission_sha256": recipe["submission_sha256"], "reference_sha256": recipe["reference_sha256"], "runtime": runtime_identity, "input_spec": recipe["input_spec"], "preprocess": recipe["preprocess"]}, "layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"} (out / "manifest.json").write_text(json.dumps(manifest, indent=2)) return manifest def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--task") ap.add_argument("--input-spec") ap.add_argument("--recipe-job", help="protected Worker calibration request, executed in the configured model environment") ap.add_argument("--out", help="new task-attempt calibration directory for --recipe-job") ap.add_argument("--model", help="Use this model's official input recipe in a separate calibration directory") ap.add_argument("--n", type=int, default=128) ap.add_argument("--from", dest="src", choices=["auto", "train", "val-holdout"], default="auto") ap.add_argument("--seed", type=int, default=20260914) a = ap.parse_args() if a.recipe_job: if not a.out: ap.error("--recipe-job requires --out") request = json.loads(Path(a.recipe_job).read_text()) out = Path(a.out) make_recipe_calibration(request["job"], out, out.parent / "calibration_calls", Path(sys.executable), request["runtime_identity"]) return 0 if not a.task or not a.input_spec: ap.error("--task and --input-spec are required") import numpy as np from harness.eval.common import preprocess_cls, letterbox from harness.adapters import calibration_dir from harness.registry import model_row row = model_row(a.model) if a.model else {"task": a.task, "input_spec": a.input_spec, "model_id": ""} if a.model and (row["task"] != a.task or row["input_spec"] != a.input_spec): raise SystemExit("--task/--input-spec must match the model recipe") ds = row.get("dataset") or TASK_DATASET[a.task] d = REG["datasets"][ds] pool = POOL / ds / d.get("version", "v1") shape = tuple(int(x) for x in a.input_spec.split("x")) out = calibration_dir(row) if a.model and out.exists(): manifest = json.loads((out / "manifest.json").read_text()) files = sorted((out / manifest["npy_dir"]).glob("*.npy")) digest = hashlib.sha256() for path in files: digest.update(path.read_bytes()) expected_recipe = row.get("preprocess") or "harness.eval.common (same as evaluator)" if (manifest.get("count") != a.n or len(files) != a.n or manifest.get("seed") != a.seed or json.dumps(manifest.get("preprocess"), sort_keys=True) != json.dumps(expected_recipe, sort_keys=True) or manifest.get("sha256") != digest.hexdigest() or (a.src == "train" and not manifest.get("source", "").startswith("train"))): raise ValueError(f"existing model calibration differs; refusing to overwrite {out}") print(f"{out} n={a.n} sha={digest.hexdigest()[:12]} (reused)") return 0 imgs, desc, source = select_images(ds, d, pool, a.n, a.seed, a.src) npy_dir = out / "npy_f32" # hb_compile 读 cal_data_dir 下的全部文件 → manifest.json 不能放同一目录;_f32 后缀是 OE 约定 if out.exists(): import shutil; shutil.rmtree(out) npy_dir.mkdir(parents=True, exist_ok=True) h = hashlib.sha256() for i, img in enumerate(imgs[:a.n]): if a.task == "cls": x = preprocess_cls(img, size=shape[-1], resize=round(shape[-1] * 256 / 224), model=a.model or "") elif a.task in ("seg", "det", "mde", "pose") and a.model: from harness.adapters import get transform = getattr(get(a.model), "preprocess", None) if a.task in ("seg", "mde") and transform is None: raise ValueError(f"no {a.task} preprocessing recipe for {a.model}") x = transform(img, a.input_spec) if transform else letterbox(img, shape[-1])[0] else: x = letterbox(img, shape[-1])[0] assert x.shape == shape, (x.shape, shape) fp = npy_dir / f"{i:05d}.npy" np.save(fp, x.astype(np.float32)) h.update(fp.read_bytes()) (out / "manifest.json").write_text(json.dumps({ "task": a.task, "input_spec": a.input_spec, "count": len(imgs[:a.n]), "seed": a.seed, "source": source, "dataset": ds, "sha256": h.hexdigest(), "files": desc[:a.n], "preprocess": row.get("preprocess") or "harness.eval.common (same as evaluator)", "layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"}, indent=2)) print(f"{out} n={len(imgs[:a.n])} source={source} sha={h.hexdigest()[:12]}") return 0 if __name__ == "__main__": sys.exit(main())