"""T1 smoke 子集:从 pool 固定 seed 抽样 → data/t1//.tar(单包,避开 JuiceFS 小文件)。 python scripts/make_t1.py --dataset imagenet1k [--dataset coco2017] python scripts/make_t1.py --dataset imagenet1k --n 100 --seed 20260914 显式指定 n/seed 时默认写入独立的 n-s/ 目录;--out-dir 可指定该目录。 不传覆盖参数时仍使用 registry 的规格及旧路径。 tar 内:manifest.json(items 清单 + 内容 sha)、images/…、(det/iseg/pose)annotations.json(COCO 子集)。 tar 确定性:条目按名排序、mtime=0、uid/gid=0 → 同一 pool 同一 seed 产出同一 sha。 外部 data/t1//manifest.json 记 tar 文件 sha256(job.json 钉的就是它)。 """ from __future__ import annotations import argparse import hashlib import io import json import random import sys import tarfile from collections import defaultdict from pathlib import Path import yaml BENCH = Path(__file__).resolve().parent.parent REG = yaml.safe_load((BENCH / "registry" / "datasets.yaml").read_text()) POOL = BENCH / REG.get("pool_root", "data/pool") T1 = BENCH / "data" / "t1" def _add(tar: tarfile.TarFile, name: str, data: bytes) -> None: ti = tarfile.TarInfo(name) ti.size = len(data); ti.mtime = 0; ti.uid = ti.gid = 0; ti.uname = ti.gname = "" tar.addfile(ti, io.BytesIO(data)) def items_imagenet(pool: Path, n: int, seed: int, stratify: str) -> list[dict]: labels = json.loads((pool / "labels.json").read_text()) _check_count(n, len(labels)) by = defaultdict(list) for fn, lab in labels.items(): by[lab].append(fn) rng = random.Random(seed) items = [] if stratify == "label": per = n // len(by) remaining = {} for lab in sorted(by): fs = sorted(by[lab]); rng.shuffle(fs) items += [{"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": lab} for f in fs[:per]] remaining[lab] = fs[per:] # 不足一轮时随机选择类别;余数及样本不足的类别也按随机轮次补齐。 # n 可整除类别数且各类充足时,保持旧 2000 张的选样和顺序。 while len(items) < n: labs = [lab for lab, fs in remaining.items() if fs] rng.shuffle(labs) for lab in labs[:n - len(items)]: f = remaining[lab].pop() items.append({"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": lab}) else: fs = sorted(labels); rng.shuffle(fs) items = [{"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": labels[f]} for f in fs[:n]] return items def _check_count(n: int, available: int) -> None: if not isinstance(n, int) or isinstance(n, bool) or n < 1: raise ValueError("n must be a positive integer") if n > available: raise ValueError(f"requested {n} samples, only {available} available") def items_coco(pool: Path, n: int, seed: int) -> tuple[list[dict], dict]: ann_p = next(pool.rglob("instances_val2017.json")) ann = json.loads(ann_p.read_text()) imgs = sorted(ann["images"], key=lambda x: x["id"]) _check_count(n, len(imgs)) rng = random.Random(seed); rng.shuffle(imgs) sel = sorted(imgs[:n], key=lambda x: x["id"]) ids = {im["id"] for im in sel} sub = {"images": sel, "annotations": [a for a in ann["annotations"] if a["image_id"] in ids], "categories": ann["categories"], "info": ann.get("info", {}), "licenses": ann.get("licenses", [])} # 关键点标注也带上(pose 共用) kp = next(pool.rglob("person_keypoints_val2017.json"), None) if kp: k = json.loads(kp.read_text()) sub["keypoints"] = {"annotations": [a for a in k["annotations"] if a["image_id"] in ids], "categories": k["categories"]} img_dir = next(pool.rglob("val2017")) items = [{"id": im["id"], "file": f"images/{im['file_name']}", "src": str((img_dir / im["file_name"]).relative_to(pool))} for im in sel] cat_map = {str(i): c["id"] for i, c in enumerate(sorted(ann["categories"], key=lambda c: c["id"]))} return items, {"annotations": sub, "cat_map": cat_map} def items_segmentation(pool: Path, ds: str, n: int, seed: int) -> list[dict]: """Keep image/label pairs together; never sample PNG labels as input images.""" if ds == "cityscapes": files = sorted(pool.glob("leftImg8bit/val/*/*_leftImg8bit.png")) pairs = [(p, pool / "gtFine/val" / p.parent.name / p.name.replace("_leftImg8bit.png", "_gtFine_labelIds.png")) for p in files] else: pairs = [(p, pool / "annotations/validation" / (p.stem + ".png")) for p in sorted((pool / "images/validation").glob("*.jpg"))] _check_count(n, len(pairs)) random.Random(seed).shuffle(pairs) items = [] for image, label in sorted(pairs[:n]): if not label.is_file(): raise FileNotFoundError(f"missing segmentation label: {label}") src, label_src = str(image.relative_to(pool)), str(label.relative_to(pool)) items.append({"id": image.stem, "file": "images/" + image.name, "src": src, "label_file": "labels/" + label.name, "label_src": label_src, "label_format": ds + "_labelIds"}) return items def build(ds: str, *, n: int | None = None, seed: int | None = None, out_dir: Path | None = None) -> Path: d = REG["datasets"][ds] ver = d.get("version", "v1") pool = POOL / ds / ver pm = json.loads((pool / "manifest.json").read_text()) t1 = d.get("t1") or {} custom = n is not None or seed is not None or out_dir is not None if n is not None: _check_count(n, n) n = t1.get("n") if n is None else n seed = t1.get("seed", 20260914) if seed is None else seed legacy_dir = T1 / ds destination = Path(out_dir) if out_dir is not None else ( legacy_dir / f"n{n}-s{seed}" if custom else legacy_dir) if custom and destination.resolve() == legacy_dir.resolve(): raise ValueError("custom subsets require a separate output directory; preserve the legacy T1") extra = {} if ds == "imagenet1k": items = items_imagenet(pool, n, seed, t1.get("stratify", "label")) elif ds == "coco2017": items, extra = items_coco(pool, n, seed) elif ds in ("cityscapes", "ade20k"): items = items_segmentation(pool, ds, n, seed) else: # 通用:pool 下全部文件(n=all)或随机 n 个;标注交给各任务评测器按 src 定位 files = sorted(str(p.relative_to(pool)) for p in pool.rglob("*") if p.is_file() and p.name != "manifest.json") if n not in (None, "all", 0): _check_count(n, len(files)) random.Random(seed).shuffle(files); files = sorted(files[:n]) items = [{"id": f, "file": f"files/{f}", "src": f} for f in files] content_sha = hashlib.sha256(json.dumps([i["src"] for i in items]).encode() + pm["sha256"].encode()).hexdigest() man = {"dataset": ds, "version": ver, "seed": seed, "n": len(items), "pool_sha256": pm["sha256"], "sha256": content_sha, "items": items, **({"cat_map": extra["cat_map"]} if "cat_map" in extra else {})} identity = {"eval_set_id": f"{ds}-n{len(items)}-{content_sha[:12]}"} if custom else {} man.update(identity) out = destination / f"{seed}.tar" manifest_path = destination / "manifest.json" if custom and manifest_path.exists(): previous = json.loads(manifest_path.read_text()) if previous.get("content_sha256") != content_sha or previous.get("seed") != seed: raise FileExistsError(f"{destination} already contains a different subset; choose a new directory") if (previous.get("tar") != out.name or previous.get("dataset") != ds or previous.get("n") != len(items)): raise ValueError(f"existing subset manifest is inconsistent: {manifest_path}") if not out.is_file(): raise ValueError(f"existing subset package is missing: {out}") fsha = hashlib.sha256(out.read_bytes()).hexdigest() if fsha != previous.get("sha256"): raise ValueError(f"existing subset package hash mismatch: {out}") print(f"{ds}: {out} n={len(items)} sha={fsha[:12]} (reused)") return out if custom and out.exists(): raise FileExistsError(f"subset package exists without its manifest: {out}") out.parent.mkdir(parents=True, exist_ok=True) with tarfile.open(out, "w") as tar: _add(tar, "manifest.json", json.dumps(man, ensure_ascii=False).encode()) if "annotations" in extra: _add(tar, "annotations.json", json.dumps(extra["annotations"]).encode()) for it in items: _add(tar, it["file"], (pool / it["src"]).read_bytes()) if "label_file" in it: _add(tar, it["label_file"], (pool / it["label_src"]).read_bytes()) fsha = hashlib.sha256(out.read_bytes()).hexdigest() manifest_path.write_text(json.dumps({"dataset": ds, "seed": seed, "n": len(items), "tar": out.name, "sha256": fsha, "content_sha256": content_sha, "pool_sha256": pm["sha256"], **identity}, indent=2)) print(f"{ds}: {out} n={len(items)} sha={fsha[:12]}") return out def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--dataset", action="append", required=True) ap.add_argument("--n", type=int, help="exact sample count; registry default when omitted") ap.add_argument("--seed", type=int, help="fixed sampling seed; registry default when omitted") ap.add_argument("--out-dir", type=Path, help="independent directory for tar and manifest (one dataset)") a = ap.parse_args() if a.out_dir is not None and len(a.dataset) != 1: ap.error("--out-dir requires exactly one --dataset") for ds in a.dataset: build(ds, n=a.n, seed=a.seed, out_dir=a.out_dir) return 0 if __name__ == "__main__": sys.exit(main())