205 lines
9.9 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""T1 smoke 子集:从 pool 固定 seed 抽样 → data/t1/<ds>/<seed>.tar(单包,避开 JuiceFS 小文件)。
python scripts/make_t1.py --dataset imagenet1k [--dataset coco2017]
python scripts/make_t1.py --dataset imagenet1k --n 100 --seed 20260914
显式指定 n/seed 时默认写入独立的 n<N>-s<seed>/ 目录;--out-dir 可指定该目录。
不传覆盖参数时仍使用 registry 的规格及旧路径。
tar 内:manifest.json(items 清单 + 内容 sha)、images/…、(det/iseg/pose)annotations.json(COCO 子集)。
tar 确定性:条目按名排序、mtime=0、uid/gid=0 → 同一 pool 同一 seed 产出同一 sha。
外部 data/t1/<ds>/manifest.json 记 tar 文件 sha256(job.json 钉的就是它)。
"""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import random
import sys
import tarfile
from collections import defaultdict
from pathlib import Path
import yaml
BENCH = Path(__file__).resolve().parent.parent
REG = yaml.safe_load((BENCH / "registry" / "datasets.yaml").read_text())
POOL = BENCH / REG.get("pool_root", "data/pool")
T1 = BENCH / "data" / "t1"
def _add(tar: tarfile.TarFile, name: str, data: bytes) -> None:
ti = tarfile.TarInfo(name)
ti.size = len(data); ti.mtime = 0; ti.uid = ti.gid = 0; ti.uname = ti.gname = ""
tar.addfile(ti, io.BytesIO(data))
def items_imagenet(pool: Path, n: int, seed: int, stratify: str) -> list[dict]:
labels = json.loads((pool / "labels.json").read_text())
_check_count(n, len(labels))
by = defaultdict(list)
for fn, lab in labels.items():
by[lab].append(fn)
rng = random.Random(seed)
items = []
if stratify == "label":
per = n // len(by)
remaining = {}
for lab in sorted(by):
fs = sorted(by[lab]); rng.shuffle(fs)
items += [{"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": lab} for f in fs[:per]]
remaining[lab] = fs[per:]
# 不足一轮时随机选择类别;余数及样本不足的类别也按随机轮次补齐。
# n 可整除类别数且各类充足时,保持旧 2000 张的选样和顺序。
while len(items) < n:
labs = [lab for lab, fs in remaining.items() if fs]
rng.shuffle(labs)
for lab in labs[:n - len(items)]:
f = remaining[lab].pop()
items.append({"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": lab})
else:
fs = sorted(labels); rng.shuffle(fs)
items = [{"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": labels[f]} for f in fs[:n]]
return items
def _check_count(n: int, available: int) -> None:
if not isinstance(n, int) or isinstance(n, bool) or n < 1:
raise ValueError("n must be a positive integer")
if n > available:
raise ValueError(f"requested {n} samples, only {available} available")
def items_coco(pool: Path, n: int, seed: int) -> tuple[list[dict], dict]:
ann_p = next(pool.rglob("instances_val2017.json"))
ann = json.loads(ann_p.read_text())
imgs = sorted(ann["images"], key=lambda x: x["id"])
_check_count(n, len(imgs))
rng = random.Random(seed); rng.shuffle(imgs)
sel = sorted(imgs[:n], key=lambda x: x["id"])
ids = {im["id"] for im in sel}
sub = {"images": sel, "annotations": [a for a in ann["annotations"] if a["image_id"] in ids],
"categories": ann["categories"], "info": ann.get("info", {}), "licenses": ann.get("licenses", [])}
# 关键点标注也带上(pose 共用)
kp = next(pool.rglob("person_keypoints_val2017.json"), None)
if kp:
k = json.loads(kp.read_text())
sub["keypoints"] = {"annotations": [a for a in k["annotations"] if a["image_id"] in ids], "categories": k["categories"]}
img_dir = next(pool.rglob("val2017"))
items = [{"id": im["id"], "file": f"images/{im['file_name']}", "src": str((img_dir / im["file_name"]).relative_to(pool))} for im in sel]
cat_map = {str(i): c["id"] for i, c in enumerate(sorted(ann["categories"], key=lambda c: c["id"]))}
return items, {"annotations": sub, "cat_map": cat_map}
def items_segmentation(pool: Path, ds: str, n: int, seed: int) -> list[dict]:
"""Keep image/label pairs together; never sample PNG labels as input images."""
if ds == "cityscapes":
files = sorted(pool.glob("leftImg8bit/val/*/*_leftImg8bit.png"))
pairs = [(p, pool / "gtFine/val" / p.parent.name / p.name.replace("_leftImg8bit.png", "_gtFine_labelIds.png"))
for p in files]
else:
pairs = [(p, pool / "annotations/validation" / (p.stem + ".png"))
for p in sorted((pool / "images/validation").glob("*.jpg"))]
_check_count(n, len(pairs))
random.Random(seed).shuffle(pairs)
items = []
for image, label in sorted(pairs[:n]):
if not label.is_file():
raise FileNotFoundError(f"missing segmentation label: {label}")
src, label_src = str(image.relative_to(pool)), str(label.relative_to(pool))
items.append({"id": image.stem, "file": "images/" + image.name, "src": src,
"label_file": "labels/" + label.name, "label_src": label_src,
"label_format": ds + "_labelIds"})
return items
def build(ds: str, *, n: int | None = None, seed: int | None = None,
out_dir: Path | None = None) -> Path:
d = REG["datasets"][ds]
ver = d.get("version", "v1")
pool = POOL / ds / ver
pm = json.loads((pool / "manifest.json").read_text())
t1 = d.get("t1") or {}
custom = n is not None or seed is not None or out_dir is not None
if n is not None:
_check_count(n, n)
n = t1.get("n") if n is None else n
seed = t1.get("seed", 20260914) if seed is None else seed
legacy_dir = T1 / ds
destination = Path(out_dir) if out_dir is not None else (
legacy_dir / f"n{n}-s{seed}" if custom else legacy_dir)
if custom and destination.resolve() == legacy_dir.resolve():
raise ValueError("custom subsets require a separate output directory; preserve the legacy T1")
extra = {}
if ds == "imagenet1k":
items = items_imagenet(pool, n, seed, t1.get("stratify", "label"))
elif ds == "coco2017":
items, extra = items_coco(pool, n, seed)
elif ds in ("cityscapes", "ade20k"):
items = items_segmentation(pool, ds, n, seed)
else:
# 通用:pool 下全部文件(n=all)或随机 n 个;标注交给各任务评测器按 src 定位
files = sorted(str(p.relative_to(pool)) for p in pool.rglob("*") if p.is_file() and p.name != "manifest.json")
if n not in (None, "all", 0):
_check_count(n, len(files))
random.Random(seed).shuffle(files); files = sorted(files[:n])
items = [{"id": f, "file": f"files/{f}", "src": f} for f in files]
content_sha = hashlib.sha256(json.dumps([i["src"] for i in items]).encode() + pm["sha256"].encode()).hexdigest()
man = {"dataset": ds, "version": ver, "seed": seed, "n": len(items), "pool_sha256": pm["sha256"],
"sha256": content_sha, "items": items, **({"cat_map": extra["cat_map"]} if "cat_map" in extra else {})}
identity = {"eval_set_id": f"{ds}-n{len(items)}-{content_sha[:12]}"} if custom else {}
man.update(identity)
out = destination / f"{seed}.tar"
manifest_path = destination / "manifest.json"
if custom and manifest_path.exists():
previous = json.loads(manifest_path.read_text())
if previous.get("content_sha256") != content_sha or previous.get("seed") != seed:
raise FileExistsError(f"{destination} already contains a different subset; choose a new directory")
if (previous.get("tar") != out.name or previous.get("dataset") != ds
or previous.get("n") != len(items)):
raise ValueError(f"existing subset manifest is inconsistent: {manifest_path}")
if not out.is_file():
raise ValueError(f"existing subset package is missing: {out}")
fsha = hashlib.sha256(out.read_bytes()).hexdigest()
if fsha != previous.get("sha256"):
raise ValueError(f"existing subset package hash mismatch: {out}")
print(f"{ds}: {out} n={len(items)} sha={fsha[:12]} (reused)")
return out
if custom and out.exists():
raise FileExistsError(f"subset package exists without its manifest: {out}")
out.parent.mkdir(parents=True, exist_ok=True)
with tarfile.open(out, "w") as tar:
_add(tar, "manifest.json", json.dumps(man, ensure_ascii=False).encode())
if "annotations" in extra:
_add(tar, "annotations.json", json.dumps(extra["annotations"]).encode())
for it in items:
_add(tar, it["file"], (pool / it["src"]).read_bytes())
if "label_file" in it:
_add(tar, it["label_file"], (pool / it["label_src"]).read_bytes())
fsha = hashlib.sha256(out.read_bytes()).hexdigest()
manifest_path.write_text(json.dumps({"dataset": ds, "seed": seed, "n": len(items), "tar": out.name,
"sha256": fsha, "content_sha256": content_sha,
"pool_sha256": pm["sha256"], **identity}, indent=2))
print(f"{ds}: {out} n={len(items)} sha={fsha[:12]}")
return out
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--dataset", action="append", required=True)
ap.add_argument("--n", type=int, help="exact sample count; registry default when omitted")
ap.add_argument("--seed", type=int, help="fixed sampling seed; registry default when omitted")
ap.add_argument("--out-dir", type=Path, help="independent directory for tar and manifest (one dataset)")
a = ap.parse_args()
if a.out_dir is not None and len(a.dataset) != 1:
ap.error("--out-dir requires exactly one --dataset")
for ds in a.dataset:
build(ds, n=a.n, seed=a.seed, out_dir=a.out_dir)
return 0
if __name__ == "__main__":
sys.exit(main())