205 lines
9.9 KiB
Python
205 lines
9.9 KiB
Python
"""T1 smoke 子集:从 pool 固定 seed 抽样 → data/t1/<ds>/<seed>.tar(单包,避开 JuiceFS 小文件)。
|
||
|
||
python scripts/make_t1.py --dataset imagenet1k [--dataset coco2017]
|
||
python scripts/make_t1.py --dataset imagenet1k --n 100 --seed 20260914
|
||
|
||
显式指定 n/seed 时默认写入独立的 n<N>-s<seed>/ 目录;--out-dir 可指定该目录。
|
||
不传覆盖参数时仍使用 registry 的规格及旧路径。
|
||
|
||
tar 内:manifest.json(items 清单 + 内容 sha)、images/…、(det/iseg/pose)annotations.json(COCO 子集)。
|
||
tar 确定性:条目按名排序、mtime=0、uid/gid=0 → 同一 pool 同一 seed 产出同一 sha。
|
||
外部 data/t1/<ds>/manifest.json 记 tar 文件 sha256(job.json 钉的就是它)。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import random
|
||
import sys
|
||
import tarfile
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
|
||
import yaml
|
||
|
||
BENCH = Path(__file__).resolve().parent.parent
|
||
REG = yaml.safe_load((BENCH / "registry" / "datasets.yaml").read_text())
|
||
POOL = BENCH / REG.get("pool_root", "data/pool")
|
||
T1 = BENCH / "data" / "t1"
|
||
|
||
|
||
def _add(tar: tarfile.TarFile, name: str, data: bytes) -> None:
|
||
ti = tarfile.TarInfo(name)
|
||
ti.size = len(data); ti.mtime = 0; ti.uid = ti.gid = 0; ti.uname = ti.gname = ""
|
||
tar.addfile(ti, io.BytesIO(data))
|
||
|
||
|
||
def items_imagenet(pool: Path, n: int, seed: int, stratify: str) -> list[dict]:
|
||
labels = json.loads((pool / "labels.json").read_text())
|
||
_check_count(n, len(labels))
|
||
by = defaultdict(list)
|
||
for fn, lab in labels.items():
|
||
by[lab].append(fn)
|
||
rng = random.Random(seed)
|
||
items = []
|
||
if stratify == "label":
|
||
per = n // len(by)
|
||
remaining = {}
|
||
for lab in sorted(by):
|
||
fs = sorted(by[lab]); rng.shuffle(fs)
|
||
items += [{"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": lab} for f in fs[:per]]
|
||
remaining[lab] = fs[per:]
|
||
# 不足一轮时随机选择类别;余数及样本不足的类别也按随机轮次补齐。
|
||
# n 可整除类别数且各类充足时,保持旧 2000 张的选样和顺序。
|
||
while len(items) < n:
|
||
labs = [lab for lab, fs in remaining.items() if fs]
|
||
rng.shuffle(labs)
|
||
for lab in labs[:n - len(items)]:
|
||
f = remaining[lab].pop()
|
||
items.append({"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": lab})
|
||
else:
|
||
fs = sorted(labels); rng.shuffle(fs)
|
||
items = [{"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": labels[f]} for f in fs[:n]]
|
||
return items
|
||
|
||
|
||
def _check_count(n: int, available: int) -> None:
|
||
if not isinstance(n, int) or isinstance(n, bool) or n < 1:
|
||
raise ValueError("n must be a positive integer")
|
||
if n > available:
|
||
raise ValueError(f"requested {n} samples, only {available} available")
|
||
|
||
|
||
def items_coco(pool: Path, n: int, seed: int) -> tuple[list[dict], dict]:
|
||
ann_p = next(pool.rglob("instances_val2017.json"))
|
||
ann = json.loads(ann_p.read_text())
|
||
imgs = sorted(ann["images"], key=lambda x: x["id"])
|
||
_check_count(n, len(imgs))
|
||
rng = random.Random(seed); rng.shuffle(imgs)
|
||
sel = sorted(imgs[:n], key=lambda x: x["id"])
|
||
ids = {im["id"] for im in sel}
|
||
sub = {"images": sel, "annotations": [a for a in ann["annotations"] if a["image_id"] in ids],
|
||
"categories": ann["categories"], "info": ann.get("info", {}), "licenses": ann.get("licenses", [])}
|
||
# 关键点标注也带上(pose 共用)
|
||
kp = next(pool.rglob("person_keypoints_val2017.json"), None)
|
||
if kp:
|
||
k = json.loads(kp.read_text())
|
||
sub["keypoints"] = {"annotations": [a for a in k["annotations"] if a["image_id"] in ids], "categories": k["categories"]}
|
||
img_dir = next(pool.rglob("val2017"))
|
||
items = [{"id": im["id"], "file": f"images/{im['file_name']}", "src": str((img_dir / im["file_name"]).relative_to(pool))} for im in sel]
|
||
cat_map = {str(i): c["id"] for i, c in enumerate(sorted(ann["categories"], key=lambda c: c["id"]))}
|
||
return items, {"annotations": sub, "cat_map": cat_map}
|
||
|
||
|
||
def items_segmentation(pool: Path, ds: str, n: int, seed: int) -> list[dict]:
|
||
"""Keep image/label pairs together; never sample PNG labels as input images."""
|
||
if ds == "cityscapes":
|
||
files = sorted(pool.glob("leftImg8bit/val/*/*_leftImg8bit.png"))
|
||
pairs = [(p, pool / "gtFine/val" / p.parent.name / p.name.replace("_leftImg8bit.png", "_gtFine_labelIds.png"))
|
||
for p in files]
|
||
else:
|
||
pairs = [(p, pool / "annotations/validation" / (p.stem + ".png"))
|
||
for p in sorted((pool / "images/validation").glob("*.jpg"))]
|
||
_check_count(n, len(pairs))
|
||
random.Random(seed).shuffle(pairs)
|
||
items = []
|
||
for image, label in sorted(pairs[:n]):
|
||
if not label.is_file():
|
||
raise FileNotFoundError(f"missing segmentation label: {label}")
|
||
src, label_src = str(image.relative_to(pool)), str(label.relative_to(pool))
|
||
items.append({"id": image.stem, "file": "images/" + image.name, "src": src,
|
||
"label_file": "labels/" + label.name, "label_src": label_src,
|
||
"label_format": ds + "_labelIds"})
|
||
return items
|
||
|
||
|
||
def build(ds: str, *, n: int | None = None, seed: int | None = None,
|
||
out_dir: Path | None = None) -> Path:
|
||
d = REG["datasets"][ds]
|
||
ver = d.get("version", "v1")
|
||
pool = POOL / ds / ver
|
||
pm = json.loads((pool / "manifest.json").read_text())
|
||
t1 = d.get("t1") or {}
|
||
custom = n is not None or seed is not None or out_dir is not None
|
||
if n is not None:
|
||
_check_count(n, n)
|
||
n = t1.get("n") if n is None else n
|
||
seed = t1.get("seed", 20260914) if seed is None else seed
|
||
legacy_dir = T1 / ds
|
||
destination = Path(out_dir) if out_dir is not None else (
|
||
legacy_dir / f"n{n}-s{seed}" if custom else legacy_dir)
|
||
if custom and destination.resolve() == legacy_dir.resolve():
|
||
raise ValueError("custom subsets require a separate output directory; preserve the legacy T1")
|
||
extra = {}
|
||
if ds == "imagenet1k":
|
||
items = items_imagenet(pool, n, seed, t1.get("stratify", "label"))
|
||
elif ds == "coco2017":
|
||
items, extra = items_coco(pool, n, seed)
|
||
elif ds in ("cityscapes", "ade20k"):
|
||
items = items_segmentation(pool, ds, n, seed)
|
||
else:
|
||
# 通用:pool 下全部文件(n=all)或随机 n 个;标注交给各任务评测器按 src 定位
|
||
files = sorted(str(p.relative_to(pool)) for p in pool.rglob("*") if p.is_file() and p.name != "manifest.json")
|
||
if n not in (None, "all", 0):
|
||
_check_count(n, len(files))
|
||
random.Random(seed).shuffle(files); files = sorted(files[:n])
|
||
items = [{"id": f, "file": f"files/{f}", "src": f} for f in files]
|
||
content_sha = hashlib.sha256(json.dumps([i["src"] for i in items]).encode() + pm["sha256"].encode()).hexdigest()
|
||
man = {"dataset": ds, "version": ver, "seed": seed, "n": len(items), "pool_sha256": pm["sha256"],
|
||
"sha256": content_sha, "items": items, **({"cat_map": extra["cat_map"]} if "cat_map" in extra else {})}
|
||
identity = {"eval_set_id": f"{ds}-n{len(items)}-{content_sha[:12]}"} if custom else {}
|
||
man.update(identity)
|
||
out = destination / f"{seed}.tar"
|
||
manifest_path = destination / "manifest.json"
|
||
if custom and manifest_path.exists():
|
||
previous = json.loads(manifest_path.read_text())
|
||
if previous.get("content_sha256") != content_sha or previous.get("seed") != seed:
|
||
raise FileExistsError(f"{destination} already contains a different subset; choose a new directory")
|
||
if (previous.get("tar") != out.name or previous.get("dataset") != ds
|
||
or previous.get("n") != len(items)):
|
||
raise ValueError(f"existing subset manifest is inconsistent: {manifest_path}")
|
||
if not out.is_file():
|
||
raise ValueError(f"existing subset package is missing: {out}")
|
||
fsha = hashlib.sha256(out.read_bytes()).hexdigest()
|
||
if fsha != previous.get("sha256"):
|
||
raise ValueError(f"existing subset package hash mismatch: {out}")
|
||
print(f"{ds}: {out} n={len(items)} sha={fsha[:12]} (reused)")
|
||
return out
|
||
if custom and out.exists():
|
||
raise FileExistsError(f"subset package exists without its manifest: {out}")
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
with tarfile.open(out, "w") as tar:
|
||
_add(tar, "manifest.json", json.dumps(man, ensure_ascii=False).encode())
|
||
if "annotations" in extra:
|
||
_add(tar, "annotations.json", json.dumps(extra["annotations"]).encode())
|
||
for it in items:
|
||
_add(tar, it["file"], (pool / it["src"]).read_bytes())
|
||
if "label_file" in it:
|
||
_add(tar, it["label_file"], (pool / it["label_src"]).read_bytes())
|
||
fsha = hashlib.sha256(out.read_bytes()).hexdigest()
|
||
manifest_path.write_text(json.dumps({"dataset": ds, "seed": seed, "n": len(items), "tar": out.name,
|
||
"sha256": fsha, "content_sha256": content_sha,
|
||
"pool_sha256": pm["sha256"], **identity}, indent=2))
|
||
print(f"{ds}: {out} n={len(items)} sha={fsha[:12]}")
|
||
return out
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--dataset", action="append", required=True)
|
||
ap.add_argument("--n", type=int, help="exact sample count; registry default when omitted")
|
||
ap.add_argument("--seed", type=int, help="fixed sampling seed; registry default when omitted")
|
||
ap.add_argument("--out-dir", type=Path, help="independent directory for tar and manifest (one dataset)")
|
||
a = ap.parse_args()
|
||
if a.out_dir is not None and len(a.dataset) != 1:
|
||
ap.error("--out-dir requires exactly one --dataset")
|
||
for ds in a.dataset:
|
||
build(ds, n=a.n, seed=a.seed, out_dir=a.out_dir)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|