data-gen-bpu/scripts/make_t1.py

121 lines
5.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""T1 smoke 子集:从 pool 固定 seed 抽样 → data/t1/<ds>/<seed>.tar(单包,避开 JuiceFS 小文件)。
python scripts/make_t1.py --dataset imagenet1k [--dataset coco2017]
tar 内:manifest.json(items 清单 + 内容 sha)、images/…、(det/iseg/pose)annotations.json(COCO 子集)。
tar 确定性:条目按名排序、mtime=0、uid/gid=0 → 同一 pool 同一 seed 产出同一 sha。
外部 data/t1/<ds>/manifest.json 记 tar 文件 sha256(job.json 钉的就是它)。
"""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import random
import sys
import tarfile
from collections import defaultdict
from pathlib import Path
import yaml
BENCH = Path(__file__).resolve().parent.parent
REG = yaml.safe_load((BENCH / "registry" / "datasets.yaml").read_text())
POOL = BENCH / REG.get("pool_root", "data/pool")
T1 = BENCH / "data" / "t1"
def _add(tar: tarfile.TarFile, name: str, data: bytes) -> None:
ti = tarfile.TarInfo(name)
ti.size = len(data); ti.mtime = 0; ti.uid = ti.gid = 0; ti.uname = ti.gname = ""
tar.addfile(ti, io.BytesIO(data))
def items_imagenet(pool: Path, n: int, seed: int, stratify: str) -> list[dict]:
labels = json.loads((pool / "labels.json").read_text())
by = defaultdict(list)
for fn, lab in labels.items():
by[lab].append(fn)
rng = random.Random(seed)
items = []
if stratify == "label":
per = max(1, n // len(by))
for lab in sorted(by):
fs = sorted(by[lab]); rng.shuffle(fs)
items += [{"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": lab} for f in fs[:per]]
else:
fs = sorted(labels); rng.shuffle(fs)
items = [{"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": labels[f]} for f in fs[:n]]
return items[:n]
def items_coco(pool: Path, n: int, seed: int) -> tuple[list[dict], dict]:
ann_p = next(pool.rglob("instances_val2017.json"))
ann = json.loads(ann_p.read_text())
imgs = sorted(ann["images"], key=lambda x: x["id"])
rng = random.Random(seed); rng.shuffle(imgs)
sel = sorted(imgs[:n], key=lambda x: x["id"])
ids = {im["id"] for im in sel}
sub = {"images": sel, "annotations": [a for a in ann["annotations"] if a["image_id"] in ids],
"categories": ann["categories"], "info": ann.get("info", {}), "licenses": ann.get("licenses", [])}
# 关键点标注也带上(pose 共用)
kp = next(pool.rglob("person_keypoints_val2017.json"), None)
if kp:
k = json.loads(kp.read_text())
sub["keypoints"] = {"annotations": [a for a in k["annotations"] if a["image_id"] in ids], "categories": k["categories"]}
img_dir = next(pool.rglob("val2017"))
items = [{"id": im["id"], "file": f"images/{im['file_name']}", "src": str((img_dir / im["file_name"]).relative_to(pool))} for im in sel]
cat_map = {str(i): c["id"] for i, c in enumerate(sorted(ann["categories"], key=lambda c: c["id"]))}
return items, {"annotations": sub, "cat_map": cat_map}
def build(ds: str) -> Path:
d = REG["datasets"][ds]
ver = d.get("version", "v1")
pool = POOL / ds / ver
pm = json.loads((pool / "manifest.json").read_text())
t1 = d.get("t1") or {}
n, seed = t1.get("n"), t1.get("seed", 20260914)
extra = {}
if ds == "imagenet1k":
items = items_imagenet(pool, n, seed, t1.get("stratify", "label"))
elif ds == "coco2017":
items, extra = items_coco(pool, n, seed)
else:
# 通用:pool 下全部文件(n=all)或随机 n 个;标注交给各任务评测器按 src 定位
files = sorted(str(p.relative_to(pool)) for p in pool.rglob("*") if p.is_file() and p.name != "manifest.json")
if n not in (None, "all", 0):
random.Random(seed).shuffle(files); files = sorted(files[:n])
items = [{"id": f, "file": f"files/{f}", "src": f} for f in files]
content_sha = hashlib.sha256(json.dumps([i["src"] for i in items]).encode() + pm["sha256"].encode()).hexdigest()
man = {"dataset": ds, "version": ver, "seed": seed, "n": len(items), "pool_sha256": pm["sha256"],
"sha256": content_sha, "items": items, **({"cat_map": extra["cat_map"]} if "cat_map" in extra else {})}
out = T1 / ds / f"{seed}.tar"
out.parent.mkdir(parents=True, exist_ok=True)
with tarfile.open(out, "w") as tar:
_add(tar, "manifest.json", json.dumps(man, ensure_ascii=False).encode())
if "annotations" in extra:
_add(tar, "annotations.json", json.dumps(extra["annotations"]).encode())
for it in items:
_add(tar, it["file"], (pool / it["src"]).read_bytes())
fsha = hashlib.sha256(out.read_bytes()).hexdigest()
(out.parent / "manifest.json").write_text(json.dumps({"dataset": ds, "seed": seed, "n": len(items), "tar": out.name,
"sha256": fsha, "content_sha256": content_sha,
"pool_sha256": pm["sha256"]}, indent=2))
print(f"{ds}: {out} n={len(items)} sha={fsha[:12]}")
return out
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--dataset", action="append", required=True)
a = ap.parse_args()
for ds in a.dataset:
build(ds)
return 0
if __name__ == "__main__":
sys.exit(main())