data-gen-bpu/scripts/ingest_pool.py

224 lines
9.5 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""数据入池:按 registry/datasets.yaml 的有序来源,把数据集复制进 data/pool/<ds>/<version>/,写 manifest.json,只读。
python scripts/ingest_pool.py --dataset imagenet1k [--dataset coco2017 …] [--all-local]
python scripts/ingest_pool.py --dataset imagenet1k --verify # 只校验已入池的 manifest
复制,不软链——别人的目录随时可能被清。入池后 chmod -R a-w。
**data/ 必须在本地盘**:JuiceFS 小文件写入实测 ~0.5 s/文件,5 万张要 7 小时;本地盘几分钟。
bench/data -> /home/<user>/bench-data 软链(README §0)。"复制不软链"说的是不要软链到别人的目录。
pool 布局(各任务评测器依赖):
imagenet1k/<ver>/images/<idx>_<label>.JPEG + labels.json {file: label}
coco2017/<ver>/val2017/*.jpg + annotations/instances_val2017.json, person_keypoints_val2017.json
cityscapes/<ver>/leftImg8bit/val/… + gtFine/val/…
kitti2015/<ver>/training/{image_2,image_3,disp_occ_0,flow_occ}/…
"""
from __future__ import annotations
import argparse
import fnmatch
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tarfile
import time
import zipfile
from pathlib import Path
import yaml
BENCH = Path(__file__).resolve().parent.parent
REG = yaml.safe_load((BENCH / "registry" / "datasets.yaml").read_text())
LOCAL_ROOT = Path(REG.get("local_root", "/mnt/data"))
POOL = BENCH / REG.get("pool_root", "data/pool")
def sha256_tree(root: Path) -> tuple[str, int]:
"""确定性:按相对路径排序,hash(path + 文件 sha256)。返回 (sha, 文件数)。"""
h = hashlib.sha256()
n = 0
for p in sorted(x for x in root.rglob("*") if x.is_file() and x.name != "manifest.json"):
rel = str(p.relative_to(root)).encode()
fh = hashlib.sha256()
with open(p, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
fh.update(chunk)
h.update(rel); h.update(fh.digest()); n += 1
return h.hexdigest(), n
def write_manifest(dst: Path, ds: str, ver: str, source: dict, extra: dict | None = None) -> dict:
sha, n = sha256_tree(dst)
man = {"dataset": ds, "version": ver, "sha256": sha, "count": n, "source": source,
"ingested_at": time.strftime("%Y-%m-%d %H:%M:%S"), **(extra or {})}
(dst / "manifest.json").write_text(json.dumps(man, indent=2, ensure_ascii=False))
subprocess.run(["chmod", "-R", "a-w", str(dst)], check=False)
return man
# ---------------------------------------------------------------- sources
def src_local_parquet(src: dict, dst: Path) -> dict:
"""HF parquet(image bytes + label)→ images/ + labels.json。imagenet 专用。"""
import io
try:
import pyarrow.parquet as pq
from PIL import Image
except ImportError:
sys.exit("需要 pyarrow + pillow:在任一 venv 里跑本脚本,或 pip install pyarrow pillow")
root = LOCAL_ROOT / src["path"]
shards = sorted(root.glob(src["glob"]))
if src.get("expect_shards") and len(shards) != src["expect_shards"]:
raise RuntimeError(f"parquet 片数 {len(shards)} != {src['expect_shards']}:{root}")
(dst / "images").mkdir(parents=True, exist_ok=True)
labels, idx = {}, 0
for sh in shards:
t = pq.read_table(sh)
cols = t.column_names
img_col = "image" if "image" in cols else cols[0]
lab_col = "label" if "label" in cols else cols[1]
for img, lab in zip(t.column(img_col).to_pylist(), t.column(lab_col).to_pylist()):
data = img.get("bytes") if isinstance(img, dict) else img
if data is None and isinstance(img, dict) and img.get("path"):
data = Path(img["path"]).read_bytes()
fn = f"{idx:06d}_{int(lab):04d}.JPEG"
# 原样落盘(parquet 里就是原始 JPEG 字节):不重编码,不改变像素;评测器 Image.open().convert("RGB") 统一处理 CMYK/PNG 伪装
(dst / "images" / fn).write_bytes(data)
labels[fn] = int(lab); idx += 1
print(f" {sh.name}: 累计 {idx}", file=sys.stderr)
(dst / "labels.json").write_text(json.dumps(labels))
return {"kind": "local_parquet", "path": str(root), "shards": len(shards), "images": idx}
def src_local_tar(src: dict, dst: Path) -> dict:
p = LOCAL_ROOT / src["path"]
print(f" 校验 {p} …", file=sys.stderr)
n = int(subprocess.run(f"tar -tzf '{p}' | grep -c -v '/$'", shell=True, capture_output=True, text=True).stdout.strip() or 0)
if src.get("expect") and n != src["expect"]:
raise RuntimeError(f"tar 条目 {n} != {src['expect']}(旁边有 .tmp 残片,可能下载不完整)")
(dst / "images").mkdir(parents=True, exist_ok=True)
with tarfile.open(p) as t:
t.extractall(dst / "images")
return {"kind": "local_tar", "path": str(p), "entries": n, "note": "labels.json 需从 classes.py / 官方 devkit 生成"}
def _annotations(src: dict, dst: Path) -> None:
for ann in src.get("annotations") or []:
ap = LOCAL_ROOT / ann
if ap.suffix == ".zip":
with zipfile.ZipFile(ap) as z:
for m in z.namelist():
if m.endswith(".json") and "val2017" in m:
z.extract(m, dst)
else:
shutil.copy2(ap, dst / ap.name)
def src_local_dir(src: dict, dst: Path) -> dict:
"""注意:JuiceFS 上按文件 rsync 实测只有 5–10 文件/s;有 zip 就优先 zip(--prefer local_zip)。"""
p = LOCAL_ROOT / src["path"]
if not p.exists():
raise FileNotFoundError(p)
tgt = dst / p.name
print(f" 复制 {p} → {tgt}", file=sys.stderr)
subprocess.run(["rsync", "-a", f"{p}/", f"{tgt}/"], check=True)
_annotations(src, dst)
return {"kind": "local_dir", "path": str(p), "annotations": src.get("annotations")}
def src_local_zip(src: dict, dst: Path) -> dict:
items = [src] + list(src.get("extra") or [])
done = []
for it in items:
p = LOCAL_ROOT / it["path"]
pat = it.get("extract_only")
print(f" 解压 {p} ({pat or 'all'})", file=sys.stderr)
with zipfile.ZipFile(p) as z:
names = [m for m in z.namelist() if not pat or fnmatch.fnmatch(m, pat)]
for m in names:
z.extract(m, dst)
done.append({"path": str(p), "n": len(names)})
_annotations(src, dst)
return {"kind": "local_zip", "items": done}
def src_hf(src: dict, dst: Path) -> dict:
if not src.get("repo"):
raise RuntimeError("hf 源没有 repo")
print(f" hf download {src['repo']} → {dst}", file=sys.stderr)
subprocess.run(["hf", "download", src["repo"], "--repo-type", "dataset", "--local-dir", str(dst)], check=True)
return {"kind": "hf", "repo": src["repo"]}
HANDLERS = {"local_parquet": src_local_parquet, "local_tar": src_local_tar, "local_dir": src_local_dir,
"local_zip": src_local_zip, "hf": src_hf}
def ingest(ds: str, force: bool = False, prefer: str | None = None) -> None:
d = REG["datasets"][ds]
ver = d.get("version", "v1")
dst = POOL / ds / ver
if (dst / "manifest.json").exists() and not force:
print(f"{ds}/{ver} 已入池(--force 重做)", file=sys.stderr); return
if dst.exists():
subprocess.run(["chmod", "-R", "u+w", str(dst)], check=False)
shutil.rmtree(dst)
dst.mkdir(parents=True)
errors = []
sources = sorted(d["sources"], key=lambda s: 0 if s["kind"] == prefer else 1) if prefer else d["sources"]
for src in sources:
h = HANDLERS.get(src["kind"])
if not h:
errors.append(f"{src['kind']}: 未实现(github/official 请手动下载到 {dst} 后 --verify)"); continue
try:
print(f"[{ds}] 尝试来源 {src['kind']} …", file=sys.stderr)
info = h(src, dst)
man = write_manifest(dst, ds, ver, info)
if d.get("expect_count") and man["count"] < d["expect_count"]:
print(f" WARN count {man['count']} < expect {d['expect_count']}", file=sys.stderr)
print(f"[{ds}] 入池完成 sha={man['sha256'][:12]} count={man['count']}", file=sys.stderr)
return
except Exception as e:
errors.append(f"{src['kind']}: {e}")
print(f" 失败:{e}", file=sys.stderr)
for x in dst.iterdir():
shutil.rmtree(x) if x.is_dir() else x.unlink()
raise SystemExit(f"[{ds}] 所有来源失败:\n " + "\n ".join(errors))
def verify(ds: str) -> bool:
d = REG["datasets"][ds]
dst = POOL / ds / d.get("version", "v1")
m = json.loads((dst / "manifest.json").read_text())
sha, n = sha256_tree(dst)
ok = sha == m["sha256"] and n == m["count"]
print(f"{ds}: {'OK' if ok else 'MISMATCH'} sha={sha[:12]} count={n}")
return ok
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--dataset", action="append", default=[])
ap.add_argument("--all-local", action="store_true", help="所有第一来源是 local_* 的数据集")
ap.add_argument("--verify", action="store_true")
ap.add_argument("--force", action="store_true")
ap.add_argument("--prefer", help="优先尝试的来源 kind,如 local_zip")
a = ap.parse_args()
names = list(a.dataset)
if a.all_local:
names += [k for k, v in REG["datasets"].items() if v["sources"][0]["kind"].startswith("local_") and k not in names]
if not names:
ap.error("--dataset 或 --all-local")
for ds in names:
if a.verify:
verify(ds)
else:
ingest(ds, a.force, a.prefer)
return 0
if __name__ == "__main__":
sys.exit(main())