224 lines
9.5 KiB
Python
224 lines
9.5 KiB
Python
"""数据入池:按 registry/datasets.yaml 的有序来源,把数据集复制进 data/pool/<ds>/<version>/,写 manifest.json,只读。
|
||
|
||
python scripts/ingest_pool.py --dataset imagenet1k [--dataset coco2017 …] [--all-local]
|
||
python scripts/ingest_pool.py --dataset imagenet1k --verify # 只校验已入池的 manifest
|
||
|
||
复制,不软链——别人的目录随时可能被清。入池后 chmod -R a-w。
|
||
**data/ 必须在本地盘**:JuiceFS 小文件写入实测 ~0.5 s/文件,5 万张要 7 小时;本地盘几分钟。
|
||
bench/data -> /home/<user>/bench-data 软链(README §0)。"复制不软链"说的是不要软链到别人的目录。
|
||
pool 布局(各任务评测器依赖):
|
||
imagenet1k/<ver>/images/<idx>_<label>.JPEG + labels.json {file: label}
|
||
coco2017/<ver>/val2017/*.jpg + annotations/instances_val2017.json, person_keypoints_val2017.json
|
||
cityscapes/<ver>/leftImg8bit/val/… + gtFine/val/…
|
||
kitti2015/<ver>/training/{image_2,image_3,disp_occ_0,flow_occ}/…
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import fnmatch
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tarfile
|
||
import time
|
||
import zipfile
|
||
from pathlib import Path
|
||
|
||
import yaml
|
||
|
||
BENCH = Path(__file__).resolve().parent.parent
|
||
REG = yaml.safe_load((BENCH / "registry" / "datasets.yaml").read_text())
|
||
LOCAL_ROOT = Path(REG.get("local_root", "/mnt/data"))
|
||
POOL = BENCH / REG.get("pool_root", "data/pool")
|
||
|
||
|
||
def sha256_tree(root: Path) -> tuple[str, int]:
|
||
"""确定性:按相对路径排序,hash(path + 文件 sha256)。返回 (sha, 文件数)。"""
|
||
h = hashlib.sha256()
|
||
n = 0
|
||
for p in sorted(x for x in root.rglob("*") if x.is_file() and x.name != "manifest.json"):
|
||
rel = str(p.relative_to(root)).encode()
|
||
fh = hashlib.sha256()
|
||
with open(p, "rb") as f:
|
||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||
fh.update(chunk)
|
||
h.update(rel); h.update(fh.digest()); n += 1
|
||
return h.hexdigest(), n
|
||
|
||
|
||
def write_manifest(dst: Path, ds: str, ver: str, source: dict, extra: dict | None = None) -> dict:
|
||
sha, n = sha256_tree(dst)
|
||
man = {"dataset": ds, "version": ver, "sha256": sha, "count": n, "source": source,
|
||
"ingested_at": time.strftime("%Y-%m-%d %H:%M:%S"), **(extra or {})}
|
||
(dst / "manifest.json").write_text(json.dumps(man, indent=2, ensure_ascii=False))
|
||
subprocess.run(["chmod", "-R", "a-w", str(dst)], check=False)
|
||
return man
|
||
|
||
|
||
# ---------------------------------------------------------------- sources
|
||
|
||
def src_local_parquet(src: dict, dst: Path) -> dict:
|
||
"""HF parquet(image bytes + label)→ images/ + labels.json。imagenet 专用。"""
|
||
import io
|
||
try:
|
||
import pyarrow.parquet as pq
|
||
from PIL import Image
|
||
except ImportError:
|
||
sys.exit("需要 pyarrow + pillow:在任一 venv 里跑本脚本,或 pip install pyarrow pillow")
|
||
root = LOCAL_ROOT / src["path"]
|
||
shards = sorted(root.glob(src["glob"]))
|
||
if src.get("expect_shards") and len(shards) != src["expect_shards"]:
|
||
raise RuntimeError(f"parquet 片数 {len(shards)} != {src['expect_shards']}:{root}")
|
||
(dst / "images").mkdir(parents=True, exist_ok=True)
|
||
labels, idx = {}, 0
|
||
for sh in shards:
|
||
t = pq.read_table(sh)
|
||
cols = t.column_names
|
||
img_col = "image" if "image" in cols else cols[0]
|
||
lab_col = "label" if "label" in cols else cols[1]
|
||
for img, lab in zip(t.column(img_col).to_pylist(), t.column(lab_col).to_pylist()):
|
||
data = img.get("bytes") if isinstance(img, dict) else img
|
||
if data is None and isinstance(img, dict) and img.get("path"):
|
||
data = Path(img["path"]).read_bytes()
|
||
fn = f"{idx:06d}_{int(lab):04d}.JPEG"
|
||
# 原样落盘(parquet 里就是原始 JPEG 字节):不重编码,不改变像素;评测器 Image.open().convert("RGB") 统一处理 CMYK/PNG 伪装
|
||
(dst / "images" / fn).write_bytes(data)
|
||
labels[fn] = int(lab); idx += 1
|
||
print(f" {sh.name}: 累计 {idx}", file=sys.stderr)
|
||
(dst / "labels.json").write_text(json.dumps(labels))
|
||
return {"kind": "local_parquet", "path": str(root), "shards": len(shards), "images": idx}
|
||
|
||
|
||
def src_local_tar(src: dict, dst: Path) -> dict:
|
||
p = LOCAL_ROOT / src["path"]
|
||
print(f" 校验 {p} …", file=sys.stderr)
|
||
n = int(subprocess.run(f"tar -tzf '{p}' | grep -c -v '/$'", shell=True, capture_output=True, text=True).stdout.strip() or 0)
|
||
if src.get("expect") and n != src["expect"]:
|
||
raise RuntimeError(f"tar 条目 {n} != {src['expect']}(旁边有 .tmp 残片,可能下载不完整)")
|
||
(dst / "images").mkdir(parents=True, exist_ok=True)
|
||
with tarfile.open(p) as t:
|
||
t.extractall(dst / "images")
|
||
return {"kind": "local_tar", "path": str(p), "entries": n, "note": "labels.json 需从 classes.py / 官方 devkit 生成"}
|
||
|
||
|
||
def _annotations(src: dict, dst: Path) -> None:
|
||
for ann in src.get("annotations") or []:
|
||
ap = LOCAL_ROOT / ann
|
||
if ap.suffix == ".zip":
|
||
with zipfile.ZipFile(ap) as z:
|
||
for m in z.namelist():
|
||
if m.endswith(".json") and "val2017" in m:
|
||
z.extract(m, dst)
|
||
else:
|
||
shutil.copy2(ap, dst / ap.name)
|
||
|
||
|
||
def src_local_dir(src: dict, dst: Path) -> dict:
|
||
"""注意:JuiceFS 上按文件 rsync 实测只有 5–10 文件/s;有 zip 就优先 zip(--prefer local_zip)。"""
|
||
p = LOCAL_ROOT / src["path"]
|
||
if not p.exists():
|
||
raise FileNotFoundError(p)
|
||
tgt = dst / p.name
|
||
print(f" 复制 {p} → {tgt}", file=sys.stderr)
|
||
subprocess.run(["rsync", "-a", f"{p}/", f"{tgt}/"], check=True)
|
||
_annotations(src, dst)
|
||
return {"kind": "local_dir", "path": str(p), "annotations": src.get("annotations")}
|
||
|
||
|
||
def src_local_zip(src: dict, dst: Path) -> dict:
|
||
items = [src] + list(src.get("extra") or [])
|
||
done = []
|
||
for it in items:
|
||
p = LOCAL_ROOT / it["path"]
|
||
pat = it.get("extract_only")
|
||
print(f" 解压 {p} ({pat or 'all'})", file=sys.stderr)
|
||
with zipfile.ZipFile(p) as z:
|
||
names = [m for m in z.namelist() if not pat or fnmatch.fnmatch(m, pat)]
|
||
for m in names:
|
||
z.extract(m, dst)
|
||
done.append({"path": str(p), "n": len(names)})
|
||
_annotations(src, dst)
|
||
return {"kind": "local_zip", "items": done}
|
||
|
||
|
||
def src_hf(src: dict, dst: Path) -> dict:
|
||
if not src.get("repo"):
|
||
raise RuntimeError("hf 源没有 repo")
|
||
print(f" hf download {src['repo']} → {dst}", file=sys.stderr)
|
||
subprocess.run(["hf", "download", src["repo"], "--repo-type", "dataset", "--local-dir", str(dst)], check=True)
|
||
return {"kind": "hf", "repo": src["repo"]}
|
||
|
||
|
||
HANDLERS = {"local_parquet": src_local_parquet, "local_tar": src_local_tar, "local_dir": src_local_dir,
|
||
"local_zip": src_local_zip, "hf": src_hf}
|
||
|
||
|
||
def ingest(ds: str, force: bool = False, prefer: str | None = None) -> None:
|
||
d = REG["datasets"][ds]
|
||
ver = d.get("version", "v1")
|
||
dst = POOL / ds / ver
|
||
if (dst / "manifest.json").exists() and not force:
|
||
print(f"{ds}/{ver} 已入池(--force 重做)", file=sys.stderr); return
|
||
if dst.exists():
|
||
subprocess.run(["chmod", "-R", "u+w", str(dst)], check=False)
|
||
shutil.rmtree(dst)
|
||
dst.mkdir(parents=True)
|
||
errors = []
|
||
sources = sorted(d["sources"], key=lambda s: 0 if s["kind"] == prefer else 1) if prefer else d["sources"]
|
||
for src in sources:
|
||
h = HANDLERS.get(src["kind"])
|
||
if not h:
|
||
errors.append(f"{src['kind']}: 未实现(github/official 请手动下载到 {dst} 后 --verify)"); continue
|
||
try:
|
||
print(f"[{ds}] 尝试来源 {src['kind']} …", file=sys.stderr)
|
||
info = h(src, dst)
|
||
man = write_manifest(dst, ds, ver, info)
|
||
if d.get("expect_count") and man["count"] < d["expect_count"]:
|
||
print(f" WARN count {man['count']} < expect {d['expect_count']}", file=sys.stderr)
|
||
print(f"[{ds}] 入池完成 sha={man['sha256'][:12]} count={man['count']}", file=sys.stderr)
|
||
return
|
||
except Exception as e:
|
||
errors.append(f"{src['kind']}: {e}")
|
||
print(f" 失败:{e}", file=sys.stderr)
|
||
for x in dst.iterdir():
|
||
shutil.rmtree(x) if x.is_dir() else x.unlink()
|
||
raise SystemExit(f"[{ds}] 所有来源失败:\n " + "\n ".join(errors))
|
||
|
||
|
||
def verify(ds: str) -> bool:
|
||
d = REG["datasets"][ds]
|
||
dst = POOL / ds / d.get("version", "v1")
|
||
m = json.loads((dst / "manifest.json").read_text())
|
||
sha, n = sha256_tree(dst)
|
||
ok = sha == m["sha256"] and n == m["count"]
|
||
print(f"{ds}: {'OK' if ok else 'MISMATCH'} sha={sha[:12]} count={n}")
|
||
return ok
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--dataset", action="append", default=[])
|
||
ap.add_argument("--all-local", action="store_true", help="所有第一来源是 local_* 的数据集")
|
||
ap.add_argument("--verify", action="store_true")
|
||
ap.add_argument("--force", action="store_true")
|
||
ap.add_argument("--prefer", help="优先尝试的来源 kind,如 local_zip")
|
||
a = ap.parse_args()
|
||
names = list(a.dataset)
|
||
if a.all_local:
|
||
names += [k for k, v in REG["datasets"].items() if v["sources"][0]["kind"].startswith("local_") and k not in names]
|
||
if not names:
|
||
ap.error("--dataset 或 --all-local")
|
||
for ds in names:
|
||
if a.verify:
|
||
verify(ds)
|
||
else:
|
||
ingest(ds, a.force, a.prefer)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|