data-gen-bpu/scripts/test_eval_subset.py

226 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Small T1 subsets stay reproducible and cannot displace legacy data/cache."""
from collections import Counter
from contextlib import ExitStack, redirect_stdout
import copy
import hashlib
import io
import json
import os
from pathlib import Path
import random
import tarfile
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from harness import paths
from harness.eval import EVALUATOR_VERSION
from harness.stages.base import StageContext
from harness.stages.s02_baseline import Baseline, baseline_dir
from harness.stages.s08_eval import Eval
from scripts import make_t1
class EvalSubsetTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name)
self.pool = self.root / "pool" / "imagenet1k" / "v1"
self.pool.mkdir(parents=True)
(self.pool / "manifest.json").write_text(json.dumps({"sha256": "a" * 64}))
self.stack = ExitStack()
self.addCleanup(self.stack.close)
self.stack.enter_context(patch.object(make_t1, "POOL", self.root / "pool"))
self.stack.enter_context(patch.object(make_t1, "T1", self.root / "t1"))
self.stack.enter_context(patch.object(make_t1, "REG", {"datasets": {
"imagenet1k": {"version": "v1", "t1": {"n": 2000, "seed": 20260914,
"stratify": "label"}}}}))
self.stack.enter_context(patch.object(paths, "CACHE", self.root / "cache"))
def labels(self, sizes):
labels = {f"{label}-{i}.jpg": label for label, size in enumerate(sizes) for i in range(size)}
(self.pool / "labels.json").write_text(json.dumps(labels))
return labels
def images(self, labels):
(self.pool / "images").mkdir()
for filename in labels:
(self.pool / "images" / filename).write_bytes(filename.encode())
def test_small_subset_samples_classes_without_sorted_prefix_bias(self):
self.labels([3] * 1000)
items = make_t1.items_imagenet(self.pool, 100, 20260914, "label")
self.assertEqual(len(items), 100)
selected = {item["label"] for item in items}
self.assertEqual(len(selected), 100)
self.assertNotEqual(selected, set(range(100)))
self.assertTrue(any(label >= 900 for label in selected))
self.assertEqual(items, make_t1.items_imagenet(self.pool, 100, 20260914, "label"))
self.assertNotEqual(selected, {item["label"] for item in
make_t1.items_imagenet(self.pool, 100, 20260915, "label")})
def test_cityscapes_keeps_image_label_pairs_and_excludes_training(self):
pool = self.root / "cityscapes"
for i in range(4):
for split in ("val", "train"):
image = pool / f"leftImg8bit/{split}/city/city_{i}_leftImg8bit.png"
label = pool / f"gtFine/{split}/city/city_{i}_gtFine_labelIds.png"
image.parent.mkdir(parents=True, exist_ok=True); image.write_bytes(b"image")
label.parent.mkdir(parents=True, exist_ok=True); label.write_bytes(b"label")
items = make_t1.items_segmentation(pool, "cityscapes", 3, 17)
self.assertEqual(items, make_t1.items_segmentation(pool, "cityscapes", 3, 17))
self.assertEqual(len(items), 3)
self.assertTrue(all("/val/" in item["src"] and "/val/" in item["label_src"] for item in items))
self.assertTrue(all(item["label_format"] == "cityscapes_labelIds" for item in items))
(pool / items[0]["label_src"]).unlink()
with self.assertRaises(FileNotFoundError):
make_t1.items_segmentation(pool, "cityscapes", 3, 17)
def test_exact_size_with_remainder_and_short_classes(self):
self.labels([1, 2, 9, 9])
for count in (1, 3, 7, 18, 21):
with self.subTest(count=count):
items = make_t1.items_imagenet(self.pool, count, 37, "label")
self.assertEqual(len(items), count)
self.assertEqual(len({item["id"] for item in items}), count)
self.labels([4] * 5)
counts = Counter(item["label"] for item in make_t1.items_imagenet(self.pool, 13, 37, "label"))
self.assertEqual(sorted(counts.values()), [2, 2, 3, 3, 3])
def test_insufficient_samples_fail_instead_of_silently_shrinking(self):
self.labels([2, 3])
for stratify in ("label", "none"):
with self.assertRaisesRegex(ValueError, "only 5 available"):
make_t1.items_imagenet(self.pool, 6, 1, stratify)
for count in (0, -1):
with self.assertRaisesRegex(ValueError, "positive integer"):
make_t1.items_imagenet(self.pool, count, 1, "label")
(self.pool / "instances_val2017.json").write_text(json.dumps({"images": [{"id": 1}]}))
with self.assertRaisesRegex(ValueError, "only 1 available"):
make_t1.items_coco(self.pool, 2, 1)
def test_default_2000_selection_is_unchanged(self):
labels = self.labels([3] * 1000)
rng = random.Random(20260914)
expected = []
for label in range(1000):
filenames = sorted(filename for filename, value in labels.items() if value == label)
rng.shuffle(filenames)
expected.extend(filenames[:2])
actual = make_t1.items_imagenet(self.pool, 2000, 20260914, "label")
self.assertEqual([item["id"] for item in actual], expected)
def test_tar_is_deterministic_and_legacy_files_remain_untouched(self):
self.images(self.labels([2] * 1000))
with redirect_stdout(io.StringIO()):
legacy = make_t1.build("imagenet1k")
legacy_bytes = legacy.read_bytes()
legacy_manifest = legacy.with_name("manifest.json").read_bytes()
subset = make_t1.build("imagenet1k", n=100)
subset_bytes = subset.read_bytes()
subset_manifest = subset.with_name("manifest.json")
manifest_bytes = subset_manifest.read_bytes()
for file in (subset, subset_manifest):
os.utime(file, ns=(1_000_000_000, 1_000_000_000))
self.assertEqual(make_t1.build("imagenet1k", n=100).read_bytes(), subset_bytes)
self.assertEqual(subset_manifest.read_bytes(), manifest_bytes)
self.assertEqual(subset.stat().st_mtime_ns, 1_000_000_000)
self.assertEqual(subset_manifest.stat().st_mtime_ns, 1_000_000_000)
self.assertEqual(legacy, self.root / "t1/imagenet1k/20260914.tar")
self.assertEqual(subset, self.root / "t1/imagenet1k/n100-s20260914/20260914.tar")
self.assertEqual(legacy.read_bytes(), legacy_bytes)
self.assertEqual(legacy.with_name("manifest.json").read_bytes(), legacy_manifest)
external = json.loads(subset.with_name("manifest.json").read_text())
self.assertEqual(external["sha256"], hashlib.sha256(subset_bytes).hexdigest())
with tarfile.open(subset) as archive:
internal = json.load(archive.extractfile("manifest.json"))
self.assertEqual(len(internal["items"]), 100)
self.assertEqual(len(archive.getmembers()), 101)
self.assertEqual(internal["sha256"], external["content_sha256"])
self.assertEqual(internal["eval_set_id"], external["eval_set_id"])
with self.assertRaisesRegex(ValueError, "preserve the legacy"):
make_t1.build("imagenet1k", n=100, out_dir=legacy.parent)
with self.assertRaisesRegex(FileExistsError, "different subset"):
make_t1.build("imagenet1k", n=99, out_dir=subset.parent)
self.assertEqual(subset.read_bytes(), subset_bytes)
def test_existing_custom_package_is_never_repaired_or_overwritten_silently(self):
self.images(self.labels([2, 2, 2]))
with redirect_stdout(io.StringIO()):
subset = make_t1.build("imagenet1k", n=2)
manifest = subset.with_name("manifest.json")
manifest_bytes = manifest.read_bytes()
subset.write_bytes(b"corrupted existing package")
with self.assertRaisesRegex(ValueError, "package hash mismatch"):
make_t1.build("imagenet1k", n=2)
self.assertEqual(subset.read_bytes(), b"corrupted existing package")
subset.unlink()
with self.assertRaisesRegex(ValueError, "package is missing"):
make_t1.build("imagenet1k", n=2)
self.assertFalse(subset.exists())
self.assertEqual(manifest.read_bytes(), manifest_bytes)
def test_explicit_cache_uses_full_content_hash_and_version(self):
job = {"model": "resnet18", "evaluator_version": EVALUATOR_VERSION,
"data": {"dataset": "imagenet1k", "t1": {"content_sha": "a" * 64}}}
legacy = baseline_dir(job)
legacy.mkdir(parents=True)
legacy_metrics = {"t1_manifest_sha": "a" * 64, "evaluator_version": EVALUATOR_VERSION,
"metrics": {"top1": 80}}
(legacy / "metrics.json").write_text(json.dumps(legacy_metrics))
for filename in ("g2_inputs.npz", "g2_fp32.npz"):
(legacy / filename).write_bytes(b"legacy G2")
self.assertIsNotNone(Baseline().cached(SimpleNamespace(job=job)))
subset = copy.deepcopy(job)
subset["data"]["t1"]["eval_set_id"] = "screen-100"
directory = baseline_dir(subset)
self.assertEqual(directory, legacy / ("a" * 64) / EVALUATOR_VERSION)
self.assertIsNone(Baseline().cached(SimpleNamespace(job=subset)))
directory.mkdir(parents=True)
(directory / "metrics.json").write_text(json.dumps(legacy_metrics))
for filename in ("g2_inputs.npz", "g2_fp32.npz"):
(directory / filename).write_bytes(b"new subset G2")
self.assertIsNotNone(Baseline().cached(SimpleNamespace(job=subset)))
# 同名、同样本量、哈希前缀相同,也不能共用缓存。
subset["data"]["t1"]["content_sha"] = "a" * 63 + "b"
self.assertNotEqual(baseline_dir(subset), directory)
subset["data"]["t1"]["content_sha"] = "a" * 64
subset["evaluator_version"] = "0.4.0"
self.assertNotEqual(baseline_dir(subset), directory)
self.assertEqual(json.loads((legacy / "metrics.json").read_text()), legacy_metrics)
for filename in ("g2_inputs.npz", "g2_fp32.npz"):
self.assertEqual((legacy / filename).read_bytes(), b"legacy G2")
subset["data"]["t1"]["content_sha"] = None
with self.assertRaisesRegex(ValueError, "full SHA256"):
baseline_dir(subset)
def test_fp32_and_board_commands_share_subset_and_g2_cache(self):
job = {"job_id": "resnet18-test", "model": "resnet18", "task": "cls", "backend": "ptq",
"evaluator_version": EVALUATOR_VERSION,
"model_spec": {"source": "torchvision", "input_spec": "1x3x224x224", "resize": 256},
"gates": {"G2": {"n_samples": 32}},
"data": {"dataset": "imagenet1k", "t1": {
"path": str(self.root / "t1/imagenet1k/n100-s20260914/20260914.tar"),
"n": 100, "eval_set_id": "screen-100", "content_sha": "a" * 64,
"manifest_sha": "b" * 64}}}
ctx = StageContext("test", job, self.root / "job", self.root / "stages", self.root / "artifacts")
with patch.object(paths, "venv_python", return_value=Path("/test/python")), \
patch("harness.stages.s02_baseline.weights_file", return_value=self.root / "weights"), \
patch("harness.stages.s08_eval.onnx_path", return_value=self.root / "model.onnx"), \
patch("harness.stages.s08_eval.selected_board", return_value={"runtime_dir": "/test/board"}):
fp32, board = Baseline().command(ctx), Eval().command(ctx)
for flag, expected in (("--t1", job["data"]["t1"]["path"]), ("--t1-sha", "b" * 64),
("--g2-samples", "32"), ("--input-spec", "1x3x224x224"), ("--resize", "256")):
self.assertEqual(fp32[fp32.index(flag) + 1], expected)
self.assertEqual(board[board.index(flag) + 1], expected)
# 评测器将两个 G2 npz 与 metrics.json 都写到 --out;INT8 全部从 --baseline 读取。
self.assertEqual(fp32[fp32.index("--out") + 1], board[board.index("--baseline") + 1])
self.assertEqual(Path(board[board.index("--baseline") + 1]),
self.root / "cache/baseline/resnet18--imagenet1k" / ("a" * 64) / EVALUATOR_VERSION)
if __name__ == "__main__":
unittest.main()