209 lines
11 KiB
Python
209 lines
11 KiB
Python
"""Small T1 subsets stay reproducible and cannot displace legacy data/cache."""
|
||
from collections import Counter
|
||
from contextlib import ExitStack, redirect_stdout
|
||
import copy
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import random
|
||
import tarfile
|
||
import tempfile
|
||
from types import SimpleNamespace
|
||
import unittest
|
||
from unittest.mock import patch
|
||
|
||
from harness import paths
|
||
from harness.eval import EVALUATOR_VERSION
|
||
from harness.stages.base import StageContext
|
||
from harness.stages.s02_baseline import Baseline, baseline_dir
|
||
from harness.stages.s08_eval import Eval
|
||
from scripts import make_t1
|
||
|
||
|
||
class EvalSubsetTests(unittest.TestCase):
|
||
def setUp(self):
|
||
temporary = tempfile.TemporaryDirectory()
|
||
self.addCleanup(temporary.cleanup)
|
||
self.root = Path(temporary.name)
|
||
self.pool = self.root / "pool" / "imagenet1k" / "v1"
|
||
self.pool.mkdir(parents=True)
|
||
(self.pool / "manifest.json").write_text(json.dumps({"sha256": "a" * 64}))
|
||
self.stack = ExitStack()
|
||
self.addCleanup(self.stack.close)
|
||
self.stack.enter_context(patch.object(make_t1, "POOL", self.root / "pool"))
|
||
self.stack.enter_context(patch.object(make_t1, "T1", self.root / "t1"))
|
||
self.stack.enter_context(patch.object(make_t1, "REG", {"datasets": {
|
||
"imagenet1k": {"version": "v1", "t1": {"n": 2000, "seed": 20260914,
|
||
"stratify": "label"}}}}))
|
||
self.stack.enter_context(patch.object(paths, "CACHE", self.root / "cache"))
|
||
|
||
def labels(self, sizes):
|
||
labels = {f"{label}-{i}.jpg": label for label, size in enumerate(sizes) for i in range(size)}
|
||
(self.pool / "labels.json").write_text(json.dumps(labels))
|
||
return labels
|
||
|
||
def images(self, labels):
|
||
(self.pool / "images").mkdir()
|
||
for filename in labels:
|
||
(self.pool / "images" / filename).write_bytes(filename.encode())
|
||
|
||
def test_small_subset_samples_classes_without_sorted_prefix_bias(self):
|
||
self.labels([3] * 1000)
|
||
items = make_t1.items_imagenet(self.pool, 100, 20260914, "label")
|
||
self.assertEqual(len(items), 100)
|
||
selected = {item["label"] for item in items}
|
||
self.assertEqual(len(selected), 100)
|
||
self.assertNotEqual(selected, set(range(100)))
|
||
self.assertTrue(any(label >= 900 for label in selected))
|
||
self.assertEqual(items, make_t1.items_imagenet(self.pool, 100, 20260914, "label"))
|
||
self.assertNotEqual(selected, {item["label"] for item in
|
||
make_t1.items_imagenet(self.pool, 100, 20260915, "label")})
|
||
|
||
def test_exact_size_with_remainder_and_short_classes(self):
|
||
self.labels([1, 2, 9, 9])
|
||
for count in (1, 3, 7, 18, 21):
|
||
with self.subTest(count=count):
|
||
items = make_t1.items_imagenet(self.pool, count, 37, "label")
|
||
self.assertEqual(len(items), count)
|
||
self.assertEqual(len({item["id"] for item in items}), count)
|
||
self.labels([4] * 5)
|
||
counts = Counter(item["label"] for item in make_t1.items_imagenet(self.pool, 13, 37, "label"))
|
||
self.assertEqual(sorted(counts.values()), [2, 2, 3, 3, 3])
|
||
|
||
def test_insufficient_samples_fail_instead_of_silently_shrinking(self):
|
||
self.labels([2, 3])
|
||
for stratify in ("label", "none"):
|
||
with self.assertRaisesRegex(ValueError, "only 5 available"):
|
||
make_t1.items_imagenet(self.pool, 6, 1, stratify)
|
||
for count in (0, -1):
|
||
with self.assertRaisesRegex(ValueError, "positive integer"):
|
||
make_t1.items_imagenet(self.pool, count, 1, "label")
|
||
(self.pool / "instances_val2017.json").write_text(json.dumps({"images": [{"id": 1}]}))
|
||
with self.assertRaisesRegex(ValueError, "only 1 available"):
|
||
make_t1.items_coco(self.pool, 2, 1)
|
||
|
||
def test_default_2000_selection_is_unchanged(self):
|
||
labels = self.labels([3] * 1000)
|
||
rng = random.Random(20260914)
|
||
expected = []
|
||
for label in range(1000):
|
||
filenames = sorted(filename for filename, value in labels.items() if value == label)
|
||
rng.shuffle(filenames)
|
||
expected.extend(filenames[:2])
|
||
actual = make_t1.items_imagenet(self.pool, 2000, 20260914, "label")
|
||
self.assertEqual([item["id"] for item in actual], expected)
|
||
|
||
def test_tar_is_deterministic_and_legacy_files_remain_untouched(self):
|
||
self.images(self.labels([2] * 1000))
|
||
with redirect_stdout(io.StringIO()):
|
||
legacy = make_t1.build("imagenet1k")
|
||
legacy_bytes = legacy.read_bytes()
|
||
legacy_manifest = legacy.with_name("manifest.json").read_bytes()
|
||
subset = make_t1.build("imagenet1k", n=100)
|
||
subset_bytes = subset.read_bytes()
|
||
subset_manifest = subset.with_name("manifest.json")
|
||
manifest_bytes = subset_manifest.read_bytes()
|
||
for file in (subset, subset_manifest):
|
||
os.utime(file, ns=(1_000_000_000, 1_000_000_000))
|
||
self.assertEqual(make_t1.build("imagenet1k", n=100).read_bytes(), subset_bytes)
|
||
self.assertEqual(subset_manifest.read_bytes(), manifest_bytes)
|
||
self.assertEqual(subset.stat().st_mtime_ns, 1_000_000_000)
|
||
self.assertEqual(subset_manifest.stat().st_mtime_ns, 1_000_000_000)
|
||
self.assertEqual(legacy, self.root / "t1/imagenet1k/20260914.tar")
|
||
self.assertEqual(subset, self.root / "t1/imagenet1k/n100-s20260914/20260914.tar")
|
||
self.assertEqual(legacy.read_bytes(), legacy_bytes)
|
||
self.assertEqual(legacy.with_name("manifest.json").read_bytes(), legacy_manifest)
|
||
external = json.loads(subset.with_name("manifest.json").read_text())
|
||
self.assertEqual(external["sha256"], hashlib.sha256(subset_bytes).hexdigest())
|
||
with tarfile.open(subset) as archive:
|
||
internal = json.load(archive.extractfile("manifest.json"))
|
||
self.assertEqual(len(internal["items"]), 100)
|
||
self.assertEqual(len(archive.getmembers()), 101)
|
||
self.assertEqual(internal["sha256"], external["content_sha256"])
|
||
self.assertEqual(internal["eval_set_id"], external["eval_set_id"])
|
||
with self.assertRaisesRegex(ValueError, "preserve the legacy"):
|
||
make_t1.build("imagenet1k", n=100, out_dir=legacy.parent)
|
||
with self.assertRaisesRegex(FileExistsError, "different subset"):
|
||
make_t1.build("imagenet1k", n=99, out_dir=subset.parent)
|
||
self.assertEqual(subset.read_bytes(), subset_bytes)
|
||
|
||
def test_existing_custom_package_is_never_repaired_or_overwritten_silently(self):
|
||
self.images(self.labels([2, 2, 2]))
|
||
with redirect_stdout(io.StringIO()):
|
||
subset = make_t1.build("imagenet1k", n=2)
|
||
manifest = subset.with_name("manifest.json")
|
||
manifest_bytes = manifest.read_bytes()
|
||
subset.write_bytes(b"corrupted existing package")
|
||
with self.assertRaisesRegex(ValueError, "package hash mismatch"):
|
||
make_t1.build("imagenet1k", n=2)
|
||
self.assertEqual(subset.read_bytes(), b"corrupted existing package")
|
||
subset.unlink()
|
||
with self.assertRaisesRegex(ValueError, "package is missing"):
|
||
make_t1.build("imagenet1k", n=2)
|
||
self.assertFalse(subset.exists())
|
||
self.assertEqual(manifest.read_bytes(), manifest_bytes)
|
||
|
||
def test_explicit_cache_uses_full_content_hash_and_version(self):
|
||
job = {"model": "resnet18", "evaluator_version": EVALUATOR_VERSION,
|
||
"data": {"dataset": "imagenet1k", "t1": {"content_sha": "a" * 64}}}
|
||
legacy = baseline_dir(job)
|
||
legacy.mkdir(parents=True)
|
||
legacy_metrics = {"t1_manifest_sha": "a" * 64, "evaluator_version": EVALUATOR_VERSION,
|
||
"metrics": {"top1": 80}}
|
||
(legacy / "metrics.json").write_text(json.dumps(legacy_metrics))
|
||
for filename in ("g2_inputs.npz", "g2_fp32.npz"):
|
||
(legacy / filename).write_bytes(b"legacy G2")
|
||
self.assertIsNotNone(Baseline().cached(SimpleNamespace(job=job)))
|
||
subset = copy.deepcopy(job)
|
||
subset["data"]["t1"]["eval_set_id"] = "screen-100"
|
||
directory = baseline_dir(subset)
|
||
self.assertEqual(directory, legacy / ("a" * 64) / EVALUATOR_VERSION)
|
||
self.assertIsNone(Baseline().cached(SimpleNamespace(job=subset)))
|
||
directory.mkdir(parents=True)
|
||
(directory / "metrics.json").write_text(json.dumps(legacy_metrics))
|
||
for filename in ("g2_inputs.npz", "g2_fp32.npz"):
|
||
(directory / filename).write_bytes(b"new subset G2")
|
||
self.assertIsNotNone(Baseline().cached(SimpleNamespace(job=subset)))
|
||
# 同名、同样本量、哈希前缀相同,也不能共用缓存。
|
||
subset["data"]["t1"]["content_sha"] = "a" * 63 + "b"
|
||
self.assertNotEqual(baseline_dir(subset), directory)
|
||
subset["data"]["t1"]["content_sha"] = "a" * 64
|
||
subset["evaluator_version"] = "0.4.0"
|
||
self.assertNotEqual(baseline_dir(subset), directory)
|
||
self.assertEqual(json.loads((legacy / "metrics.json").read_text()), legacy_metrics)
|
||
for filename in ("g2_inputs.npz", "g2_fp32.npz"):
|
||
self.assertEqual((legacy / filename).read_bytes(), b"legacy G2")
|
||
subset["data"]["t1"]["content_sha"] = None
|
||
with self.assertRaisesRegex(ValueError, "full SHA256"):
|
||
baseline_dir(subset)
|
||
|
||
def test_fp32_and_board_commands_share_subset_and_g2_cache(self):
|
||
job = {"job_id": "resnet18-test", "model": "resnet18", "task": "cls", "backend": "ptq",
|
||
"evaluator_version": EVALUATOR_VERSION,
|
||
"model_spec": {"source": "torchvision", "input_spec": "1x3x224x224", "resize": 256},
|
||
"gates": {"G2": {"n_samples": 32}},
|
||
"data": {"dataset": "imagenet1k", "t1": {
|
||
"path": str(self.root / "t1/imagenet1k/n100-s20260914/20260914.tar"),
|
||
"n": 100, "eval_set_id": "screen-100", "content_sha": "a" * 64,
|
||
"manifest_sha": "b" * 64}}}
|
||
ctx = StageContext("test", job, self.root / "job", self.root / "stages", self.root / "artifacts")
|
||
with patch.object(paths, "venv_python", return_value=Path("/test/python")), \
|
||
patch("harness.stages.s02_baseline.weights_file", return_value=self.root / "weights"), \
|
||
patch("harness.stages.s08_eval.onnx_path", return_value=self.root / "model.onnx"), \
|
||
patch("harness.stages.s08_eval.selected_board", return_value={"runtime_dir": "/test/board"}):
|
||
fp32, board = Baseline().command(ctx), Eval().command(ctx)
|
||
for flag, expected in (("--t1", job["data"]["t1"]["path"]), ("--t1-sha", "b" * 64),
|
||
("--g2-samples", "32"), ("--input-spec", "1x3x224x224"), ("--resize", "256")):
|
||
self.assertEqual(fp32[fp32.index(flag) + 1], expected)
|
||
self.assertEqual(board[board.index(flag) + 1], expected)
|
||
# 评测器将两个 G2 npz 与 metrics.json 都写到 --out;INT8 全部从 --baseline 读取。
|
||
self.assertEqual(fp32[fp32.index("--out") + 1], board[board.index("--baseline") + 1])
|
||
self.assertEqual(Path(board[board.index("--baseline") + 1]),
|
||
self.root / "cache/baseline/resnet18--imagenet1k" / ("a" * 64) / EVALUATOR_VERSION)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|