"""Small T1 subsets stay reproducible and cannot displace legacy data/cache.""" from collections import Counter from contextlib import ExitStack, redirect_stdout import copy import hashlib import io import json import os from pathlib import Path import random import tarfile import tempfile from types import SimpleNamespace import unittest from unittest.mock import patch from harness import paths from harness.eval import EVALUATOR_VERSION from harness.stages.base import StageContext from harness.stages.s02_baseline import Baseline, baseline_dir from harness.stages.s08_eval import Eval from scripts import make_t1 class EvalSubsetTests(unittest.TestCase): def setUp(self): temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup) self.root = Path(temporary.name) self.pool = self.root / "pool" / "imagenet1k" / "v1" self.pool.mkdir(parents=True) (self.pool / "manifest.json").write_text(json.dumps({"sha256": "a" * 64})) self.stack = ExitStack() self.addCleanup(self.stack.close) self.stack.enter_context(patch.object(make_t1, "POOL", self.root / "pool")) self.stack.enter_context(patch.object(make_t1, "T1", self.root / "t1")) self.stack.enter_context(patch.object(make_t1, "REG", {"datasets": { "imagenet1k": {"version": "v1", "t1": {"n": 2000, "seed": 20260914, "stratify": "label"}}}})) self.stack.enter_context(patch.object(paths, "CACHE", self.root / "cache")) def labels(self, sizes): labels = {f"{label}-{i}.jpg": label for label, size in enumerate(sizes) for i in range(size)} (self.pool / "labels.json").write_text(json.dumps(labels)) return labels def images(self, labels): (self.pool / "images").mkdir() for filename in labels: (self.pool / "images" / filename).write_bytes(filename.encode()) def test_small_subset_samples_classes_without_sorted_prefix_bias(self): self.labels([3] * 1000) items = make_t1.items_imagenet(self.pool, 100, 20260914, "label") self.assertEqual(len(items), 100) selected = {item["label"] for item in items} self.assertEqual(len(selected), 100) self.assertNotEqual(selected, set(range(100))) self.assertTrue(any(label >= 900 for label in selected)) self.assertEqual(items, make_t1.items_imagenet(self.pool, 100, 20260914, "label")) self.assertNotEqual(selected, {item["label"] for item in make_t1.items_imagenet(self.pool, 100, 20260915, "label")}) def test_exact_size_with_remainder_and_short_classes(self): self.labels([1, 2, 9, 9]) for count in (1, 3, 7, 18, 21): with self.subTest(count=count): items = make_t1.items_imagenet(self.pool, count, 37, "label") self.assertEqual(len(items), count) self.assertEqual(len({item["id"] for item in items}), count) self.labels([4] * 5) counts = Counter(item["label"] for item in make_t1.items_imagenet(self.pool, 13, 37, "label")) self.assertEqual(sorted(counts.values()), [2, 2, 3, 3, 3]) def test_insufficient_samples_fail_instead_of_silently_shrinking(self): self.labels([2, 3]) for stratify in ("label", "none"): with self.assertRaisesRegex(ValueError, "only 5 available"): make_t1.items_imagenet(self.pool, 6, 1, stratify) for count in (0, -1): with self.assertRaisesRegex(ValueError, "positive integer"): make_t1.items_imagenet(self.pool, count, 1, "label") (self.pool / "instances_val2017.json").write_text(json.dumps({"images": [{"id": 1}]})) with self.assertRaisesRegex(ValueError, "only 1 available"): make_t1.items_coco(self.pool, 2, 1) def test_default_2000_selection_is_unchanged(self): labels = self.labels([3] * 1000) rng = random.Random(20260914) expected = [] for label in range(1000): filenames = sorted(filename for filename, value in labels.items() if value == label) rng.shuffle(filenames) expected.extend(filenames[:2]) actual = make_t1.items_imagenet(self.pool, 2000, 20260914, "label") self.assertEqual([item["id"] for item in actual], expected) def test_tar_is_deterministic_and_legacy_files_remain_untouched(self): self.images(self.labels([2] * 1000)) with redirect_stdout(io.StringIO()): legacy = make_t1.build("imagenet1k") legacy_bytes = legacy.read_bytes() legacy_manifest = legacy.with_name("manifest.json").read_bytes() subset = make_t1.build("imagenet1k", n=100) subset_bytes = subset.read_bytes() subset_manifest = subset.with_name("manifest.json") manifest_bytes = subset_manifest.read_bytes() for file in (subset, subset_manifest): os.utime(file, ns=(1_000_000_000, 1_000_000_000)) self.assertEqual(make_t1.build("imagenet1k", n=100).read_bytes(), subset_bytes) self.assertEqual(subset_manifest.read_bytes(), manifest_bytes) self.assertEqual(subset.stat().st_mtime_ns, 1_000_000_000) self.assertEqual(subset_manifest.stat().st_mtime_ns, 1_000_000_000) self.assertEqual(legacy, self.root / "t1/imagenet1k/20260914.tar") self.assertEqual(subset, self.root / "t1/imagenet1k/n100-s20260914/20260914.tar") self.assertEqual(legacy.read_bytes(), legacy_bytes) self.assertEqual(legacy.with_name("manifest.json").read_bytes(), legacy_manifest) external = json.loads(subset.with_name("manifest.json").read_text()) self.assertEqual(external["sha256"], hashlib.sha256(subset_bytes).hexdigest()) with tarfile.open(subset) as archive: internal = json.load(archive.extractfile("manifest.json")) self.assertEqual(len(internal["items"]), 100) self.assertEqual(len(archive.getmembers()), 101) self.assertEqual(internal["sha256"], external["content_sha256"]) self.assertEqual(internal["eval_set_id"], external["eval_set_id"]) with self.assertRaisesRegex(ValueError, "preserve the legacy"): make_t1.build("imagenet1k", n=100, out_dir=legacy.parent) with self.assertRaisesRegex(FileExistsError, "different subset"): make_t1.build("imagenet1k", n=99, out_dir=subset.parent) self.assertEqual(subset.read_bytes(), subset_bytes) def test_existing_custom_package_is_never_repaired_or_overwritten_silently(self): self.images(self.labels([2, 2, 2])) with redirect_stdout(io.StringIO()): subset = make_t1.build("imagenet1k", n=2) manifest = subset.with_name("manifest.json") manifest_bytes = manifest.read_bytes() subset.write_bytes(b"corrupted existing package") with self.assertRaisesRegex(ValueError, "package hash mismatch"): make_t1.build("imagenet1k", n=2) self.assertEqual(subset.read_bytes(), b"corrupted existing package") subset.unlink() with self.assertRaisesRegex(ValueError, "package is missing"): make_t1.build("imagenet1k", n=2) self.assertFalse(subset.exists()) self.assertEqual(manifest.read_bytes(), manifest_bytes) def test_explicit_cache_uses_full_content_hash_and_version(self): job = {"model": "resnet18", "evaluator_version": EVALUATOR_VERSION, "data": {"dataset": "imagenet1k", "t1": {"content_sha": "a" * 64}}} legacy = baseline_dir(job) legacy.mkdir(parents=True) legacy_metrics = {"t1_manifest_sha": "a" * 64, "evaluator_version": EVALUATOR_VERSION, "metrics": {"top1": 80}} (legacy / "metrics.json").write_text(json.dumps(legacy_metrics)) for filename in ("g2_inputs.npz", "g2_fp32.npz"): (legacy / filename).write_bytes(b"legacy G2") self.assertIsNotNone(Baseline().cached(SimpleNamespace(job=job))) subset = copy.deepcopy(job) subset["data"]["t1"]["eval_set_id"] = "screen-100" directory = baseline_dir(subset) self.assertEqual(directory, legacy / ("a" * 64) / EVALUATOR_VERSION) self.assertIsNone(Baseline().cached(SimpleNamespace(job=subset))) directory.mkdir(parents=True) (directory / "metrics.json").write_text(json.dumps(legacy_metrics)) for filename in ("g2_inputs.npz", "g2_fp32.npz"): (directory / filename).write_bytes(b"new subset G2") self.assertIsNotNone(Baseline().cached(SimpleNamespace(job=subset))) # 同名、同样本量、哈希前缀相同,也不能共用缓存。 subset["data"]["t1"]["content_sha"] = "a" * 63 + "b" self.assertNotEqual(baseline_dir(subset), directory) subset["data"]["t1"]["content_sha"] = "a" * 64 subset["evaluator_version"] = "0.4.0" self.assertNotEqual(baseline_dir(subset), directory) self.assertEqual(json.loads((legacy / "metrics.json").read_text()), legacy_metrics) for filename in ("g2_inputs.npz", "g2_fp32.npz"): self.assertEqual((legacy / filename).read_bytes(), b"legacy G2") subset["data"]["t1"]["content_sha"] = None with self.assertRaisesRegex(ValueError, "full SHA256"): baseline_dir(subset) def test_fp32_and_board_commands_share_subset_and_g2_cache(self): job = {"job_id": "resnet18-test", "model": "resnet18", "task": "cls", "backend": "ptq", "evaluator_version": EVALUATOR_VERSION, "model_spec": {"source": "torchvision", "input_spec": "1x3x224x224", "resize": 256}, "gates": {"G2": {"n_samples": 32}}, "data": {"dataset": "imagenet1k", "t1": { "path": str(self.root / "t1/imagenet1k/n100-s20260914/20260914.tar"), "n": 100, "eval_set_id": "screen-100", "content_sha": "a" * 64, "manifest_sha": "b" * 64}}} ctx = StageContext("test", job, self.root / "job", self.root / "stages", self.root / "artifacts") with patch.object(paths, "venv_python", return_value=Path("/test/python")), \ patch("harness.stages.s02_baseline.weights_file", return_value=self.root / "weights"), \ patch("harness.stages.s08_eval.onnx_path", return_value=self.root / "model.onnx"), \ patch("harness.stages.s08_eval.selected_board", return_value={"runtime_dir": "/test/board"}): fp32, board = Baseline().command(ctx), Eval().command(ctx) for flag, expected in (("--t1", job["data"]["t1"]["path"]), ("--t1-sha", "b" * 64), ("--g2-samples", "32"), ("--input-spec", "1x3x224x224"), ("--resize", "256")): self.assertEqual(fp32[fp32.index(flag) + 1], expected) self.assertEqual(board[board.index(flag) + 1], expected) # 评测器将两个 G2 npz 与 metrics.json 都写到 --out;INT8 全部从 --baseline 读取。 self.assertEqual(fp32[fp32.index("--out") + 1], board[board.index("--baseline") + 1]) self.assertEqual(Path(board[board.index("--baseline") + 1]), self.root / "cache/baseline/resnet18--imagenet1k" / ("a" * 64) / EVALUATOR_VERSION) if __name__ == "__main__": unittest.main()