"""Model caches keep frozen runtime/task identities separate without moving legacy artifacts.""" import copy from contextlib import ExitStack import json import os from pathlib import Path import tempfile from types import SimpleNamespace import unittest from unittest.mock import patch from harness import paths from harness.backends import ptq, qat from harness.eval import task_contract from harness.stages import s01_fetch from harness.stages.s02_baseline import Baseline, baseline_dir from harness.stages.s03_export import Export, onnx_path class CacheIdentity(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.root = Path(self.tmp.name).resolve() self.stack = ExitStack() self.addCleanup(self.stack.close) self.cfg = {"paths": {"venv": {"bench": str(self.root / "runtime-a")}}} self.stack.enter_context(patch.object(paths, "config", return_value=self.cfg)) self.stack.enter_context(patch.object(paths, "CACHE", self.root / "cache")) self.stack.enter_context(patch.object(paths, "RUNS", self.root / "runs")) self.stack.enter_context(patch.object(s01_fetch, "INDEX", self.root / "cache/weights/index.json")) s01_fetch.INDEX.parent.mkdir(parents=True) s01_fetch.INDEX.write_text(json.dumps({"toy": {"sha256": "b" * 64}})) self.exporter = self.stack.enter_context(patch("harness.tools.export_onnx.exporter_version", return_value="fixture-1")) def job(self, runtime="a"): python = str(self.root / f"runtime-{runtime}/bin/python") return {"model": "toy", "job_id": "toy-job", "task": "cls", "backend": "ptq", "model_spec": {"source": "custom", "input_spec": "1x3x4x4", "resize": 4}, "task_contract": task_contract("cls"), "evaluator_version": "0.5.0", "model_runtime": {"configured": python, "python": python}, "cache_contract": {"version": 1, "model_python": python, "exporter_version": "fixture-1"}, "data": {"dataset": "fixture", "t1": {"content_sha": "a" * 64, "manifest_sha": "c" * 64, "path": str(self.root / "t1.tar")}, "calib": {"path": str(self.root / "calib")}}, "gates": {"G2": {"n_samples": 4}, "G3": {"metric": "top1", "direction": "higher"}}} def write_baseline(self, job, *, stamp=True): directory = baseline_dir(job) directory.mkdir(parents=True, exist_ok=True) (directory / "metrics.json").write_text(json.dumps({"t1_manifest_sha": "a" * 64, "evaluator_version": "0.5.0", "metrics": {"top1": 80}})) if stamp: Baseline().measure(SimpleNamespace(job=job), None, None) return directory / "metrics.json" def write_onnx(self, job, *, stamp=True): path = onnx_path(job) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(b"fixture ONNX") if stamp: Export().measure(SimpleNamespace(job=job), None, None) else: (path.parent / "meta.json").write_text(json.dumps({"exporter_version": "fixture-1"})) return path def test_distinct_runtime_paths_never_overwrite_other_or_legacy_cache(self): a, b = self.job("a"), self.job("b") legacy = copy.deepcopy(a); legacy.pop("cache_contract") old_base = self.write_baseline(legacy) old_onnx = self.write_onnx(legacy) before = (old_base.read_bytes(), old_onnx.read_bytes()) self.assertNotEqual(baseline_dir(a), baseline_dir(b)) self.assertNotEqual(onnx_path(a), onnx_path(b)) self.assertNotEqual(baseline_dir(a), baseline_dir(legacy)) self.assertNotEqual(onnx_path(a), onnx_path(legacy)) self.write_baseline(a); self.write_baseline(b) self.write_onnx(a); self.write_onnx(b) self.assertEqual((old_base.read_bytes(), old_onnx.read_bytes()), before) def test_new_identity_requires_matching_metadata_and_reuses_same_identity(self): job = self.job() self.write_baseline(job, stamp=False); self.write_onnx(job, stamp=False) ctx = SimpleNamespace(job=job) self.assertIsNone(Baseline().cached(ctx)) self.assertIsNone(Export().cached(ctx)) baseline_before = (baseline_dir(job) / "metrics.json").read_bytes() Baseline().cached(ctx) self.assertEqual((baseline_dir(job) / "metrics.json").read_bytes(), baseline_before) self.write_baseline(job); self.write_onnx(job) self.assertIsNotNone(Baseline().cached(ctx)) self.assertIsNotNone(Export().cached(ctx)) self.assertEqual(Baseline().cached(ctx)["cache_identity"], json.loads((baseline_dir(job) / "metrics.json").read_text())["cache_identity"]) def test_default_runtime_is_frozen_and_direct_cached_calls_reject_drift(self): job = self.job() job["model_runtime"] = None job["cache_contract"] = paths.freeze_cache_contract(job) self.write_baseline(job); self.write_onnx(job) self.cfg["paths"]["venv"]["bench"] = str(self.root / "runtime-b") for stage in (Baseline(), Export()): with self.subTest(stage=stage.name), self.assertRaisesRegex(ValueError, "cache runtime"): stage.cached(SimpleNamespace(job=job)) def test_baseline_task_and_sampling_change_identity_but_not_onnx(self): original = self.job() for change in ("task", "g2_samples", "t1"): altered = copy.deepcopy(original) if change == "task": altered["task_contract"]["source_sha256"] = "changed" elif change == "g2_samples": altered["gates"]["G2"]["n_samples"] = 8 else: altered["data"]["t1"]["content_sha"] = "d" * 64 with self.subTest(change=change): self.assertNotEqual(baseline_dir(original), baseline_dir(altered)) self.assertEqual(onnx_path(original), onnx_path(altered)) original["gates"]["G3"]["direction"] = "lower" self.assertEqual(onnx_path(original), onnx_path(self.job())) self.assertEqual(baseline_dir(original), baseline_dir(self.job())) def test_changed_input_and_exporter_have_distinct_onnx_paths(self): original = self.job() for key in ("input", "exporter"): changed = copy.deepcopy(original) if key == "input": changed["model_spec"]["input_spec"] = "1x3x8x8" else: changed["cache_contract"]["exporter_version"] = "fixture-2" with self.subTest(change=key): self.assertNotEqual(onnx_path(original), onnx_path(changed)) def test_cached_rejects_task_and_exporter_drift_before_using_bytes(self): job = self.job(); self.write_baseline(job); self.write_onnx(job) with patch("harness.eval.task_contract", return_value={"source_sha256": "changed"}): with self.assertRaisesRegex(ValueError, "task"): Baseline().cached(SimpleNamespace(job=job)) self.assertIsNotNone(Export().cached(SimpleNamespace(job=job))) self.exporter.return_value = "fixture-2" with self.assertRaisesRegex(ValueError, "exporter"): Export().cached(SimpleNamespace(job=job)) def test_legacy_path_and_explicit_subset_are_preserved(self): job = self.job(); job.pop("cache_contract") directory = paths.CACHE / "baseline/toy--fixture" self.assertEqual(baseline_dir(job), directory) self.assertEqual(onnx_path(job), paths.CACHE / "onnx" / ("b" * 64) / "toy.onnx") job["data"]["t1"]["eval_set_id"] = "screen" self.assertEqual(baseline_dir(job), directory / ("a" * 64) / "0.5.0") def test_qat_and_ptq_export_use_same_model_python(self): job = self.job("b") for backend in (ptq, qat): self.assertEqual(backend.export_cmd(job, self.root / "out.onnx", self.root / "weights")[0], job["model_runtime"]["python"]) def test_relative_runtime_preserves_venv_launch_path_instead_of_python_symlink_target(self): shared = self.root / "shared-python" shared.write_bytes(b"fixture, never executed") launch = self.root / "runtime-a/bin/python" launch.parent.mkdir(parents=True) launch.symlink_to(shared) job = self.job(); job.pop("cache_contract") job["model_runtime"] = None job["model_spec"]["runtime_python"] = "runtime-a/bin/python" with patch.object(paths, "BENCH_ROOT", self.root): contract = paths.freeze_cache_contract(job) self.assertEqual(contract["model_python"], str(launch)) job["model_runtime"] = {"configured": "runtime-a/bin/python", "python": str(launch)} job["cache_contract"] = contract self.assertEqual(paths.model_python(job), launch) paths.cache_contract(job, validate=True) def test_oldest_relative_runtime_keeps_old_interpreter_semantics(self): shared = self.root / "shared-python" shared.write_bytes(b"fixture, never executed") launch = self.root / "legacy-venv/bin/python" launch.parent.mkdir(parents=True) launch.symlink_to(shared) job = self.job(); job.pop("cache_contract"); job.pop("model_runtime") job["model_spec"]["runtime_python"] = os.path.relpath(launch, paths.BENCH_ROOT) self.assertEqual(paths.model_python(job), shared) job["cache_contract"] = paths.freeze_cache_contract(job) self.assertEqual(job["cache_contract"]["model_python"], str(launch)) self.assertEqual(paths.model_python(job), launch) def test_repeat_publication_does_not_move_legacy_but_cannot_change_new_contract(self): from harness.pipeline import _check_frozen_job job = self.job() old = copy.deepcopy(job); old.pop("cache_contract") path = self.root / "job.json" path.write_text(json.dumps(old)) before = path.read_bytes() _check_frozen_job(path, job) self.assertEqual(path.read_bytes(), before) self.assertEqual(baseline_dir(json.loads(path.read_text())), baseline_dir(old)) path.write_text(json.dumps(job)) changed = copy.deepcopy(job) changed["cache_contract"]["model_python"] = "/different/bin/python" with self.assertRaisesRegex(ValueError, "content conflict"): _check_frozen_job(path, changed) def test_unknown_or_explicit_null_cache_contract_is_rejected(self): for contract in (None, {}, {"version": 2}, {"version": True}): job = self.job(); job["cache_contract"] = contract with self.subTest(contract=contract), self.assertRaisesRegex(ValueError, "cache contract"): Baseline().cached(SimpleNamespace(job=job)) def test_new_job_builder_freezes_cache_contract(self): from harness import registry spec = registry.JobSpec(job_id="toy-job", task="cls", model="toy", march="nash-m", backend="ptq", qcfg="int8", priority="A", mode="oracle") with patch.object(registry, "model_row", return_value=self.job()["model_spec"]), \ patch.object(registry, "board_for_march", return_value=None), \ patch.object(registry, "data_snapshot", return_value=self.job()["data"]), \ patch("harness.toolchains.resolve", return_value={}), \ patch.object(registry, "registry_fingerprint", return_value="fixture"): job = registry.build_job_json(spec, "run", "0.5.0") self.assertEqual(job["cache_contract"], {"version": 1, "model_python": str(self.root / "runtime-a/bin/python"), "exporter_version": "fixture-1"}) if __name__ == "__main__": unittest.main()