data-gen-bpu/scripts/test_cache_identity.py

226 lines
12 KiB
Python

"""Model caches keep frozen runtime/task identities separate without moving legacy artifacts."""
import copy
from contextlib import ExitStack
import json
import os
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from harness import paths
from harness.backends import ptq, qat
from harness.eval import task_contract
from harness.stages import s01_fetch
from harness.stages.s02_baseline import Baseline, baseline_dir
from harness.stages.s03_export import Export, onnx_path
class CacheIdentity(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name).resolve()
self.stack = ExitStack()
self.addCleanup(self.stack.close)
self.cfg = {"paths": {"venv": {"bench": str(self.root / "runtime-a")}}}
self.stack.enter_context(patch.object(paths, "config", return_value=self.cfg))
self.stack.enter_context(patch.object(paths, "CACHE", self.root / "cache"))
self.stack.enter_context(patch.object(paths, "RUNS", self.root / "runs"))
self.stack.enter_context(patch.object(s01_fetch, "INDEX", self.root / "cache/weights/index.json"))
s01_fetch.INDEX.parent.mkdir(parents=True)
s01_fetch.INDEX.write_text(json.dumps({"toy": {"sha256": "b" * 64}}))
self.exporter = self.stack.enter_context(patch("harness.tools.export_onnx.exporter_version", return_value="fixture-1"))
def job(self, runtime="a"):
python = str(self.root / f"runtime-{runtime}/bin/python")
return {"model": "toy", "job_id": "toy-job", "task": "cls", "backend": "ptq",
"model_spec": {"source": "custom", "input_spec": "1x3x4x4", "resize": 4},
"task_contract": task_contract("cls"), "evaluator_version": "0.5.0",
"model_runtime": {"configured": python, "python": python},
"cache_contract": {"version": 1, "model_python": python, "exporter_version": "fixture-1"},
"data": {"dataset": "fixture", "t1": {"content_sha": "a" * 64,
"manifest_sha": "c" * 64, "path": str(self.root / "t1.tar")},
"calib": {"path": str(self.root / "calib")}},
"gates": {"G2": {"n_samples": 4}, "G3": {"metric": "top1", "direction": "higher"}}}
def write_baseline(self, job, *, stamp=True):
directory = baseline_dir(job)
directory.mkdir(parents=True, exist_ok=True)
(directory / "metrics.json").write_text(json.dumps({"t1_manifest_sha": "a" * 64,
"evaluator_version": "0.5.0", "metrics": {"top1": 80}}))
if stamp:
Baseline().measure(SimpleNamespace(job=job), None, None)
return directory / "metrics.json"
def write_onnx(self, job, *, stamp=True):
path = onnx_path(job)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"fixture ONNX")
if stamp:
Export().measure(SimpleNamespace(job=job), None, None)
else:
(path.parent / "meta.json").write_text(json.dumps({"exporter_version": "fixture-1"}))
return path
def test_distinct_runtime_paths_never_overwrite_other_or_legacy_cache(self):
a, b = self.job("a"), self.job("b")
legacy = copy.deepcopy(a); legacy.pop("cache_contract")
old_base = self.write_baseline(legacy)
old_onnx = self.write_onnx(legacy)
before = (old_base.read_bytes(), old_onnx.read_bytes())
self.assertNotEqual(baseline_dir(a), baseline_dir(b))
self.assertNotEqual(onnx_path(a), onnx_path(b))
self.assertNotEqual(baseline_dir(a), baseline_dir(legacy))
self.assertNotEqual(onnx_path(a), onnx_path(legacy))
self.write_baseline(a); self.write_baseline(b)
self.write_onnx(a); self.write_onnx(b)
self.assertEqual((old_base.read_bytes(), old_onnx.read_bytes()), before)
def test_new_identity_requires_matching_metadata_and_reuses_same_identity(self):
job = self.job()
self.write_baseline(job, stamp=False); self.write_onnx(job, stamp=False)
ctx = SimpleNamespace(job=job)
self.assertIsNone(Baseline().cached(ctx))
self.assertIsNone(Export().cached(ctx))
baseline_before = (baseline_dir(job) / "metrics.json").read_bytes()
Baseline().cached(ctx)
self.assertEqual((baseline_dir(job) / "metrics.json").read_bytes(), baseline_before)
self.write_baseline(job); self.write_onnx(job)
self.assertIsNotNone(Baseline().cached(ctx))
self.assertIsNotNone(Export().cached(ctx))
self.assertEqual(Baseline().cached(ctx)["cache_identity"],
json.loads((baseline_dir(job) / "metrics.json").read_text())["cache_identity"])
def test_default_runtime_is_frozen_and_direct_cached_calls_reject_drift(self):
job = self.job()
job["model_runtime"] = None
job["cache_contract"] = paths.freeze_cache_contract(job)
self.write_baseline(job); self.write_onnx(job)
self.cfg["paths"]["venv"]["bench"] = str(self.root / "runtime-b")
for stage in (Baseline(), Export()):
with self.subTest(stage=stage.name), self.assertRaisesRegex(ValueError, "cache runtime"):
stage.cached(SimpleNamespace(job=job))
def test_baseline_task_and_sampling_change_identity_but_not_onnx(self):
original = self.job()
for change in ("task", "g2_samples", "t1"):
altered = copy.deepcopy(original)
if change == "task":
altered["task_contract"]["source_sha256"] = "changed"
elif change == "g2_samples":
altered["gates"]["G2"]["n_samples"] = 8
else:
altered["data"]["t1"]["content_sha"] = "d" * 64
with self.subTest(change=change):
self.assertNotEqual(baseline_dir(original), baseline_dir(altered))
self.assertEqual(onnx_path(original), onnx_path(altered))
original["gates"]["G3"]["direction"] = "lower"
self.assertEqual(onnx_path(original), onnx_path(self.job()))
self.assertEqual(baseline_dir(original), baseline_dir(self.job()))
def test_changed_input_and_exporter_have_distinct_onnx_paths(self):
original = self.job()
for key in ("input", "exporter"):
changed = copy.deepcopy(original)
if key == "input":
changed["model_spec"]["input_spec"] = "1x3x8x8"
else:
changed["cache_contract"]["exporter_version"] = "fixture-2"
with self.subTest(change=key):
self.assertNotEqual(onnx_path(original), onnx_path(changed))
def test_cached_rejects_task_and_exporter_drift_before_using_bytes(self):
job = self.job(); self.write_baseline(job); self.write_onnx(job)
with patch("harness.eval.task_contract", return_value={"source_sha256": "changed"}):
with self.assertRaisesRegex(ValueError, "task"):
Baseline().cached(SimpleNamespace(job=job))
self.assertIsNotNone(Export().cached(SimpleNamespace(job=job)))
self.exporter.return_value = "fixture-2"
with self.assertRaisesRegex(ValueError, "exporter"):
Export().cached(SimpleNamespace(job=job))
def test_legacy_path_and_explicit_subset_are_preserved(self):
job = self.job(); job.pop("cache_contract")
directory = paths.CACHE / "baseline/toy--fixture"
self.assertEqual(baseline_dir(job), directory)
self.assertEqual(onnx_path(job), paths.CACHE / "onnx" / ("b" * 64) / "toy.onnx")
job["data"]["t1"]["eval_set_id"] = "screen"
self.assertEqual(baseline_dir(job), directory / ("a" * 64) / "0.5.0")
def test_qat_and_ptq_export_use_same_model_python(self):
job = self.job("b")
for backend in (ptq, qat):
self.assertEqual(backend.export_cmd(job, self.root / "out.onnx", self.root / "weights")[0],
job["model_runtime"]["python"])
def test_relative_runtime_preserves_venv_launch_path_instead_of_python_symlink_target(self):
shared = self.root / "shared-python"
shared.write_bytes(b"fixture, never executed")
launch = self.root / "runtime-a/bin/python"
launch.parent.mkdir(parents=True)
launch.symlink_to(shared)
job = self.job(); job.pop("cache_contract")
job["model_runtime"] = None
job["model_spec"]["runtime_python"] = "runtime-a/bin/python"
with patch.object(paths, "BENCH_ROOT", self.root):
contract = paths.freeze_cache_contract(job)
self.assertEqual(contract["model_python"], str(launch))
job["model_runtime"] = {"configured": "runtime-a/bin/python", "python": str(launch)}
job["cache_contract"] = contract
self.assertEqual(paths.model_python(job), launch)
paths.cache_contract(job, validate=True)
def test_oldest_relative_runtime_keeps_old_interpreter_semantics(self):
shared = self.root / "shared-python"
shared.write_bytes(b"fixture, never executed")
launch = self.root / "legacy-venv/bin/python"
launch.parent.mkdir(parents=True)
launch.symlink_to(shared)
job = self.job(); job.pop("cache_contract"); job.pop("model_runtime")
job["model_spec"]["runtime_python"] = os.path.relpath(launch, paths.BENCH_ROOT)
self.assertEqual(paths.model_python(job), shared)
job["cache_contract"] = paths.freeze_cache_contract(job)
self.assertEqual(job["cache_contract"]["model_python"], str(launch))
self.assertEqual(paths.model_python(job), launch)
def test_repeat_publication_does_not_move_legacy_but_cannot_change_new_contract(self):
from harness.pipeline import _check_frozen_job
job = self.job()
old = copy.deepcopy(job); old.pop("cache_contract")
path = self.root / "job.json"
path.write_text(json.dumps(old))
before = path.read_bytes()
_check_frozen_job(path, job)
self.assertEqual(path.read_bytes(), before)
self.assertEqual(baseline_dir(json.loads(path.read_text())), baseline_dir(old))
path.write_text(json.dumps(job))
changed = copy.deepcopy(job)
changed["cache_contract"]["model_python"] = "/different/bin/python"
with self.assertRaisesRegex(ValueError, "content conflict"):
_check_frozen_job(path, changed)
def test_unknown_or_explicit_null_cache_contract_is_rejected(self):
for contract in (None, {}, {"version": 2}, {"version": True}):
job = self.job(); job["cache_contract"] = contract
with self.subTest(contract=contract), self.assertRaisesRegex(ValueError, "cache contract"):
Baseline().cached(SimpleNamespace(job=job))
def test_new_job_builder_freezes_cache_contract(self):
from harness import registry
spec = registry.JobSpec(job_id="toy-job", task="cls", model="toy", march="nash-m",
backend="ptq", qcfg="int8", priority="A", mode="oracle")
with patch.object(registry, "model_row", return_value=self.job()["model_spec"]), \
patch.object(registry, "board_for_march", return_value=None), \
patch.object(registry, "data_snapshot", return_value=self.job()["data"]), \
patch("harness.toolchains.resolve", return_value={}), \
patch.object(registry, "registry_fingerprint", return_value="fixture"):
job = registry.build_job_json(spec, "run", "0.5.0")
self.assertEqual(job["cache_contract"], {"version": 1,
"model_python": str(self.root / "runtime-a/bin/python"), "exporter_version": "fixture-1"})
if __name__ == "__main__":
unittest.main()