226 lines
12 KiB
Python
226 lines
12 KiB
Python
"""Model caches keep frozen runtime/task identities separate without moving legacy artifacts."""
|
|
import copy
|
|
from contextlib import ExitStack
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import paths
|
|
from harness.backends import ptq, qat
|
|
from harness.eval import task_contract
|
|
from harness.stages import s01_fetch
|
|
from harness.stages.s02_baseline import Baseline, baseline_dir
|
|
from harness.stages.s03_export import Export, onnx_path
|
|
|
|
|
|
class CacheIdentity(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.root = Path(self.tmp.name).resolve()
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
self.cfg = {"paths": {"venv": {"bench": str(self.root / "runtime-a")}}}
|
|
self.stack.enter_context(patch.object(paths, "config", return_value=self.cfg))
|
|
self.stack.enter_context(patch.object(paths, "CACHE", self.root / "cache"))
|
|
self.stack.enter_context(patch.object(paths, "RUNS", self.root / "runs"))
|
|
self.stack.enter_context(patch.object(s01_fetch, "INDEX", self.root / "cache/weights/index.json"))
|
|
s01_fetch.INDEX.parent.mkdir(parents=True)
|
|
s01_fetch.INDEX.write_text(json.dumps({"toy": {"sha256": "b" * 64}}))
|
|
self.exporter = self.stack.enter_context(patch("harness.tools.export_onnx.exporter_version", return_value="fixture-1"))
|
|
|
|
def job(self, runtime="a"):
|
|
python = str(self.root / f"runtime-{runtime}/bin/python")
|
|
return {"model": "toy", "job_id": "toy-job", "task": "cls", "backend": "ptq",
|
|
"model_spec": {"source": "custom", "input_spec": "1x3x4x4", "resize": 4},
|
|
"task_contract": task_contract("cls"), "evaluator_version": "0.5.0",
|
|
"model_runtime": {"configured": python, "python": python},
|
|
"cache_contract": {"version": 1, "model_python": python, "exporter_version": "fixture-1"},
|
|
"data": {"dataset": "fixture", "t1": {"content_sha": "a" * 64,
|
|
"manifest_sha": "c" * 64, "path": str(self.root / "t1.tar")},
|
|
"calib": {"path": str(self.root / "calib")}},
|
|
"gates": {"G2": {"n_samples": 4}, "G3": {"metric": "top1", "direction": "higher"}}}
|
|
|
|
def write_baseline(self, job, *, stamp=True):
|
|
directory = baseline_dir(job)
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
(directory / "metrics.json").write_text(json.dumps({"t1_manifest_sha": "a" * 64,
|
|
"evaluator_version": "0.5.0", "metrics": {"top1": 80}}))
|
|
if stamp:
|
|
Baseline().measure(SimpleNamespace(job=job), None, None)
|
|
return directory / "metrics.json"
|
|
|
|
def write_onnx(self, job, *, stamp=True):
|
|
path = onnx_path(job)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(b"fixture ONNX")
|
|
if stamp:
|
|
Export().measure(SimpleNamespace(job=job), None, None)
|
|
else:
|
|
(path.parent / "meta.json").write_text(json.dumps({"exporter_version": "fixture-1"}))
|
|
return path
|
|
|
|
def test_distinct_runtime_paths_never_overwrite_other_or_legacy_cache(self):
|
|
a, b = self.job("a"), self.job("b")
|
|
legacy = copy.deepcopy(a); legacy.pop("cache_contract")
|
|
old_base = self.write_baseline(legacy)
|
|
old_onnx = self.write_onnx(legacy)
|
|
before = (old_base.read_bytes(), old_onnx.read_bytes())
|
|
self.assertNotEqual(baseline_dir(a), baseline_dir(b))
|
|
self.assertNotEqual(onnx_path(a), onnx_path(b))
|
|
self.assertNotEqual(baseline_dir(a), baseline_dir(legacy))
|
|
self.assertNotEqual(onnx_path(a), onnx_path(legacy))
|
|
self.write_baseline(a); self.write_baseline(b)
|
|
self.write_onnx(a); self.write_onnx(b)
|
|
self.assertEqual((old_base.read_bytes(), old_onnx.read_bytes()), before)
|
|
|
|
def test_new_identity_requires_matching_metadata_and_reuses_same_identity(self):
|
|
job = self.job()
|
|
self.write_baseline(job, stamp=False); self.write_onnx(job, stamp=False)
|
|
ctx = SimpleNamespace(job=job)
|
|
self.assertIsNone(Baseline().cached(ctx))
|
|
self.assertIsNone(Export().cached(ctx))
|
|
baseline_before = (baseline_dir(job) / "metrics.json").read_bytes()
|
|
Baseline().cached(ctx)
|
|
self.assertEqual((baseline_dir(job) / "metrics.json").read_bytes(), baseline_before)
|
|
self.write_baseline(job); self.write_onnx(job)
|
|
self.assertIsNotNone(Baseline().cached(ctx))
|
|
self.assertIsNotNone(Export().cached(ctx))
|
|
self.assertEqual(Baseline().cached(ctx)["cache_identity"],
|
|
json.loads((baseline_dir(job) / "metrics.json").read_text())["cache_identity"])
|
|
|
|
def test_default_runtime_is_frozen_and_direct_cached_calls_reject_drift(self):
|
|
job = self.job()
|
|
job["model_runtime"] = None
|
|
job["cache_contract"] = paths.freeze_cache_contract(job)
|
|
self.write_baseline(job); self.write_onnx(job)
|
|
self.cfg["paths"]["venv"]["bench"] = str(self.root / "runtime-b")
|
|
for stage in (Baseline(), Export()):
|
|
with self.subTest(stage=stage.name), self.assertRaisesRegex(ValueError, "cache runtime"):
|
|
stage.cached(SimpleNamespace(job=job))
|
|
|
|
def test_baseline_task_and_sampling_change_identity_but_not_onnx(self):
|
|
original = self.job()
|
|
for change in ("task", "g2_samples", "t1"):
|
|
altered = copy.deepcopy(original)
|
|
if change == "task":
|
|
altered["task_contract"]["source_sha256"] = "changed"
|
|
elif change == "g2_samples":
|
|
altered["gates"]["G2"]["n_samples"] = 8
|
|
else:
|
|
altered["data"]["t1"]["content_sha"] = "d" * 64
|
|
with self.subTest(change=change):
|
|
self.assertNotEqual(baseline_dir(original), baseline_dir(altered))
|
|
self.assertEqual(onnx_path(original), onnx_path(altered))
|
|
original["gates"]["G3"]["direction"] = "lower"
|
|
self.assertEqual(onnx_path(original), onnx_path(self.job()))
|
|
self.assertEqual(baseline_dir(original), baseline_dir(self.job()))
|
|
|
|
def test_changed_input_and_exporter_have_distinct_onnx_paths(self):
|
|
original = self.job()
|
|
for key in ("input", "exporter"):
|
|
changed = copy.deepcopy(original)
|
|
if key == "input":
|
|
changed["model_spec"]["input_spec"] = "1x3x8x8"
|
|
else:
|
|
changed["cache_contract"]["exporter_version"] = "fixture-2"
|
|
with self.subTest(change=key):
|
|
self.assertNotEqual(onnx_path(original), onnx_path(changed))
|
|
|
|
def test_cached_rejects_task_and_exporter_drift_before_using_bytes(self):
|
|
job = self.job(); self.write_baseline(job); self.write_onnx(job)
|
|
with patch("harness.eval.task_contract", return_value={"source_sha256": "changed"}):
|
|
with self.assertRaisesRegex(ValueError, "task"):
|
|
Baseline().cached(SimpleNamespace(job=job))
|
|
self.assertIsNotNone(Export().cached(SimpleNamespace(job=job)))
|
|
self.exporter.return_value = "fixture-2"
|
|
with self.assertRaisesRegex(ValueError, "exporter"):
|
|
Export().cached(SimpleNamespace(job=job))
|
|
|
|
def test_legacy_path_and_explicit_subset_are_preserved(self):
|
|
job = self.job(); job.pop("cache_contract")
|
|
directory = paths.CACHE / "baseline/toy--fixture"
|
|
self.assertEqual(baseline_dir(job), directory)
|
|
self.assertEqual(onnx_path(job), paths.CACHE / "onnx" / ("b" * 64) / "toy.onnx")
|
|
job["data"]["t1"]["eval_set_id"] = "screen"
|
|
self.assertEqual(baseline_dir(job), directory / ("a" * 64) / "0.5.0")
|
|
|
|
def test_qat_and_ptq_export_use_same_model_python(self):
|
|
job = self.job("b")
|
|
for backend in (ptq, qat):
|
|
self.assertEqual(backend.export_cmd(job, self.root / "out.onnx", self.root / "weights")[0],
|
|
job["model_runtime"]["python"])
|
|
|
|
def test_relative_runtime_preserves_venv_launch_path_instead_of_python_symlink_target(self):
|
|
shared = self.root / "shared-python"
|
|
shared.write_bytes(b"fixture, never executed")
|
|
launch = self.root / "runtime-a/bin/python"
|
|
launch.parent.mkdir(parents=True)
|
|
launch.symlink_to(shared)
|
|
job = self.job(); job.pop("cache_contract")
|
|
job["model_runtime"] = None
|
|
job["model_spec"]["runtime_python"] = "runtime-a/bin/python"
|
|
with patch.object(paths, "BENCH_ROOT", self.root):
|
|
contract = paths.freeze_cache_contract(job)
|
|
self.assertEqual(contract["model_python"], str(launch))
|
|
job["model_runtime"] = {"configured": "runtime-a/bin/python", "python": str(launch)}
|
|
job["cache_contract"] = contract
|
|
self.assertEqual(paths.model_python(job), launch)
|
|
paths.cache_contract(job, validate=True)
|
|
|
|
def test_oldest_relative_runtime_keeps_old_interpreter_semantics(self):
|
|
shared = self.root / "shared-python"
|
|
shared.write_bytes(b"fixture, never executed")
|
|
launch = self.root / "legacy-venv/bin/python"
|
|
launch.parent.mkdir(parents=True)
|
|
launch.symlink_to(shared)
|
|
job = self.job(); job.pop("cache_contract"); job.pop("model_runtime")
|
|
job["model_spec"]["runtime_python"] = os.path.relpath(launch, paths.BENCH_ROOT)
|
|
self.assertEqual(paths.model_python(job), shared)
|
|
job["cache_contract"] = paths.freeze_cache_contract(job)
|
|
self.assertEqual(job["cache_contract"]["model_python"], str(launch))
|
|
self.assertEqual(paths.model_python(job), launch)
|
|
|
|
def test_repeat_publication_does_not_move_legacy_but_cannot_change_new_contract(self):
|
|
from harness.pipeline import _check_frozen_job
|
|
job = self.job()
|
|
old = copy.deepcopy(job); old.pop("cache_contract")
|
|
path = self.root / "job.json"
|
|
path.write_text(json.dumps(old))
|
|
before = path.read_bytes()
|
|
_check_frozen_job(path, job)
|
|
self.assertEqual(path.read_bytes(), before)
|
|
self.assertEqual(baseline_dir(json.loads(path.read_text())), baseline_dir(old))
|
|
path.write_text(json.dumps(job))
|
|
changed = copy.deepcopy(job)
|
|
changed["cache_contract"]["model_python"] = "/different/bin/python"
|
|
with self.assertRaisesRegex(ValueError, "content conflict"):
|
|
_check_frozen_job(path, changed)
|
|
|
|
def test_unknown_or_explicit_null_cache_contract_is_rejected(self):
|
|
for contract in (None, {}, {"version": 2}, {"version": True}):
|
|
job = self.job(); job["cache_contract"] = contract
|
|
with self.subTest(contract=contract), self.assertRaisesRegex(ValueError, "cache contract"):
|
|
Baseline().cached(SimpleNamespace(job=job))
|
|
|
|
def test_new_job_builder_freezes_cache_contract(self):
|
|
from harness import registry
|
|
spec = registry.JobSpec(job_id="toy-job", task="cls", model="toy", march="nash-m",
|
|
backend="ptq", qcfg="int8", priority="A", mode="oracle")
|
|
with patch.object(registry, "model_row", return_value=self.job()["model_spec"]), \
|
|
patch.object(registry, "board_for_march", return_value=None), \
|
|
patch.object(registry, "data_snapshot", return_value=self.job()["data"]), \
|
|
patch("harness.toolchains.resolve", return_value={}), \
|
|
patch.object(registry, "registry_fingerprint", return_value="fixture"):
|
|
job = registry.build_job_json(spec, "run", "0.5.0")
|
|
self.assertEqual(job["cache_contract"], {"version": 1,
|
|
"model_python": str(self.root / "runtime-a/bin/python"), "exporter_version": "fixture-1"})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|