data-gen-bpu/scripts/test_cache_identity.py
bench 6ab088d7ee 评测分批不建新批次也能上:冻结规则增加逐对批准的等价改写,任务文件换成分批版本
- harness/eval/__init__.py:EQUIVALENT_SOURCES 列出精确的「冻结指纹 → 现指纹」对,contract_holds 判断现文件
  仍实现冻结合同(完全相同,或同一模块的已批准改写)。阶段检查、基线检查、expand 比对三处都走它。
- 批准的只有 f56ac2f 的 5 个任务文件:上板由逐张改为按实测字节分批,板上仍逐个样本推理
  (board_bench.infer_inputs),每张的预处理、后处理和指标一字未改。
- 批次的冻结记录不动,基线缓存按冻结指纹寻址,照常命中,不用重算。
- 用户 10-15:不建新批次。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 15:10:04 +08:00

239 lines
13 KiB
Python

"""Model caches keep frozen runtime/task identities separate without moving legacy artifacts."""
import copy
from contextlib import ExitStack
import json
import os
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from harness import paths
from harness.backends import ptq, qat
from harness.eval import task_contract
from harness.stages import s01_fetch
from harness.stages.s02_baseline import Baseline, baseline_dir
from harness.stages.s03_export import Export, onnx_path
class CacheIdentity(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name).resolve()
self.stack = ExitStack()
self.addCleanup(self.stack.close)
self.cfg = {"paths": {"venv": {"bench": str(self.root / "runtime-a")}}}
self.stack.enter_context(patch.object(paths, "config", return_value=self.cfg))
self.stack.enter_context(patch.object(paths, "CACHE", self.root / "cache"))
self.stack.enter_context(patch.object(paths, "RUNS", self.root / "runs"))
self.stack.enter_context(patch.object(s01_fetch, "INDEX", self.root / "cache/weights/index.json"))
s01_fetch.INDEX.parent.mkdir(parents=True)
s01_fetch.INDEX.write_text(json.dumps({"toy": {"sha256": "b" * 64}}))
self.exporter = self.stack.enter_context(patch("harness.tools.export_onnx.exporter_version", return_value="fixture-1"))
def job(self, runtime="a"):
python = str(self.root / f"runtime-{runtime}/bin/python")
return {"model": "toy", "job_id": "toy-job", "task": "cls", "backend": "ptq",
"model_spec": {"source": "custom", "input_spec": "1x3x4x4", "resize": 4},
"task_contract": task_contract("cls"), "evaluator_version": "0.5.0",
"model_runtime": {"configured": python, "python": python},
"cache_contract": {"version": 1, "model_python": python, "exporter_version": "fixture-1"},
"data": {"dataset": "fixture", "t1": {"content_sha": "a" * 64,
"manifest_sha": "c" * 64, "path": str(self.root / "t1.tar")},
"calib": {"path": str(self.root / "calib")}},
"gates": {"G2": {"n_samples": 4}, "G3": {"metric": "top1", "direction": "higher"}}}
def write_baseline(self, job, *, stamp=True):
directory = baseline_dir(job)
directory.mkdir(parents=True, exist_ok=True)
(directory / "metrics.json").write_text(json.dumps({"t1_manifest_sha": "a" * 64,
"evaluator_version": "0.5.0", "metrics": {"top1": 80}}))
if stamp:
Baseline().measure(SimpleNamespace(job=job), None, None)
return directory / "metrics.json"
def write_onnx(self, job, *, stamp=True):
path = onnx_path(job)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"fixture ONNX")
if stamp:
Export().measure(SimpleNamespace(job=job), None, None)
else:
(path.parent / "meta.json").write_text(json.dumps({"exporter_version": "fixture-1"}))
return path
def test_distinct_runtime_paths_never_overwrite_other_or_legacy_cache(self):
a, b = self.job("a"), self.job("b")
legacy = copy.deepcopy(a); legacy.pop("cache_contract")
old_base = self.write_baseline(legacy)
old_onnx = self.write_onnx(legacy)
before = (old_base.read_bytes(), old_onnx.read_bytes())
self.assertNotEqual(baseline_dir(a), baseline_dir(b))
self.assertNotEqual(onnx_path(a), onnx_path(b))
self.assertNotEqual(baseline_dir(a), baseline_dir(legacy))
self.assertNotEqual(onnx_path(a), onnx_path(legacy))
self.write_baseline(a); self.write_baseline(b)
self.write_onnx(a); self.write_onnx(b)
self.assertEqual((old_base.read_bytes(), old_onnx.read_bytes()), before)
def test_new_identity_requires_matching_metadata_and_reuses_same_identity(self):
job = self.job()
self.write_baseline(job, stamp=False); self.write_onnx(job, stamp=False)
ctx = SimpleNamespace(job=job)
self.assertIsNone(Baseline().cached(ctx))
self.assertIsNone(Export().cached(ctx))
baseline_before = (baseline_dir(job) / "metrics.json").read_bytes()
Baseline().cached(ctx)
self.assertEqual((baseline_dir(job) / "metrics.json").read_bytes(), baseline_before)
self.write_baseline(job); self.write_onnx(job)
self.assertIsNotNone(Baseline().cached(ctx))
self.assertIsNotNone(Export().cached(ctx))
self.assertEqual(Baseline().cached(ctx)["cache_identity"],
json.loads((baseline_dir(job) / "metrics.json").read_text())["cache_identity"])
def test_default_runtime_is_frozen_and_direct_cached_calls_reject_drift(self):
job = self.job()
job["model_runtime"] = None
job["cache_contract"] = paths.freeze_cache_contract(job)
self.write_baseline(job); self.write_onnx(job)
self.cfg["paths"]["venv"]["bench"] = str(self.root / "runtime-b")
for stage in (Baseline(), Export()):
with self.subTest(stage=stage.name), self.assertRaisesRegex(ValueError, "cache runtime"):
stage.cached(SimpleNamespace(job=job))
def test_baseline_task_and_sampling_change_identity_but_not_onnx(self):
original = self.job()
for change in ("task", "g2_samples", "t1"):
altered = copy.deepcopy(original)
if change == "task":
altered["task_contract"]["source_sha256"] = "changed"
elif change == "g2_samples":
altered["gates"]["G2"]["n_samples"] = 8
else:
altered["data"]["t1"]["content_sha"] = "d" * 64
with self.subTest(change=change):
self.assertNotEqual(baseline_dir(original), baseline_dir(altered))
self.assertEqual(onnx_path(original), onnx_path(altered))
original["gates"]["G3"]["direction"] = "lower"
self.assertEqual(onnx_path(original), onnx_path(self.job()))
self.assertEqual(baseline_dir(original), baseline_dir(self.job()))
def test_changed_input_and_exporter_have_distinct_onnx_paths(self):
original = self.job()
for key in ("input", "exporter"):
changed = copy.deepcopy(original)
if key == "input":
changed["model_spec"]["input_spec"] = "1x3x8x8"
else:
changed["cache_contract"]["exporter_version"] = "fixture-2"
with self.subTest(change=key):
self.assertNotEqual(onnx_path(original), onnx_path(changed))
def test_only_approved_task_rewrites_keep_the_frozen_contract(self):
# 10-15: board-call batching was approved as an exact rewrite, so running runs keep their contract and cached
# baselines; any other change, a reverse step or another module still fails.
from harness.eval import EQUIVALENT_SOURCES, contract_holds
old, new = sorted(EQUIVALENT_SOURCES)[0]
frozen = {**task_contract("seg"), "source_sha256": old}
self.assertTrue(contract_holds(frozen, {**frozen, "source_sha256": new}))
self.assertFalse(contract_holds(frozen, {**frozen, "source_sha256": "other"}))
self.assertFalse(contract_holds({**frozen, "source_sha256": new}, frozen))
self.assertFalse(contract_holds(frozen, {**frozen, "source_sha256": new, "module": "harness.eval.det"}))
current = {task_contract(t)["source_sha256"] for t in ("seg", "mde", "flow", "feat", "pose")}
self.assertEqual(current, {b for _, b in EQUIVALENT_SOURCES})
def test_cached_rejects_task_and_exporter_drift_before_using_bytes(self):
job = self.job(); self.write_baseline(job); self.write_onnx(job)
with patch("harness.eval.task_contract", return_value={"source_sha256": "changed"}):
with self.assertRaisesRegex(ValueError, "task"):
Baseline().cached(SimpleNamespace(job=job))
self.assertIsNotNone(Export().cached(SimpleNamespace(job=job)))
self.exporter.return_value = "fixture-2"
with self.assertRaisesRegex(ValueError, "exporter"):
Export().cached(SimpleNamespace(job=job))
def test_legacy_path_and_explicit_subset_are_preserved(self):
job = self.job(); job.pop("cache_contract")
directory = paths.CACHE / "baseline/toy--fixture"
self.assertEqual(baseline_dir(job), directory)
self.assertEqual(onnx_path(job), paths.CACHE / "onnx" / ("b" * 64) / "toy.onnx")
job["data"]["t1"]["eval_set_id"] = "screen"
self.assertEqual(baseline_dir(job), directory / ("a" * 64) / "0.5.0")
def test_qat_and_ptq_export_use_same_model_python(self):
job = self.job("b")
for backend in (ptq, qat):
self.assertEqual(backend.export_cmd(job, self.root / "out.onnx", self.root / "weights")[0],
job["model_runtime"]["python"])
def test_relative_runtime_preserves_venv_launch_path_instead_of_python_symlink_target(self):
shared = self.root / "shared-python"
shared.write_bytes(b"fixture, never executed")
launch = self.root / "runtime-a/bin/python"
launch.parent.mkdir(parents=True)
launch.symlink_to(shared)
job = self.job(); job.pop("cache_contract")
job["model_runtime"] = None
job["model_spec"]["runtime_python"] = "runtime-a/bin/python"
with patch.object(paths, "BENCH_ROOT", self.root):
contract = paths.freeze_cache_contract(job)
self.assertEqual(contract["model_python"], str(launch))
job["model_runtime"] = {"configured": "runtime-a/bin/python", "python": str(launch)}
job["cache_contract"] = contract
self.assertEqual(paths.model_python(job), launch)
paths.cache_contract(job, validate=True)
def test_oldest_relative_runtime_keeps_old_interpreter_semantics(self):
shared = self.root / "shared-python"
shared.write_bytes(b"fixture, never executed")
launch = self.root / "legacy-venv/bin/python"
launch.parent.mkdir(parents=True)
launch.symlink_to(shared)
job = self.job(); job.pop("cache_contract"); job.pop("model_runtime")
job["model_spec"]["runtime_python"] = os.path.relpath(launch, paths.BENCH_ROOT)
self.assertEqual(paths.model_python(job), shared)
job["cache_contract"] = paths.freeze_cache_contract(job)
self.assertEqual(job["cache_contract"]["model_python"], str(launch))
self.assertEqual(paths.model_python(job), launch)
def test_repeat_publication_does_not_move_legacy_but_cannot_change_new_contract(self):
from harness.pipeline import _check_frozen_job
job = self.job()
old = copy.deepcopy(job); old.pop("cache_contract")
path = self.root / "job.json"
path.write_text(json.dumps(old))
before = path.read_bytes()
_check_frozen_job(path, job)
self.assertEqual(path.read_bytes(), before)
self.assertEqual(baseline_dir(json.loads(path.read_text())), baseline_dir(old))
path.write_text(json.dumps(job))
changed = copy.deepcopy(job)
changed["cache_contract"]["model_python"] = "/different/bin/python"
with self.assertRaisesRegex(ValueError, "content conflict"):
_check_frozen_job(path, changed)
def test_unknown_or_explicit_null_cache_contract_is_rejected(self):
for contract in (None, {}, {"version": 2}, {"version": True}):
job = self.job(); job["cache_contract"] = contract
with self.subTest(contract=contract), self.assertRaisesRegex(ValueError, "cache contract"):
Baseline().cached(SimpleNamespace(job=job))
def test_new_job_builder_freezes_cache_contract(self):
from harness import registry
spec = registry.JobSpec(job_id="toy-job", task="cls", model="toy", march="nash-m",
backend="ptq", qcfg="int8", priority="A", mode="oracle")
with patch.object(registry, "model_row", return_value=self.job()["model_spec"]), \
patch.object(registry, "board_for_march", return_value=None), \
patch.object(registry, "data_snapshot", return_value=self.job()["data"]), \
patch("harness.toolchains.resolve", return_value={}), \
patch.object(registry, "registry_fingerprint", return_value="fixture"):
job = registry.build_job_json(spec, "run", "0.5.0")
self.assertEqual(job["cache_contract"], {"version": 1,
"model_python": str(self.root / "runtime-a/bin/python"), "exporter_version": "fixture-1"})
if __name__ == "__main__":
unittest.main()