data-gen-bpu/scripts/test_recipe_provenance.py

275 lines
15 KiB
Python

"""Real-file counterexamples for recipe attempt, parent and artifact identity."""
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from types import SimpleNamespace
from harness import paths, verify
from harness.stages.base import Stage, StageContext, StageOutcome, recipe_identity, recipe_output, recipe_receipt, recipe_artifact, work_dir
from harness.stages.s04_calibrate import Calibrate
from harness.stages.s05_compile import Compile
from harness.stages.s06_hbm import Hbm
class RecipeProvenance(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name).resolve()
self.rd = self.root / "rollouts/00"
self.rd.mkdir(parents=True)
self.job = {"job_id": "fixture", "run_id": "run", "task": "seg", "backend": "ptq", "march": "nash-m", "qcfg": "int8",
"recipe": {"submission_sha256": "a" * 64, "reference_sha256": "b" * 64},
"model_spec": {"input_spec": "1x3x4x4", "preprocess": {"resize": 4}},
"gates": {"G2": {"n_samples": 2}}, "data": {"t1": {"content_sha": "t1"},
"calib": {"path": str(self.root / "calib")}}, "evaluator_version": "fixture"}
self.ctx = StageContext("run", self.job, self.root, self.rd / "stages", self.rd / "artifacts", rollout=0,
extra={"runtime_identity": {"python": "fixture-runtime"}, "recipe_parents": {}})
self.clock = 0
self.addCleanup(patch.stopall)
patch.object(paths, "config", return_value={"paths": {"work_root": str(self.root / "work")}}).start()
def file(self, name, content=b"fixture"):
path = self.root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
return path
def record(self, stage, attempt=0, *, state="OK", outputs=None, parents=(), measurements=None, **extra):
self.ctx.extra["recipe_stage"] = stage
self.ctx.extra["recipe_parents"] = {}
for name, reference in parents:
recipe_receipt(self.ctx, name, reference)
self.clock += 1
meta = {"stage": stage, "attempt": attempt, "state": state, "started_at": self.clock,
"recipe": recipe_identity(self.ctx), "recipe_parents": dict(self.ctx.extra["recipe_parents"]),
"recipe_outputs": {name: recipe_output(path) for name, path in (outputs or {}).items()},
"measurements": measurements or {}, **extra}
sd = self.ctx.stage_root / stage / f"a{attempt:02d}"
sd.mkdir(parents=True, exist_ok=True)
(sd / "meta.json").write_text(json.dumps(meta))
(sd.parent / "meta.json").write_text(json.dumps(meta))
return meta
def export(self, attempt=0):
return self.record("03-export", attempt, outputs={"onnx": self.file(f"graph-{attempt}.onnx")},
recipe_input={"input_spec": "1x3x4x4", "preprocess": {"resize": 4}})
def test_reference_cache_excludes_candidate_but_includes_runtime_and_evaluation(self):
first = recipe_identity(self.ctx)
self.job["recipe"]["submission_sha256"] = "c" * 64
second = recipe_identity(self.ctx)
self.assertNotEqual(first["submission"], second["submission"])
self.assertEqual(first["reference"], second["reference"])
self.ctx.extra["runtime_identity"]["python"] = "different-runtime"
self.assertNotEqual(second["reference"], recipe_identity(self.ctx)["reference"])
self.ctx.extra["runtime_identity"]["python"] = "fixture-runtime"
self.job["data"]["t1"]["content_sha"] = "different-subset"
self.assertNotEqual(second["reference"], recipe_identity(self.ctx)["reference"])
def test_failed_newest_attempt_never_falls_back_to_old_success(self):
self.export()
self.record("03-export", 1, state="FAIL")
with self.assertRaisesRegex(ValueError, "did not succeed"):
recipe_artifact(self.ctx, "03-export", "onnx")
def test_historical_parent_remains_valid_but_artifact_or_meta_drift_fails(self):
graph = self.export()
self.record("06-hbm", outputs={"hbm": self.file("first.hbm")}, parents=(("03-export", False),))
self.assertEqual(recipe_artifact(self.ctx, "06-hbm", "hbm"), self.root / "first.hbm")
Path(graph["recipe_outputs"]["onnx"]["path"]).write_bytes(b"changed")
with self.assertRaisesRegex(ValueError, "artifact changed"):
recipe_artifact(self.ctx, "06-hbm", "hbm")
Path(graph["recipe_outputs"]["onnx"]["path"]).write_bytes(b"fixture")
self.export(1)
self.assertEqual(recipe_artifact(self.ctx, "06-hbm", "hbm"), self.root / "first.hbm")
self.record("03-export", 2, state="FAIL")
self.assertEqual(recipe_artifact(self.ctx, "06-hbm", "hbm"), self.root / "first.hbm")
with self.assertRaisesRegex(ValueError, "did not succeed"):
recipe_artifact(self.ctx, "03-export", "onnx")
old_meta = self.ctx.stage_root / "03-export/a00/meta.json"
old_meta.write_text(old_meta.read_text() + " ")
with self.assertRaisesRegex(ValueError, "parent changed"):
recipe_artifact(self.ctx, "06-hbm", "hbm")
def test_selected_parent_returns_its_graph_not_a_newer_export(self):
self.export()
parent = self.record("04-calibrate", parents=(("03-export", False),))
self.export(1)
self.assertEqual(recipe_artifact(self.ctx, "03-export", "onnx", parent=parent), self.root / "graph-0.onnx")
self.assertEqual(recipe_artifact(self.ctx, "03-export", "onnx"), self.root / "graph-1.onnx")
def test_one_verification_traversal_hashes_shared_ancestors_once(self):
self.export()
self.record("04-calibrate", outputs={"calibration": self.file("calib.npy")}, parents=(("03-export", False),))
self.record("05-compile", outputs={"hbm": self.file("compiled.hbm")},
parents=(("03-export", False), ("04-calibrate", False)))
self.record("06-hbm", outputs={"hbm": self.file("final.hbm")},
parents=(("03-export", False), ("04-calibrate", False), ("05-compile", False)))
with patch("harness.stages.base.recipe_output", wraps=recipe_output) as describe:
recipe_receipt(self.ctx, "06-hbm")
self.assertEqual(describe.call_count, 4)
self.file("graph-0.onnx", b"tampered after verification")
with self.assertRaisesRegex(ValueError, "artifact changed"):
recipe_receipt(self.ctx, "06-hbm")
def test_graph_only_calibration_keeps_original_bytes_and_accepts_numeric_overrides(self):
import yaml
self.export()
original = {"count": 1, "source": "existing-platform", "preprocessing_identity": {"resize": 999}}
self.file("calib/manifest.json", json.dumps(original).encode())
self.file("calib/npy_f32/000.npy")
self.ctx.agent_config = self.file("config.yaml", b"input_parameters:\n mean_value: '1 2 3'\n")
self.ctx.extra.update(recipe_stage="04-calibrate", compile_attempt=3)
sd = self.ctx.stage_root / "04-calibrate/a00"
sd.mkdir(parents=True)
with patch("harness.stages.s04_calibrate.watchdog.run", side_effect=AssertionError("graph-only must not execute model code")):
result = Calibrate().native(self.ctx, sd)
self.assertEqual(result.state, "OK")
self.assertEqual(result.meta["calibration_mode"], "platform")
self.assertEqual(result.meta["calibration_manifest"], original)
self.assertEqual(result.meta["recipe_outputs"]["calibration"], recipe_output(self.root / "calib"))
config = yaml.safe_load((sd / "compile.yaml").read_text())
self.assertEqual(config["input_parameters"]["mean_value"], "1 2 3")
self.assertEqual(config["calibration_parameters"]["cal_data_dir"], str(self.root / "calib/npy_f32"))
def test_all_output_descriptors_are_checked_including_external_tensor_bundle(self):
graph = self.file("bundle/model.onnx")
tensor = self.file("bundle/weights.bin")
self.record("03-export", outputs={"onnx": graph, "onnx_bundle": graph.parent})
tensor.write_bytes(b"different external tensor")
with self.assertRaisesRegex(ValueError, "artifact changed"):
recipe_artifact(self.ctx, "03-export", "onnx")
def test_descriptor_rejects_symlinks_and_hardlinks(self):
original = self.file("original")
link = self.root / "link"
link.symlink_to(original)
with self.assertRaisesRegex(ValueError, "symlink"):
recipe_output(link)
link.unlink()
os.link(original, link)
with self.assertRaisesRegex(ValueError, "regular file"):
recipe_output(original)
def test_verifier_does_not_mix_new_graph_with_old_hbm_or_scores(self):
self.export()
self.record("06-hbm", outputs={"hbm": self.file("first.hbm")}, parents=(("03-export", False),),
measurements={"G1": {"hbm_exists": True}})
self.record("07-board", parents=(("06-hbm", False),), measurements={"G4": {"latency_ms_mean": 1}})
self.assertIn("G4", verify.collect_measurements(self.rd)[0])
self.job["recipe"]["submission_sha256"] = "c" * 64
self.export(1)
meas, summary = verify.collect_measurements(self.rd)
self.assertNotIn("G4", meas)
self.assertEqual(summary["06-hbm"]["state"], "UNVERIFIED")
self.file("rollouts/00/artifacts/fixture.hbm", b"legacy fallback must not be used")
self.assertFalse(verify.rollout_hbm_path(self.rd, "fixture").exists())
def test_accepted_submission_without_stage_output_invalidates_older_scores(self):
self.export()
self.record("06-hbm", outputs={"hbm": self.file("first.hbm")}, measurements={"G1": {"hbm_exists": True}})
frozen = self.file("queued/recipe.json", json.dumps({"submission_sha256": "c" * 64}).encode())
request = self.ctx.stage_root / "_queue/new/request.json"
request.parent.mkdir(parents=True)
request.write_text(json.dumps({"received_at": 10, "recipe": str(frozen)}))
self.assertFalse(verify.collect_measurements(self.rd)[0]["G1"]["hbm_exists"])
def test_calibration_metadata_cannot_replace_frozen_training_sources(self):
self.export()
self.file("calib/manifest.json", b'{"count":1}')
self.file("calib/npy_f32/000.npy")
self.ctx.extra.update(recipe_stage="04-calibrate", compile_attempt=3)
sd = self.ctx.stage_root / "04-calibrate/a00"
sd.mkdir(parents=True)
self.job["recipe"]["entrypoint"] = "candidate.py"
self.job["data"]["calib"]["preprocessing_identity"] = {"input_spec": "1x3x4x4", "preprocess": {"resize": 4}}
with patch.object(paths, "model_python", return_value=Path("/fixture/python")), \
self.assertRaisesRegex(ValueError, "frozen dataset training sources"):
Calibrate().native(self.ctx, sd)
def test_compile_requires_unique_hbm_and_records_specific_attempt_files(self):
self.ctx.extra["recipe_stage"] = "05-compile"
self.ctx.attempt = 2
wd = work_dir(self.ctx)
self.assertTrue(str(wd).endswith("05-compile/a02"))
(wd / "one.hbm").write_bytes(b"hbm")
(wd / "one_quantized_model.bc").write_bytes(b"bc")
(wd / "one_ptq_model.onnx").write_bytes(b"onnx")
out = StageOutcome("OK")
Compile().measure(self.ctx, self.ctx.stage_root, out)
self.assertEqual(set(out.meta["recipe_outputs"]), {"hbm", "bc", "ptq_onnx"})
(wd / "other.hbm").write_bytes(b"other")
with self.assertRaisesRegex(ValueError, "exactly one HBM"):
Compile().measure(self.ctx, self.ctx.stage_root, out)
def test_hbm_uses_declared_compile_output_and_only_that_attempt_logs(self):
self.export()
wd = self.root / "compile-attempt"
source = self.file("compile-attempt/declared.hbm", b"declared HBM")
bc = self.file("compile-attempt/quantized_model.bc", b"BC")
onnx = self.file("compile-attempt/model_ptq_model.onnx", b"PTQ ONNX")
calibration = self.file("calib/npy_f32/000.npy").parent.parent
self.record("04-calibrate", outputs={"calibration": calibration}, parents=(("03-export", False),))
self.record("05-compile", 1, outputs={"hbm": source, "bc": bc, "ptq_onnx": onnx},
parents=(("03-export", False), ("04-calibrate", False)), work_dir=str(wd))
self.file("compile-attempt/newer-unrelated.hbm", b"must not choose by mtime")
self.file("rollouts/00/stages/05-compile/a00/stderr.log", b"ERROR stale failed attempt")
self.file("rollouts/00/stages/05-compile/a01/stderr.log", b"INFO current compile")
self.job["gates"]["G1"] = {}
self.ctx.extra.update(recipe_stage="06-hbm", recipe_parents={})
sd = self.ctx.stage_root / "06-hbm/a00"
sd.mkdir(parents=True)
def run(command, **kw):
kw["stdout"].write_text("march: nash-m\ninput input [1, 3, 4, 4] FLOAT32\n")
kw["stderr"].write_text("")
return SimpleNamespace(state="OK", exit_code=0)
backend = SimpleNamespace(model_info_cmd=lambda *a, **kw: ["fixture-info"],
verifier_cmd=lambda *a, **kw: ["fixture-verifier"])
with patch("harness.stages.s06_hbm.backend", return_value=backend), \
patch("harness.stages.s06_hbm.watchdog.run", side_effect=run), \
patch("harness.stages.s06_hbm._hbm_bc_spotcheck", return_value=1.0), \
patch.object(self.ctx, "watchdog_params", return_value={"stall_s": 10}):
result = Hbm().native(self.ctx, sd)
self.assertEqual(result.state, "OK")
self.assertEqual(Path(result.meta["hbm"]).read_bytes(), b"declared HBM")
self.assertFalse(result.measurements["G1"]["log_has_error"])
self.assertTrue(onnx.exists()) # Receipt-bound intermediate bytes remain available for verification.
self.assertEqual(set(result.meta["recipe_outputs"]), {"hbm", "bc"})
def test_stage_cache_preserves_outputs_and_runtime_drift_fails_closed(self):
baseline = self.file("baseline/metrics.json", b'{"metric":1}')
saved = self.record("02-baseline", outputs={"baseline": baseline.parent}, measurements={"baseline": {"metric": 1}})
class Cached(Stage):
name = "02-baseline"
def cached(inner, ctx):
return saved
self.ctx.attempt = 1
with patch("harness.recipe_runtime.runtime_identity", return_value=self.ctx.extra["runtime_identity"]), \
patch.object(paths, "model_python", return_value=Path("/fixture/python")):
outcome = Cached().run(self.ctx)
self.assertEqual(outcome.state, "SKIPPED")
current = recipe_receipt(self.ctx, "02-baseline", True)
self.assertEqual(current["recipe_outputs"], saved["recipe_outputs"])
self.assertEqual(current["measurements"], saved["measurements"])
with self.assertRaisesRegex(ValueError, "already exists"):
Cached().run(self.ctx)
self.ctx.attempt = 2
with patch("harness.recipe_runtime.runtime_identity", return_value={"python": "changed"}), \
patch.object(paths, "model_python", return_value=Path("/fixture/python")):
outcome = Cached().run(self.ctx)
self.assertEqual(outcome.state, "FAIL")
with self.assertRaisesRegex(ValueError, "did not succeed"):
recipe_receipt(self.ctx, "02-baseline", True)
if __name__ == "__main__":
unittest.main()