275 lines
15 KiB
Python
275 lines
15 KiB
Python
"""Real-file counterexamples for recipe attempt, parent and artifact identity."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from types import SimpleNamespace
|
|
|
|
from harness import paths, verify
|
|
from harness.stages.base import Stage, StageContext, StageOutcome, recipe_identity, recipe_output, recipe_receipt, recipe_artifact, work_dir
|
|
from harness.stages.s04_calibrate import Calibrate
|
|
from harness.stages.s05_compile import Compile
|
|
from harness.stages.s06_hbm import Hbm
|
|
|
|
|
|
class RecipeProvenance(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.root = Path(self.tmp.name).resolve()
|
|
self.rd = self.root / "rollouts/00"
|
|
self.rd.mkdir(parents=True)
|
|
self.job = {"job_id": "fixture", "run_id": "run", "task": "seg", "backend": "ptq", "march": "nash-m", "qcfg": "int8",
|
|
"recipe": {"submission_sha256": "a" * 64, "reference_sha256": "b" * 64},
|
|
"model_spec": {"input_spec": "1x3x4x4", "preprocess": {"resize": 4}},
|
|
"gates": {"G2": {"n_samples": 2}}, "data": {"t1": {"content_sha": "t1"},
|
|
"calib": {"path": str(self.root / "calib")}}, "evaluator_version": "fixture"}
|
|
self.ctx = StageContext("run", self.job, self.root, self.rd / "stages", self.rd / "artifacts", rollout=0,
|
|
extra={"runtime_identity": {"python": "fixture-runtime"}, "recipe_parents": {}})
|
|
self.clock = 0
|
|
self.addCleanup(patch.stopall)
|
|
patch.object(paths, "config", return_value={"paths": {"work_root": str(self.root / "work")}}).start()
|
|
|
|
def file(self, name, content=b"fixture"):
|
|
path = self.root / name
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(content)
|
|
return path
|
|
|
|
def record(self, stage, attempt=0, *, state="OK", outputs=None, parents=(), measurements=None, **extra):
|
|
self.ctx.extra["recipe_stage"] = stage
|
|
self.ctx.extra["recipe_parents"] = {}
|
|
for name, reference in parents:
|
|
recipe_receipt(self.ctx, name, reference)
|
|
self.clock += 1
|
|
meta = {"stage": stage, "attempt": attempt, "state": state, "started_at": self.clock,
|
|
"recipe": recipe_identity(self.ctx), "recipe_parents": dict(self.ctx.extra["recipe_parents"]),
|
|
"recipe_outputs": {name: recipe_output(path) for name, path in (outputs or {}).items()},
|
|
"measurements": measurements or {}, **extra}
|
|
sd = self.ctx.stage_root / stage / f"a{attempt:02d}"
|
|
sd.mkdir(parents=True, exist_ok=True)
|
|
(sd / "meta.json").write_text(json.dumps(meta))
|
|
(sd.parent / "meta.json").write_text(json.dumps(meta))
|
|
return meta
|
|
|
|
def export(self, attempt=0):
|
|
return self.record("03-export", attempt, outputs={"onnx": self.file(f"graph-{attempt}.onnx")},
|
|
recipe_input={"input_spec": "1x3x4x4", "preprocess": {"resize": 4}})
|
|
|
|
def test_reference_cache_excludes_candidate_but_includes_runtime_and_evaluation(self):
|
|
first = recipe_identity(self.ctx)
|
|
self.job["recipe"]["submission_sha256"] = "c" * 64
|
|
second = recipe_identity(self.ctx)
|
|
self.assertNotEqual(first["submission"], second["submission"])
|
|
self.assertEqual(first["reference"], second["reference"])
|
|
self.ctx.extra["runtime_identity"]["python"] = "different-runtime"
|
|
self.assertNotEqual(second["reference"], recipe_identity(self.ctx)["reference"])
|
|
self.ctx.extra["runtime_identity"]["python"] = "fixture-runtime"
|
|
self.job["data"]["t1"]["content_sha"] = "different-subset"
|
|
self.assertNotEqual(second["reference"], recipe_identity(self.ctx)["reference"])
|
|
|
|
def test_failed_newest_attempt_never_falls_back_to_old_success(self):
|
|
self.export()
|
|
self.record("03-export", 1, state="FAIL")
|
|
with self.assertRaisesRegex(ValueError, "did not succeed"):
|
|
recipe_artifact(self.ctx, "03-export", "onnx")
|
|
|
|
def test_historical_parent_remains_valid_but_artifact_or_meta_drift_fails(self):
|
|
graph = self.export()
|
|
self.record("06-hbm", outputs={"hbm": self.file("first.hbm")}, parents=(("03-export", False),))
|
|
self.assertEqual(recipe_artifact(self.ctx, "06-hbm", "hbm"), self.root / "first.hbm")
|
|
Path(graph["recipe_outputs"]["onnx"]["path"]).write_bytes(b"changed")
|
|
with self.assertRaisesRegex(ValueError, "artifact changed"):
|
|
recipe_artifact(self.ctx, "06-hbm", "hbm")
|
|
Path(graph["recipe_outputs"]["onnx"]["path"]).write_bytes(b"fixture")
|
|
self.export(1)
|
|
self.assertEqual(recipe_artifact(self.ctx, "06-hbm", "hbm"), self.root / "first.hbm")
|
|
self.record("03-export", 2, state="FAIL")
|
|
self.assertEqual(recipe_artifact(self.ctx, "06-hbm", "hbm"), self.root / "first.hbm")
|
|
with self.assertRaisesRegex(ValueError, "did not succeed"):
|
|
recipe_artifact(self.ctx, "03-export", "onnx")
|
|
old_meta = self.ctx.stage_root / "03-export/a00/meta.json"
|
|
old_meta.write_text(old_meta.read_text() + " ")
|
|
with self.assertRaisesRegex(ValueError, "parent changed"):
|
|
recipe_artifact(self.ctx, "06-hbm", "hbm")
|
|
|
|
def test_selected_parent_returns_its_graph_not_a_newer_export(self):
|
|
self.export()
|
|
parent = self.record("04-calibrate", parents=(("03-export", False),))
|
|
self.export(1)
|
|
self.assertEqual(recipe_artifact(self.ctx, "03-export", "onnx", parent=parent), self.root / "graph-0.onnx")
|
|
self.assertEqual(recipe_artifact(self.ctx, "03-export", "onnx"), self.root / "graph-1.onnx")
|
|
|
|
def test_one_verification_traversal_hashes_shared_ancestors_once(self):
|
|
self.export()
|
|
self.record("04-calibrate", outputs={"calibration": self.file("calib.npy")}, parents=(("03-export", False),))
|
|
self.record("05-compile", outputs={"hbm": self.file("compiled.hbm")},
|
|
parents=(("03-export", False), ("04-calibrate", False)))
|
|
self.record("06-hbm", outputs={"hbm": self.file("final.hbm")},
|
|
parents=(("03-export", False), ("04-calibrate", False), ("05-compile", False)))
|
|
with patch("harness.stages.base.recipe_output", wraps=recipe_output) as describe:
|
|
recipe_receipt(self.ctx, "06-hbm")
|
|
self.assertEqual(describe.call_count, 4)
|
|
self.file("graph-0.onnx", b"tampered after verification")
|
|
with self.assertRaisesRegex(ValueError, "artifact changed"):
|
|
recipe_receipt(self.ctx, "06-hbm")
|
|
|
|
def test_graph_only_calibration_keeps_original_bytes_and_accepts_numeric_overrides(self):
|
|
import yaml
|
|
self.export()
|
|
original = {"count": 1, "source": "existing-platform", "preprocessing_identity": {"resize": 999}}
|
|
self.file("calib/manifest.json", json.dumps(original).encode())
|
|
self.file("calib/npy_f32/000.npy")
|
|
self.ctx.agent_config = self.file("config.yaml", b"input_parameters:\n mean_value: '1 2 3'\n")
|
|
self.ctx.extra.update(recipe_stage="04-calibrate", compile_attempt=3)
|
|
sd = self.ctx.stage_root / "04-calibrate/a00"
|
|
sd.mkdir(parents=True)
|
|
with patch("harness.stages.s04_calibrate.watchdog.run", side_effect=AssertionError("graph-only must not execute model code")):
|
|
result = Calibrate().native(self.ctx, sd)
|
|
self.assertEqual(result.state, "OK")
|
|
self.assertEqual(result.meta["calibration_mode"], "platform")
|
|
self.assertEqual(result.meta["calibration_manifest"], original)
|
|
self.assertEqual(result.meta["recipe_outputs"]["calibration"], recipe_output(self.root / "calib"))
|
|
config = yaml.safe_load((sd / "compile.yaml").read_text())
|
|
self.assertEqual(config["input_parameters"]["mean_value"], "1 2 3")
|
|
self.assertEqual(config["calibration_parameters"]["cal_data_dir"], str(self.root / "calib/npy_f32"))
|
|
|
|
def test_all_output_descriptors_are_checked_including_external_tensor_bundle(self):
|
|
graph = self.file("bundle/model.onnx")
|
|
tensor = self.file("bundle/weights.bin")
|
|
self.record("03-export", outputs={"onnx": graph, "onnx_bundle": graph.parent})
|
|
tensor.write_bytes(b"different external tensor")
|
|
with self.assertRaisesRegex(ValueError, "artifact changed"):
|
|
recipe_artifact(self.ctx, "03-export", "onnx")
|
|
|
|
def test_descriptor_rejects_symlinks_and_hardlinks(self):
|
|
original = self.file("original")
|
|
link = self.root / "link"
|
|
link.symlink_to(original)
|
|
with self.assertRaisesRegex(ValueError, "symlink"):
|
|
recipe_output(link)
|
|
link.unlink()
|
|
os.link(original, link)
|
|
with self.assertRaisesRegex(ValueError, "regular file"):
|
|
recipe_output(original)
|
|
|
|
def test_verifier_does_not_mix_new_graph_with_old_hbm_or_scores(self):
|
|
self.export()
|
|
self.record("06-hbm", outputs={"hbm": self.file("first.hbm")}, parents=(("03-export", False),),
|
|
measurements={"G1": {"hbm_exists": True}})
|
|
self.record("07-board", parents=(("06-hbm", False),), measurements={"G4": {"latency_ms_mean": 1}})
|
|
self.assertIn("G4", verify.collect_measurements(self.rd)[0])
|
|
self.job["recipe"]["submission_sha256"] = "c" * 64
|
|
self.export(1)
|
|
meas, summary = verify.collect_measurements(self.rd)
|
|
self.assertNotIn("G4", meas)
|
|
self.assertEqual(summary["06-hbm"]["state"], "UNVERIFIED")
|
|
self.file("rollouts/00/artifacts/fixture.hbm", b"legacy fallback must not be used")
|
|
self.assertFalse(verify.rollout_hbm_path(self.rd, "fixture").exists())
|
|
|
|
def test_accepted_submission_without_stage_output_invalidates_older_scores(self):
|
|
self.export()
|
|
self.record("06-hbm", outputs={"hbm": self.file("first.hbm")}, measurements={"G1": {"hbm_exists": True}})
|
|
frozen = self.file("queued/recipe.json", json.dumps({"submission_sha256": "c" * 64}).encode())
|
|
request = self.ctx.stage_root / "_queue/new/request.json"
|
|
request.parent.mkdir(parents=True)
|
|
request.write_text(json.dumps({"received_at": 10, "recipe": str(frozen)}))
|
|
self.assertFalse(verify.collect_measurements(self.rd)[0]["G1"]["hbm_exists"])
|
|
|
|
def test_calibration_metadata_cannot_replace_frozen_training_sources(self):
|
|
self.export()
|
|
self.file("calib/manifest.json", b'{"count":1}')
|
|
self.file("calib/npy_f32/000.npy")
|
|
self.ctx.extra.update(recipe_stage="04-calibrate", compile_attempt=3)
|
|
sd = self.ctx.stage_root / "04-calibrate/a00"
|
|
sd.mkdir(parents=True)
|
|
self.job["recipe"]["entrypoint"] = "candidate.py"
|
|
self.job["data"]["calib"]["preprocessing_identity"] = {"input_spec": "1x3x4x4", "preprocess": {"resize": 4}}
|
|
with patch.object(paths, "model_python", return_value=Path("/fixture/python")), \
|
|
self.assertRaisesRegex(ValueError, "frozen dataset training sources"):
|
|
Calibrate().native(self.ctx, sd)
|
|
|
|
def test_compile_requires_unique_hbm_and_records_specific_attempt_files(self):
|
|
self.ctx.extra["recipe_stage"] = "05-compile"
|
|
self.ctx.attempt = 2
|
|
wd = work_dir(self.ctx)
|
|
self.assertTrue(str(wd).endswith("05-compile/a02"))
|
|
(wd / "one.hbm").write_bytes(b"hbm")
|
|
(wd / "one_quantized_model.bc").write_bytes(b"bc")
|
|
(wd / "one_ptq_model.onnx").write_bytes(b"onnx")
|
|
out = StageOutcome("OK")
|
|
Compile().measure(self.ctx, self.ctx.stage_root, out)
|
|
self.assertEqual(set(out.meta["recipe_outputs"]), {"hbm", "bc", "ptq_onnx"})
|
|
(wd / "other.hbm").write_bytes(b"other")
|
|
with self.assertRaisesRegex(ValueError, "exactly one HBM"):
|
|
Compile().measure(self.ctx, self.ctx.stage_root, out)
|
|
|
|
def test_hbm_uses_declared_compile_output_and_only_that_attempt_logs(self):
|
|
self.export()
|
|
wd = self.root / "compile-attempt"
|
|
source = self.file("compile-attempt/declared.hbm", b"declared HBM")
|
|
bc = self.file("compile-attempt/quantized_model.bc", b"BC")
|
|
onnx = self.file("compile-attempt/model_ptq_model.onnx", b"PTQ ONNX")
|
|
calibration = self.file("calib/npy_f32/000.npy").parent.parent
|
|
self.record("04-calibrate", outputs={"calibration": calibration}, parents=(("03-export", False),))
|
|
self.record("05-compile", 1, outputs={"hbm": source, "bc": bc, "ptq_onnx": onnx},
|
|
parents=(("03-export", False), ("04-calibrate", False)), work_dir=str(wd))
|
|
self.file("compile-attempt/newer-unrelated.hbm", b"must not choose by mtime")
|
|
self.file("rollouts/00/stages/05-compile/a00/stderr.log", b"ERROR stale failed attempt")
|
|
self.file("rollouts/00/stages/05-compile/a01/stderr.log", b"INFO current compile")
|
|
self.job["gates"]["G1"] = {}
|
|
self.ctx.extra.update(recipe_stage="06-hbm", recipe_parents={})
|
|
sd = self.ctx.stage_root / "06-hbm/a00"
|
|
sd.mkdir(parents=True)
|
|
|
|
def run(command, **kw):
|
|
kw["stdout"].write_text("march: nash-m\ninput input [1, 3, 4, 4] FLOAT32\n")
|
|
kw["stderr"].write_text("")
|
|
return SimpleNamespace(state="OK", exit_code=0)
|
|
|
|
backend = SimpleNamespace(model_info_cmd=lambda *a, **kw: ["fixture-info"],
|
|
verifier_cmd=lambda *a, **kw: ["fixture-verifier"])
|
|
with patch("harness.stages.s06_hbm.backend", return_value=backend), \
|
|
patch("harness.stages.s06_hbm.watchdog.run", side_effect=run), \
|
|
patch("harness.stages.s06_hbm._hbm_bc_spotcheck", return_value=1.0), \
|
|
patch.object(self.ctx, "watchdog_params", return_value={"stall_s": 10}):
|
|
result = Hbm().native(self.ctx, sd)
|
|
self.assertEqual(result.state, "OK")
|
|
self.assertEqual(Path(result.meta["hbm"]).read_bytes(), b"declared HBM")
|
|
self.assertFalse(result.measurements["G1"]["log_has_error"])
|
|
self.assertTrue(onnx.exists()) # Receipt-bound intermediate bytes remain available for verification.
|
|
self.assertEqual(set(result.meta["recipe_outputs"]), {"hbm", "bc"})
|
|
|
|
def test_stage_cache_preserves_outputs_and_runtime_drift_fails_closed(self):
|
|
baseline = self.file("baseline/metrics.json", b'{"metric":1}')
|
|
saved = self.record("02-baseline", outputs={"baseline": baseline.parent}, measurements={"baseline": {"metric": 1}})
|
|
|
|
class Cached(Stage):
|
|
name = "02-baseline"
|
|
def cached(inner, ctx):
|
|
return saved
|
|
|
|
self.ctx.attempt = 1
|
|
with patch("harness.recipe_runtime.runtime_identity", return_value=self.ctx.extra["runtime_identity"]), \
|
|
patch.object(paths, "model_python", return_value=Path("/fixture/python")):
|
|
outcome = Cached().run(self.ctx)
|
|
self.assertEqual(outcome.state, "SKIPPED")
|
|
current = recipe_receipt(self.ctx, "02-baseline", True)
|
|
self.assertEqual(current["recipe_outputs"], saved["recipe_outputs"])
|
|
self.assertEqual(current["measurements"], saved["measurements"])
|
|
with self.assertRaisesRegex(ValueError, "already exists"):
|
|
Cached().run(self.ctx)
|
|
self.ctx.attempt = 2
|
|
with patch("harness.recipe_runtime.runtime_identity", return_value={"python": "changed"}), \
|
|
patch.object(paths, "model_python", return_value=Path("/fixture/python")):
|
|
outcome = Cached().run(self.ctx)
|
|
self.assertEqual(outcome.state, "FAIL")
|
|
with self.assertRaisesRegex(ValueError, "did not succeed"):
|
|
recipe_receipt(self.ctx, "02-baseline", True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|