data-gen-bpu/scripts/test_hbm_provenance.py

241 lines
13 KiB
Python

"""HBM identity is a requirement of accepted evidence, not just a recorded label."""
from contextlib import nullcontext
import copy
import hashlib
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from harness import db, paths, pipeline, scheduler, verify
from harness.stages.base import StageOutcome
from harness.stages import Baseline, Board, Eval
class HbmProvenance(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.root = Path(tmp.name)
self.jd = self.root / "runs/r/job"
self.rd = self.jd / "rollouts/00"
self.hbm = self.rd / "artifacts/job.hbm"
self.hbm.parent.mkdir(parents=True)
self.hbm.write_bytes(b"model B")
self.sha = hashlib.sha256(self.hbm.read_bytes()).hexdigest()
self.other_sha = hashlib.sha256(b"model A").hexdigest()
self.job = {"run_id": "r", "job_id": "job", "model": "fixture", "task": "cls",
"gates": {"required": ["G1", "G3"], "G1": {}, "G2": {},
"G3": {"kind": "rel_pct", "tol": 5, "metric": "top1", "require_source": "board"},
"G4": {"record": ["latency_ms_mean"]}}}
self.meas = {
"G1": {"hbm_exists": True, "hbm_sha256": self.sha, "march_ok": True,
"io_ok": True, "verifier_ok": True, "log_has_error": False},
"G3": {"fp32": 80, "quant": 80, "metric": "top1", "source": "board", "hbm_sha256": self.sha},
"G4": {"latency_ms_mean": 1, "hbm_sha256": self.sha}}
self.save(self.jd / "job.json", self.job)
self.save(self.rd / "rollout.json", {"state": "DONE"})
@staticmethod
def save(path, value):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value))
def stage_records(self, root=None):
root = root or self.rd / "stages"
for stage, gate in (("06-hbm", "G1"), ("07-board", "G4"), ("08-eval", "G3")):
self.save(root / stage / "meta.json", {"state": "OK", "measurements": {gate: self.meas.get(gate)}})
def test_mixed_model_accuracy_cannot_pass_complete_verifier(self):
self.meas["G3"]["hbm_sha256"] = self.other_sha
self.stage_records()
result = verify.verify_rollout(self.rd, write=False)
self.assertEqual(result["measured_terminal"], "UNVERIFIED")
self.assertEqual(result["gates"]["G3"]["verdict"], "unverified")
self.assertFalse(result["completion"]["workflow_completed"])
self.assertIn("differs", result["hbm_provenance"]["gates"]["G3"]["reason"])
def test_same_artifact_passes_in_protected_and_legacy_layouts(self):
self.stage_records()
for layout in ("artifacts", "workspace/artifacts"):
with self.subTest(layout=layout):
destination = self.rd / layout / "job.hbm"
destination.parent.mkdir(parents=True, exist_ok=True)
self.hbm.rename(destination)
self.hbm = destination
result = verify.verify_rollout(self.rd, write=False)
self.assertEqual(result["terminal"], "PASS")
self.assertTrue(result["completion"]["workflow_completed"])
self.assertEqual(result["hbm_provenance"]["sha256"], self.sha)
def test_replaced_and_missing_file_cannot_pass(self):
self.stage_records()
for content in (b"replaced since 06", None):
with self.subTest(content=content):
if content is None:
self.hbm.unlink()
else:
self.hbm.write_bytes(content)
result = verify.verify_rollout(self.rd, write=False)
self.assertEqual(result["terminal"], "UNVERIFIED")
self.assertFalse(result["completion"]["workflow_completed"])
def test_missing_or_invalid_required_sha_is_unverified(self):
for gate in ("G1", "G3"):
for value in (None, "not-a-digest"):
with self.subTest(gate=gate, value=value):
self.meas[gate]["hbm_sha256"] = value
self.stage_records()
result = verify.verify_rollout(self.rd, write=False)
self.assertEqual(result["terminal"], "UNVERIFIED")
self.meas[gate]["hbm_sha256"] = self.sha
def test_optional_invalid_perf_is_discarded_without_failing_accuracy(self):
for value in (None, self.other_sha):
with self.subTest(value=value):
self.meas["G4"]["hbm_sha256"] = value
self.stage_records()
result = verify.verify_rollout(self.rd, write=False)
self.assertEqual(result["terminal"], "PASS")
self.assertIsNone(result["gates"]["G4"]["latency_ms_mean"])
self.assertEqual(result["hbm_provenance"]["gates"]["G4"]["status"], "unverified")
def test_g4_only_still_requires_current_06_identity(self):
self.job["gates"]["required"] = ["G4"]
for budget in (None, 10):
self.job["gates"]["G4"]["latency_budget_ms"] = budget
self.save(self.jd / "job.json", self.job)
for gate in ("G1", "G4"):
with self.subTest(gate=gate, budget=budget):
self.meas[gate]["hbm_sha256"] = self.other_sha
self.stage_records()
result = verify.verify_rollout(self.rd, write=False)
self.assertEqual(result["terminal"], "UNVERIFIED")
self.meas[gate]["hbm_sha256"] = self.sha
def test_latency_budget_makes_g4_required_even_when_not_in_explicit_list(self):
self.job["gates"]["G4"]["latency_budget_ms"] = 10
self.save(self.jd / "job.json", self.job)
self.meas["G4"]["hbm_sha256"] = self.other_sha
self.stage_records()
result = verify.verify_rollout(self.rd, write=False)
self.assertEqual(result["terminal"], "UNVERIFIED")
self.assertIn("G4", result["required"])
def test_readonly_verification_keeps_raw_and_stored_verdict(self):
self.meas["G1"].pop("hbm_sha256")
self.stage_records()
self.save(self.rd / "verdict.json", {"terminal": "PASS", "historical": True})
before = {p: p.read_bytes() for p in self.jd.rglob("*") if p.is_file()}
original = copy.deepcopy(self.meas)
accepted, _ = verify.validate_hbm_measurements(self.meas, self.hbm)
self.assertIsNone(accepted["G3"])
self.assertEqual(self.meas, original)
self.assertEqual(verify.verify_rollout(self.rd, write=False)["terminal"], "UNVERIFIED")
self.assertEqual(before, {p: p.read_bytes() for p in before})
def oracle_artifact(self):
destination = self.jd / "oracle/artifacts/job.hbm"
destination.parent.mkdir(parents=True)
destination.write_bytes(self.hbm.read_bytes())
return destination
def test_rejudge_checks_current_artifact_and_old_g4_fallback(self):
self.oracle_artifact()
self.meas["G3"]["hbm_sha256"] = self.other_sha
self.meas["G4"] = None
self.stage_records(self.jd / "oracle/stages")
result_path = self.jd / "oracle/result.json"
self.save(result_path, {"terminal": "PASS", "completion": {"workflow_completed": True},
"measurements": {"G4": {"latency_ms_mean": 2, "hbm_sha256": self.other_sha}}})
with patch.object(paths, "RUNS", self.root / "runs"), \
patch.object(db, "connect", return_value=nullcontext(None)), patch.object(db, "set_job_state"):
self.assertEqual(pipeline.rejudge("r", "job")["terminal"], "UNVERIFIED")
result = json.loads(result_path.read_text())
self.assertFalse(result["completion"]["workflow_completed"])
self.assertEqual(result["measurements"]["G4"]["hbm_sha256"], self.other_sha) # raw fallback retained
self.assertIsNone(result["gates"]["G4"]["latency_ms_mean"])
self.assertEqual(result["hbm_provenance"]["gates"]["G4"]["status"], "unverified")
def run_oracle(self, failed_board=False, **kwargs):
order = []
for stage_name, gate in (("06-hbm", "G1"), ("07-board", "G4"), ("08-eval", "G3")):
outcome = StageOutcome("OK", measurements={gate: self.meas[gate]})
if failed_board and gate == "G4":
outcome = StageOutcome("FAIL", error_class="board_offline")
order.append(type(stage_name, (), {"name": stage_name, "level": "target", "pool": "cpu",
"run": lambda self, ctx, out=outcome: out}))
with patch.object(paths, "RUNS", self.root / "runs"), patch.object(paths, "STATE", self.root), \
patch.object(paths, "config", return_value={"retry": {"stalled": 0, "transient": 0}}), \
patch.object(pipeline, "ORDER", order), \
patch.object(db, "connect", side_effect=lambda: nullcontext(None)), \
patch.object(db, "claim_job", return_value=True), patch.object(db, "stage_start"), \
patch.object(db, "stage_finish"), patch.object(db, "set_job_state"), \
patch.object(scheduler, "acquire_stage", side_effect=lambda *args: nullcontext(0)):
return pipeline.run_oracle_job("r", "job", log=lambda _: None, **kwargs)
def test_oracle_memory_measurements_cannot_bypass_identity(self):
self.oracle_artifact()
self.meas["G3"]["hbm_sha256"] = self.other_sha
result = self.run_oracle()
self.assertEqual(result["terminal"], "UNVERIFIED")
self.assertFalse(result["completion"]["accuracy_evaluated"])
def test_failed_optional_board_stage_allows_matching_accuracy(self):
self.oracle_artifact()
result = self.run_oracle(failed_board=True)
self.assertEqual(result["terminal"], "PASS")
self.assertIsNone(result["measurements"]["G4"])
self.assertTrue(result["completion"]["workflow_completed"])
def test_dry_run_and_preparation_are_never_verified_pass(self):
self.oracle_artifact()
self.assertEqual(self.run_oracle(dry_run=True)["terminal"], "UNVERIFIED")
self.assertEqual(self.run_oracle(prepare_only=True)["terminal"], "PREPARED")
def run_backfill(self):
with patch.object(paths, "RUNS", self.root / "runs"), \
patch.object(db, "connect", side_effect=lambda: nullcontext(None)), \
patch.object(db, "list_jobs", return_value=[{"job_id": "job", "state": "PASS"}]), \
patch.object(db, "stage_start"), patch.object(db, "stage_finish"), patch.object(db, "set_job_state"), \
patch("harness.stages.s07_board.selected_board", return_value={"name": "fixture"}), \
patch.object(scheduler, "acquire_stage", side_effect=lambda *args: nullcontext(0)):
return pipeline.board_backfill("r", log=lambda _: None)
def test_oracle_backfill_requires_06_revalidation_for_legacy_or_replaced_hbm(self):
artifact = self.oracle_artifact()
for kind in ("missing_sha", "replaced"):
with self.subTest(kind=kind):
self.meas["G1"]["hbm_sha256"] = None if kind == "missing_sha" else self.sha
artifact.write_bytes(b"replaced" if kind == "replaced" else b"model B")
self.stage_records(self.jd / "oracle/stages")
with patch.object(Board, "run") as board, patch.object(Eval, "run") as evaluate:
result = self.run_backfill()
self.assertEqual(result, {"backfilled": 0, "skipped": {"needs_hbm_revalidation": 1}})
board.assert_not_called()
evaluate.assert_not_called()
self.assertFalse((self.jd / "oracle/result.json").exists())
def test_oracle_backfill_does_not_skip_stale_board_accuracy(self):
self.oracle_artifact()
self.meas["G3"]["hbm_sha256"] = self.other_sha
self.stage_records(self.jd / "oracle/stages")
def evaluate(stage, ctx):
measurement = {**self.meas["G3"], "hbm_sha256": self.sha}
self.save(ctx.stage_root / "08-eval/meta.json", {"state": "OK", "measurements": {"G3": measurement}})
return StageOutcome("OK", measurements={"G3": measurement})
with patch.object(Board, "run") as board, patch.object(Baseline, "cached", return_value=StageOutcome("SKIPPED")), \
patch.object(Eval, "run", autospec=True, side_effect=evaluate) as evaluation:
result = self.run_backfill()
self.assertEqual(result, {"backfilled": 1, "skipped": {}})
board.assert_not_called()
evaluation.assert_called_once()
self.assertEqual(json.loads((self.jd / "oracle/result.json").read_text())["terminal"], "PASS")
if __name__ == "__main__":
unittest.main()