241 lines
13 KiB
Python
241 lines
13 KiB
Python
"""HBM identity is a requirement of accepted evidence, not just a recorded label."""
|
|
from contextlib import nullcontext
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import db, paths, pipeline, scheduler, verify
|
|
from harness.stages.base import StageOutcome
|
|
from harness.stages import Baseline, Board, Eval
|
|
|
|
|
|
class HbmProvenance(unittest.TestCase):
|
|
def setUp(self):
|
|
tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(tmp.cleanup)
|
|
self.root = Path(tmp.name)
|
|
self.jd = self.root / "runs/r/job"
|
|
self.rd = self.jd / "rollouts/00"
|
|
self.hbm = self.rd / "artifacts/job.hbm"
|
|
self.hbm.parent.mkdir(parents=True)
|
|
self.hbm.write_bytes(b"model B")
|
|
self.sha = hashlib.sha256(self.hbm.read_bytes()).hexdigest()
|
|
self.other_sha = hashlib.sha256(b"model A").hexdigest()
|
|
self.job = {"run_id": "r", "job_id": "job", "model": "fixture", "task": "cls",
|
|
"gates": {"required": ["G1", "G3"], "G1": {}, "G2": {},
|
|
"G3": {"kind": "rel_pct", "tol": 5, "metric": "top1", "require_source": "board"},
|
|
"G4": {"record": ["latency_ms_mean"]}}}
|
|
self.meas = {
|
|
"G1": {"hbm_exists": True, "hbm_sha256": self.sha, "march_ok": True,
|
|
"io_ok": True, "verifier_ok": True, "log_has_error": False},
|
|
"G3": {"fp32": 80, "quant": 80, "metric": "top1", "source": "board", "hbm_sha256": self.sha},
|
|
"G4": {"latency_ms_mean": 1, "hbm_sha256": self.sha}}
|
|
self.save(self.jd / "job.json", self.job)
|
|
self.save(self.rd / "rollout.json", {"state": "DONE"})
|
|
|
|
@staticmethod
|
|
def save(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value))
|
|
|
|
def stage_records(self, root=None):
|
|
root = root or self.rd / "stages"
|
|
for stage, gate in (("06-hbm", "G1"), ("07-board", "G4"), ("08-eval", "G3")):
|
|
self.save(root / stage / "meta.json", {"state": "OK", "measurements": {gate: self.meas.get(gate)}})
|
|
|
|
def test_mixed_model_accuracy_cannot_pass_complete_verifier(self):
|
|
self.meas["G3"]["hbm_sha256"] = self.other_sha
|
|
self.stage_records()
|
|
result = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(result["measured_terminal"], "UNVERIFIED")
|
|
self.assertEqual(result["gates"]["G3"]["verdict"], "unverified")
|
|
self.assertFalse(result["completion"]["workflow_completed"])
|
|
self.assertIn("differs", result["hbm_provenance"]["gates"]["G3"]["reason"])
|
|
|
|
def test_same_artifact_passes_in_protected_and_legacy_layouts(self):
|
|
self.stage_records()
|
|
for layout in ("artifacts", "workspace/artifacts"):
|
|
with self.subTest(layout=layout):
|
|
destination = self.rd / layout / "job.hbm"
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
self.hbm.rename(destination)
|
|
self.hbm = destination
|
|
result = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(result["terminal"], "PASS")
|
|
self.assertTrue(result["completion"]["workflow_completed"])
|
|
self.assertEqual(result["hbm_provenance"]["sha256"], self.sha)
|
|
|
|
def test_replaced_and_missing_file_cannot_pass(self):
|
|
self.stage_records()
|
|
for content in (b"replaced since 06", None):
|
|
with self.subTest(content=content):
|
|
if content is None:
|
|
self.hbm.unlink()
|
|
else:
|
|
self.hbm.write_bytes(content)
|
|
result = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(result["terminal"], "UNVERIFIED")
|
|
self.assertFalse(result["completion"]["workflow_completed"])
|
|
|
|
def test_missing_or_invalid_required_sha_is_unverified(self):
|
|
for gate in ("G1", "G3"):
|
|
for value in (None, "not-a-digest"):
|
|
with self.subTest(gate=gate, value=value):
|
|
self.meas[gate]["hbm_sha256"] = value
|
|
self.stage_records()
|
|
result = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(result["terminal"], "UNVERIFIED")
|
|
self.meas[gate]["hbm_sha256"] = self.sha
|
|
|
|
def test_optional_invalid_perf_is_discarded_without_failing_accuracy(self):
|
|
for value in (None, self.other_sha):
|
|
with self.subTest(value=value):
|
|
self.meas["G4"]["hbm_sha256"] = value
|
|
self.stage_records()
|
|
result = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(result["terminal"], "PASS")
|
|
self.assertIsNone(result["gates"]["G4"]["latency_ms_mean"])
|
|
self.assertEqual(result["hbm_provenance"]["gates"]["G4"]["status"], "unverified")
|
|
|
|
def test_g4_only_still_requires_current_06_identity(self):
|
|
self.job["gates"]["required"] = ["G4"]
|
|
for budget in (None, 10):
|
|
self.job["gates"]["G4"]["latency_budget_ms"] = budget
|
|
self.save(self.jd / "job.json", self.job)
|
|
for gate in ("G1", "G4"):
|
|
with self.subTest(gate=gate, budget=budget):
|
|
self.meas[gate]["hbm_sha256"] = self.other_sha
|
|
self.stage_records()
|
|
result = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(result["terminal"], "UNVERIFIED")
|
|
self.meas[gate]["hbm_sha256"] = self.sha
|
|
|
|
def test_latency_budget_makes_g4_required_even_when_not_in_explicit_list(self):
|
|
self.job["gates"]["G4"]["latency_budget_ms"] = 10
|
|
self.save(self.jd / "job.json", self.job)
|
|
self.meas["G4"]["hbm_sha256"] = self.other_sha
|
|
self.stage_records()
|
|
result = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(result["terminal"], "UNVERIFIED")
|
|
self.assertIn("G4", result["required"])
|
|
|
|
def test_readonly_verification_keeps_raw_and_stored_verdict(self):
|
|
self.meas["G1"].pop("hbm_sha256")
|
|
self.stage_records()
|
|
self.save(self.rd / "verdict.json", {"terminal": "PASS", "historical": True})
|
|
before = {p: p.read_bytes() for p in self.jd.rglob("*") if p.is_file()}
|
|
original = copy.deepcopy(self.meas)
|
|
accepted, _ = verify.validate_hbm_measurements(self.meas, self.hbm)
|
|
self.assertIsNone(accepted["G3"])
|
|
self.assertEqual(self.meas, original)
|
|
self.assertEqual(verify.verify_rollout(self.rd, write=False)["terminal"], "UNVERIFIED")
|
|
self.assertEqual(before, {p: p.read_bytes() for p in before})
|
|
|
|
def oracle_artifact(self):
|
|
destination = self.jd / "oracle/artifacts/job.hbm"
|
|
destination.parent.mkdir(parents=True)
|
|
destination.write_bytes(self.hbm.read_bytes())
|
|
return destination
|
|
|
|
def test_rejudge_checks_current_artifact_and_old_g4_fallback(self):
|
|
self.oracle_artifact()
|
|
self.meas["G3"]["hbm_sha256"] = self.other_sha
|
|
self.meas["G4"] = None
|
|
self.stage_records(self.jd / "oracle/stages")
|
|
result_path = self.jd / "oracle/result.json"
|
|
self.save(result_path, {"terminal": "PASS", "completion": {"workflow_completed": True},
|
|
"measurements": {"G4": {"latency_ms_mean": 2, "hbm_sha256": self.other_sha}}})
|
|
with patch.object(paths, "RUNS", self.root / "runs"), \
|
|
patch.object(db, "connect", return_value=nullcontext(None)), patch.object(db, "set_job_state"):
|
|
self.assertEqual(pipeline.rejudge("r", "job")["terminal"], "UNVERIFIED")
|
|
result = json.loads(result_path.read_text())
|
|
self.assertFalse(result["completion"]["workflow_completed"])
|
|
self.assertEqual(result["measurements"]["G4"]["hbm_sha256"], self.other_sha) # raw fallback retained
|
|
self.assertIsNone(result["gates"]["G4"]["latency_ms_mean"])
|
|
self.assertEqual(result["hbm_provenance"]["gates"]["G4"]["status"], "unverified")
|
|
|
|
def run_oracle(self, failed_board=False, **kwargs):
|
|
order = []
|
|
for stage_name, gate in (("06-hbm", "G1"), ("07-board", "G4"), ("08-eval", "G3")):
|
|
outcome = StageOutcome("OK", measurements={gate: self.meas[gate]})
|
|
if failed_board and gate == "G4":
|
|
outcome = StageOutcome("FAIL", error_class="board_offline")
|
|
order.append(type(stage_name, (), {"name": stage_name, "level": "target", "pool": "cpu",
|
|
"run": lambda self, ctx, out=outcome: out}))
|
|
with patch.object(paths, "RUNS", self.root / "runs"), patch.object(paths, "STATE", self.root), \
|
|
patch.object(paths, "config", return_value={"retry": {"stalled": 0, "transient": 0}}), \
|
|
patch.object(pipeline, "ORDER", order), \
|
|
patch.object(db, "connect", side_effect=lambda: nullcontext(None)), \
|
|
patch.object(db, "claim_job", return_value=True), patch.object(db, "stage_start"), \
|
|
patch.object(db, "stage_finish"), patch.object(db, "set_job_state"), \
|
|
patch.object(scheduler, "acquire_stage", side_effect=lambda *args: nullcontext(0)):
|
|
return pipeline.run_oracle_job("r", "job", log=lambda _: None, **kwargs)
|
|
|
|
def test_oracle_memory_measurements_cannot_bypass_identity(self):
|
|
self.oracle_artifact()
|
|
self.meas["G3"]["hbm_sha256"] = self.other_sha
|
|
result = self.run_oracle()
|
|
self.assertEqual(result["terminal"], "UNVERIFIED")
|
|
self.assertFalse(result["completion"]["accuracy_evaluated"])
|
|
|
|
def test_failed_optional_board_stage_allows_matching_accuracy(self):
|
|
self.oracle_artifact()
|
|
result = self.run_oracle(failed_board=True)
|
|
self.assertEqual(result["terminal"], "PASS")
|
|
self.assertIsNone(result["measurements"]["G4"])
|
|
self.assertTrue(result["completion"]["workflow_completed"])
|
|
|
|
def test_dry_run_and_preparation_are_never_verified_pass(self):
|
|
self.oracle_artifact()
|
|
self.assertEqual(self.run_oracle(dry_run=True)["terminal"], "UNVERIFIED")
|
|
self.assertEqual(self.run_oracle(prepare_only=True)["terminal"], "PREPARED")
|
|
|
|
def run_backfill(self):
|
|
with patch.object(paths, "RUNS", self.root / "runs"), \
|
|
patch.object(db, "connect", side_effect=lambda: nullcontext(None)), \
|
|
patch.object(db, "list_jobs", return_value=[{"job_id": "job", "state": "PASS"}]), \
|
|
patch.object(db, "stage_start"), patch.object(db, "stage_finish"), patch.object(db, "set_job_state"), \
|
|
patch("harness.stages.s07_board.selected_board", return_value={"name": "fixture"}), \
|
|
patch.object(scheduler, "acquire_stage", side_effect=lambda *args: nullcontext(0)):
|
|
return pipeline.board_backfill("r", log=lambda _: None)
|
|
|
|
def test_oracle_backfill_requires_06_revalidation_for_legacy_or_replaced_hbm(self):
|
|
artifact = self.oracle_artifact()
|
|
for kind in ("missing_sha", "replaced"):
|
|
with self.subTest(kind=kind):
|
|
self.meas["G1"]["hbm_sha256"] = None if kind == "missing_sha" else self.sha
|
|
artifact.write_bytes(b"replaced" if kind == "replaced" else b"model B")
|
|
self.stage_records(self.jd / "oracle/stages")
|
|
with patch.object(Board, "run") as board, patch.object(Eval, "run") as evaluate:
|
|
result = self.run_backfill()
|
|
self.assertEqual(result, {"backfilled": 0, "skipped": {"needs_hbm_revalidation": 1}})
|
|
board.assert_not_called()
|
|
evaluate.assert_not_called()
|
|
self.assertFalse((self.jd / "oracle/result.json").exists())
|
|
|
|
def test_oracle_backfill_does_not_skip_stale_board_accuracy(self):
|
|
self.oracle_artifact()
|
|
self.meas["G3"]["hbm_sha256"] = self.other_sha
|
|
self.stage_records(self.jd / "oracle/stages")
|
|
|
|
def evaluate(stage, ctx):
|
|
measurement = {**self.meas["G3"], "hbm_sha256": self.sha}
|
|
self.save(ctx.stage_root / "08-eval/meta.json", {"state": "OK", "measurements": {"G3": measurement}})
|
|
return StageOutcome("OK", measurements={"G3": measurement})
|
|
|
|
with patch.object(Board, "run") as board, patch.object(Baseline, "cached", return_value=StageOutcome("SKIPPED")), \
|
|
patch.object(Eval, "run", autospec=True, side_effect=evaluate) as evaluation:
|
|
result = self.run_backfill()
|
|
self.assertEqual(result, {"backfilled": 1, "skipped": {}})
|
|
board.assert_not_called()
|
|
evaluation.assert_called_once()
|
|
self.assertEqual(json.loads((self.jd / "oracle/result.json").read_text())["terminal"], "PASS")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|