"""Budget termination must not erase completed model deployment evidence.""" import json import tempfile import unittest from pathlib import Path from unittest.mock import patch from harness import registry, verify, pipeline class CompletionTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.jd = Path(self.tmp.name) / "runs/r/job" self.rd = self.jd / "rollouts/00" self.rd.mkdir(parents=True) spec = registry.JobSpec(job_id="job", task="cls", model="example", march="nash-m", backend="ptq", qcfg="int8", priority="A", mode="oracle") self.cfg = registry.gate_snapshot(spec) (self.jd / "job.json").write_text(json.dumps({"job_id": "job", "run_id": "r", "gates": self.cfg})) (self.rd / "rollout.json").write_text(json.dumps({"state": "TIMEOUT", "timeout_reason": "budget", "mode": "e2e", "policy": "teacher"})) (self.rd / "claim.json").write_text('{"terminal":"PASS"}') def stage(self, name, measurements=None, state="OK", error=None): d = self.rd / "stages" / name d.mkdir(parents=True, exist_ok=True) (d / "meta.json").write_text(json.dumps({"state": state, "measurements": measurements or {}, "error_class": error, "attempt": 0})) def complete(self, quant=79): self.stage("05-compile") self.stage("06-hbm", {"G1": dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False)}) self.stage("07-board", {"G4": {"latency_ms_mean": 1.0}}) self.stage("08-eval", {"G3": {"fp32": 80, "quant": quant, "source": "board", "tier": "t1", "n": 2000}}) def test_timeout_with_real_pass_preserves_success(self): self.complete() v = verify.verify_rollout(self.rd, write=False) self.assertEqual(v["terminal"], "PASS") self.assertEqual(v["execution_terminal"], "TIMEOUT") self.assertEqual(v["timeout_reason"], "budget") self.assertTrue(v["claim_consistent"]) self.assertTrue(v["completion"]["workflow_completed"]) def test_timeout_with_measured_accuracy_drop_keeps_reason(self): self.complete(68) (self.rd / "claim.json").write_text('{"terminal":"FAIL"}') v = verify.verify_rollout(self.rd, write=False) self.assertEqual(v["terminal"], "FAIL") self.assertEqual(v["failed_gate"], "G3") self.assertEqual(v["execution_terminal"], "TIMEOUT") self.assertTrue(v["completion"]["board_completed"]) self.assertTrue(v["completion"]["workflow_completed"]) self.assertFalse(v["completion"]["accuracy_passed"]) self.assertTrue(v["claim_consistent"]) def test_compile_only_is_unverified_not_budget_failure(self): self.stage("05-compile") (self.rd / "claim.json").unlink() v = verify.verify_rollout(self.rd, write=False) self.assertEqual(v["terminal"], "UNVERIFIED") self.assertTrue(v["completion"]["compiled"]) self.assertFalse(v["completion"]["hbm_generated"]) def test_board_success_without_accuracy_is_incomplete(self): self.complete() (self.rd / "stages/08-eval/meta.json").unlink() (self.rd / "claim.json").unlink() v = verify.verify_rollout(self.rd, write=False) self.assertEqual(v["terminal"], "UNVERIFIED") self.assertTrue(v["completion"]["board_completed"]) self.assertFalse(v["completion"]["accuracy_evaluated"]) def test_real_compile_failure_remains_failure(self): self.stage("05-compile", state="FAIL", error="unsupported_op") (self.rd / "claim.json").unlink() v = verify.verify_rollout(self.rd, write=False) self.assertEqual(v["terminal"], "FAIL") self.assertEqual(v["error_class"], "unsupported_op") def test_board_or_eval_runtime_failure_is_not_accuracy_failure(self): for failed in ("07-board", "08-eval"): with self.subTest(stage=failed): self.complete() (self.rd / "claim.json").unlink(missing_ok=True) self.stage(failed, state="FAIL", error="board_offline") if failed == "07-board": (self.rd / "stages/08-eval/meta.json").unlink() v = verify.verify_rollout(self.rd, write=False) self.assertEqual(v["terminal"], "UNVERIFIED") self.assertEqual(v["error_class"], "board_offline") self.assertFalse(v["completion"]["accuracy_evaluated"]) def test_blocked_claim_cannot_override_actual_pass(self): self.complete() (self.rd / "claim.json").write_text('{"terminal":"BLOCKED"}') v = verify.verify_rollout(self.rd, write=False) self.assertEqual(v["terminal"], "PASS") self.assertFalse(v["claim_consistent"]) def test_statistics_distinguish_completion_from_acceptance(self): self.complete(68) v = verify.verify_rollout(self.rd, write=False) c = verify.summarize_results([json.dumps(v), None]) self.assertEqual(c["recorded"], 1) self.assertEqual(c["workflow_completed"], 1) self.assertEqual(c["accuracy_passed"], 0) self.assertEqual(c["agent_timeout"], 1) def test_export_error_does_not_roll_back_model_result(self): with patch("export.to_agentrl.materialize", side_effect=OSError("disk full")), patch.object(pipeline, "_stamped") as log: self.assertIsNone(pipeline.refresh_run_dataset("r")) self.assertIn("bench export --run r", log.call_args[0][0]) with self.assertRaises(OSError): pipeline.refresh_run_dataset("r", strict=True) if __name__ == "__main__": unittest.main()