137 lines
6.6 KiB
Python
137 lines
6.6 KiB
Python
"""Budget termination must not erase completed model deployment evidence."""
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from harness import registry, verify, pipeline
|
|
|
|
|
|
class CompletionTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.jd = Path(self.tmp.name) / "runs/r/job"
|
|
self.rd = self.jd / "rollouts/00"
|
|
self.rd.mkdir(parents=True)
|
|
spec = registry.JobSpec(job_id="job", task="cls", model="example", march="nash-m",
|
|
backend="ptq", qcfg="int8", priority="A", mode="oracle")
|
|
self.cfg = registry.gate_snapshot(spec)
|
|
(self.jd / "job.json").write_text(json.dumps({"job_id": "job", "run_id": "r", "gates": self.cfg}))
|
|
(self.rd / "rollout.json").write_text(json.dumps({"state": "TIMEOUT", "timeout_reason": "budget",
|
|
"mode": "e2e", "policy": "teacher"}))
|
|
(self.rd / "claim.json").write_text('{"terminal":"PASS"}')
|
|
|
|
def stage(self, name, measurements=None, state="OK", error=None):
|
|
d = self.rd / "stages" / name
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
(d / "meta.json").write_text(json.dumps({"state": state, "measurements": measurements or {},
|
|
"error_class": error, "attempt": 0}))
|
|
|
|
def complete(self, quant=79):
|
|
self.stage("05-compile")
|
|
self.stage("06-hbm", {"G1": dict(hbm_exists=True, march_ok=True, io_ok=True,
|
|
verifier_ok=True, log_has_error=False)})
|
|
self.stage("07-board", {"G4": {"latency_ms_mean": 1.0}})
|
|
self.stage("08-eval", {"G3": {"fp32": 80, "quant": quant, "source": "board", "tier": "t1", "n": 2000}})
|
|
|
|
def test_timeout_with_real_pass_preserves_success(self):
|
|
self.complete()
|
|
v = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(v["terminal"], "PASS")
|
|
self.assertEqual(v["execution_terminal"], "TIMEOUT")
|
|
self.assertEqual(v["timeout_reason"], "budget")
|
|
self.assertTrue(v["claim_consistent"])
|
|
self.assertTrue(v["completion"]["workflow_completed"])
|
|
|
|
def test_timeout_with_measured_accuracy_drop_keeps_reason(self):
|
|
self.complete(68)
|
|
(self.rd / "claim.json").write_text('{"terminal":"FAIL"}')
|
|
v = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(v["terminal"], "FAIL")
|
|
self.assertEqual(v["failed_gate"], "G3")
|
|
self.assertEqual(v["execution_terminal"], "TIMEOUT")
|
|
self.assertTrue(v["completion"]["board_completed"])
|
|
self.assertTrue(v["completion"]["workflow_completed"])
|
|
self.assertFalse(v["completion"]["accuracy_passed"])
|
|
self.assertTrue(v["claim_consistent"])
|
|
|
|
def test_compile_only_is_unverified_not_budget_failure(self):
|
|
self.stage("05-compile")
|
|
(self.rd / "claim.json").unlink()
|
|
v = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(v["terminal"], "UNVERIFIED")
|
|
self.assertTrue(v["completion"]["compiled"])
|
|
self.assertFalse(v["completion"]["hbm_generated"])
|
|
|
|
def test_board_success_without_accuracy_is_incomplete(self):
|
|
self.complete()
|
|
(self.rd / "stages/08-eval/meta.json").unlink()
|
|
(self.rd / "claim.json").unlink()
|
|
v = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(v["terminal"], "UNVERIFIED")
|
|
self.assertTrue(v["completion"]["board_completed"])
|
|
self.assertFalse(v["completion"]["accuracy_evaluated"])
|
|
|
|
def test_real_compile_failure_remains_failure(self):
|
|
self.stage("05-compile", state="FAIL", error="unsupported_op")
|
|
(self.rd / "claim.json").unlink()
|
|
v = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(v["terminal"], "FAIL")
|
|
self.assertEqual(v["error_class"], "unsupported_op")
|
|
|
|
def test_board_or_eval_runtime_failure_is_not_accuracy_failure(self):
|
|
for failed in ("07-board", "08-eval"):
|
|
with self.subTest(stage=failed):
|
|
self.complete()
|
|
(self.rd / "claim.json").unlink(missing_ok=True)
|
|
self.stage(failed, state="FAIL", error="board_offline")
|
|
if failed == "07-board":
|
|
(self.rd / "stages/08-eval/meta.json").unlink()
|
|
v = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(v["terminal"], "UNVERIFIED")
|
|
self.assertEqual(v["error_class"], "board_offline")
|
|
self.assertFalse(v["completion"]["accuracy_evaluated"])
|
|
|
|
def test_blocked_claim_cannot_override_actual_pass(self):
|
|
self.complete()
|
|
(self.rd / "claim.json").write_text('{"terminal":"BLOCKED"}')
|
|
v = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(v["terminal"], "PASS")
|
|
self.assertFalse(v["claim_consistent"])
|
|
|
|
def test_blocked_is_preserved_as_unproven_self_report(self):
|
|
self.stage("05-compile", state="FAIL", error="unsupported_op")
|
|
(self.rd / "claim.json").write_text('{"terminal":"BLOCKED","stage":"05-compile","error_class":"unsupported_op"}')
|
|
od = self.jd / "oracle"
|
|
od.mkdir()
|
|
(od / "result.json").write_text('{"terminal":"FAIL","stage_failed":"05-compile","error_class":"unsupported_op"}')
|
|
v = verify.verify_rollout(self.rd, write=False)
|
|
self.assertEqual(v["terminal"], "BLOCKED")
|
|
self.assertEqual(v["measured_terminal"], "FAIL")
|
|
self.assertIsNone(v["blocked_correct"])
|
|
self.assertEqual(v["claim"]["terminal"], "BLOCKED")
|
|
self.assertEqual(v["error_class"], "unsupported_op")
|
|
self.assertNotIn("oracle_ref", v)
|
|
|
|
def test_statistics_distinguish_completion_from_acceptance(self):
|
|
self.complete(68)
|
|
v = verify.verify_rollout(self.rd, write=False)
|
|
c = verify.summarize_results([json.dumps(v), None])
|
|
self.assertEqual(c["recorded"], 1)
|
|
self.assertEqual(c["workflow_completed"], 1)
|
|
self.assertEqual(c["accuracy_passed"], 0)
|
|
self.assertEqual(c["agent_timeout"], 1)
|
|
|
|
def test_export_error_does_not_roll_back_model_result(self):
|
|
with patch("export.to_agentrl.materialize", side_effect=OSError("disk full")), patch.object(pipeline, "_stamped") as log:
|
|
self.assertIsNone(pipeline.refresh_run_dataset("r"))
|
|
self.assertIn("bench export --run r", log.call_args[0][0])
|
|
with self.assertRaises(OSError):
|
|
pipeline.refresh_run_dataset("r", strict=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|