data-gen-bpu/scripts/test_completion.py

123 lines
5.8 KiB
Python

"""Budget termination must not erase completed model deployment evidence."""
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from harness import registry, verify, pipeline
class CompletionTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.jd = Path(self.tmp.name) / "runs/r/job"
self.rd = self.jd / "rollouts/00"
self.rd.mkdir(parents=True)
spec = registry.JobSpec(job_id="job", task="cls", model="example", march="nash-m",
backend="ptq", qcfg="int8", priority="A", mode="oracle")
self.cfg = registry.gate_snapshot(spec)
(self.jd / "job.json").write_text(json.dumps({"job_id": "job", "run_id": "r", "gates": self.cfg}))
(self.rd / "rollout.json").write_text(json.dumps({"state": "TIMEOUT", "timeout_reason": "budget",
"mode": "e2e", "policy": "teacher"}))
(self.rd / "claim.json").write_text('{"terminal":"PASS"}')
def stage(self, name, measurements=None, state="OK", error=None):
d = self.rd / "stages" / name
d.mkdir(parents=True, exist_ok=True)
(d / "meta.json").write_text(json.dumps({"state": state, "measurements": measurements or {},
"error_class": error, "attempt": 0}))
def complete(self, quant=79):
self.stage("05-compile")
self.stage("06-hbm", {"G1": dict(hbm_exists=True, march_ok=True, io_ok=True,
verifier_ok=True, log_has_error=False)})
self.stage("07-board", {"G4": {"latency_ms_mean": 1.0}})
self.stage("08-eval", {"G3": {"fp32": 80, "quant": quant, "source": "board", "tier": "t1", "n": 2000}})
def test_timeout_with_real_pass_preserves_success(self):
self.complete()
v = verify.verify_rollout(self.rd, write=False)
self.assertEqual(v["terminal"], "PASS")
self.assertEqual(v["execution_terminal"], "TIMEOUT")
self.assertEqual(v["timeout_reason"], "budget")
self.assertTrue(v["claim_consistent"])
self.assertTrue(v["completion"]["workflow_completed"])
def test_timeout_with_measured_accuracy_drop_keeps_reason(self):
self.complete(68)
(self.rd / "claim.json").write_text('{"terminal":"FAIL"}')
v = verify.verify_rollout(self.rd, write=False)
self.assertEqual(v["terminal"], "FAIL")
self.assertEqual(v["failed_gate"], "G3")
self.assertEqual(v["execution_terminal"], "TIMEOUT")
self.assertTrue(v["completion"]["board_completed"])
self.assertTrue(v["completion"]["workflow_completed"])
self.assertFalse(v["completion"]["accuracy_passed"])
self.assertTrue(v["claim_consistent"])
def test_compile_only_is_unverified_not_budget_failure(self):
self.stage("05-compile")
(self.rd / "claim.json").unlink()
v = verify.verify_rollout(self.rd, write=False)
self.assertEqual(v["terminal"], "UNVERIFIED")
self.assertTrue(v["completion"]["compiled"])
self.assertFalse(v["completion"]["hbm_generated"])
def test_board_success_without_accuracy_is_incomplete(self):
self.complete()
(self.rd / "stages/08-eval/meta.json").unlink()
(self.rd / "claim.json").unlink()
v = verify.verify_rollout(self.rd, write=False)
self.assertEqual(v["terminal"], "UNVERIFIED")
self.assertTrue(v["completion"]["board_completed"])
self.assertFalse(v["completion"]["accuracy_evaluated"])
def test_real_compile_failure_remains_failure(self):
self.stage("05-compile", state="FAIL", error="unsupported_op")
(self.rd / "claim.json").unlink()
v = verify.verify_rollout(self.rd, write=False)
self.assertEqual(v["terminal"], "FAIL")
self.assertEqual(v["error_class"], "unsupported_op")
def test_board_or_eval_runtime_failure_is_not_accuracy_failure(self):
for failed in ("07-board", "08-eval"):
with self.subTest(stage=failed):
self.complete()
(self.rd / "claim.json").unlink(missing_ok=True)
self.stage(failed, state="FAIL", error="board_offline")
if failed == "07-board":
(self.rd / "stages/08-eval/meta.json").unlink()
v = verify.verify_rollout(self.rd, write=False)
self.assertEqual(v["terminal"], "UNVERIFIED")
self.assertEqual(v["error_class"], "board_offline")
self.assertFalse(v["completion"]["accuracy_evaluated"])
def test_blocked_claim_cannot_override_actual_pass(self):
self.complete()
(self.rd / "claim.json").write_text('{"terminal":"BLOCKED"}')
v = verify.verify_rollout(self.rd, write=False)
self.assertEqual(v["terminal"], "PASS")
self.assertFalse(v["claim_consistent"])
def test_statistics_distinguish_completion_from_acceptance(self):
self.complete(68)
v = verify.verify_rollout(self.rd, write=False)
c = verify.summarize_results([json.dumps(v), None])
self.assertEqual(c["recorded"], 1)
self.assertEqual(c["workflow_completed"], 1)
self.assertEqual(c["accuracy_passed"], 0)
self.assertEqual(c["agent_timeout"], 1)
def test_export_error_does_not_roll_back_model_result(self):
with patch("export.to_agentrl.materialize", side_effect=OSError("disk full")), patch.object(pipeline, "_stamped") as log:
self.assertIsNone(pipeline.refresh_run_dataset("r"))
self.assertIn("bench export --run r", log.call_args[0][0])
with self.assertRaises(OSError):
pipeline.refresh_run_dataset("r", strict=True)
if __name__ == "__main__":
unittest.main()