"""Accuracy acceptance boundaries, frozen policies, and reward direction.""" import importlib.util from pathlib import Path import unittest from harness import gates, registry ROOT = Path(__file__).resolve().parents[1] _spec = importlib.util.spec_from_file_location("reward", ROOT / "export/reward.py") reward = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(reward) class AccuracyPolicy(unittest.TestCase): def snapshot(self, task="cls"): spec = registry.JobSpec( job_id=f"{task}--example--nash-m--ptq--int8", task=task, model="example", march="nash-m", backend="ptq", qcfg="int8", priority="A", mode="oracle", ) return registry.gate_snapshot(spec) def measurement(self, quant, fp32=80.0, **extra): return {"fp32": fp32, "quant": quant, "tier": "t1", "source": "board", **extra} def test_new_jobs_require_board_task_accuracy(self): cfg = self.snapshot() self.assertEqual(cfg["required"], ["G1", "G3"]) self.assertEqual(cfg["G3"]["kind"], "rel_pct") self.assertEqual(cfg["G3"]["tol"], 5) self.assertEqual(cfg["G3"]["require_source"], "board") self.assertEqual(cfg["G3"]["t1_screen_widen_se"], 0) for source in (None, "simulator"): result = gates.g3(self.measurement(80, source=source), cfg["G3"]) self.assertEqual(result["verdict"], "unverified") def test_relative_five_percent_boundary_and_improvement(self): cfg = self.snapshot()["G3"] for quant, verdict in ((76, "pass-screen"), (75.999, "fail"), (85, "pass-screen")): with self.subTest(quant=quant): self.assertEqual(gates.g3(self.measurement(quant, se=10), cfg)["verdict"], verdict) # The boundary must tolerate floating point representation of 5%. self.assertEqual(gates.g3(self.measurement(.57, fp32=.6), cfg)["verdict"], "pass-screen") def test_lower_is_better_metrics(self): # Exercise scalar direction here; test_depth_gate covers the independent # required alignment and delta1 measurements for real MDE acceptance. cfg = {**self.snapshot("mde")["G3"], "extra": {}} for quant, verdict in ((.21, "pass-screen"), (.21001, "fail"), (.15, "pass-screen")): with self.subTest(quant=quant): self.assertEqual(gates.g3(self.measurement(quant, fp32=.2), cfg)["verdict"], verdict) def test_invalid_or_missing_metrics_are_unverified(self): cfg = self.snapshot()["G3"] for m in (None, self.measurement(None), self.measurement(0, fp32=0), self.measurement(float("inf")), self.measurement(float("nan"))): with self.subTest(measurement=m): self.assertEqual(gates.g3(m, cfg)["verdict"], "unverified") def test_flow_requires_both_original_metrics(self): cfg = self.snapshot("flow")["G3"] measurement = self.measurement(1, fp32=1, per_metric={ "epe2d": {"fp32": 1, "quant": 1}, "fl_all": {"fp32": 10, "quant": 10.6}}) result = gates.g3(measurement, cfg) self.assertEqual(result["verdict"], "fail") self.assertEqual(result["metric"], "fl_all") self.assertEqual(result["per_metric"]["epe2d"]["verdict"], "pass-screen") measurement["per_metric"]["fl_all"]["quant"] = 10.5 self.assertEqual(gates.g3(measurement, cfg)["verdict"], "pass-screen") del measurement["per_metric"]["fl_all"] self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified") measurement["source"] = "sim" self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified") def test_cosine_and_performance_do_not_block_new_jobs(self): cfg = self.snapshot() measurements = { "G1": dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False), "G2": {"cos_min": .5}, "G3": self.measurement(76), "G4": {"latency_ms_mean": 10000}, } result = gates.evaluate(measurements, cfg) self.assertEqual(result["gates"]["G2"]["verdict"], "fail") self.assertEqual(result["gates"]["G4"]["latency_ms_mean"], 10000) self.assertEqual(result["gates"]["G4"]["verdict"], "na") self.assertEqual(result["terminal"], "PASS") measurements.pop("G2") measurements.pop("G4") self.assertEqual(gates.evaluate(measurements, cfg)["terminal"], "PASS") measurements.pop("G3") self.assertEqual(gates.evaluate(measurements, cfg)["terminal"], "UNVERIFIED") def test_frozen_absolute_policy_uses_fixed_tolerance(self): cfg = {"kind": "abs_pp", "tol": -1, "metric": "top1", "t1_screen_widen_se": 2} # Legacy snapshots keep their metric, signed absolute threshold and # required gates, but no longer widen acceptance using standard error. for se in (0, .5, 10): for quant, verdict in ((79, "pass-screen"), (78.999, "fail"), (78.5, "fail"), (81, "pass-screen")): with self.subTest(se=se, quant=quant): result = gates.g3(self.measurement(quant, source=None, se=se), cfg) self.assertEqual(result["verdict"], verdict) self.assertEqual(result["effective_tolerance"], -1) self.assertEqual(result["scoring_rule"], gates.SCORING_RULE) # An old job which requires G2 must still fail G2 on rejudge. frozen = self.snapshot() frozen["G3"] = {"verdict": "na"} frozen["required"] = ["G1", "G2"] measurements = {"G1": dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False), "G2": {"cos_min": .5}} self.assertEqual(gates.evaluate(measurements, frozen)["terminal"], "FAIL") def test_expanded_default_jobs_use_diagnostic_cosine(self): jobs = registry.expand_matrix(modes=["oracle"]) self.assertTrue(jobs) self.assertTrue(all(j.gates_required == ["G1", "G3"] for j in jobs if "gates" not in j.overrides)) def test_reward_uses_absolute_gate_evidence_for_both_kinds(self): for kind in ("rel_pct", "abs_pp"): with self.subTest(kind=kind): verdict = {"gates": {"G3": {"verdict": "pass-screen", "kind": kind, "delta": 2, "tolerance": 5}}, "oracle_ref": {"G3_delta": 99, "G3_kind": kind}} expected = reward.compute(verdict) self.assertEqual(expected["terms"]["G3"], 2.0) verdict["oracle_ref"]["G3_delta"] = -99 self.assertEqual(reward.compute(verdict), expected) verdict["gates"]["G3"]["verdict"] = "fail" self.assertEqual(reward.compute(verdict)["terms"]["G3"], 0.0) if __name__ == "__main__": unittest.main()