data-gen-bpu/scripts/test_accuracy_policy.py

132 lines
6.7 KiB
Python

"""Accuracy acceptance boundaries, frozen policies, and reward direction."""
import importlib.util
from pathlib import Path
import unittest
from harness import gates, registry
ROOT = Path(__file__).resolve().parents[1]
_spec = importlib.util.spec_from_file_location("reward", ROOT / "export/reward.py")
reward = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(reward)
class AccuracyPolicy(unittest.TestCase):
def snapshot(self, task="cls"):
spec = registry.JobSpec(
job_id=f"{task}--example--nash-m--ptq--int8", task=task, model="example",
march="nash-m", backend="ptq", qcfg="int8", priority="A", mode="oracle",
)
return registry.gate_snapshot(spec)
def measurement(self, quant, fp32=80.0, **extra):
return {"fp32": fp32, "quant": quant, "tier": "t1", "source": "board", **extra}
def test_new_jobs_require_board_task_accuracy(self):
cfg = self.snapshot()
self.assertEqual(cfg["required"], ["G1", "G3"])
self.assertEqual(cfg["G3"]["kind"], "rel_pct")
self.assertEqual(cfg["G3"]["tol"], 5)
self.assertEqual(cfg["G3"]["require_source"], "board")
self.assertEqual(cfg["G3"]["t1_screen_widen_se"], 0)
for source in (None, "simulator"):
result = gates.g3(self.measurement(80, source=source), cfg["G3"])
self.assertEqual(result["verdict"], "unverified")
def test_relative_five_percent_boundary_and_improvement(self):
cfg = self.snapshot()["G3"]
for quant, verdict in ((76, "pass-screen"), (75.999, "fail"), (85, "pass-screen")):
with self.subTest(quant=quant):
self.assertEqual(gates.g3(self.measurement(quant, se=10), cfg)["verdict"], verdict)
# The boundary must tolerate floating point representation of 5%.
self.assertEqual(gates.g3(self.measurement(.57, fp32=.6), cfg)["verdict"], "pass-screen")
def test_lower_is_better_metrics(self):
# Exercise scalar direction here; test_depth_gate covers the independent
# required alignment and delta1 measurements for real MDE acceptance.
cfg = {**self.snapshot("mde")["G3"], "extra": {}}
for quant, verdict in ((.21, "pass-screen"), (.21001, "fail"), (.15, "pass-screen")):
with self.subTest(quant=quant):
self.assertEqual(gates.g3(self.measurement(quant, fp32=.2), cfg)["verdict"], verdict)
def test_invalid_or_missing_metrics_are_unverified(self):
cfg = self.snapshot()["G3"]
for m in (None, self.measurement(None), self.measurement(0, fp32=0),
self.measurement(float("inf")), self.measurement(float("nan"))):
with self.subTest(measurement=m):
self.assertEqual(gates.g3(m, cfg)["verdict"], "unverified")
def test_flow_requires_both_original_metrics(self):
cfg = self.snapshot("flow")["G3"]
measurement = self.measurement(1, fp32=1, per_metric={
"epe2d": {"fp32": 1, "quant": 1}, "fl_all": {"fp32": 10, "quant": 10.6}})
result = gates.g3(measurement, cfg)
self.assertEqual(result["verdict"], "fail")
self.assertEqual(result["metric"], "fl_all")
self.assertEqual(result["per_metric"]["epe2d"]["verdict"], "pass-screen")
measurement["per_metric"]["fl_all"]["quant"] = 10.5
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "pass-screen")
del measurement["per_metric"]["fl_all"]
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified")
measurement["source"] = "sim"
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified")
def test_cosine_and_performance_do_not_block_new_jobs(self):
cfg = self.snapshot()
measurements = {
"G1": dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False),
"G2": {"cos_min": .5}, "G3": self.measurement(76),
"G4": {"latency_ms_mean": 10000},
}
result = gates.evaluate(measurements, cfg)
self.assertEqual(result["gates"]["G2"]["verdict"], "fail")
self.assertEqual(result["gates"]["G4"]["latency_ms_mean"], 10000)
self.assertEqual(result["gates"]["G4"]["verdict"], "na")
self.assertEqual(result["terminal"], "PASS")
measurements.pop("G2")
measurements.pop("G4")
self.assertEqual(gates.evaluate(measurements, cfg)["terminal"], "PASS")
measurements.pop("G3")
self.assertEqual(gates.evaluate(measurements, cfg)["terminal"], "UNVERIFIED")
def test_frozen_absolute_policy_uses_fixed_tolerance(self):
cfg = {"kind": "abs_pp", "tol": -1, "metric": "top1", "t1_screen_widen_se": 2}
# Legacy snapshots keep their metric, signed absolute threshold and
# required gates, but no longer widen acceptance using standard error.
for se in (0, .5, 10):
for quant, verdict in ((79, "pass-screen"), (78.999, "fail"),
(78.5, "fail"), (81, "pass-screen")):
with self.subTest(se=se, quant=quant):
result = gates.g3(self.measurement(quant, source=None, se=se), cfg)
self.assertEqual(result["verdict"], verdict)
self.assertEqual(result["effective_tolerance"], -1)
self.assertEqual(result["scoring_rule"], gates.SCORING_RULE)
# An old job which requires G2 must still fail G2 on rejudge.
frozen = self.snapshot()
frozen["G3"] = {"verdict": "na"}
frozen["required"] = ["G1", "G2"]
measurements = {"G1": dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False),
"G2": {"cos_min": .5}}
self.assertEqual(gates.evaluate(measurements, frozen)["terminal"], "FAIL")
def test_expanded_default_jobs_use_diagnostic_cosine(self):
jobs = registry.expand_matrix(modes=["oracle"])
self.assertTrue(jobs)
self.assertTrue(all(j.gates_required == ["G1", "G3"] for j in jobs if "gates" not in j.overrides))
def test_reward_uses_absolute_gate_evidence_for_both_kinds(self):
for kind in ("rel_pct", "abs_pp"):
with self.subTest(kind=kind):
verdict = {"gates": {"G3": {"verdict": "pass-screen", "kind": kind, "delta": 2, "tolerance": 5}},
"oracle_ref": {"G3_delta": 99, "G3_kind": kind}}
expected = reward.compute(verdict)
self.assertEqual(expected["terms"]["G3"], 2.0)
verdict["oracle_ref"]["G3_delta"] = -99
self.assertEqual(reward.compute(verdict), expected)
verdict["gates"]["G3"]["verdict"] = "fail"
self.assertEqual(reward.compute(verdict)["terms"]["G3"], 0.0)
if __name__ == "__main__":
unittest.main()