132 lines
6.7 KiB
Python
132 lines
6.7 KiB
Python
"""Accuracy acceptance boundaries, frozen policies, and reward direction."""
|
|
import importlib.util
|
|
from pathlib import Path
|
|
import unittest
|
|
|
|
from harness import gates, registry
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
_spec = importlib.util.spec_from_file_location("reward", ROOT / "export/reward.py")
|
|
reward = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(reward)
|
|
|
|
|
|
class AccuracyPolicy(unittest.TestCase):
|
|
def snapshot(self, task="cls"):
|
|
spec = registry.JobSpec(
|
|
job_id=f"{task}--example--nash-m--ptq--int8", task=task, model="example",
|
|
march="nash-m", backend="ptq", qcfg="int8", priority="A", mode="oracle",
|
|
)
|
|
return registry.gate_snapshot(spec)
|
|
|
|
def measurement(self, quant, fp32=80.0, **extra):
|
|
return {"fp32": fp32, "quant": quant, "tier": "t1", "source": "board", **extra}
|
|
|
|
def test_new_jobs_require_board_task_accuracy(self):
|
|
cfg = self.snapshot()
|
|
self.assertEqual(cfg["required"], ["G1", "G3"])
|
|
self.assertEqual(cfg["G3"]["kind"], "rel_pct")
|
|
self.assertEqual(cfg["G3"]["tol"], 5)
|
|
self.assertEqual(cfg["G3"]["require_source"], "board")
|
|
self.assertEqual(cfg["G3"]["t1_screen_widen_se"], 0)
|
|
for source in (None, "simulator"):
|
|
result = gates.g3(self.measurement(80, source=source), cfg["G3"])
|
|
self.assertEqual(result["verdict"], "unverified")
|
|
|
|
def test_relative_five_percent_boundary_and_improvement(self):
|
|
cfg = self.snapshot()["G3"]
|
|
for quant, verdict in ((76, "pass-screen"), (75.999, "fail"), (85, "pass-screen")):
|
|
with self.subTest(quant=quant):
|
|
self.assertEqual(gates.g3(self.measurement(quant, se=10), cfg)["verdict"], verdict)
|
|
# The boundary must tolerate floating point representation of 5%.
|
|
self.assertEqual(gates.g3(self.measurement(.57, fp32=.6), cfg)["verdict"], "pass-screen")
|
|
|
|
def test_lower_is_better_metrics(self):
|
|
# Exercise scalar direction here; test_depth_gate covers the independent
|
|
# required alignment and delta1 measurements for real MDE acceptance.
|
|
cfg = {**self.snapshot("mde")["G3"], "extra": {}}
|
|
for quant, verdict in ((.21, "pass-screen"), (.21001, "fail"), (.15, "pass-screen")):
|
|
with self.subTest(quant=quant):
|
|
self.assertEqual(gates.g3(self.measurement(quant, fp32=.2), cfg)["verdict"], verdict)
|
|
|
|
def test_invalid_or_missing_metrics_are_unverified(self):
|
|
cfg = self.snapshot()["G3"]
|
|
for m in (None, self.measurement(None), self.measurement(0, fp32=0),
|
|
self.measurement(float("inf")), self.measurement(float("nan"))):
|
|
with self.subTest(measurement=m):
|
|
self.assertEqual(gates.g3(m, cfg)["verdict"], "unverified")
|
|
|
|
def test_flow_requires_both_original_metrics(self):
|
|
cfg = self.snapshot("flow")["G3"]
|
|
measurement = self.measurement(1, fp32=1, per_metric={
|
|
"epe2d": {"fp32": 1, "quant": 1}, "fl_all": {"fp32": 10, "quant": 10.6}})
|
|
result = gates.g3(measurement, cfg)
|
|
self.assertEqual(result["verdict"], "fail")
|
|
self.assertEqual(result["metric"], "fl_all")
|
|
self.assertEqual(result["per_metric"]["epe2d"]["verdict"], "pass-screen")
|
|
measurement["per_metric"]["fl_all"]["quant"] = 10.5
|
|
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "pass-screen")
|
|
del measurement["per_metric"]["fl_all"]
|
|
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified")
|
|
measurement["source"] = "sim"
|
|
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified")
|
|
|
|
def test_cosine_and_performance_do_not_block_new_jobs(self):
|
|
cfg = self.snapshot()
|
|
measurements = {
|
|
"G1": dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False),
|
|
"G2": {"cos_min": .5}, "G3": self.measurement(76),
|
|
"G4": {"latency_ms_mean": 10000},
|
|
}
|
|
result = gates.evaluate(measurements, cfg)
|
|
self.assertEqual(result["gates"]["G2"]["verdict"], "fail")
|
|
self.assertEqual(result["gates"]["G4"]["latency_ms_mean"], 10000)
|
|
self.assertEqual(result["gates"]["G4"]["verdict"], "na")
|
|
self.assertEqual(result["terminal"], "PASS")
|
|
measurements.pop("G2")
|
|
measurements.pop("G4")
|
|
self.assertEqual(gates.evaluate(measurements, cfg)["terminal"], "PASS")
|
|
measurements.pop("G3")
|
|
self.assertEqual(gates.evaluate(measurements, cfg)["terminal"], "UNVERIFIED")
|
|
|
|
def test_frozen_absolute_policy_uses_fixed_tolerance(self):
|
|
cfg = {"kind": "abs_pp", "tol": -1, "metric": "top1", "t1_screen_widen_se": 2}
|
|
# Legacy snapshots keep their metric, signed absolute threshold and
|
|
# required gates, but no longer widen acceptance using standard error.
|
|
for se in (0, .5, 10):
|
|
for quant, verdict in ((79, "pass-screen"), (78.999, "fail"),
|
|
(78.5, "fail"), (81, "pass-screen")):
|
|
with self.subTest(se=se, quant=quant):
|
|
result = gates.g3(self.measurement(quant, source=None, se=se), cfg)
|
|
self.assertEqual(result["verdict"], verdict)
|
|
self.assertEqual(result["effective_tolerance"], -1)
|
|
self.assertEqual(result["scoring_rule"], gates.SCORING_RULE)
|
|
# An old job which requires G2 must still fail G2 on rejudge.
|
|
frozen = self.snapshot()
|
|
frozen["G3"] = {"verdict": "na"}
|
|
frozen["required"] = ["G1", "G2"]
|
|
measurements = {"G1": dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False),
|
|
"G2": {"cos_min": .5}}
|
|
self.assertEqual(gates.evaluate(measurements, frozen)["terminal"], "FAIL")
|
|
|
|
def test_expanded_default_jobs_use_diagnostic_cosine(self):
|
|
jobs = registry.expand_matrix(modes=["oracle"])
|
|
self.assertTrue(jobs)
|
|
self.assertTrue(all(j.gates_required == ["G1", "G3"] for j in jobs if "gates" not in j.overrides))
|
|
|
|
def test_reward_uses_absolute_gate_evidence_for_both_kinds(self):
|
|
for kind in ("rel_pct", "abs_pp"):
|
|
with self.subTest(kind=kind):
|
|
verdict = {"gates": {"G3": {"verdict": "pass-screen", "kind": kind, "delta": 2, "tolerance": 5}},
|
|
"oracle_ref": {"G3_delta": 99, "G3_kind": kind}}
|
|
expected = reward.compute(verdict)
|
|
self.assertEqual(expected["terms"]["G3"], 2.0)
|
|
verdict["oracle_ref"]["G3_delta"] = -99
|
|
self.assertEqual(reward.compute(verdict), expected)
|
|
verdict["gates"]["G3"]["verdict"] = "fail"
|
|
self.assertEqual(reward.compute(verdict)["terms"]["G3"], 0.0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|