data-gen-bpu/scripts/test_metric_direction.py

145 lines
7.9 KiB
Python

"""Frozen G3 metric direction: new error metrics must not silently become higher-is-better."""
import copy
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from harness import gates, registry
class MetricDirection(unittest.TestCase):
def config(self, **overrides):
return {"metric": "mae", "direction": "lower", "kind": "rel_pct", "tol": 5, **overrides}
def measurement(self, quant, metric="mae"):
return {"fp32": 1, "quant": quant, "metric": metric, "source": "board"}
def snapshot(self, task_config, overrides=None):
base = copy.deepcopy(registry.gates())
base["G3"]["tasks"]["fixture"] = task_config
spec = registry.JobSpec(job_id="fixture", task="fixture", model="fixture", march="nash-m",
backend="ptq", qcfg="int8", priority="A", mode="oracle",
overrides={"g3": overrides} if overrides else {})
with patch.object(registry, "gates", return_value=base):
return registry.gate_snapshot(spec)["G3"]
def test_mae_deterioration_fails_and_improvement_passes(self):
for quant, verdict in ((2, "fail"), (.5, "pass")):
with self.subTest(quant=quant):
self.assertEqual(gates.g3(self.measurement(quant), self.config())["verdict"], verdict)
def test_mixed_metrics_each_use_their_frozen_direction(self):
cfg = self.config(metric=["top1", "mae"], direction={"top1": "higher", "mae": "lower"})
measurement = {"source": "board", "per_metric": {
"top1": {"fp32": 80, "quant": 80}, "mae": {"fp32": 1, "quant": 2}}}
result = gates.g3(measurement, cfg)
self.assertEqual(result["verdict"], "fail")
self.assertEqual(result["per_metric"]["top1"]["verdict"], "pass")
self.assertEqual(result["per_metric"]["mae"]["verdict"], "fail")
measurement["per_metric"]["mae"]["quant"] = .5
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "pass")
measurement["per_metric"]["top1"]["quant"] = 60
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "fail")
measurement["per_metric"]["top1"]["quant"] = 80
del measurement["per_metric"]["mae"]
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified")
def test_scalar_and_nested_metric_conflicts_are_unverified(self):
self.assertEqual(gates.g3(self.measurement(.5, "top1"), self.config())["verdict"], "unverified")
cfg = self.config(metric=["mae"], direction={"mae": "lower"})
measurement = {"source": "board", "per_metric": {"mae": self.measurement(.5, "top1")}}
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified")
measurement["metric"] = "unrelated"
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified")
def test_unknown_metric_missing_direction_is_rejected_at_snapshot(self):
cfg = self.config()
del cfg["direction"]
with self.assertRaisesRegex(ValueError, "direction"):
self.snapshot(cfg)
def test_invalid_and_incomplete_direction_are_rejected(self):
for direction in (None, "", "sideways", True, {}, {"mae": "sideways"}):
with self.subTest(direction=direction), self.assertRaisesRegex(ValueError, "direction"):
self.snapshot(self.config(direction=direction))
with self.assertRaisesRegex(ValueError, "direction"):
self.snapshot(self.config(metric=["mae", "top1"], direction={"mae": "lower"}))
def test_override_metric_cannot_inherit_unrelated_direction(self):
base = self.config(metric="top1", direction="higher")
with self.assertRaisesRegex(ValueError, "direction"):
self.snapshot(base, {"metric": "mae"})
cfg = self.snapshot(base, {"metric": "mae", "direction": "lower"})
self.assertEqual(cfg["direction"], "lower")
self.assertEqual(gates.g3(self.measurement(2), cfg)["verdict"], "fail")
def test_new_snapshots_freeze_scalar_and_per_metric_directions(self):
self.assertEqual(self.snapshot(self.config())["direction"], "lower")
cfg = self.config(metric=["epe", "d1_all"])
del cfg["direction"]
self.assertEqual(self.snapshot(cfg)["direction"], {"epe": "lower", "d1_all": "lower"})
base = self.config(metric="top1")
del base["direction"]
self.assertEqual(self.snapshot(base)["direction"], "higher")
self.assertEqual(self.snapshot(self.config(metric=["epe", "d1_all"]))["direction"],
{"epe": "lower", "d1_all": "lower"})
def test_known_direction_freezing_preserves_tolerances_and_standard_error(self):
for metric, direction in (("top1", "higher"), ("absrel", "lower")):
for kind, tol in (("rel_pct", 5), ("abs_pp", -1)):
for quant in (77.5, 78, 80, 82, 86):
with self.subTest(metric=metric, kind=kind, quant=quant):
cfg = {"metric": metric, "kind": kind, "tol": tol, "t1_screen_widen_se": 2}
measurement = {"metric": metric, "fp32": 80, "quant": quant, "tier": "t1", "se": .5}
self.assertEqual(gates.g3(measurement, cfg),
gates.g3(measurement, {**cfg, "direction": direction}))
def test_invalid_frozen_direction_is_unverified_instead_of_guessing(self):
self.assertEqual(gates.g3(self.measurement(.5), self.config(direction="wrong"))["verdict"], "unverified")
def test_legacy_fallback_is_limited_to_known_metrics(self):
cfg = self.config()
del cfg["direction"]
self.assertEqual(gates.g3(self.measurement(.5), cfg)["verdict"], "unverified")
for metric, quant in (("top1", 2), ("absrel", .5)):
with self.subTest(metric=metric):
self.assertEqual(gates.g3(self.measurement(quant, metric), {**cfg, "metric": metric})["verdict"], "pass")
def test_equivalent_legacy_snapshot_comparison_preserves_both_inputs(self):
from harness.pipeline import _check_frozen_job
for metric, direction in (("top1", "higher"), (["epe", "d1_all"], {"epe": "lower", "d1_all": "lower"})):
with self.subTest(metric=metric), tempfile.TemporaryDirectory() as directory:
old = {"task": "cls", "rollout_plans": [], "gates": {"G3": {
"metric": metric, "kind": "rel_pct", "tol": 5}}}
requested = copy.deepcopy(old)
requested["gates"]["G3"]["direction"] = direction
expected = copy.deepcopy(requested)
path = Path(directory) / "job.json"
path.write_text(json.dumps(old, indent=3))
before = path.read_bytes()
_check_frozen_job(path, requested)
self.assertEqual(path.read_bytes(), before)
self.assertEqual(requested, expected)
def test_legacy_snapshot_comparison_rejects_unknown_reversed_or_changed_rules(self):
from harness.pipeline import _check_frozen_job
for metric, direction, tol in (("mae", "lower", 5), ("top1", "lower", 5),
("top1", None, 5), ("top1", "wrong", 5), ("top1", "higher", 6)):
with self.subTest(metric=metric, direction=direction, tol=tol), tempfile.TemporaryDirectory() as directory:
old = {"task": "cls", "rollout_plans": [], "gates": {"G3": {
"metric": metric, "kind": "rel_pct", "tol": 5}}}
requested = copy.deepcopy(old)
requested["gates"]["G3"].update(direction=direction, tol=tol)
path = Path(directory) / "job.json"
path.write_text(json.dumps(old))
before = path.read_bytes()
with self.assertRaisesRegex(ValueError, "content conflict"):
_check_frozen_job(path, requested)
self.assertEqual(path.read_bytes(), before)
if __name__ == "__main__":
unittest.main()