145 lines
7.9 KiB
Python
145 lines
7.9 KiB
Python
"""Frozen G3 metric direction: new error metrics must not silently become higher-is-better."""
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import gates, registry
|
|
|
|
|
|
class MetricDirection(unittest.TestCase):
|
|
def config(self, **overrides):
|
|
return {"metric": "mae", "direction": "lower", "kind": "rel_pct", "tol": 5, **overrides}
|
|
|
|
def measurement(self, quant, metric="mae"):
|
|
return {"fp32": 1, "quant": quant, "metric": metric, "source": "board"}
|
|
|
|
def snapshot(self, task_config, overrides=None):
|
|
base = copy.deepcopy(registry.gates())
|
|
base["G3"]["tasks"]["fixture"] = task_config
|
|
spec = registry.JobSpec(job_id="fixture", task="fixture", model="fixture", march="nash-m",
|
|
backend="ptq", qcfg="int8", priority="A", mode="oracle",
|
|
overrides={"g3": overrides} if overrides else {})
|
|
with patch.object(registry, "gates", return_value=base):
|
|
return registry.gate_snapshot(spec)["G3"]
|
|
|
|
def test_mae_deterioration_fails_and_improvement_passes(self):
|
|
for quant, verdict in ((2, "fail"), (.5, "pass")):
|
|
with self.subTest(quant=quant):
|
|
self.assertEqual(gates.g3(self.measurement(quant), self.config())["verdict"], verdict)
|
|
|
|
def test_mixed_metrics_each_use_their_frozen_direction(self):
|
|
cfg = self.config(metric=["top1", "mae"], direction={"top1": "higher", "mae": "lower"})
|
|
measurement = {"source": "board", "per_metric": {
|
|
"top1": {"fp32": 80, "quant": 80}, "mae": {"fp32": 1, "quant": 2}}}
|
|
result = gates.g3(measurement, cfg)
|
|
self.assertEqual(result["verdict"], "fail")
|
|
self.assertEqual(result["per_metric"]["top1"]["verdict"], "pass")
|
|
self.assertEqual(result["per_metric"]["mae"]["verdict"], "fail")
|
|
measurement["per_metric"]["mae"]["quant"] = .5
|
|
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "pass")
|
|
measurement["per_metric"]["top1"]["quant"] = 60
|
|
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "fail")
|
|
measurement["per_metric"]["top1"]["quant"] = 80
|
|
del measurement["per_metric"]["mae"]
|
|
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified")
|
|
|
|
def test_scalar_and_nested_metric_conflicts_are_unverified(self):
|
|
self.assertEqual(gates.g3(self.measurement(.5, "top1"), self.config())["verdict"], "unverified")
|
|
cfg = self.config(metric=["mae"], direction={"mae": "lower"})
|
|
measurement = {"source": "board", "per_metric": {"mae": self.measurement(.5, "top1")}}
|
|
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified")
|
|
measurement["metric"] = "unrelated"
|
|
self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified")
|
|
|
|
def test_unknown_metric_missing_direction_is_rejected_at_snapshot(self):
|
|
cfg = self.config()
|
|
del cfg["direction"]
|
|
with self.assertRaisesRegex(ValueError, "direction"):
|
|
self.snapshot(cfg)
|
|
|
|
def test_invalid_and_incomplete_direction_are_rejected(self):
|
|
for direction in (None, "", "sideways", True, {}, {"mae": "sideways"}):
|
|
with self.subTest(direction=direction), self.assertRaisesRegex(ValueError, "direction"):
|
|
self.snapshot(self.config(direction=direction))
|
|
with self.assertRaisesRegex(ValueError, "direction"):
|
|
self.snapshot(self.config(metric=["mae", "top1"], direction={"mae": "lower"}))
|
|
|
|
def test_override_metric_cannot_inherit_unrelated_direction(self):
|
|
base = self.config(metric="top1", direction="higher")
|
|
with self.assertRaisesRegex(ValueError, "direction"):
|
|
self.snapshot(base, {"metric": "mae"})
|
|
cfg = self.snapshot(base, {"metric": "mae", "direction": "lower"})
|
|
self.assertEqual(cfg["direction"], "lower")
|
|
self.assertEqual(gates.g3(self.measurement(2), cfg)["verdict"], "fail")
|
|
|
|
def test_new_snapshots_freeze_scalar_and_per_metric_directions(self):
|
|
self.assertEqual(self.snapshot(self.config())["direction"], "lower")
|
|
cfg = self.config(metric=["epe", "d1_all"])
|
|
del cfg["direction"]
|
|
self.assertEqual(self.snapshot(cfg)["direction"], {"epe": "lower", "d1_all": "lower"})
|
|
base = self.config(metric="top1")
|
|
del base["direction"]
|
|
self.assertEqual(self.snapshot(base)["direction"], "higher")
|
|
self.assertEqual(self.snapshot(self.config(metric=["epe", "d1_all"]))["direction"],
|
|
{"epe": "lower", "d1_all": "lower"})
|
|
|
|
def test_known_direction_freezing_preserves_tolerances_and_standard_error(self):
|
|
for metric, direction in (("top1", "higher"), ("absrel", "lower")):
|
|
for kind, tol in (("rel_pct", 5), ("abs_pp", -1)):
|
|
for quant in (77.5, 78, 80, 82, 86):
|
|
with self.subTest(metric=metric, kind=kind, quant=quant):
|
|
cfg = {"metric": metric, "kind": kind, "tol": tol, "t1_screen_widen_se": 2}
|
|
measurement = {"metric": metric, "fp32": 80, "quant": quant, "tier": "t1", "se": .5}
|
|
self.assertEqual(gates.g3(measurement, cfg),
|
|
gates.g3(measurement, {**cfg, "direction": direction}))
|
|
|
|
def test_invalid_frozen_direction_is_unverified_instead_of_guessing(self):
|
|
self.assertEqual(gates.g3(self.measurement(.5), self.config(direction="wrong"))["verdict"], "unverified")
|
|
|
|
def test_legacy_fallback_is_limited_to_known_metrics(self):
|
|
cfg = self.config()
|
|
del cfg["direction"]
|
|
self.assertEqual(gates.g3(self.measurement(.5), cfg)["verdict"], "unverified")
|
|
for metric, quant in (("top1", 2), ("absrel", .5)):
|
|
with self.subTest(metric=metric):
|
|
self.assertEqual(gates.g3(self.measurement(quant, metric), {**cfg, "metric": metric})["verdict"], "pass")
|
|
|
|
def test_equivalent_legacy_snapshot_comparison_preserves_both_inputs(self):
|
|
from harness.pipeline import _check_frozen_job
|
|
for metric, direction in (("top1", "higher"), (["epe", "d1_all"], {"epe": "lower", "d1_all": "lower"})):
|
|
with self.subTest(metric=metric), tempfile.TemporaryDirectory() as directory:
|
|
old = {"task": "cls", "rollout_plans": [], "gates": {"G3": {
|
|
"metric": metric, "kind": "rel_pct", "tol": 5}}}
|
|
requested = copy.deepcopy(old)
|
|
requested["gates"]["G3"]["direction"] = direction
|
|
expected = copy.deepcopy(requested)
|
|
path = Path(directory) / "job.json"
|
|
path.write_text(json.dumps(old, indent=3))
|
|
before = path.read_bytes()
|
|
_check_frozen_job(path, requested)
|
|
self.assertEqual(path.read_bytes(), before)
|
|
self.assertEqual(requested, expected)
|
|
|
|
def test_legacy_snapshot_comparison_rejects_unknown_reversed_or_changed_rules(self):
|
|
from harness.pipeline import _check_frozen_job
|
|
for metric, direction, tol in (("mae", "lower", 5), ("top1", "lower", 5),
|
|
("top1", None, 5), ("top1", "wrong", 5), ("top1", "higher", 6)):
|
|
with self.subTest(metric=metric, direction=direction, tol=tol), tempfile.TemporaryDirectory() as directory:
|
|
old = {"task": "cls", "rollout_plans": [], "gates": {"G3": {
|
|
"metric": metric, "kind": "rel_pct", "tol": 5}}}
|
|
requested = copy.deepcopy(old)
|
|
requested["gates"]["G3"].update(direction=direction, tol=tol)
|
|
path = Path(directory) / "job.json"
|
|
path.write_text(json.dumps(old))
|
|
before = path.read_bytes()
|
|
with self.assertRaisesRegex(ValueError, "content conflict"):
|
|
_check_frozen_job(path, requested)
|
|
self.assertEqual(path.read_bytes(), before)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|