60 lines
3.0 KiB
Python
60 lines
3.0 KiB
Python
"""Depth accuracy needs both the original delta1 limit and real alignment."""
|
|
from copy import deepcopy
|
|
import unittest
|
|
|
|
from harness.gates import g3
|
|
|
|
|
|
class DepthGate(unittest.TestCase):
|
|
def setUp(self):
|
|
self.cfg = {"metric": "absrel", "kind": "rel_pct", "tol": 5, "require_source": "board",
|
|
"extra": {"delta1_abs_pp_tol": -1.0, "scale_alignment": "required"}}
|
|
self.measurement = {"metric": "absrel", "fp32": .1, "quant": .104, "source": "board",
|
|
"tier": "t1", "n": 1, "extra": {
|
|
"delta1": {"status": "measured", "unit": "percent", "fp32": 90., "quant": 89., "delta_pp": -1.},
|
|
"scale_alignment": {"status": "measured", "protocol": {"id": "relative-inverse-ls"}, "n": 1,
|
|
"samples": [{"id": "image1", "fp32_scale": 2., "fp32_shift": .1,
|
|
"quant_scale": 1.9, "quant_shift": .11}]}}}
|
|
|
|
def test_existing_threshold_boundary_passes(self):
|
|
self.assertEqual(g3(self.measurement, self.cfg)["verdict"], "pass-screen")
|
|
|
|
def test_absrel_pass_does_not_hide_delta1_failure(self):
|
|
measurement = deepcopy(self.measurement)
|
|
measurement["extra"]["delta1"].update(quant=88.9, delta_pp=-1.1)
|
|
result = g3(measurement, self.cfg)
|
|
self.assertEqual(result["verdict"], "fail")
|
|
self.assertIn("delta1", result["reason"])
|
|
|
|
def test_status_without_measured_alignment_is_unverified(self):
|
|
measurement = deepcopy(self.measurement)
|
|
del measurement["extra"]["scale_alignment"]["samples"]
|
|
self.assertEqual(g3(measurement, self.cfg)["verdict"], "unverified")
|
|
|
|
def test_invalid_alignment_and_inconsistent_delta_are_unverified(self):
|
|
for field, value in (("quant_scale", float("inf")), ("quant_shift", float("nan"))):
|
|
with self.subTest(field=field):
|
|
measurement = deepcopy(self.measurement)
|
|
measurement["extra"]["scale_alignment"]["samples"][0][field] = value
|
|
self.assertEqual(g3(measurement, self.cfg)["verdict"], "unverified")
|
|
measurement = deepcopy(self.measurement)
|
|
measurement["extra"]["delta1"]["delta_pp"] = 0
|
|
self.assertEqual(g3(measurement, self.cfg)["verdict"], "unverified")
|
|
|
|
def test_affine_least_squares_keeps_a_finite_signed_solution(self):
|
|
import numpy as np
|
|
from harness.eval.mde import aligned_metrics
|
|
prediction = np.array([[1., 2.]])
|
|
depth = np.array([[.5, 1.]])
|
|
result = aligned_metrics(prediction, depth, np.ones_like(depth, dtype=bool))
|
|
self.assertEqual(result["scale"], -1.)
|
|
self.assertEqual(result["shift"], 3.)
|
|
self.assertEqual(result["absrel"], 0.)
|
|
measurement = deepcopy(self.measurement)
|
|
measurement["extra"]["scale_alignment"]["samples"][0].update(fp32_scale=-1., quant_scale=-1.)
|
|
self.assertEqual(g3(measurement, self.cfg)["verdict"], "pass-screen")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|