297 lines
16 KiB
Python
297 lines
16 KiB
Python
"""Run in bench-venv: python -m unittest scripts.test_pose_gate scripts.test_coco_extra scripts.test_accuracy_policy.
|
|
|
|
Geometry fixtures use actual decoding/letterbox and real COCOeval; no mocked AP.
|
|
These are CPU contract tests, not evidence of a board model passing its gate.
|
|
"""
|
|
import copy
|
|
import io
|
|
import json
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
|
|
import numpy as np
|
|
|
|
from harness import gates
|
|
from harness.eval.coco_extra import (
|
|
measure_pose_shift, pose_snapshot, pose_shift_summary, predict, run_fp32, run_quant,
|
|
)
|
|
|
|
|
|
def detection(x=0, shift=(0, 0), confidence=.9, score=.9):
|
|
points = [[x + 10 + i * 2 + shift[0], 10 + i + shift[1], confidence] for i in range(17)]
|
|
return {"category_id": 1, "bbox_xyxy": [x, 0, x + 100, 100], "score": score, "keypoints": points}
|
|
|
|
|
|
def snapshot(*image_detections):
|
|
t1 = SimpleNamespace(manifest={"sha256": "test-t1"}, items=[{"id": i + 1} for i in range(len(image_detections))])
|
|
result = pose_snapshot(t1, "1x3x640x640")
|
|
result["images"] = [
|
|
{"image_id": i + 1, "image_size": [640, 480], "scale": 1.0, "pad": [0, 80], "detections": ds}
|
|
for i, ds in enumerate(image_detections)
|
|
]
|
|
return result
|
|
|
|
|
|
def measurement(shift, quant=80):
|
|
return {"metric": "oks_ap", "fp32": 80, "quant": quant, "tier": "t1", "source": "board", "n": 1,
|
|
"extra": {"keypoint_mean_shift_px": shift["keypoint_mean_shift_px"], "keypoint_shift": pose_shift_summary(shift)}}
|
|
|
|
|
|
CFG = {"metric": "oks_ap", "kind": "rel_pct", "tol": 5, "require_source": "board",
|
|
"t1_screen_widen_se": 0, "extra": {"keypoint_mean_shift_px_max": 2}}
|
|
|
|
|
|
class PoseGeometry(unittest.TestCase):
|
|
def test_topdown_uses_actual_affine_without_fabricated_letterbox(self):
|
|
reference, measured = snapshot([detection()]), snapshot([detection(shift=(3, 4))])
|
|
for record in (reference, measured):
|
|
image = record["images"][0]
|
|
image.pop("scale"); image.pop("pad")
|
|
image["transform"] = "topdown_affine"
|
|
image["detections"][0]["affine"] = [[.5, 0., 7.], [0., .5, 11.]]
|
|
result = measure_pose_shift(reference, measured)
|
|
self.assertEqual(result["keypoint_mean_shift_px"], 5)
|
|
self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "fail")
|
|
measured["images"][0]["detections"][0]["affine"][0][2] += 1
|
|
self.assertEqual(measure_pose_shift(reference, measured)["status"], "unverified")
|
|
measured["images"][0]["detections"][0]["affine"] = [[0., 0., 0.], [0., 0., 0.]]
|
|
self.assertEqual(measure_pose_shift(reference, measured)["status"], "unverified")
|
|
|
|
def test_euclidean_distance_and_replayable_evidence(self):
|
|
result = measure_pose_shift(snapshot([detection()]), snapshot([detection(shift=(3, 4))]))
|
|
self.assertEqual(result["status"], "measured")
|
|
self.assertEqual(result["keypoint_mean_shift_px"], 5)
|
|
self.assertEqual((result["n_matched"], result["n_keypoints"]), (1, 17))
|
|
pair = json.loads(json.dumps(result, allow_nan=False))["matches"][0]
|
|
diff = np.asarray(pair["quant"]["keypoints"])[:, :2] - np.asarray(pair["fp32"]["keypoints"])[:, :2]
|
|
np.testing.assert_allclose(np.linalg.norm(diff, axis=1), pair["distances_px"])
|
|
|
|
def test_metric_is_mean_of_lengths_not_max_rms_or_length_of_mean(self):
|
|
a, b = detection(), detection()
|
|
b["keypoints"][0][0] += 17
|
|
b["keypoints"][1][0] -= 17
|
|
result = measure_pose_shift(snapshot([a]), snapshot([b]))
|
|
self.assertEqual(result["keypoint_mean_shift_px"], 2)
|
|
self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "pass-screen")
|
|
|
|
def test_order_and_score_changes_do_not_pair_different_people(self):
|
|
a = snapshot([detection(), detection(x=200)])
|
|
b = snapshot([detection(x=200, shift=(0, 2), score=.1), detection(shift=(2, 0), score=.99)])
|
|
result = measure_pose_shift(a, b)
|
|
self.assertEqual(result["keypoint_mean_shift_px"], 2)
|
|
self.assertEqual([(p["fp32_index"], p["quant_index"]) for p in result["matches"]], [(0, 1), (1, 0)])
|
|
|
|
def test_matching_uses_boxes_not_nearest_keypoints(self):
|
|
# Deliberately swapped skeletons: matching by keypoints would falsely give zero.
|
|
a = snapshot([detection(), detection(x=200)])
|
|
b = snapshot([detection(shift=(200, 0)), detection(x=200, shift=(-200, 0))])
|
|
result = measure_pose_shift(a, b)
|
|
self.assertEqual(result["keypoint_mean_shift_px"], 200)
|
|
|
|
def test_one_to_one_and_deterministic_iou_tie(self):
|
|
result = measure_pose_shift(snapshot([detection(), detection()]), snapshot([detection()]))
|
|
self.assertEqual(result["n_matched"], 1)
|
|
self.assertEqual(result["matches"][0]["fp32_index"], 0)
|
|
self.assertEqual(result["unmatched_fp32"], [{"image_id": 1, "index": 1}])
|
|
|
|
def test_iou_matching_boundary(self):
|
|
a, b = detection(), detection()
|
|
a["bbox_xyxy"], b["bbox_xyxy"] = [0, 0, 3, 3], [1, 0, 4, 3]
|
|
result = measure_pose_shift(snapshot([a]), snapshot([b]))
|
|
self.assertEqual(result["matches"][0]["bbox_iou"], .5)
|
|
b["bbox_xyxy"] = [1.001, 0, 4.001, 3]
|
|
self.assertEqual(measure_pose_shift(snapshot([a]), snapshot([b]))["status"], "unverified")
|
|
|
|
def test_no_cross_image_matching(self):
|
|
result = measure_pose_shift(snapshot([detection()], []), snapshot([], [detection()]))
|
|
self.assertEqual(result["status"], "unverified")
|
|
self.assertEqual(result["n_matched"], 0)
|
|
self.assertIsNone(result["keypoint_mean_shift_px"])
|
|
|
|
def test_unmatched_detections_are_counted_not_zero_filled(self):
|
|
result = measure_pose_shift(snapshot([detection(), detection(x=200)]),
|
|
snapshot([detection(shift=(3, 4))]))
|
|
self.assertEqual(result["keypoint_mean_shift_px"], 5)
|
|
self.assertEqual((result["n_fp32"], result["n_quant"], result["n_matched"]), (2, 1, 1))
|
|
self.assertEqual(gates.g3(measurement(result), CFG)["extra"]["keypoint_shift"]["n_unmatched_fp32"], 1)
|
|
|
|
def test_joint_confidence_cannot_hide_shift(self):
|
|
result = measure_pose_shift(snapshot([detection(confidence=0)]),
|
|
snapshot([detection(shift=(3, 4), confidence=0)]))
|
|
self.assertEqual(result["keypoint_mean_shift_px"], 5)
|
|
self.assertEqual(result["n_keypoints"], 17)
|
|
|
|
def test_empty_missing_and_unmatched_are_unverified(self):
|
|
for a, b in ((None, snapshot([])), (snapshot([]), None), (snapshot([]), snapshot([])),
|
|
(snapshot([detection()]), snapshot([])),
|
|
(snapshot([detection()]), snapshot([detection(x=200)]))):
|
|
with self.subTest(fp32=a, quant=b):
|
|
result = measure_pose_shift(a, b)
|
|
self.assertEqual(result["status"], "unverified")
|
|
self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "unverified")
|
|
|
|
def test_invalid_geometry_or_incomplete_evidence_never_measures(self):
|
|
reference = snapshot([detection()])
|
|
bad = []
|
|
for value in (float("nan"), float("inf")):
|
|
other = copy.deepcopy(reference)
|
|
other["images"][0]["detections"][0]["keypoints"][0][0] = value
|
|
bad.append(other)
|
|
for field, value in (("schema", "older"), ("input_spec", "1x3x320x320"),
|
|
("t1_manifest_sha", "other"), ("image_ids", [2]), ("images", [])):
|
|
bad.append({**reference, field: value})
|
|
other = copy.deepcopy(reference)
|
|
other["images"][0]["pad"] = [0, 79]
|
|
bad.append(other)
|
|
other = copy.deepcopy(reference)
|
|
other["images"][0]["detections"][0]["keypoints"].pop()
|
|
bad.append(other)
|
|
other = copy.deepcopy(reference)
|
|
other["images"][0]["detections"][0]["category_id"] = 2
|
|
bad.append(other)
|
|
for other in bad:
|
|
with self.subTest(other=other):
|
|
self.assertEqual(measure_pose_shift(reference, other)["status"], "unverified")
|
|
|
|
|
|
class PoseGate(unittest.TestCase):
|
|
def shifted(self, pixels):
|
|
return measurement(measure_pose_shift(snapshot([detection()]), snapshot([detection(shift=(pixels, 0))])))
|
|
|
|
def test_two_pixel_inclusive_boundary(self):
|
|
for pixels, verdict in ((0, "pass-screen"), (1.9999, "pass-screen"), (2, "pass-screen"), (2.0001, "fail")):
|
|
with self.subTest(pixels=pixels):
|
|
result = gates.g3(self.shifted(pixels), CFG)
|
|
self.assertEqual(result["verdict"], verdict)
|
|
self.assertEqual(result["extra"]["keypoint_shift"]["threshold_px"], 2)
|
|
|
|
def test_both_ap_and_geometry_must_pass(self):
|
|
for quant, pixels, verdict in ((76, 2, "pass-screen"), (75.999, 0, "fail"), (85, 3, "fail")):
|
|
m = self.shifted(pixels)
|
|
m["quant"] = quant
|
|
self.assertEqual(gates.g3(m, CFG)["verdict"], verdict)
|
|
|
|
def test_missing_geometry_cannot_pass_on_oks_alone_even_with_ap_failure(self):
|
|
for quant in (80, 0):
|
|
m = {"fp32": 80, "quant": quant, "source": "board", "tier": "t1"}
|
|
self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified")
|
|
m["extra"] = {"keypoint_mean_shift_px": 0}
|
|
self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified")
|
|
|
|
def test_bad_counts_values_and_status_only_are_unverified(self):
|
|
bad = []
|
|
for value in (None, -1, True, "0", float("nan"), float("inf")):
|
|
m = self.shifted(0)
|
|
m["extra"]["keypoint_mean_shift_px"] = value
|
|
bad.append(m)
|
|
for key, value in (("n_matched", 0), ("n_keypoints", 0), ("n_fp32", 99), ("distance_sum_px", None),
|
|
("method", "manual-label"), ("keypoint_mean_shift_px", 1)):
|
|
m = self.shifted(0)
|
|
m["extra"]["keypoint_shift"][key] = value
|
|
bad.append(m)
|
|
m = self.shifted(0)
|
|
m["extra"]["keypoint_shift"]["distance_sum_px"] = float("nan")
|
|
bad.append(m)
|
|
for m in bad:
|
|
with self.subTest(measurement=m):
|
|
self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified")
|
|
|
|
def test_t1_se_does_not_widen_pixel_threshold(self):
|
|
m = self.shifted(2.1)
|
|
m["se"] = 100
|
|
self.assertEqual(gates.g3(m, {**CFG, "t1_screen_widen_se": 2})["verdict"], "fail")
|
|
|
|
def test_source_na_and_other_task_thresholds_preserved(self):
|
|
m = self.shifted(0)
|
|
m["source"] = "sim"
|
|
self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified")
|
|
self.assertEqual(gates.g3(None, {**CFG, "verdict": "na"})["verdict"], "na")
|
|
for metric in ("top1", "map_50_95", "mask_ap"):
|
|
cfg = {k: v for k, v in CFG.items() if k != "extra"}
|
|
cfg["metric"] = metric
|
|
for quant, verdict in ((76, "pass-screen"), (75.999, "fail")):
|
|
self.assertEqual(gates.g3({"fp32": 80, "quant": quant, "source": "board", "tier": "t1"}, cfg)["verdict"], verdict)
|
|
|
|
|
|
def t1_fixture(image_size=(640, 480)):
|
|
from PIL import Image
|
|
points = detection()["keypoints"]
|
|
annotation = {"id": 1, "image_id": 1, "category_id": 1, "bbox": [0, 0, 100, 100], "area": 10000,
|
|
"iscrowd": 0, "num_keypoints": 17, "keypoints": [v for x, y, _ in points for v in (x, y, 2)]}
|
|
category = {"id": 1, "name": "person", "keypoints": [str(i) for i in range(17)], "skeleton": []}
|
|
ann = {"info": {}, "images": [{"id": 1, "width": image_size[0], "height": image_size[1]}],
|
|
"annotations": [], "categories": [], "keypoints": {"annotations": [annotation], "categories": [category]}}
|
|
return SimpleNamespace(manifest={"sha256": "test-t1"}, items=[{"id": 1}],
|
|
image=lambda _: Image.new("RGB", image_size),
|
|
tar=SimpleNamespace(extractfile=lambda _: io.BytesIO(json.dumps(ann).encode())))
|
|
|
|
|
|
def head_runner(image_size, input_size, shift=(0, 0), points=None, score=.9):
|
|
scale = input_size / max(image_size)
|
|
pad = [(input_size - round(s * scale)) // 2 for s in image_size]
|
|
head = np.zeros((1, 56, 1), dtype=np.float32)
|
|
head[0, :4, 0] = [50 * scale + pad[0], 50 * scale + pad[1], 100 * scale, 100 * scale]
|
|
head[0, 4, 0] = score
|
|
keypoints = np.asarray(points if points is not None else detection()["keypoints"], dtype=float)
|
|
keypoints[:, :2] = (keypoints[:, :2] + np.asarray(shift)) * scale + pad
|
|
head[0, 5:, 0] = keypoints.reshape(-1)
|
|
return lambda _: [head.copy()]
|
|
|
|
|
|
class PoseIntegration(unittest.TestCase):
|
|
def test_real_decode_undoes_resize_and_asymmetric_letterbox(self):
|
|
t1 = t1_fixture((5, 3))
|
|
points = [[1, 1, .8] for _ in range(17)]
|
|
a, b = pose_snapshot(t1, "1x3x8x8"), pose_snapshot(t1, "1x3x8x8")
|
|
predict(head_runner((5, 3), 8, points=points), t1, "1x3x8x8", "pose", a)
|
|
predict(head_runner((5, 3), 8, shift=(0, 2), points=points), t1, "1x3x8x8", "pose", b)
|
|
self.assertEqual(a["images"][0]["pad"], [0, 1])
|
|
self.assertAlmostEqual(a["images"][0]["detections"][0]["keypoints"][0][1], 1, places=6)
|
|
self.assertAlmostEqual(measure_pose_shift(a, b)["keypoint_mean_shift_px"], 2, places=6)
|
|
|
|
def test_unclipped_evidence_does_not_hide_edge_displacement(self):
|
|
t1 = t1_fixture()
|
|
points = [[-5, 20, .8] for _ in range(17)]
|
|
a, b = pose_snapshot(t1, "1x3x640x640"), pose_snapshot(t1, "1x3x640x640")
|
|
pa = predict(head_runner((640, 480), 640, points=points), t1, "1x3x640x640", "pose", a)
|
|
pb = predict(head_runner((640, 480), 640, shift=(-3, 0), points=points), t1, "1x3x640x640", "pose", b)
|
|
self.assertEqual(pa[0]["keypoints"], pb[0]["keypoints"]) # unchanged AP clipping
|
|
self.assertEqual(measure_pose_shift(a, b)["keypoint_mean_shift_px"], 3)
|
|
|
|
def test_real_cocoeval_and_json_baseline_quant_wiring(self):
|
|
t1 = t1_fixture()
|
|
fp = {"task": "pose", "model": "fixture"}
|
|
run_fp32(head_runner((640, 480), 640), t1, "1x3x640x640", fp)
|
|
self.assertAlmostEqual(fp["metrics"]["oks_ap"], 100)
|
|
baseline = json.loads(json.dumps(fp, allow_nan=False))
|
|
quant = {"task": "pose", "model": "fixture"}
|
|
run_quant(head_runner((640, 480), 640, shift=(2, 0)), t1, "1x3x640x640", baseline, quant)
|
|
self.assertEqual(fp, baseline) # quant cannot mutate the saved reference
|
|
self.assertEqual(quant["G3"]["extra"]["keypoint_mean_shift_px"], 2)
|
|
self.assertEqual(quant["pose_shift_evidence"]["matches"][0]["distances_px"], [2] * 17)
|
|
self.assertNotIn("matches", quant["G3"]["extra"]["keypoint_shift"])
|
|
self.assertGreaterEqual(quant["metrics"]["oks_ap"], 95)
|
|
quant["G3"]["source"] = "board" # gate source fixture; runner here is synthetic
|
|
self.assertEqual(gates.g3(quant["G3"], CFG)["verdict"], "pass-screen")
|
|
json.dumps(quant, allow_nan=False)
|
|
|
|
def test_legacy_baseline_keeps_ap_but_shift_is_unverified(self):
|
|
t1 = t1_fixture()
|
|
out = {"task": "pose", "model": "fixture"}
|
|
run_quant(head_runner((640, 480), 640), t1, "1x3x640x640",
|
|
{"model": "fixture", "metrics": {"oks_ap": 100}}, out)
|
|
self.assertAlmostEqual(out["metrics"]["oks_ap"], 100)
|
|
out["G3"]["source"] = "board"
|
|
self.assertEqual(gates.g3(out["G3"], CFG)["verdict"], "unverified")
|
|
|
|
def test_no_detections_produces_complete_empty_image_evidence(self):
|
|
t1 = t1_fixture()
|
|
evidence = pose_snapshot(t1, "1x3x640x640")
|
|
self.assertEqual(predict(head_runner((640, 480), 640, score=0), t1, "1x3x640x640", "pose", evidence), [])
|
|
self.assertEqual(evidence["images"][0]["detections"], [])
|
|
self.assertEqual(measure_pose_shift(evidence, evidence)["status"], "unverified")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|