"""Run in bench-venv: python -m unittest scripts.test_pose_gate scripts.test_coco_extra scripts.test_accuracy_policy. Geometry fixtures use actual decoding/letterbox and real COCOeval; no mocked AP. These are CPU contract tests, not evidence of a board model passing its gate. """ import copy import io import json from types import SimpleNamespace import unittest import numpy as np from harness import gates from harness.eval.coco_extra import ( measure_pose_shift, pose_snapshot, pose_shift_summary, predict, run_fp32, run_quant, ) def detection(x=0, shift=(0, 0), confidence=.9, score=.9): points = [[x + 10 + i * 2 + shift[0], 10 + i + shift[1], confidence] for i in range(17)] return {"category_id": 1, "bbox_xyxy": [x, 0, x + 100, 100], "score": score, "keypoints": points} def snapshot(*image_detections): t1 = SimpleNamespace(manifest={"sha256": "test-t1"}, items=[{"id": i + 1} for i in range(len(image_detections))]) result = pose_snapshot(t1, "1x3x640x640") result["images"] = [ {"image_id": i + 1, "image_size": [640, 480], "scale": 1.0, "pad": [0, 80], "detections": ds} for i, ds in enumerate(image_detections) ] return result def measurement(shift, quant=80): return {"metric": "oks_ap", "fp32": 80, "quant": quant, "tier": "t1", "source": "board", "n": 1, "extra": {"keypoint_mean_shift_px": shift["keypoint_mean_shift_px"], "keypoint_shift": pose_shift_summary(shift)}} CFG = {"metric": "oks_ap", "kind": "rel_pct", "tol": 5, "require_source": "board", "t1_screen_widen_se": 0, "extra": {"keypoint_mean_shift_px_max": 2}} class PoseGeometry(unittest.TestCase): @staticmethod def topdown(*detections): value = snapshot(list(detections)) value.update(boxes_file='fixed.json', boxes_sha256='fixed-hash', pose_protocol='fixed-topdown', official_model='hrnet') image = value['images'][0] image['transform'] = 'topdown_affine' for index, item in enumerate(image['detections']): item.setdefault('source_index', index) item.setdefault('affine', [[.5, 0., 7. + index], [0., .5, 11.]]) return value def test_topdown_nms_difference_never_pairs_different_crop_ids(self): reference = self.topdown(detection(), detection()) measured = self.topdown(detection(shift=(2, 0)), detection()) measured['images'][0]['detections'][1]['source_index'] = 2 measured['images'][0]['detections'][1]['affine'][0][2] += 1 result = measure_pose_shift(reference, measured) self.assertEqual(result['method'], 'coco-pose-source-id-v1') self.assertEqual((result['n_matched'], result['n_keypoints']), (1, 17)) self.assertEqual(result['keypoint_mean_shift_px'], 2) self.assertEqual(gates.g3(measurement(result), CFG)['verdict'], 'pass-screen') self.assertEqual(len(result['unmatched_fp32']), 1) self.assertEqual(len(result['unmatched_quant']), 1) def test_topdown_identity_is_required_and_early_abort_counts_are_true(self): reference = self.topdown(detection(), detection()) for defect in ('missing', 'duplicate', 'bbox', 'protocol'): measured = copy.deepcopy(reference) second = measured['images'][0]['detections'][1] if defect == 'missing': del second['source_index'] elif defect == 'duplicate': second['source_index'] = 0 elif defect == 'bbox': second['bbox_xyxy'][0] += 1 else: measured['boxes_sha256'] = 'other' result = measure_pose_shift(reference, measured) self.assertEqual(result['status'], 'unverified') self.assertFalse(result['complete']) self.assertEqual(result['n_matched'], len(result['matches'])) if defect == 'bbox': self.assertEqual((result['n_matched'], result['n_keypoints'], result['n_fp32']), (1, 17, 2)) def test_topdown_uses_actual_affine_without_fabricated_letterbox(self): reference, measured = self.topdown(detection()), self.topdown(detection(shift=(3, 4))) for record in (reference, measured): image = record["images"][0] image.pop("scale"); image.pop("pad") image["transform"] = "topdown_affine" image["detections"][0]["affine"] = [[.5, 0., 7.], [0., .5, 11.]] result = measure_pose_shift(reference, measured) self.assertEqual(result["keypoint_mean_shift_px"], 5) self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "fail") measured["images"][0]["detections"][0]["affine"][0][2] += 1 self.assertEqual(measure_pose_shift(reference, measured)["status"], "unverified") measured["images"][0]["detections"][0]["affine"] = [[0., 0., 0.], [0., 0., 0.]] self.assertEqual(measure_pose_shift(reference, measured)["status"], "unverified") def test_euclidean_distance_and_replayable_evidence(self): result = measure_pose_shift(snapshot([detection()]), snapshot([detection(shift=(3, 4))])) self.assertEqual(result["status"], "measured") self.assertEqual(result["keypoint_mean_shift_px"], 5) self.assertEqual((result["n_matched"], result["n_keypoints"]), (1, 17)) pair = json.loads(json.dumps(result, allow_nan=False))["matches"][0] diff = np.asarray(pair["quant"]["keypoints"])[:, :2] - np.asarray(pair["fp32"]["keypoints"])[:, :2] np.testing.assert_allclose(np.linalg.norm(diff, axis=1), pair["distances_px"]) def test_metric_is_mean_of_lengths_not_max_rms_or_length_of_mean(self): a, b = detection(), detection() b["keypoints"][0][0] += 17 b["keypoints"][1][0] -= 17 result = measure_pose_shift(snapshot([a]), snapshot([b])) self.assertEqual(result["keypoint_mean_shift_px"], 2) self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "pass-screen") def test_order_and_score_changes_do_not_pair_different_people(self): a = snapshot([detection(), detection(x=200)]) b = snapshot([detection(x=200, shift=(0, 2), score=.1), detection(shift=(2, 0), score=.99)]) result = measure_pose_shift(a, b) self.assertEqual(result["keypoint_mean_shift_px"], 2) self.assertEqual([(p["fp32_index"], p["quant_index"]) for p in result["matches"]], [(0, 1), (1, 0)]) def test_matching_uses_boxes_not_nearest_keypoints(self): # Deliberately swapped skeletons: matching by keypoints would falsely give zero. a = snapshot([detection(), detection(x=200)]) b = snapshot([detection(shift=(200, 0)), detection(x=200, shift=(-200, 0))]) result = measure_pose_shift(a, b) self.assertEqual(result["keypoint_mean_shift_px"], 200) def test_one_to_one_and_deterministic_iou_tie(self): result = measure_pose_shift(snapshot([detection(), detection()]), snapshot([detection()])) self.assertEqual(result["n_matched"], 1) self.assertEqual(result["matches"][0]["fp32_index"], 0) self.assertEqual(result["unmatched_fp32"], [{"image_id": 1, "index": 1}]) def test_iou_matching_boundary(self): a, b = detection(), detection() a["bbox_xyxy"], b["bbox_xyxy"] = [0, 0, 3, 3], [1, 0, 4, 3] result = measure_pose_shift(snapshot([a]), snapshot([b])) self.assertEqual(result["matches"][0]["bbox_iou"], .5) b["bbox_xyxy"] = [1.001, 0, 4.001, 3] self.assertEqual(measure_pose_shift(snapshot([a]), snapshot([b]))["status"], "unverified") def test_no_cross_image_matching(self): result = measure_pose_shift(snapshot([detection()], []), snapshot([], [detection()])) self.assertEqual(result["status"], "unverified") self.assertEqual(result["n_matched"], 0) self.assertIsNone(result["keypoint_mean_shift_px"]) def test_unmatched_detections_are_counted_not_zero_filled(self): result = measure_pose_shift(snapshot([detection(), detection(x=200)]), snapshot([detection(shift=(3, 4))])) self.assertEqual(result["keypoint_mean_shift_px"], 5) self.assertEqual((result["n_fp32"], result["n_quant"], result["n_matched"]), (2, 1, 1)) self.assertEqual(gates.g3(measurement(result), CFG)["extra"]["keypoint_shift"]["n_unmatched_fp32"], 1) def test_joint_confidence_cannot_hide_shift(self): result = measure_pose_shift(snapshot([detection(confidence=0)]), snapshot([detection(shift=(3, 4), confidence=0)])) self.assertEqual(result["keypoint_mean_shift_px"], 5) self.assertEqual(result["n_keypoints"], 17) def test_empty_missing_and_unmatched_are_unverified(self): for a, b in ((None, snapshot([])), (snapshot([]), None), (snapshot([]), snapshot([])), (snapshot([detection()]), snapshot([])), (snapshot([detection()]), snapshot([detection(x=200)]))): with self.subTest(fp32=a, quant=b): result = measure_pose_shift(a, b) self.assertEqual(result["status"], "unverified") self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "unverified") def test_invalid_geometry_or_incomplete_evidence_never_measures(self): reference = snapshot([detection()]) bad = [] for value in (float("nan"), float("inf")): other = copy.deepcopy(reference) other["images"][0]["detections"][0]["keypoints"][0][0] = value bad.append(other) for field, value in (("schema", "older"), ("input_spec", "1x3x320x320"), ("t1_manifest_sha", "other"), ("image_ids", [2]), ("images", [])): bad.append({**reference, field: value}) other = copy.deepcopy(reference) other["images"][0]["pad"] = [0, 79] bad.append(other) other = copy.deepcopy(reference) other["images"][0]["detections"][0]["keypoints"].pop() bad.append(other) other = copy.deepcopy(reference) other["images"][0]["detections"][0]["category_id"] = 2 bad.append(other) for other in bad: with self.subTest(other=other): self.assertEqual(measure_pose_shift(reference, other)["status"], "unverified") class PoseGate(unittest.TestCase): def shifted(self, pixels): return measurement(measure_pose_shift(snapshot([detection()]), snapshot([detection(shift=(pixels, 0))]))) def test_two_pixel_inclusive_boundary(self): for pixels, verdict in ((0, "pass-screen"), (1.9999, "pass-screen"), (2, "pass-screen"), (2.0001, "fail")): with self.subTest(pixels=pixels): result = gates.g3(self.shifted(pixels), CFG) self.assertEqual(result["verdict"], verdict) self.assertEqual(result["extra"]["keypoint_shift"]["threshold_px"], 2) def test_both_ap_and_geometry_must_pass(self): for quant, pixels, verdict in ((76, 2, "pass-screen"), (75.999, 0, "fail"), (85, 3, "fail")): m = self.shifted(pixels) m["quant"] = quant self.assertEqual(gates.g3(m, CFG)["verdict"], verdict) def test_missing_geometry_cannot_pass_on_oks_alone_even_with_ap_failure(self): for quant in (80, 0): m = {"fp32": 80, "quant": quant, "source": "board", "tier": "t1"} self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified") m["extra"] = {"keypoint_mean_shift_px": 0} self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified") def test_bad_counts_values_and_status_only_are_unverified(self): bad = [] for value in (None, -1, True, "0", float("nan"), float("inf")): m = self.shifted(0) m["extra"]["keypoint_mean_shift_px"] = value bad.append(m) for key, value in (("n_matched", 0), ("n_keypoints", 0), ("n_fp32", 99), ("distance_sum_px", None), ("method", "manual-label"), ("keypoint_mean_shift_px", 1)): m = self.shifted(0) m["extra"]["keypoint_shift"][key] = value bad.append(m) m = self.shifted(0) m["extra"]["keypoint_shift"]["distance_sum_px"] = float("nan") bad.append(m) for m in bad: with self.subTest(measurement=m): self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified") def test_t1_se_does_not_widen_pixel_threshold(self): m = self.shifted(2.1) m["se"] = 100 self.assertEqual(gates.g3(m, {**CFG, "t1_screen_widen_se": 2})["verdict"], "fail") def test_source_na_and_other_task_thresholds_preserved(self): m = self.shifted(0) m["source"] = "sim" self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified") self.assertEqual(gates.g3(None, {**CFG, "verdict": "na"})["verdict"], "na") for metric in ("top1", "map_50_95", "mask_ap"): cfg = {k: v for k, v in CFG.items() if k != "extra"} cfg["metric"] = metric for quant, verdict in ((76, "pass-screen"), (75.999, "fail")): self.assertEqual(gates.g3({"fp32": 80, "quant": quant, "source": "board", "tier": "t1"}, cfg)["verdict"], verdict) def t1_fixture(image_size=(640, 480)): from PIL import Image points = detection()["keypoints"] annotation = {"id": 1, "image_id": 1, "category_id": 1, "bbox": [0, 0, 100, 100], "area": 10000, "iscrowd": 0, "num_keypoints": 17, "keypoints": [v for x, y, _ in points for v in (x, y, 2)]} category = {"id": 1, "name": "person", "keypoints": [str(i) for i in range(17)], "skeleton": []} ann = {"info": {}, "images": [{"id": 1, "width": image_size[0], "height": image_size[1]}], "annotations": [], "categories": [], "keypoints": {"annotations": [annotation], "categories": [category]}} return SimpleNamespace(manifest={"sha256": "test-t1"}, items=[{"id": 1}], image=lambda _: Image.new("RGB", image_size), tar=SimpleNamespace(extractfile=lambda _: io.BytesIO(json.dumps(ann).encode()))) def head_runner(image_size, input_size, shift=(0, 0), points=None, score=.9): scale = input_size / max(image_size) pad = [(input_size - round(s * scale)) // 2 for s in image_size] head = np.zeros((1, 56, 1), dtype=np.float32) head[0, :4, 0] = [50 * scale + pad[0], 50 * scale + pad[1], 100 * scale, 100 * scale] head[0, 4, 0] = score keypoints = np.asarray(points if points is not None else detection()["keypoints"], dtype=float) keypoints[:, :2] = (keypoints[:, :2] + np.asarray(shift)) * scale + pad head[0, 5:, 0] = keypoints.reshape(-1) return lambda _: [head.copy()] class PoseIntegration(unittest.TestCase): def test_real_decode_undoes_resize_and_asymmetric_letterbox(self): t1 = t1_fixture((5, 3)) points = [[1, 1, .8] for _ in range(17)] a, b = pose_snapshot(t1, "1x3x8x8"), pose_snapshot(t1, "1x3x8x8") predict(head_runner((5, 3), 8, points=points), t1, "1x3x8x8", "pose", a) predict(head_runner((5, 3), 8, shift=(0, 2), points=points), t1, "1x3x8x8", "pose", b) self.assertEqual(a["images"][0]["pad"], [0, 1]) self.assertAlmostEqual(a["images"][0]["detections"][0]["keypoints"][0][1], 1, places=6) self.assertAlmostEqual(measure_pose_shift(a, b)["keypoint_mean_shift_px"], 2, places=6) def test_unclipped_evidence_does_not_hide_edge_displacement(self): t1 = t1_fixture() points = [[-5, 20, .8] for _ in range(17)] a, b = pose_snapshot(t1, "1x3x640x640"), pose_snapshot(t1, "1x3x640x640") pa = predict(head_runner((640, 480), 640, points=points), t1, "1x3x640x640", "pose", a) pb = predict(head_runner((640, 480), 640, shift=(-3, 0), points=points), t1, "1x3x640x640", "pose", b) self.assertEqual(pa[0]["keypoints"], pb[0]["keypoints"]) # unchanged AP clipping self.assertEqual(measure_pose_shift(a, b)["keypoint_mean_shift_px"], 3) def test_real_cocoeval_and_json_baseline_quant_wiring(self): t1 = t1_fixture() fp = {"task": "pose", "model": "fixture"} run_fp32(head_runner((640, 480), 640), t1, "1x3x640x640", fp) self.assertAlmostEqual(fp["metrics"]["oks_ap"], 100) baseline = json.loads(json.dumps(fp, allow_nan=False)) quant = {"task": "pose", "model": "fixture"} run_quant(head_runner((640, 480), 640, shift=(2, 0)), t1, "1x3x640x640", baseline, quant) self.assertEqual(fp, baseline) # quant cannot mutate the saved reference self.assertEqual(quant["G3"]["extra"]["keypoint_mean_shift_px"], 2) self.assertEqual(quant["pose_shift_evidence"]["matches"][0]["distances_px"], [2] * 17) self.assertNotIn("matches", quant["G3"]["extra"]["keypoint_shift"]) self.assertGreaterEqual(quant["metrics"]["oks_ap"], 95) quant["G3"]["source"] = "board" # gate source fixture; runner here is synthetic self.assertEqual(gates.g3(quant["G3"], CFG)["verdict"], "pass-screen") json.dumps(quant, allow_nan=False) def test_legacy_baseline_keeps_ap_but_shift_is_unverified(self): t1 = t1_fixture() out = {"task": "pose", "model": "fixture"} run_quant(head_runner((640, 480), 640), t1, "1x3x640x640", {"model": "fixture", "metrics": {"oks_ap": 100}}, out) self.assertAlmostEqual(out["metrics"]["oks_ap"], 100) out["G3"]["source"] = "board" self.assertEqual(gates.g3(out["G3"], CFG)["verdict"], "unverified") def test_no_detections_produces_complete_empty_image_evidence(self): t1 = t1_fixture() evidence = pose_snapshot(t1, "1x3x640x640") self.assertEqual(predict(head_runner((640, 480), 640, score=0), t1, "1x3x640x640", "pose", evidence), []) self.assertEqual(evidence["images"][0]["detections"], []) self.assertEqual(measure_pose_shift(evidence, evidence)["status"], "unverified") if __name__ == "__main__": unittest.main()