data-gen-bpu/scripts/test_pose_gate.py
bench ff3c8926df Revert the pose evaluator edit from 8a1ee24: running batches froze its checksum
harness/eval/coco_extra.py is frozen per batch by task_contract (only listed equivalent rewrites may replace it).
The 8a1ee24 edit made every pose stage of the pilot batches fail the contract check (Qwen rtmpose.s never got a
formal stage). The keypoint shift gate it fed was removed in e0c224f, so the shift reaching 'unverified' no longer
decides anything; back to the approved version 505d37.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-24 00:15:37 +08:00

350 lines
19 KiB
Python

"""Run in bench-venv: python -m unittest scripts.test_pose_gate scripts.test_coco_extra scripts.test_accuracy_policy.
Geometry fixtures use actual decoding/letterbox and real COCOeval; no mocked AP.
These are CPU contract tests, not evidence of a board model passing its gate.
"""
import copy
import io
import json
from types import SimpleNamespace
import unittest
import numpy as np
from harness import gates
from harness.eval.coco_extra import (
measure_pose_shift, pose_snapshot, pose_shift_summary, predict, run_fp32, run_quant,
)
def detection(x=0, shift=(0, 0), confidence=.9, score=.9):
points = [[x + 10 + i * 2 + shift[0], 10 + i + shift[1], confidence] for i in range(17)]
return {"category_id": 1, "bbox_xyxy": [x, 0, x + 100, 100], "score": score, "keypoints": points}
def snapshot(*image_detections):
t1 = SimpleNamespace(manifest={"sha256": "test-t1"}, items=[{"id": i + 1} for i in range(len(image_detections))])
result = pose_snapshot(t1, "1x3x640x640")
result["images"] = [
{"image_id": i + 1, "image_size": [640, 480], "scale": 1.0, "pad": [0, 80], "detections": ds}
for i, ds in enumerate(image_detections)
]
return result
def measurement(shift, quant=80):
return {"metric": "oks_ap", "fp32": 80, "quant": quant, "tier": "t1", "source": "board", "n": 1,
"extra": {"keypoint_mean_shift_px": shift["keypoint_mean_shift_px"], "keypoint_shift": pose_shift_summary(shift)}}
CFG = {"metric": "oks_ap", "kind": "rel_pct", "tol": 5, "require_source": "board",
"t1_screen_widen_se": 0, "extra": {"keypoint_mean_shift_px_max": 2}}
class RegistryPoseGate(unittest.TestCase):
def test_keypoint_shift_is_recorded_not_gated(self):
"""10-22: SimpleBaseline AP 68.34 vs FP32 68.22 failed on a 3.3 px mean shift of all 17 joints in raw image
pixels (invisible joints included, not scale-normalized). OKS-AP already scores the task; new runs drop it."""
import yaml
from pathlib import Path
tasks = yaml.safe_load((Path(__file__).resolve().parents[1] / "registry/gates.yaml").read_text())["G3"]["tasks"]
self.assertNotIn("keypoint_mean_shift_px_max", tasks["pose"].get("extra") or {})
m = {"metric": "oks_ap", "fp32": 68.2, "quant": 68.3, "tier": "t1", "source": "board", "n": 1,
"extra": {"keypoint_mean_shift_px": 3.3}}
self.assertEqual(gates.g3(m, {**tasks["pose"], "require_source": "board"})["verdict"], "pass-screen")
class PoseGeometry(unittest.TestCase):
@staticmethod
def topdown(*detections):
value = snapshot(list(detections))
value.update(boxes_file='fixed.json', boxes_sha256='fixed-hash', pose_protocol='fixed-topdown', official_model='hrnet')
image = value['images'][0]
image['transform'] = 'topdown_affine'
for index, item in enumerate(image['detections']):
item.setdefault('source_index', index)
item.setdefault('affine', [[.5, 0., 7. + index], [0., .5, 11.]])
return value
def test_topdown_nms_difference_never_pairs_different_crop_ids(self):
reference = self.topdown(detection(), detection())
measured = self.topdown(detection(shift=(2, 0)), detection())
measured['images'][0]['detections'][1]['source_index'] = 2
measured['images'][0]['detections'][1]['affine'][0][2] += 1
result = measure_pose_shift(reference, measured)
self.assertEqual(result['method'], 'coco-pose-source-id-v1')
self.assertEqual((result['n_matched'], result['n_keypoints']), (1, 17))
self.assertEqual(result['keypoint_mean_shift_px'], 2)
self.assertEqual(gates.g3(measurement(result), CFG)['verdict'], 'pass-screen')
self.assertEqual(len(result['unmatched_fp32']), 1)
self.assertEqual(len(result['unmatched_quant']), 1)
def test_topdown_identity_is_required_and_early_abort_counts_are_true(self):
reference = self.topdown(detection(), detection())
for defect in ('missing', 'duplicate', 'bbox', 'protocol'):
measured = copy.deepcopy(reference)
second = measured['images'][0]['detections'][1]
if defect == 'missing': del second['source_index']
elif defect == 'duplicate': second['source_index'] = 0
elif defect == 'bbox': second['bbox_xyxy'][0] += 1
else: measured['boxes_sha256'] = 'other'
result = measure_pose_shift(reference, measured)
self.assertEqual(result['status'], 'unverified')
self.assertFalse(result['complete'])
self.assertEqual(result['n_matched'], len(result['matches']))
if defect == 'bbox':
self.assertEqual((result['n_matched'], result['n_keypoints'], result['n_fp32']), (1, 17, 2))
def test_topdown_uses_actual_affine_without_fabricated_letterbox(self):
reference, measured = self.topdown(detection()), self.topdown(detection(shift=(3, 4)))
for record in (reference, measured):
image = record["images"][0]
image.pop("scale"); image.pop("pad")
image["transform"] = "topdown_affine"
image["detections"][0]["affine"] = [[.5, 0., 7.], [0., .5, 11.]]
result = measure_pose_shift(reference, measured)
self.assertEqual(result["keypoint_mean_shift_px"], 5)
self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "fail")
measured["images"][0]["detections"][0]["affine"][0][2] += 1
self.assertEqual(measure_pose_shift(reference, measured)["status"], "unverified")
measured["images"][0]["detections"][0]["affine"] = [[0., 0., 0.], [0., 0., 0.]]
self.assertEqual(measure_pose_shift(reference, measured)["status"], "unverified")
def test_euclidean_distance_and_replayable_evidence(self):
result = measure_pose_shift(snapshot([detection()]), snapshot([detection(shift=(3, 4))]))
self.assertEqual(result["status"], "measured")
self.assertEqual(result["keypoint_mean_shift_px"], 5)
self.assertEqual((result["n_matched"], result["n_keypoints"]), (1, 17))
pair = json.loads(json.dumps(result, allow_nan=False))["matches"][0]
diff = np.asarray(pair["quant"]["keypoints"])[:, :2] - np.asarray(pair["fp32"]["keypoints"])[:, :2]
np.testing.assert_allclose(np.linalg.norm(diff, axis=1), pair["distances_px"])
def test_metric_is_mean_of_lengths_not_max_rms_or_length_of_mean(self):
a, b = detection(), detection()
b["keypoints"][0][0] += 17
b["keypoints"][1][0] -= 17
result = measure_pose_shift(snapshot([a]), snapshot([b]))
self.assertEqual(result["keypoint_mean_shift_px"], 2)
self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "pass-screen")
def test_order_and_score_changes_do_not_pair_different_people(self):
a = snapshot([detection(), detection(x=200)])
b = snapshot([detection(x=200, shift=(0, 2), score=.1), detection(shift=(2, 0), score=.99)])
result = measure_pose_shift(a, b)
self.assertEqual(result["keypoint_mean_shift_px"], 2)
self.assertEqual([(p["fp32_index"], p["quant_index"]) for p in result["matches"]], [(0, 1), (1, 0)])
def test_matching_uses_boxes_not_nearest_keypoints(self):
# Deliberately swapped skeletons: matching by keypoints would falsely give zero.
a = snapshot([detection(), detection(x=200)])
b = snapshot([detection(shift=(200, 0)), detection(x=200, shift=(-200, 0))])
result = measure_pose_shift(a, b)
self.assertEqual(result["keypoint_mean_shift_px"], 200)
def test_one_to_one_and_deterministic_iou_tie(self):
result = measure_pose_shift(snapshot([detection(), detection()]), snapshot([detection()]))
self.assertEqual(result["n_matched"], 1)
self.assertEqual(result["matches"][0]["fp32_index"], 0)
self.assertEqual(result["unmatched_fp32"], [{"image_id": 1, "index": 1}])
def test_iou_matching_boundary(self):
a, b = detection(), detection()
a["bbox_xyxy"], b["bbox_xyxy"] = [0, 0, 3, 3], [1, 0, 4, 3]
result = measure_pose_shift(snapshot([a]), snapshot([b]))
self.assertEqual(result["matches"][0]["bbox_iou"], .5)
b["bbox_xyxy"] = [1.001, 0, 4.001, 3]
self.assertEqual(measure_pose_shift(snapshot([a]), snapshot([b]))["status"], "unverified")
def test_no_cross_image_matching(self):
result = measure_pose_shift(snapshot([detection()], []), snapshot([], [detection()]))
self.assertEqual(result["status"], "unverified")
self.assertEqual(result["n_matched"], 0)
self.assertIsNone(result["keypoint_mean_shift_px"])
def test_unmatched_detections_are_counted_not_zero_filled(self):
result = measure_pose_shift(snapshot([detection(), detection(x=200)]),
snapshot([detection(shift=(3, 4))]))
self.assertEqual(result["keypoint_mean_shift_px"], 5)
self.assertEqual((result["n_fp32"], result["n_quant"], result["n_matched"]), (2, 1, 1))
self.assertEqual(gates.g3(measurement(result), CFG)["extra"]["keypoint_shift"]["n_unmatched_fp32"], 1)
def test_joint_confidence_cannot_hide_shift(self):
result = measure_pose_shift(snapshot([detection(confidence=0)]),
snapshot([detection(shift=(3, 4), confidence=0)]))
self.assertEqual(result["keypoint_mean_shift_px"], 5)
self.assertEqual(result["n_keypoints"], 17)
def test_empty_missing_and_unmatched_are_unverified(self):
for a, b in ((None, snapshot([])), (snapshot([]), None), (snapshot([]), snapshot([])),
(snapshot([detection()]), snapshot([])),
(snapshot([detection()]), snapshot([detection(x=200)]))):
with self.subTest(fp32=a, quant=b):
result = measure_pose_shift(a, b)
self.assertEqual(result["status"], "unverified")
self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "unverified")
def test_invalid_geometry_or_incomplete_evidence_never_measures(self):
reference = snapshot([detection()])
bad = []
for value in (float("nan"), float("inf")):
other = copy.deepcopy(reference)
other["images"][0]["detections"][0]["keypoints"][0][0] = value
bad.append(other)
for field, value in (("schema", "older"), ("input_spec", "1x3x320x320"),
("t1_manifest_sha", "other"), ("image_ids", [2]), ("images", [])):
bad.append({**reference, field: value})
other = copy.deepcopy(reference)
other["images"][0]["pad"] = [0, 79]
bad.append(other)
other = copy.deepcopy(reference)
other["images"][0]["detections"][0]["keypoints"].pop()
bad.append(other)
other = copy.deepcopy(reference)
other["images"][0]["detections"][0]["category_id"] = 2
bad.append(other)
for other in bad:
with self.subTest(other=other):
self.assertEqual(measure_pose_shift(reference, other)["status"], "unverified")
class PoseGate(unittest.TestCase):
def shifted(self, pixels):
return measurement(measure_pose_shift(snapshot([detection()]), snapshot([detection(shift=(pixels, 0))])))
def test_two_pixel_inclusive_boundary(self):
for pixels, verdict in ((0, "pass-screen"), (1.9999, "pass-screen"), (2, "pass-screen"), (2.0001, "fail")):
with self.subTest(pixels=pixels):
result = gates.g3(self.shifted(pixels), CFG)
self.assertEqual(result["verdict"], verdict)
self.assertEqual(result["extra"]["keypoint_shift"]["threshold_px"], 2)
def test_both_ap_and_geometry_must_pass(self):
for quant, pixels, verdict in ((76, 2, "pass-screen"), (75.999, 0, "fail"), (85, 3, "fail")):
m = self.shifted(pixels)
m["quant"] = quant
self.assertEqual(gates.g3(m, CFG)["verdict"], verdict)
def test_missing_geometry_cannot_pass_on_oks_alone_even_with_ap_failure(self):
for quant in (80, 0):
m = {"fp32": 80, "quant": quant, "source": "board", "tier": "t1"}
self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified")
m["extra"] = {"keypoint_mean_shift_px": 0}
self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified")
def test_bad_counts_values_and_status_only_are_unverified(self):
bad = []
for value in (None, -1, True, "0", float("nan"), float("inf")):
m = self.shifted(0)
m["extra"]["keypoint_mean_shift_px"] = value
bad.append(m)
for key, value in (("n_matched", 0), ("n_keypoints", 0), ("n_fp32", 99), ("distance_sum_px", None),
("method", "manual-label"), ("keypoint_mean_shift_px", 1)):
m = self.shifted(0)
m["extra"]["keypoint_shift"][key] = value
bad.append(m)
m = self.shifted(0)
m["extra"]["keypoint_shift"]["distance_sum_px"] = float("nan")
bad.append(m)
for m in bad:
with self.subTest(measurement=m):
self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified")
def test_t1_se_does_not_widen_pixel_threshold(self):
m = self.shifted(2.1)
m["se"] = 100
self.assertEqual(gates.g3(m, {**CFG, "t1_screen_widen_se": 2})["verdict"], "fail")
def test_source_na_and_other_task_thresholds_preserved(self):
m = self.shifted(0)
m["source"] = "sim"
self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified")
self.assertEqual(gates.g3(None, {**CFG, "verdict": "na"})["verdict"], "na")
for metric in ("top1", "map_50_95", "mask_ap"):
cfg = {k: v for k, v in CFG.items() if k != "extra"}
cfg["metric"] = metric
for quant, verdict in ((76, "pass-screen"), (75.999, "fail")):
self.assertEqual(gates.g3({"fp32": 80, "quant": quant, "source": "board", "tier": "t1"}, cfg)["verdict"], verdict)
def t1_fixture(image_size=(640, 480)):
from PIL import Image
points = detection()["keypoints"]
annotation = {"id": 1, "image_id": 1, "category_id": 1, "bbox": [0, 0, 100, 100], "area": 10000,
"iscrowd": 0, "num_keypoints": 17, "keypoints": [v for x, y, _ in points for v in (x, y, 2)]}
category = {"id": 1, "name": "person", "keypoints": [str(i) for i in range(17)], "skeleton": []}
ann = {"info": {}, "images": [{"id": 1, "width": image_size[0], "height": image_size[1]}],
"annotations": [], "categories": [], "keypoints": {"annotations": [annotation], "categories": [category]}}
return SimpleNamespace(manifest={"sha256": "test-t1"}, items=[{"id": 1}],
image=lambda _: Image.new("RGB", image_size),
tar=SimpleNamespace(extractfile=lambda _: io.BytesIO(json.dumps(ann).encode())))
def head_runner(image_size, input_size, shift=(0, 0), points=None, score=.9):
scale = input_size / max(image_size)
pad = [(input_size - round(s * scale)) // 2 for s in image_size]
head = np.zeros((1, 56, 1), dtype=np.float32)
head[0, :4, 0] = [50 * scale + pad[0], 50 * scale + pad[1], 100 * scale, 100 * scale]
head[0, 4, 0] = score
keypoints = np.asarray(points if points is not None else detection()["keypoints"], dtype=float)
keypoints[:, :2] = (keypoints[:, :2] + np.asarray(shift)) * scale + pad
head[0, 5:, 0] = keypoints.reshape(-1)
return lambda _: [head.copy()]
class PoseIntegration(unittest.TestCase):
def test_real_decode_undoes_resize_and_asymmetric_letterbox(self):
t1 = t1_fixture((5, 3))
points = [[1, 1, .8] for _ in range(17)]
a, b = pose_snapshot(t1, "1x3x8x8"), pose_snapshot(t1, "1x3x8x8")
predict(head_runner((5, 3), 8, points=points), t1, "1x3x8x8", "pose", a)
predict(head_runner((5, 3), 8, shift=(0, 2), points=points), t1, "1x3x8x8", "pose", b)
self.assertEqual(a["images"][0]["pad"], [0, 1])
self.assertAlmostEqual(a["images"][0]["detections"][0]["keypoints"][0][1], 1, places=6)
self.assertAlmostEqual(measure_pose_shift(a, b)["keypoint_mean_shift_px"], 2, places=6)
def test_unclipped_evidence_does_not_hide_edge_displacement(self):
t1 = t1_fixture()
points = [[-5, 20, .8] for _ in range(17)]
a, b = pose_snapshot(t1, "1x3x640x640"), pose_snapshot(t1, "1x3x640x640")
pa = predict(head_runner((640, 480), 640, points=points), t1, "1x3x640x640", "pose", a)
pb = predict(head_runner((640, 480), 640, shift=(-3, 0), points=points), t1, "1x3x640x640", "pose", b)
self.assertEqual(pa[0]["keypoints"], pb[0]["keypoints"]) # unchanged AP clipping
self.assertEqual(measure_pose_shift(a, b)["keypoint_mean_shift_px"], 3)
def test_real_cocoeval_and_json_baseline_quant_wiring(self):
t1 = t1_fixture()
fp = {"task": "pose", "model": "fixture"}
run_fp32(head_runner((640, 480), 640), t1, "1x3x640x640", fp)
self.assertAlmostEqual(fp["metrics"]["oks_ap"], 100)
baseline = json.loads(json.dumps(fp, allow_nan=False))
quant = {"task": "pose", "model": "fixture"}
run_quant(head_runner((640, 480), 640, shift=(2, 0)), t1, "1x3x640x640", baseline, quant)
self.assertEqual(fp, baseline) # quant cannot mutate the saved reference
self.assertEqual(quant["G3"]["extra"]["keypoint_mean_shift_px"], 2)
self.assertEqual(quant["pose_shift_evidence"]["matches"][0]["distances_px"], [2] * 17)
self.assertNotIn("matches", quant["G3"]["extra"]["keypoint_shift"])
self.assertGreaterEqual(quant["metrics"]["oks_ap"], 95)
quant["G3"]["source"] = "board" # gate source fixture; runner here is synthetic
self.assertEqual(gates.g3(quant["G3"], CFG)["verdict"], "pass-screen")
json.dumps(quant, allow_nan=False)
def test_legacy_baseline_keeps_ap_but_shift_is_unverified(self):
t1 = t1_fixture()
out = {"task": "pose", "model": "fixture"}
run_quant(head_runner((640, 480), 640), t1, "1x3x640x640",
{"model": "fixture", "metrics": {"oks_ap": 100}}, out)
self.assertAlmostEqual(out["metrics"]["oks_ap"], 100)
out["G3"]["source"] = "board"
self.assertEqual(gates.g3(out["G3"], CFG)["verdict"], "unverified")
def test_no_detections_produces_complete_empty_image_evidence(self):
t1 = t1_fixture()
evidence = pose_snapshot(t1, "1x3x640x640")
self.assertEqual(predict(head_runner((640, 480), 640, score=0), t1, "1x3x640x640", "pose", evidence), [])
self.assertEqual(evidence["images"][0]["detections"], [])
self.assertEqual(measure_pose_shift(evidence, evidence)["status"], "unverified")
if __name__ == "__main__":
unittest.main()