data-gen-bpu/harness/eval/coco_extra.py
bench ff3c8926df Revert the pose evaluator edit from 8a1ee24: running batches froze its checksum
harness/eval/coco_extra.py is frozen per batch by task_contract (only listed equivalent rewrites may replace it).
The 8a1ee24 edit made every pose stage of the pilot batches fail the contract check (Qwen rtmpose.s never got a
formal stage). The keypoint shift gate it fed was removed in e0c224f, so the shift reaching 'unverified' no longer
decides anything; back to the approved version 505d37.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-24 00:15:37 +08:00

311 lines
17 KiB
Python

"""COCO mask AP / keypoint OKS AP for the existing Ultralytics exporters.
Both FP32 and board outputs use exactly the same letterbox, decoding and COCOeval.
FastSAM is deliberately excluded: its class-agnostic masks are not COCO classes.
"""
from __future__ import annotations
import json
import numpy as np
from .common import groups, letterbox, run_batched
from .det import CONF, IOU, MAX_DET, _nms
METRICS = {"iseg": "mask_ap", "pose": "oks_ap"}
POSE_SHIFT_SCHEMA = "coco-pose-bbox-iou-v1"
POSE_MATCH_IOU = 0.5
def decode(outputs, task):
nc, extra = (80, 32) if task == "iseg" else (1, 51)
candidates = [o for o in outputs if o.ndim == 3 and o.shape[1] == 4 + nc + extra]
if len(candidates) != 1:
raise ValueError(f"{task}: expected COCO YOLO head [1,{4+nc+extra},N], got {[o.shape for o in outputs]}")
p = candidates[0][0].T
labels = p[:, 4:4 + nc].argmax(1)
scores = p[:, 4:4 + nc].max(1)
selected = scores > CONF
p, labels, scores = p[selected], labels[selected], scores[selected]
boxes = np.concatenate((p[:, :2] - p[:, 2:4] / 2, p[:, :2] + p[:, 2:4] / 2), axis=1)
keep = (np.empty(0, dtype=np.intp) if task == "pose" and not len(scores)
else _nms(boxes + labels[:, None] * 4096, scores, IOU)[:MAX_DET])
proto = None
if task == "iseg":
protos = [o for o in outputs if o.ndim == 4 and o.shape[:2] == (1, extra)]
if len(protos) != 1:
raise ValueError("segmentation requires exactly one 32-channel prototype output")
proto = protos[0][0]
return boxes[keep], scores[keep], labels[keep], p[keep, 4 + nc:], proto
def masks_to_original(proto, coeff, boxes, size, image_size, scale, pad):
"""Undo our actual integer letterbox offsets, including asymmetric rounding."""
import torch
from torch.nn.functional import interpolate
from ultralytics.utils.ops import crop_mask
w, h = image_size
px, py = pad
nw, nh = round(w * scale), round(h * scale)
# Bound peak memory for 300 detections at original image resolution.
for i in range(0, len(coeff), 16):
logits = torch.from_numpy(coeff[i:i + 16]) @ torch.from_numpy(proto).reshape(proto.shape[0], -1)
logits = logits.reshape(-1, *proto.shape[1:])
logits = interpolate(logits[:, None], (size, size), mode="bilinear", align_corners=False)
logits = logits[:, :, py:py + nh, px:px + nw]
masks = interpolate(logits, (h, w), mode="bilinear", align_corners=False)[:, 0] > 0
yield from crop_mask(masks.byte(), torch.from_numpy(boxes[i:i + 16]).float()).numpy()
def predict(runner, t1, input_spec, task, pose_evidence=None, model=""):
if model:
from ..adapters import get
custom = getattr(get(model), "predict", None)
if custom is not None:
return custom(runner, t1, input_spec, pose_evidence=pose_evidence)
if task == "iseg":
from pycocotools import mask as mask_utils
size = int(input_spec.split("x")[-1])
predictions = []
for items in groups(runner, t1.items, 16):
images = [t1.image(it) for it in items]
pre = [letterbox(im, size) for im in images]
outputs = run_batched(runner, [v[0] for v in pre])
if task == "pose" and len(outputs) != len(items):
raise ValueError("pose: missing image outputs")
for it, im, (_, scale, pad), outs in zip(items, images, pre, outputs):
boxes, scores, labels, extra, proto = decode(outs, task)
if task == "pose" and pose_evidence is not None:
# Keep unclipped original-image geometry: clipping at image edges
# can hide a real shift. These records never enter COCOeval.
raw_boxes = (boxes.astype(np.float64) - np.array([*pad, *pad])) / scale
raw_points = extra.astype(np.float64).reshape(-1, 17, 3).copy()
raw_points[:, :, :2] = (raw_points[:, :, :2] - np.array(pad)) / scale
pose_evidence["images"].append({
"image_id": it["id"], "image_size": list(im.size), "scale": scale, "pad": list(pad),
"detections": [{"category_id": 1, "bbox_xyxy": box.tolist(), "score": float(score),
"keypoints": points.tolist()}
for box, score, points in zip(raw_boxes, scores, raw_points)],
})
boxes = (boxes - np.array([*pad, *pad], dtype=np.float32)) / scale
boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, im.width)
boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, im.height)
if task == "iseg":
masks = masks_to_original(proto, extra, boxes, size, im.size, scale, pad)
for mask, score, label in zip(masks, scores, labels):
rle = mask_utils.encode(np.asfortranarray(mask.astype(np.uint8)))
rle["counts"] = rle["counts"].decode("ascii")
predictions.append({"image_id": it["id"], "category_id": t1.manifest["cat_map"][str(int(label))],
"score": float(score), "segmentation": rle})
else:
points = extra.reshape(-1, 17, 3).copy()
points[:, :, :2] = (points[:, :, :2] - np.array(pad)) / scale
points[:, :, 0] = points[:, :, 0].clip(0, im.width)
points[:, :, 1] = points[:, :, 1].clip(0, im.height)
for points_one, score in zip(points, scores):
predictions.append({"image_id": it["id"], "category_id": 1, "score": float(score),
"keypoints": points_one.reshape(-1).tolist()})
return predictions
def pose_snapshot(t1, input_spec):
"""Stored in the existing metrics.json/eval.json; no separate cache or runner."""
return {"schema": POSE_SHIFT_SCHEMA, "input_spec": input_spec,
"t1_manifest_sha": t1.manifest.get("sha256"),
"image_ids": [it["id"] for it in t1.items], "images": []}
def measure_pose_shift(fp32, quant):
"""Mean L2 distance in original pixels over matched post-NMS poses, all 17 joints.
Fixed top-down crops match by source ID; other poses use box IoU >= 0.5.
Match only within an image/category, greedily by descending box IoU otherwise;
ties use the saved post-NMS indices. Each detection is used at most once.
Neither keypoint positions nor keypoint confidence select pairs/joints.
Unmatched detections do not contribute zeros; their indices and coverage are
retained. OKS AP separately judges missed/extra people. No pairs => unverified.
"""
result = {"status": "unverified", "method": POSE_SHIFT_SCHEMA,
"coordinate_space": "original_image_pixels_unclipped", "match_iou_min": POSE_MATCH_IOU,
"joint_policy": "all_17_no_confidence_filter", "keypoint_mean_shift_px": None,
"distance_sum_px": None,
"n_fp32": 0, "n_quant": 0, "n_matched": 0, "n_keypoints": 0,
"matches": [], "unmatched_fp32": [], "unmatched_quant": [], "complete": False}
if not isinstance(fp32, dict) or not isinstance(quant, dict):
return {**result, "reason": "pose reference or measurement missing; rerun FP32 to record geometry"}
for key in ("schema", "input_spec", "t1_manifest_sha", "image_ids"):
if key not in fp32 or key not in quant or fp32[key] != quant[key]:
return {**result, "reason": f"pose evidence {key} mismatch or missing"}
if fp32["schema"] != POSE_SHIFT_SCHEMA:
return {**result, "reason": "unsupported pose evidence schema"}
if not isinstance(fp32["t1_manifest_sha"], str) or not fp32["t1_manifest_sha"]:
return {**result, "reason": "pose evidence T1 identity missing"}
def checked_images(snapshot):
ids = snapshot["image_ids"]
if not isinstance(ids, list) or len(set(ids)) != len(ids):
raise ValueError("duplicate or invalid image IDs")
images = snapshot["images"]
if [im["image_id"] for im in images] != ids:
raise ValueError("missing, reordered or extra image evidence")
for im in images:
source_ids = set()
size = np.asarray(im["image_size"], dtype=float)
if size.shape != (2,) or not np.isfinite(size).all() or (size <= 0).any():
raise ValueError("invalid image geometry")
if im.get("transform") != "topdown_affine":
pad = np.asarray(im["pad"], dtype=float)
if (pad.shape != (2,) or not np.isfinite(pad).all()
or not np.isfinite(im["scale"]) or im["scale"] <= 0):
raise ValueError("invalid letterbox geometry")
for d in im["detections"]:
box, points = np.asarray(d["bbox_xyxy"], dtype=float), np.asarray(d["keypoints"], dtype=float)
if (box.shape != (4,) or points.shape != (17, 3) or not np.isfinite(box).all()
or not np.isfinite(points).all() or not np.isfinite(d["score"])
or (box[2:] <= box[:2]).any() or d["category_id"] != 1):
raise ValueError("invalid/non-finite COCO pose detection")
if im.get("transform") == "topdown_affine":
source = d.get("source_index")
if type(source) is not int or source < 0 or source in source_ids:
raise ValueError("missing, invalid or duplicate crop source_index")
source_ids.add(source)
affine = np.asarray(d["affine"], dtype=float)
if (affine.shape != (2, 3) or not np.isfinite(affine).all()
or abs(np.linalg.det(affine[:, :2])) < 1e-12):
raise ValueError("invalid measured top-down affine geometry")
return images
try:
refs, measured = checked_images(fp32), checked_images(quant)
topdown = any(im.get("transform") == "topdown_affine" for im in refs)
if topdown:
if not all(im.get("transform") == "topdown_affine" for im in refs):
raise ValueError("mixed pose transforms")
for key in ("boxes_file", "boxes_sha256", "pose_protocol", "official_model"):
if not fp32.get(key) or fp32[key] != quant.get(key):
raise ValueError(f"fixed crop {key} mismatch or missing")
result.update(method="coco-pose-source-id-v1", match_iou_min=None)
for ref, cur in zip(refs, measured):
if ref.get("transform") != cur.get("transform"):
raise ValueError("FP32/quant image transform mismatch")
keys = ("image_size",) if ref.get("transform") == "topdown_affine" else ("image_size", "scale", "pad")
if any(ref[k] != cur[k] for k in keys):
raise ValueError("FP32/quant image geometry mismatch")
except (KeyError, TypeError, ValueError, OverflowError) as e:
return {**result, "reason": str(e)}
result["n_fp32"] = sum(len(im["detections"]) for im in refs)
result["n_quant"] = sum(len(im["detections"]) for im in measured)
for ref, cur in zip(refs, measured):
a, b = ref["detections"], cur["detections"]
candidates = []
if topdown:
indices = {d["source_index"]: j for j, d in enumerate(b)}
candidates = [(-1.0, i, indices[d["source_index"]])
for i, d in enumerate(a) if d["source_index"] in indices]
elif a and b:
ab = np.asarray([d["bbox_xyxy"] for d in a], dtype=np.float64)
bb = np.asarray([d["bbox_xyxy"] for d in b], dtype=np.float64)
wh = np.maximum(0, np.minimum(ab[:, None, 2:], bb[None, :, 2:])
- np.maximum(ab[:, None, :2], bb[None, :, :2]))
inter = wh.prod(axis=2)
union = (ab[:, 2:] - ab[:, :2]).prod(axis=1)[:, None] + (bb[:, 2:] - bb[:, :2]).prod(axis=1) - inter
ious = inter / union
if not np.isfinite(ious).all():
return {**result, "reason": "non-finite box IoU"}
candidates = sorted((-float(ious[i, j]), int(i), int(j))
for i, j in np.argwhere(ious >= POSE_MATCH_IOU))
used_a, used_b = set(), set()
for neg_iou, i, j in candidates:
if i in used_a or j in used_b:
continue
if topdown and any(a[i][k] != b[j][k] for k in ("bbox_xyxy", "affine")):
return {**result, "reason": "FP32/quant matched crop geometry mismatch"}
used_a.add(i)
used_b.add(j)
delta = (np.asarray(b[j]["keypoints"], dtype=np.float64)[:, :2]
- np.asarray(a[i]["keypoints"], dtype=np.float64)[:, :2])
distances = np.hypot(delta[:, 0], delta[:, 1])
if not np.isfinite(distances).all():
return {**result, "reason": "non-finite keypoint distance"}
result["matches"].append({"image_id": ref["image_id"], "fp32_index": i, "quant_index": j,
"bbox_iou": -neg_iou, "fp32": a[i], "quant": b[j],
"distances_px": distances.tolist()})
result["n_matched"] = len(result["matches"])
result["n_keypoints"] = 17 * result["n_matched"]
for key, dets, used in (("unmatched_fp32", a, used_a), ("unmatched_quant", b, used_b)):
result[key].extend({"image_id": ref["image_id"], "index": i} for i in range(len(dets)) if i not in used)
result["complete"] = True
if not result["n_matched"]:
return {**result, "reason": "no matched FP32/quant poses"}
total = float(np.sum([d for pair in result["matches"] for d in pair["distances_px"]]))
mean = total / result["n_keypoints"]
if not np.isfinite(mean):
return {**result, "reason": "non-finite mean keypoint shift"}
return {**result, "status": "measured", "keypoint_mean_shift_px": mean, "distance_sum_px": total}
def pose_shift_summary(evidence):
# run.py prints G3 into stage logs: keep full per-pair geometry in eval.json
# at the top level, so logs/verdicts do not duplicate thousands of poses.
return {**{k: v for k, v in evidence.items() if k not in ("matches", "unmatched_fp32", "unmatched_quant")},
"n_unmatched_fp32": len(evidence["unmatched_fp32"]),
"n_unmatched_quant": len(evidence["unmatched_quant"]),
"evidence": "eval.json#/pose_shift_evidence"}
def evaluate(t1, predictions, task):
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
ann = json.load(t1.tar.extractfile("annotations.json"))
if task == "pose":
keypoints = ann.get("keypoints")
if not keypoints or not keypoints.get("annotations"):
raise ValueError("COCO T1 is missing person keypoint annotations")
ann = {**ann, "annotations": keypoints["annotations"], "categories": keypoints["categories"]}
elif not any(a.get("segmentation") for a in ann["annotations"]):
raise ValueError("COCO T1 is missing instance masks")
gt = COCO()
gt.dataset = ann
gt.createIndex()
metric = METRICS[task]
if not predictions:
return {metric: 0.0, "ap50": 0.0, "n": len(t1.items)}
ev = COCOeval(gt, gt.loadRes(predictions), "segm" if task == "iseg" else "keypoints")
ev.params.imgIds = [it["id"] for it in t1.items]
if task == "pose":
ev.params.catIds = [1]
ev.evaluate()
ev.accumulate()
ev.summarize()
if ev.stats[0] < 0:
raise ValueError("no evaluable ground truth for this COCO subset")
return {metric: float(ev.stats[0] * 100), "ap50": float(ev.stats[1] * 100), "n": len(t1.items)}
def run_fp32(runner, t1, input_spec, out, resize=0):
task = out["task"]
evidence = pose_snapshot(t1, input_spec) if task == "pose" else None
predictions = predict(runner, t1, input_spec, task, pose_evidence=evidence, model=out.get("model", ""))
if evidence is not None:
out["pose_reference"] = evidence
out["metrics"] = evaluate(t1, predictions, task)
def run_quant(runner, t1, input_spec, baseline, out, resize=0):
task = out["task"]
metric = METRICS[task]
evidence = pose_snapshot(t1, input_spec) if task == "pose" else None
predictions = predict(runner, t1, input_spec, task, pose_evidence=evidence, model=out.get("model", ""))
measured = evaluate(t1, predictions, task)
out["metrics"] = measured
out["G3"] = {"metric": metric, "fp32": baseline["metrics"][metric], "quant": measured[metric],
"tier": "t1", "n": measured["n"], "se": None, "also": {"ap50": measured["ap50"]}}
if evidence is not None:
out["pose_reference"] = evidence
shift = measure_pose_shift(baseline.get("pose_reference"), evidence)
if baseline.get("model") != out.get("model"):
shift.update(status="unverified", keypoint_mean_shift_px=None, reason="FP32/quant model mismatch")
out["pose_shift_evidence"] = shift
out["G3"]["extra"] = {"keypoint_mean_shift_px": shift["keypoint_mean_shift_px"],
"keypoint_shift": pose_shift_summary(shift)}