harness/eval/coco_extra.py is frozen per batch by task_contract (only listed equivalent rewrites may replace it). The 8a1ee24 edit made every pose stage of the pilot batches fail the contract check (Qwen rtmpose.s never got a formal stage). The keypoint shift gate it fed was removed in e0c224f, so the shift reaching 'unverified' no longer decides anything; back to the approved version 505d37. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
311 lines
17 KiB
Python
311 lines
17 KiB
Python
"""COCO mask AP / keypoint OKS AP for the existing Ultralytics exporters.
|
|
|
|
Both FP32 and board outputs use exactly the same letterbox, decoding and COCOeval.
|
|
FastSAM is deliberately excluded: its class-agnostic masks are not COCO classes.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import numpy as np
|
|
|
|
from .common import groups, letterbox, run_batched
|
|
from .det import CONF, IOU, MAX_DET, _nms
|
|
|
|
METRICS = {"iseg": "mask_ap", "pose": "oks_ap"}
|
|
POSE_SHIFT_SCHEMA = "coco-pose-bbox-iou-v1"
|
|
POSE_MATCH_IOU = 0.5
|
|
|
|
|
|
def decode(outputs, task):
|
|
nc, extra = (80, 32) if task == "iseg" else (1, 51)
|
|
candidates = [o for o in outputs if o.ndim == 3 and o.shape[1] == 4 + nc + extra]
|
|
if len(candidates) != 1:
|
|
raise ValueError(f"{task}: expected COCO YOLO head [1,{4+nc+extra},N], got {[o.shape for o in outputs]}")
|
|
p = candidates[0][0].T
|
|
labels = p[:, 4:4 + nc].argmax(1)
|
|
scores = p[:, 4:4 + nc].max(1)
|
|
selected = scores > CONF
|
|
p, labels, scores = p[selected], labels[selected], scores[selected]
|
|
boxes = np.concatenate((p[:, :2] - p[:, 2:4] / 2, p[:, :2] + p[:, 2:4] / 2), axis=1)
|
|
keep = (np.empty(0, dtype=np.intp) if task == "pose" and not len(scores)
|
|
else _nms(boxes + labels[:, None] * 4096, scores, IOU)[:MAX_DET])
|
|
proto = None
|
|
if task == "iseg":
|
|
protos = [o for o in outputs if o.ndim == 4 and o.shape[:2] == (1, extra)]
|
|
if len(protos) != 1:
|
|
raise ValueError("segmentation requires exactly one 32-channel prototype output")
|
|
proto = protos[0][0]
|
|
return boxes[keep], scores[keep], labels[keep], p[keep, 4 + nc:], proto
|
|
|
|
|
|
def masks_to_original(proto, coeff, boxes, size, image_size, scale, pad):
|
|
"""Undo our actual integer letterbox offsets, including asymmetric rounding."""
|
|
import torch
|
|
from torch.nn.functional import interpolate
|
|
from ultralytics.utils.ops import crop_mask
|
|
w, h = image_size
|
|
px, py = pad
|
|
nw, nh = round(w * scale), round(h * scale)
|
|
# Bound peak memory for 300 detections at original image resolution.
|
|
for i in range(0, len(coeff), 16):
|
|
logits = torch.from_numpy(coeff[i:i + 16]) @ torch.from_numpy(proto).reshape(proto.shape[0], -1)
|
|
logits = logits.reshape(-1, *proto.shape[1:])
|
|
logits = interpolate(logits[:, None], (size, size), mode="bilinear", align_corners=False)
|
|
logits = logits[:, :, py:py + nh, px:px + nw]
|
|
masks = interpolate(logits, (h, w), mode="bilinear", align_corners=False)[:, 0] > 0
|
|
yield from crop_mask(masks.byte(), torch.from_numpy(boxes[i:i + 16]).float()).numpy()
|
|
|
|
|
|
def predict(runner, t1, input_spec, task, pose_evidence=None, model=""):
|
|
if model:
|
|
from ..adapters import get
|
|
custom = getattr(get(model), "predict", None)
|
|
if custom is not None:
|
|
return custom(runner, t1, input_spec, pose_evidence=pose_evidence)
|
|
if task == "iseg":
|
|
from pycocotools import mask as mask_utils
|
|
size = int(input_spec.split("x")[-1])
|
|
predictions = []
|
|
for items in groups(runner, t1.items, 16):
|
|
images = [t1.image(it) for it in items]
|
|
pre = [letterbox(im, size) for im in images]
|
|
outputs = run_batched(runner, [v[0] for v in pre])
|
|
if task == "pose" and len(outputs) != len(items):
|
|
raise ValueError("pose: missing image outputs")
|
|
for it, im, (_, scale, pad), outs in zip(items, images, pre, outputs):
|
|
boxes, scores, labels, extra, proto = decode(outs, task)
|
|
if task == "pose" and pose_evidence is not None:
|
|
# Keep unclipped original-image geometry: clipping at image edges
|
|
# can hide a real shift. These records never enter COCOeval.
|
|
raw_boxes = (boxes.astype(np.float64) - np.array([*pad, *pad])) / scale
|
|
raw_points = extra.astype(np.float64).reshape(-1, 17, 3).copy()
|
|
raw_points[:, :, :2] = (raw_points[:, :, :2] - np.array(pad)) / scale
|
|
pose_evidence["images"].append({
|
|
"image_id": it["id"], "image_size": list(im.size), "scale": scale, "pad": list(pad),
|
|
"detections": [{"category_id": 1, "bbox_xyxy": box.tolist(), "score": float(score),
|
|
"keypoints": points.tolist()}
|
|
for box, score, points in zip(raw_boxes, scores, raw_points)],
|
|
})
|
|
boxes = (boxes - np.array([*pad, *pad], dtype=np.float32)) / scale
|
|
boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, im.width)
|
|
boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, im.height)
|
|
if task == "iseg":
|
|
masks = masks_to_original(proto, extra, boxes, size, im.size, scale, pad)
|
|
for mask, score, label in zip(masks, scores, labels):
|
|
rle = mask_utils.encode(np.asfortranarray(mask.astype(np.uint8)))
|
|
rle["counts"] = rle["counts"].decode("ascii")
|
|
predictions.append({"image_id": it["id"], "category_id": t1.manifest["cat_map"][str(int(label))],
|
|
"score": float(score), "segmentation": rle})
|
|
else:
|
|
points = extra.reshape(-1, 17, 3).copy()
|
|
points[:, :, :2] = (points[:, :, :2] - np.array(pad)) / scale
|
|
points[:, :, 0] = points[:, :, 0].clip(0, im.width)
|
|
points[:, :, 1] = points[:, :, 1].clip(0, im.height)
|
|
for points_one, score in zip(points, scores):
|
|
predictions.append({"image_id": it["id"], "category_id": 1, "score": float(score),
|
|
"keypoints": points_one.reshape(-1).tolist()})
|
|
return predictions
|
|
|
|
|
|
def pose_snapshot(t1, input_spec):
|
|
"""Stored in the existing metrics.json/eval.json; no separate cache or runner."""
|
|
return {"schema": POSE_SHIFT_SCHEMA, "input_spec": input_spec,
|
|
"t1_manifest_sha": t1.manifest.get("sha256"),
|
|
"image_ids": [it["id"] for it in t1.items], "images": []}
|
|
|
|
|
|
def measure_pose_shift(fp32, quant):
|
|
"""Mean L2 distance in original pixels over matched post-NMS poses, all 17 joints.
|
|
|
|
Fixed top-down crops match by source ID; other poses use box IoU >= 0.5.
|
|
Match only within an image/category, greedily by descending box IoU otherwise;
|
|
ties use the saved post-NMS indices. Each detection is used at most once.
|
|
Neither keypoint positions nor keypoint confidence select pairs/joints.
|
|
Unmatched detections do not contribute zeros; their indices and coverage are
|
|
retained. OKS AP separately judges missed/extra people. No pairs => unverified.
|
|
"""
|
|
result = {"status": "unverified", "method": POSE_SHIFT_SCHEMA,
|
|
"coordinate_space": "original_image_pixels_unclipped", "match_iou_min": POSE_MATCH_IOU,
|
|
"joint_policy": "all_17_no_confidence_filter", "keypoint_mean_shift_px": None,
|
|
"distance_sum_px": None,
|
|
"n_fp32": 0, "n_quant": 0, "n_matched": 0, "n_keypoints": 0,
|
|
"matches": [], "unmatched_fp32": [], "unmatched_quant": [], "complete": False}
|
|
if not isinstance(fp32, dict) or not isinstance(quant, dict):
|
|
return {**result, "reason": "pose reference or measurement missing; rerun FP32 to record geometry"}
|
|
for key in ("schema", "input_spec", "t1_manifest_sha", "image_ids"):
|
|
if key not in fp32 or key not in quant or fp32[key] != quant[key]:
|
|
return {**result, "reason": f"pose evidence {key} mismatch or missing"}
|
|
if fp32["schema"] != POSE_SHIFT_SCHEMA:
|
|
return {**result, "reason": "unsupported pose evidence schema"}
|
|
if not isinstance(fp32["t1_manifest_sha"], str) or not fp32["t1_manifest_sha"]:
|
|
return {**result, "reason": "pose evidence T1 identity missing"}
|
|
|
|
def checked_images(snapshot):
|
|
ids = snapshot["image_ids"]
|
|
if not isinstance(ids, list) or len(set(ids)) != len(ids):
|
|
raise ValueError("duplicate or invalid image IDs")
|
|
images = snapshot["images"]
|
|
if [im["image_id"] for im in images] != ids:
|
|
raise ValueError("missing, reordered or extra image evidence")
|
|
for im in images:
|
|
source_ids = set()
|
|
size = np.asarray(im["image_size"], dtype=float)
|
|
if size.shape != (2,) or not np.isfinite(size).all() or (size <= 0).any():
|
|
raise ValueError("invalid image geometry")
|
|
if im.get("transform") != "topdown_affine":
|
|
pad = np.asarray(im["pad"], dtype=float)
|
|
if (pad.shape != (2,) or not np.isfinite(pad).all()
|
|
or not np.isfinite(im["scale"]) or im["scale"] <= 0):
|
|
raise ValueError("invalid letterbox geometry")
|
|
for d in im["detections"]:
|
|
box, points = np.asarray(d["bbox_xyxy"], dtype=float), np.asarray(d["keypoints"], dtype=float)
|
|
if (box.shape != (4,) or points.shape != (17, 3) or not np.isfinite(box).all()
|
|
or not np.isfinite(points).all() or not np.isfinite(d["score"])
|
|
or (box[2:] <= box[:2]).any() or d["category_id"] != 1):
|
|
raise ValueError("invalid/non-finite COCO pose detection")
|
|
if im.get("transform") == "topdown_affine":
|
|
source = d.get("source_index")
|
|
if type(source) is not int or source < 0 or source in source_ids:
|
|
raise ValueError("missing, invalid or duplicate crop source_index")
|
|
source_ids.add(source)
|
|
affine = np.asarray(d["affine"], dtype=float)
|
|
if (affine.shape != (2, 3) or not np.isfinite(affine).all()
|
|
or abs(np.linalg.det(affine[:, :2])) < 1e-12):
|
|
raise ValueError("invalid measured top-down affine geometry")
|
|
return images
|
|
|
|
try:
|
|
refs, measured = checked_images(fp32), checked_images(quant)
|
|
topdown = any(im.get("transform") == "topdown_affine" for im in refs)
|
|
if topdown:
|
|
if not all(im.get("transform") == "topdown_affine" for im in refs):
|
|
raise ValueError("mixed pose transforms")
|
|
for key in ("boxes_file", "boxes_sha256", "pose_protocol", "official_model"):
|
|
if not fp32.get(key) or fp32[key] != quant.get(key):
|
|
raise ValueError(f"fixed crop {key} mismatch or missing")
|
|
result.update(method="coco-pose-source-id-v1", match_iou_min=None)
|
|
for ref, cur in zip(refs, measured):
|
|
if ref.get("transform") != cur.get("transform"):
|
|
raise ValueError("FP32/quant image transform mismatch")
|
|
keys = ("image_size",) if ref.get("transform") == "topdown_affine" else ("image_size", "scale", "pad")
|
|
if any(ref[k] != cur[k] for k in keys):
|
|
raise ValueError("FP32/quant image geometry mismatch")
|
|
except (KeyError, TypeError, ValueError, OverflowError) as e:
|
|
return {**result, "reason": str(e)}
|
|
|
|
result["n_fp32"] = sum(len(im["detections"]) for im in refs)
|
|
result["n_quant"] = sum(len(im["detections"]) for im in measured)
|
|
for ref, cur in zip(refs, measured):
|
|
a, b = ref["detections"], cur["detections"]
|
|
candidates = []
|
|
if topdown:
|
|
indices = {d["source_index"]: j for j, d in enumerate(b)}
|
|
candidates = [(-1.0, i, indices[d["source_index"]])
|
|
for i, d in enumerate(a) if d["source_index"] in indices]
|
|
elif a and b:
|
|
ab = np.asarray([d["bbox_xyxy"] for d in a], dtype=np.float64)
|
|
bb = np.asarray([d["bbox_xyxy"] for d in b], dtype=np.float64)
|
|
wh = np.maximum(0, np.minimum(ab[:, None, 2:], bb[None, :, 2:])
|
|
- np.maximum(ab[:, None, :2], bb[None, :, :2]))
|
|
inter = wh.prod(axis=2)
|
|
union = (ab[:, 2:] - ab[:, :2]).prod(axis=1)[:, None] + (bb[:, 2:] - bb[:, :2]).prod(axis=1) - inter
|
|
ious = inter / union
|
|
if not np.isfinite(ious).all():
|
|
return {**result, "reason": "non-finite box IoU"}
|
|
candidates = sorted((-float(ious[i, j]), int(i), int(j))
|
|
for i, j in np.argwhere(ious >= POSE_MATCH_IOU))
|
|
used_a, used_b = set(), set()
|
|
for neg_iou, i, j in candidates:
|
|
if i in used_a or j in used_b:
|
|
continue
|
|
if topdown and any(a[i][k] != b[j][k] for k in ("bbox_xyxy", "affine")):
|
|
return {**result, "reason": "FP32/quant matched crop geometry mismatch"}
|
|
used_a.add(i)
|
|
used_b.add(j)
|
|
delta = (np.asarray(b[j]["keypoints"], dtype=np.float64)[:, :2]
|
|
- np.asarray(a[i]["keypoints"], dtype=np.float64)[:, :2])
|
|
distances = np.hypot(delta[:, 0], delta[:, 1])
|
|
if not np.isfinite(distances).all():
|
|
return {**result, "reason": "non-finite keypoint distance"}
|
|
result["matches"].append({"image_id": ref["image_id"], "fp32_index": i, "quant_index": j,
|
|
"bbox_iou": -neg_iou, "fp32": a[i], "quant": b[j],
|
|
"distances_px": distances.tolist()})
|
|
result["n_matched"] = len(result["matches"])
|
|
result["n_keypoints"] = 17 * result["n_matched"]
|
|
for key, dets, used in (("unmatched_fp32", a, used_a), ("unmatched_quant", b, used_b)):
|
|
result[key].extend({"image_id": ref["image_id"], "index": i} for i in range(len(dets)) if i not in used)
|
|
result["complete"] = True
|
|
if not result["n_matched"]:
|
|
return {**result, "reason": "no matched FP32/quant poses"}
|
|
total = float(np.sum([d for pair in result["matches"] for d in pair["distances_px"]]))
|
|
mean = total / result["n_keypoints"]
|
|
if not np.isfinite(mean):
|
|
return {**result, "reason": "non-finite mean keypoint shift"}
|
|
return {**result, "status": "measured", "keypoint_mean_shift_px": mean, "distance_sum_px": total}
|
|
|
|
|
|
def pose_shift_summary(evidence):
|
|
# run.py prints G3 into stage logs: keep full per-pair geometry in eval.json
|
|
# at the top level, so logs/verdicts do not duplicate thousands of poses.
|
|
return {**{k: v for k, v in evidence.items() if k not in ("matches", "unmatched_fp32", "unmatched_quant")},
|
|
"n_unmatched_fp32": len(evidence["unmatched_fp32"]),
|
|
"n_unmatched_quant": len(evidence["unmatched_quant"]),
|
|
"evidence": "eval.json#/pose_shift_evidence"}
|
|
|
|
|
|
def evaluate(t1, predictions, task):
|
|
from pycocotools.coco import COCO
|
|
from pycocotools.cocoeval import COCOeval
|
|
ann = json.load(t1.tar.extractfile("annotations.json"))
|
|
if task == "pose":
|
|
keypoints = ann.get("keypoints")
|
|
if not keypoints or not keypoints.get("annotations"):
|
|
raise ValueError("COCO T1 is missing person keypoint annotations")
|
|
ann = {**ann, "annotations": keypoints["annotations"], "categories": keypoints["categories"]}
|
|
elif not any(a.get("segmentation") for a in ann["annotations"]):
|
|
raise ValueError("COCO T1 is missing instance masks")
|
|
gt = COCO()
|
|
gt.dataset = ann
|
|
gt.createIndex()
|
|
metric = METRICS[task]
|
|
if not predictions:
|
|
return {metric: 0.0, "ap50": 0.0, "n": len(t1.items)}
|
|
ev = COCOeval(gt, gt.loadRes(predictions), "segm" if task == "iseg" else "keypoints")
|
|
ev.params.imgIds = [it["id"] for it in t1.items]
|
|
if task == "pose":
|
|
ev.params.catIds = [1]
|
|
ev.evaluate()
|
|
ev.accumulate()
|
|
ev.summarize()
|
|
if ev.stats[0] < 0:
|
|
raise ValueError("no evaluable ground truth for this COCO subset")
|
|
return {metric: float(ev.stats[0] * 100), "ap50": float(ev.stats[1] * 100), "n": len(t1.items)}
|
|
|
|
|
|
def run_fp32(runner, t1, input_spec, out, resize=0):
|
|
task = out["task"]
|
|
evidence = pose_snapshot(t1, input_spec) if task == "pose" else None
|
|
predictions = predict(runner, t1, input_spec, task, pose_evidence=evidence, model=out.get("model", ""))
|
|
if evidence is not None:
|
|
out["pose_reference"] = evidence
|
|
out["metrics"] = evaluate(t1, predictions, task)
|
|
|
|
|
|
def run_quant(runner, t1, input_spec, baseline, out, resize=0):
|
|
task = out["task"]
|
|
metric = METRICS[task]
|
|
evidence = pose_snapshot(t1, input_spec) if task == "pose" else None
|
|
predictions = predict(runner, t1, input_spec, task, pose_evidence=evidence, model=out.get("model", ""))
|
|
measured = evaluate(t1, predictions, task)
|
|
out["metrics"] = measured
|
|
out["G3"] = {"metric": metric, "fp32": baseline["metrics"][metric], "quant": measured[metric],
|
|
"tier": "t1", "n": measured["n"], "se": None, "also": {"ap50": measured["ap50"]}}
|
|
if evidence is not None:
|
|
out["pose_reference"] = evidence
|
|
shift = measure_pose_shift(baseline.get("pose_reference"), evidence)
|
|
if baseline.get("model") != out.get("model"):
|
|
shift.update(status="unverified", keypoint_mean_shift_px=None, reason="FP32/quant model mismatch")
|
|
out["pose_shift_evidence"] = shift
|
|
out["G3"]["extra"] = {"keypoint_mean_shift_px": shift["keypoint_mean_shift_px"],
|
|
"keypoint_shift": pose_shift_summary(shift)}
|