131 lines
6.4 KiB
Python

"""KITTI2015 flow_occ evaluation in original image coordinates.
T1 items contain id, file (image_2/*_10.png), file2 (*_11.png), and flow
(16-bit RGB PNG: u*64+32768, v*64+32768, valid). Calibration only needs
file/file2. EPE2D and Fl-all are weighted by valid pixels across images;
per_image_epe is also recorded, matching torchvision's reference metrics.
"""
from __future__ import annotations
import numpy as np
from ..adapters import get
from ..adapters.raft_common import input_shape, preprocess
METRIC = "epe2d"
SUPPORTED_MODELS = {"raft.small", "raft.large"}
# Additive families (batch 9): pinned official flow models that reuse the
# packed 1x6x384x1024 RGB pair deployment contract. Every family dispatches to
# its own verified adapter recipe; the frozen raft recipe path stays untouched.
SUPPORTED_MODELS |= {"gmflow", "unimatch.flow", "gma", "sea.raft", "flow1d", "flowformer",
"flowformerplusplus", "flownet2"}
def inputs(t1, item: dict, input_spec: str, model: str) -> np.ndarray:
"""One source of paired inputs for FP32, G2, board evaluation and calibration."""
if model not in SUPPORTED_MODELS:
raise ValueError(f"no verified flow input recipe for {model}")
frames = (t1.image(item), t1.image({"file": item["file2"]}))
if model.startswith("raft."):
return preprocess(*frames, input_spec)
return get(model).preprocess(*frames, input_spec)
def read_kitti_flow(data: bytes) -> tuple[np.ndarray, np.ndarray]:
"""Decode RGB uint16 without the 8-bit truncation of PIL RGB conversion."""
import torch
from torchvision.io import decode_png
if data[:8] != b"\x89PNG\r\n\x1a\n" or data[24:26] != bytes((16, 2)):
raise ValueError("KITTI flow must be a 16-bit RGB PNG")
value = decode_png(torch.from_numpy(np.frombuffer(data, dtype=np.uint8).copy()))
if value.dtype != torch.uint16 or value.ndim != 3 or value.shape[0] != 3:
raise ValueError(f"invalid KITTI flow encoding: {value.dtype} {tuple(value.shape)}")
rgb = value.to(torch.float32).numpy()
if not np.isin(rgb[2], (0, 1)).all():
raise ValueError("KITTI validity channel must contain 0 or 1")
return (rgb[:2] - 32768.0) / 64.0, rgb[2].astype(bool)
def ground_truth(t1, item: dict) -> tuple[np.ndarray, np.ndarray]:
if t1.manifest.get("dataset") != "kitti2015":
raise ValueError("this evaluator currently requires real KITTI2015 flow_occ labels")
if item.get("flow_format") != "kitti_png16":
raise ValueError("T1 item must explicitly declare flow_format=kitti_png16")
with t1.tar.extractfile(item["flow"]) as stream:
truth, valid = read_kitti_flow(stream.read())
if truth.shape[-2:] != t1.image(item).size[::-1]:
raise ValueError(f"flow/image dimensions differ for {item['id']}")
return truth, valid
def restore_flow(outputs, input_spec: str, original_hw: tuple[int, int]) -> np.ndarray:
"""Reverse image resizing, including the displacement-vector scale."""
import torch
import torch.nn.functional as F
shape = input_shape(input_spec)
if len(outputs) != 1:
raise ValueError(f"RAFT must expose one final flow, got {len(outputs)} outputs")
flow = np.asarray(outputs[0], dtype=np.float32)
if flow.shape != (1, 2, *shape[-2:]) or not np.isfinite(flow).all():
raise ValueError(f"invalid final flow output: {flow.shape}")
h, w = original_hw
restored = F.interpolate(torch.from_numpy(flow), size=(h, w), mode="bilinear", align_corners=False)[0].numpy()
restored[0] *= w / shape[-1]
restored[1] *= h / shape[-2]
return restored
def sample_metrics(pred: np.ndarray, truth: np.ndarray, valid: np.ndarray) -> dict:
if pred.shape != truth.shape or valid.shape != truth.shape[-2:] or not valid.any():
raise ValueError("invalid or empty ground-truth flow/mask")
if not np.isfinite(pred).all() or not np.isfinite(truth[:, valid]).all():
raise ValueError("non-finite flow")
epe = np.linalg.norm(pred.astype(np.float64) - truth.astype(np.float64), axis=0)[valid]
magnitude = np.linalg.norm(truth.astype(np.float64), axis=0)[valid]
outlier = (epe > 3.0) & (epe > .05 * magnitude)
return {"valid_pixels": int(valid.sum()), "epe_sum": float(epe.sum()),
"outlier_pixels": int(outlier.sum()), "epe2d": float(epe.mean()),
"fl_all": float(outlier.mean() * 100)}
def aggregate(samples: list[dict]) -> dict:
count = sum(s["valid_pixels"] for s in samples)
if not samples or count == 0:
raise ValueError("no flow samples with real valid labels")
return {"epe2d": sum(s["epe_sum"] for s in samples) / count,
"fl_all": sum(s["outlier_pixels"] for s in samples) / count * 100,
"per_image_epe": float(np.mean([s["epe2d"] for s in samples])),
"valid_pixels": count, "n": len(samples)}
def evaluate(runner, t1, input_spec: str, model: str) -> tuple[dict, list[dict]]:
samples = []
for item in t1.items:
truth, valid = ground_truth(t1, item)
out = runner(inputs(t1, item, input_spec, model))
restored = restore_flow(out, input_spec, truth.shape[-2:])
samples.append({"id": item["id"], **sample_metrics(restored, truth, valid)})
return aggregate(samples), samples
def run_fp32(runner, t1, input_spec: str, out: dict, resize: int = 0) -> None:
out["metrics"], out["_flow_samples"] = evaluate(runner, t1, input_spec, out["model"])
out["flow_protocol"] = "kitti2015-flow_occ; original pixels; pixel-weighted EPE/Fl-all; >3px AND >5%"
def run_quant(runner, t1, input_spec: str, baseline: dict, out: dict, resize: int = 0) -> None:
if baseline.get("model") != out["model"]:
raise ValueError("flow baseline model mismatch")
run_fp32(runner, t1, input_spec, out, resize=resize)
if [s["id"] for s in baseline.get("_flow_samples", [])] != [s["id"] for s in out["_flow_samples"]]:
raise ValueError("flow baseline and measured sample identities differ")
out["G3"] = {"metric": METRIC, "fp32": baseline["metrics"][METRIC],
"quant": out["metrics"][METRIC], "tier": "t1", "n": out["metrics"]["n"],
"se": None, "also": {"fl_all": out["metrics"]["fl_all"],
"fp32_fl_all": baseline["metrics"]["fl_all"]},
"per_metric": {name: {"fp32": baseline["metrics"][name], "quant": out["metrics"][name]}
for name in ("epe2d", "fl_all")}}
# execution/source=board is supplied by the existing trusted eval entrypoint.