- harness/eval/__init__.py:EQUIVALENT_SOURCES 列出精确的「冻结指纹 → 现指纹」对,contract_holds 判断现文件 仍实现冻结合同(完全相同,或同一模块的已批准改写)。阶段检查、基线检查、expand 比对三处都走它。 - 批准的只有 f56ac2f 的 5 个任务文件:上板由逐张改为按实测字节分批,板上仍逐个样本推理 (board_bench.infer_inputs),每张的预处理、后处理和指标一字未改。 - 批次的冻结记录不动,基线缓存按冻结指纹寻址,照常命中,不用重算。 - 用户 10-15:不建新批次。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
132 lines
6.6 KiB
Python
132 lines
6.6 KiB
Python
"""KITTI2015 flow_occ evaluation in original image coordinates.
|
|
|
|
T1 items contain id, file (image_2/*_10.png), file2 (*_11.png), and flow
|
|
(16-bit RGB PNG: u*64+32768, v*64+32768, valid). Calibration only needs
|
|
file/file2. EPE2D and Fl-all are weighted by valid pixels across images;
|
|
per_image_epe is also recorded, matching torchvision's reference metrics.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import numpy as np
|
|
|
|
from ..adapters import get
|
|
from ..adapters.raft_common import input_shape, preprocess
|
|
|
|
METRIC = "epe2d"
|
|
SUPPORTED_MODELS = {"raft.small", "raft.large"}
|
|
# Additive families (batch 9): pinned official flow models that reuse the
|
|
# packed 1x6x384x1024 RGB pair deployment contract. Every family dispatches to
|
|
# its own verified adapter recipe; the frozen raft recipe path stays untouched.
|
|
SUPPORTED_MODELS |= {"gmflow", "unimatch.flow", "gma", "sea.raft", "flow1d", "flowformer",
|
|
"flowformerplusplus", "flownet2"}
|
|
|
|
|
|
def inputs(t1, item: dict, input_spec: str, model: str) -> np.ndarray:
|
|
"""One source of paired inputs for FP32, G2, board evaluation and calibration."""
|
|
if model not in SUPPORTED_MODELS:
|
|
raise ValueError(f"no verified flow input recipe for {model}")
|
|
frames = (t1.image(item), t1.image({"file": item["file2"]}))
|
|
if model.startswith("raft."):
|
|
return preprocess(*frames, input_spec)
|
|
return get(model).preprocess(*frames, input_spec)
|
|
|
|
|
|
def read_kitti_flow(data: bytes) -> tuple[np.ndarray, np.ndarray]:
|
|
"""Decode RGB uint16 without the 8-bit truncation of PIL RGB conversion."""
|
|
import torch
|
|
from torchvision.io import decode_png
|
|
|
|
if data[:8] != b"\x89PNG\r\n\x1a\n" or data[24:26] != bytes((16, 2)):
|
|
raise ValueError("KITTI flow must be a 16-bit RGB PNG")
|
|
value = decode_png(torch.from_numpy(np.frombuffer(data, dtype=np.uint8).copy()))
|
|
if value.dtype != torch.uint16 or value.ndim != 3 or value.shape[0] != 3:
|
|
raise ValueError(f"invalid KITTI flow encoding: {value.dtype} {tuple(value.shape)}")
|
|
rgb = value.to(torch.float32).numpy()
|
|
if not np.isin(rgb[2], (0, 1)).all():
|
|
raise ValueError("KITTI validity channel must contain 0 or 1")
|
|
return (rgb[:2] - 32768.0) / 64.0, rgb[2].astype(bool)
|
|
|
|
|
|
def ground_truth(t1, item: dict) -> tuple[np.ndarray, np.ndarray]:
|
|
if t1.manifest.get("dataset") != "kitti2015":
|
|
raise ValueError("this evaluator currently requires real KITTI2015 flow_occ labels")
|
|
if item.get("flow_format") != "kitti_png16":
|
|
raise ValueError("T1 item must explicitly declare flow_format=kitti_png16")
|
|
with t1.tar.extractfile(item["flow"]) as stream:
|
|
truth, valid = read_kitti_flow(stream.read())
|
|
if truth.shape[-2:] != t1.image(item).size[::-1]:
|
|
raise ValueError(f"flow/image dimensions differ for {item['id']}")
|
|
return truth, valid
|
|
|
|
|
|
def restore_flow(outputs, input_spec: str, original_hw: tuple[int, int]) -> np.ndarray:
|
|
"""Reverse image resizing, including the displacement-vector scale."""
|
|
import torch
|
|
import torch.nn.functional as F
|
|
|
|
shape = input_shape(input_spec)
|
|
if len(outputs) != 1:
|
|
raise ValueError(f"RAFT must expose one final flow, got {len(outputs)} outputs")
|
|
flow = np.asarray(outputs[0], dtype=np.float32)
|
|
if flow.shape != (1, 2, *shape[-2:]) or not np.isfinite(flow).all():
|
|
raise ValueError(f"invalid final flow output: {flow.shape}")
|
|
h, w = original_hw
|
|
restored = F.interpolate(torch.from_numpy(flow), size=(h, w), mode="bilinear", align_corners=False)[0].numpy()
|
|
restored[0] *= w / shape[-1]
|
|
restored[1] *= h / shape[-2]
|
|
return restored
|
|
|
|
|
|
def sample_metrics(pred: np.ndarray, truth: np.ndarray, valid: np.ndarray) -> dict:
|
|
if pred.shape != truth.shape or valid.shape != truth.shape[-2:] or not valid.any():
|
|
raise ValueError("invalid or empty ground-truth flow/mask")
|
|
if not np.isfinite(pred).all() or not np.isfinite(truth[:, valid]).all():
|
|
raise ValueError("non-finite flow")
|
|
epe = np.linalg.norm(pred.astype(np.float64) - truth.astype(np.float64), axis=0)[valid]
|
|
magnitude = np.linalg.norm(truth.astype(np.float64), axis=0)[valid]
|
|
outlier = (epe > 3.0) & (epe > .05 * magnitude)
|
|
return {"valid_pixels": int(valid.sum()), "epe_sum": float(epe.sum()),
|
|
"outlier_pixels": int(outlier.sum()), "epe2d": float(epe.mean()),
|
|
"fl_all": float(outlier.mean() * 100)}
|
|
|
|
|
|
def aggregate(samples: list[dict]) -> dict:
|
|
count = sum(s["valid_pixels"] for s in samples)
|
|
if not samples or count == 0:
|
|
raise ValueError("no flow samples with real valid labels")
|
|
return {"epe2d": sum(s["epe_sum"] for s in samples) / count,
|
|
"fl_all": sum(s["outlier_pixels"] for s in samples) / count * 100,
|
|
"per_image_epe": float(np.mean([s["epe2d"] for s in samples])),
|
|
"valid_pixels": count, "n": len(samples)}
|
|
|
|
|
|
def evaluate(runner, t1, input_spec: str, model: str) -> tuple[dict, list[dict]]:
|
|
from .common import groups, run_batched
|
|
samples = []
|
|
for items in groups(runner, t1.items):
|
|
for item, out in zip(items, run_batched(runner, [inputs(t1, item, input_spec, model) for item in items]), strict=True):
|
|
truth, valid = ground_truth(t1, item)
|
|
restored = restore_flow(out, input_spec, truth.shape[-2:])
|
|
samples.append({"id": item["id"], **sample_metrics(restored, truth, valid)})
|
|
return aggregate(samples), samples
|
|
|
|
|
|
def run_fp32(runner, t1, input_spec: str, out: dict, resize: int = 0) -> None:
|
|
out["metrics"], out["_flow_samples"] = evaluate(runner, t1, input_spec, out["model"])
|
|
out["flow_protocol"] = "kitti2015-flow_occ; original pixels; pixel-weighted EPE/Fl-all; >3px AND >5%"
|
|
|
|
|
|
def run_quant(runner, t1, input_spec: str, baseline: dict, out: dict, resize: int = 0) -> None:
|
|
if baseline.get("model") != out["model"]:
|
|
raise ValueError("flow baseline model mismatch")
|
|
run_fp32(runner, t1, input_spec, out, resize=resize)
|
|
if [s["id"] for s in baseline.get("_flow_samples", [])] != [s["id"] for s in out["_flow_samples"]]:
|
|
raise ValueError("flow baseline and measured sample identities differ")
|
|
out["G3"] = {"metric": METRIC, "fp32": baseline["metrics"][METRIC],
|
|
"quant": out["metrics"][METRIC], "tier": "t1", "n": out["metrics"]["n"],
|
|
"se": None, "also": {"fl_all": out["metrics"]["fl_all"],
|
|
"fp32_fl_all": baseline["metrics"]["fl_all"]},
|
|
"per_metric": {name: {"fp32": baseline["metrics"][name], "quant": out["metrics"][name]}
|
|
for name in ("epe2d", "fl_all")}}
|
|
# execution/source=board is supplied by the existing trusted eval entrypoint.
|