195 lines
11 KiB
Python

"""NYUv2 relative inverse-depth evaluation with an explicit alignment protocol.
This is a paired bench protocol, not a claim to reproduce a paper's scores.
GT is original NYU filled depth in metres. Model outputs are never treated as
metric predictions without the recorded per-image scale/shift alignment.
"""
from __future__ import annotations
import io
import numpy as np
from ..adapters import get
from ..adapters.depth_anything_v2_common import INPUT_SPEC
PROTOCOL = {
"id": "nyuv2-relative-inverse-ls-v1",
"prediction": "affine_invariant_inverse_depth",
"alignment": "per_image_least_squares_scale_and_shift_in_inverse_depth",
"alignment_mask": "same as score mask; independent identical fit rule for FP32 and quant",
"crop_yxyx": [45, 41, 471, 601],
"min_depth_m": 0.001, "max_depth_m": 10.0,
"valid_mask": "finite GT strictly greater than min and strictly less than max, inside crop",
"aggregation": "arithmetic mean of per-image metrics",
"delta1_unit": "percent", "metric_depth_prediction": False,
}
SUPPORTED_MODELS = {f"depth.anything.{version}.{variant}"
for version in ("v1", "v2") for variant in ("small", "base", "large")}
# Additive families (batch 7): pinned official MiDaS relative models and the
# metric Depth Anything V2 checkpoints from the same pinned upstream tree.
# Depth-Anything v1/v2 relative models keep their exact code paths below.
SUPPORTED_MODELS |= {"midas.small", "dpt.hybrid", "dpt.large"}
SUPPORTED_MODELS |= {f"depth.anything.v2.metric.{domain}.{variant}"
for domain in ("hypersim", "vkitti") for variant in ("small", "base", "large")}
# Additive families (batch 11): pinned official ZoeDepth, Metric3D v1/v2,
# UniDepth v1/v2 and DepthPro checkpoints. All existing family paths above and
# below stay byte-identical.
SUPPORTED_MODELS |= {"zoedepth.n", "zoedepth.k", "zoedepth.nk"}
SUPPORTED_MODELS |= {"metric3dv2.vit.small", "metric3dv2.vit.large", "metric3dv2.vit.giant2",
"metric3d.v1.convnext.tiny", "metric3d.v1.convnext.large"}
SUPPORTED_MODELS |= {"unidepth.v1.convnext.l", "unidepth.v1.vit.l",
"unidepth.v2.vit.s", "unidepth.v2.vit.b", "unidepth.v2.vit.l"}
SUPPORTED_MODELS |= {"depthpro"}
# New families evaluate at their own pinned static input resolution; the
# frozen Depth-Anything recipe stays 1x3x518x518.
FAMILY_INPUT_SPECS = {
"midas.small": "1x3x256x256",
"dpt.hybrid": "1x3x384x384",
"dpt.large": "1x3x384x384",
**{f"depth.anything.v2.metric.{domain}.{variant}": "1x3x518x518"
for domain in ("hypersim", "vkitti") for variant in ("small", "base", "large")},
**{m: "1x3x384x512" for m in ("zoedepth.n", "zoedepth.k", "zoedepth.nk")},
**{m: "1x3x518x518" for m in ("metric3dv2.vit.small", "metric3dv2.vit.large", "metric3dv2.vit.giant2")},
**{m: "1x3x544x1216" for m in ("metric3d.v1.convnext.tiny", "metric3d.v1.convnext.large")},
**{m: "1x3x462x616" for m in ("unidepth.v1.convnext.l", "unidepth.v1.vit.l")},
**{m: "1x3x518x518" for m in ("unidepth.v2.vit.s", "unidepth.v2.vit.b", "unidepth.v2.vit.l")},
"depthpro": "1x3x1536x1536",
}
# Additive (batch 11): families whose official network emits a spatial size that
# differs from its input spec (Metric3D v2 RAFT decoder upsamples the 129x129
# quarter-resolution field x4 -> 516). Empty for every pre-existing family, so
# the frozen restore shape check below is unchanged for them.
FAMILY_OUTPUT_SPECS = {
"metric3dv2.vit.small": "1x1x516x516",
"metric3dv2.vit.large": "1x1x516x516",
"metric3dv2.vit.giant2": "1x1x516x516",
}
def inputs(t1, item: dict, input_spec: str, model: str) -> np.ndarray:
if model not in SUPPORTED_MODELS:
raise ValueError(f"no verified MDE input recipe for {model}")
return get(model).preprocess(t1.image(item), input_spec)
def ground_truth(t1, item: dict) -> tuple[np.ndarray, np.ndarray]:
if t1.manifest.get("dataset") != "nyuv2" or item.get("depth_unit") != "m":
raise ValueError("expected NYUv2 original metre-depth labels")
if item.get("depth_format") != "npy_float32_m":
raise ValueError("depth labels must explicitly declare npy_float32_m")
with t1.tar.extractfile(item["depth"]) as stream:
depth = np.load(io.BytesIO(stream.read()), allow_pickle=False)
if depth.dtype != np.float32 or depth.shape != (480, 640) or t1.image(item).size != (640, 480):
raise ValueError("expected aligned original NYUv2 RGB/depth at 640x480")
mask = np.zeros(depth.shape, dtype=bool)
y1, x1, y2, x2 = PROTOCOL["crop_yxyx"]
mask[y1:y2, x1:x2] = True
mask &= np.isfinite(depth) & (depth > PROTOCOL["min_depth_m"]) & (depth < PROTOCOL["max_depth_m"])
if not mask.any():
raise ValueError("no valid NYUv2 depth pixels")
return depth, mask
def restore_prediction(outputs, original_hw: tuple[int, int], model: str | None = None) -> np.ndarray:
import torch
import torch.nn.functional as F
if len(outputs) != 1:
raise ValueError("expected one relative inverse-depth output")
prediction = np.asarray(outputs[0], dtype=np.float32)
# Frozen Depth-Anything shape check stays (1, 518, 518); additive families
# declare their own pinned static resolution (FAMILY_OUTPUT_SPECS only
# covers batch-11 families whose official output size differs from the
# input spec, so existing families keep the exact input-spec shape check).
family_spec = FAMILY_INPUT_SPECS.get(model) if model is not None else None
output_spec = FAMILY_OUTPUT_SPECS.get(model) if model is not None else None
expected_shape = (1, *(int(v) for v in ((output_spec or family_spec or INPUT_SPEC).split("x")[2:])))
if prediction.shape != expected_shape or not np.isfinite(prediction).all():
raise ValueError(f"invalid relative inverse-depth output: {prediction.shape}")
if np.any(prediction < 0):
raise ValueError("official Depth Anything forward must return nonnegative values")
if model is not None and model not in SUPPORTED_MODELS:
raise ValueError(f"no verified MDE restore recipe for {model}")
align_corners = getattr(get(model), "RESTORE_ALIGN_CORNERS", True) if model else True
mode = getattr(get(model), "RESTORE_INTERPOLATION", "bilinear") if model else "bilinear"
# Preserve each author's interpolation; the two-argument V2 API stays True.
return F.interpolate(torch.from_numpy(prediction[:, None]), size=original_hw,
mode=mode, align_corners=align_corners)[0, 0].numpy()
def aligned_metrics(prediction: np.ndarray, depth_m: np.ndarray, mask: np.ndarray) -> dict:
if prediction.shape != depth_m.shape or mask.shape != depth_m.shape or not mask.any():
raise ValueError("prediction/GT/mask mismatch or no valid pixels")
if not np.isfinite(prediction).all() or not np.isfinite(depth_m[mask]).all() or np.any(depth_m[mask] <= 0):
raise ValueError("non-finite prediction or invalid metric GT")
x = prediction[mask].astype(np.float64)
gt = depth_m[mask].astype(np.float64)
inverse_gt = 1.0 / gt
dx = x - x.mean()
variance_sum = float(dx @ dx)
if variance_sum <= np.finfo(np.float64).eps * max(1.0, float(x @ x)):
raise ValueError("relative-depth alignment is singular (constant/degenerate prediction)")
scale = float(dx @ (inverse_gt - inverse_gt.mean()) / variance_sum)
shift = float(inverse_gt.mean() - scale * x.mean())
# Unconstrained affine least squares has a signed solution. Positive
# definiteness applies to the normal matrix, not the fitted coefficient.
if not np.isfinite([scale, shift]).all():
raise ValueError(f"relative inverse-depth alignment is nonfinite: scale={scale}, shift={shift}")
inverse = scale * x + shift
near, far = PROTOCOL["min_depth_m"], PROTOCOL["max_depth_m"]
clipped = np.clip(inverse, 1.0 / far, 1.0 / near)
aligned = 1.0 / clipped
ratio = np.maximum(aligned / gt, gt / aligned)
return {"absrel": float(np.mean(np.abs(aligned - gt) / gt)),
"rmse": float(np.sqrt(np.mean((aligned - gt) ** 2))),
"delta1": float(np.mean(ratio < 1.25) * 100),
"valid_pixels": int(mask.sum()), "scale": scale, "shift": shift,
"clipped_inverse_pixels": int(np.count_nonzero(clipped != inverse))}
def evaluate(runner, t1, input_spec: str, model: str) -> tuple[dict, list[dict]]:
if input_spec != INPUT_SPEC and input_spec != FAMILY_INPUT_SPECS.get(model):
raise ValueError("MDE evaluation input recipe mismatch")
if t1.manifest.get("evaluation_protocol") != PROTOCOL:
raise ValueError("T1 must freeze the explicit relative-depth evaluation protocol")
samples = []
for item in t1.items:
depth, mask = ground_truth(t1, item)
prediction = restore_prediction(runner(inputs(t1, item, input_spec, model)), depth.shape, model)
samples.append({"id": item["id"], **aligned_metrics(prediction, depth, mask)})
if not samples:
raise ValueError("empty depth evaluation")
metrics = {key: float(np.mean([item[key] for item in samples])) for key in ("absrel", "rmse", "delta1")}
metrics.update(n=len(samples), valid_pixels=sum(item["valid_pixels"] for item in samples))
return metrics, samples
def run_fp32(runner, t1, input_spec: str, out: dict, resize: int = 0) -> None:
out["metrics"], out["_depth_samples"] = evaluate(runner, t1, input_spec, out["model"])
out["depth_protocol"] = PROTOCOL
def run_quant(runner, t1, input_spec: str, baseline: dict, out: dict, resize: int = 0) -> None:
if baseline.get("model") != out["model"] or baseline.get("depth_protocol") != PROTOCOL:
raise ValueError("MDE baseline model/protocol mismatch")
run_fp32(runner, t1, input_spec, out, resize)
if [s["id"] for s in baseline.get("_depth_samples", [])] != [s["id"] for s in out["_depth_samples"]]:
raise ValueError("MDE baseline and measured sample identities differ")
fp, measured = baseline["metrics"], out["metrics"]
alignment_samples = [
{"id": quant["id"], "fp32_scale": reference["scale"], "fp32_shift": reference["shift"],
"quant_scale": quant["scale"], "quant_shift": quant["shift"]}
for reference, quant in zip(baseline["_depth_samples"], out["_depth_samples"])
]
out["G3"] = {"metric": "absrel", "fp32": fp["absrel"], "quant": measured["absrel"],
"tier": "t1", "n": measured["n"], "se": None,
"also": {"rmse": measured["rmse"], "fp32_rmse": fp["rmse"],
"delta1": measured["delta1"], "fp32_delta1": fp["delta1"]},
"extra": {"scale_alignment": {"status": "measured", "protocol": PROTOCOL,
"n": measured["n"], "space": "inverse_depth",
"samples": alignment_samples},
"delta1": {"status": "measured", "unit": "percent",
"fp32": fp["delta1"], "quant": measured["delta1"],
"delta_pp": measured["delta1"] - fp["delta1"]}}}
# Trusted eval.run attaches execution provenance (board/sim); this module does not.