- harness/eval/__init__.py:EQUIVALENT_SOURCES 列出精确的「冻结指纹 → 现指纹」对,contract_holds 判断现文件 仍实现冻结合同(完全相同,或同一模块的已批准改写)。阶段检查、基线检查、expand 比对三处都走它。 - 批准的只有 f56ac2f 的 5 个任务文件:上板由逐张改为按实测字节分批,板上仍逐个样本推理 (board_bench.infer_inputs),每张的预处理、后处理和指标一字未改。 - 批次的冻结记录不动,基线缓存按冻结指纹寻址,照常命中,不用重算。 - 用户 10-15:不建新批次。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
197 lines
11 KiB
Python
197 lines
11 KiB
Python
"""NYUv2 relative inverse-depth evaluation with an explicit alignment protocol.
|
|
|
|
This is a paired bench protocol, not a claim to reproduce a paper's scores.
|
|
GT is original NYU filled depth in metres. Model outputs are never treated as
|
|
metric predictions without the recorded per-image scale/shift alignment.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import numpy as np
|
|
|
|
from ..adapters import get
|
|
from ..adapters.depth_anything_v2_common import INPUT_SPEC
|
|
|
|
PROTOCOL = {
|
|
"id": "nyuv2-relative-inverse-ls-v1",
|
|
"prediction": "affine_invariant_inverse_depth",
|
|
"alignment": "per_image_least_squares_scale_and_shift_in_inverse_depth",
|
|
"alignment_mask": "same as score mask; independent identical fit rule for FP32 and quant",
|
|
"crop_yxyx": [45, 41, 471, 601],
|
|
"min_depth_m": 0.001, "max_depth_m": 10.0,
|
|
"valid_mask": "finite GT strictly greater than min and strictly less than max, inside crop",
|
|
"aggregation": "arithmetic mean of per-image metrics",
|
|
"delta1_unit": "percent", "metric_depth_prediction": False,
|
|
}
|
|
SUPPORTED_MODELS = {f"depth.anything.{version}.{variant}"
|
|
for version in ("v1", "v2") for variant in ("small", "base", "large")}
|
|
# Additive families (batch 7): pinned official MiDaS relative models and the
|
|
# metric Depth Anything V2 checkpoints from the same pinned upstream tree.
|
|
# Depth-Anything v1/v2 relative models keep their exact code paths below.
|
|
SUPPORTED_MODELS |= {"midas.small", "dpt.hybrid", "dpt.large"}
|
|
SUPPORTED_MODELS |= {f"depth.anything.v2.metric.{domain}.{variant}"
|
|
for domain in ("hypersim", "vkitti") for variant in ("small", "base", "large")}
|
|
# Additive families (batch 11): pinned official ZoeDepth, Metric3D v1/v2,
|
|
# UniDepth v1/v2 and DepthPro checkpoints. All existing family paths above and
|
|
# below stay byte-identical.
|
|
SUPPORTED_MODELS |= {"zoedepth.n", "zoedepth.k", "zoedepth.nk"}
|
|
SUPPORTED_MODELS |= {"metric3dv2.vit.small", "metric3dv2.vit.large", "metric3dv2.vit.giant2",
|
|
"metric3d.v1.convnext.tiny", "metric3d.v1.convnext.large"}
|
|
SUPPORTED_MODELS |= {"unidepth.v1.convnext.l", "unidepth.v1.vit.l",
|
|
"unidepth.v2.vit.s", "unidepth.v2.vit.b", "unidepth.v2.vit.l"}
|
|
SUPPORTED_MODELS |= {"depthpro"}
|
|
# New families evaluate at their own pinned static input resolution; the
|
|
# frozen Depth-Anything recipe stays 1x3x518x518.
|
|
FAMILY_INPUT_SPECS = {
|
|
"midas.small": "1x3x256x256",
|
|
"dpt.hybrid": "1x3x384x384",
|
|
"dpt.large": "1x3x384x384",
|
|
**{f"depth.anything.v2.metric.{domain}.{variant}": "1x3x518x518"
|
|
for domain in ("hypersim", "vkitti") for variant in ("small", "base", "large")},
|
|
**{m: "1x3x384x512" for m in ("zoedepth.n", "zoedepth.k", "zoedepth.nk")},
|
|
**{m: "1x3x518x518" for m in ("metric3dv2.vit.small", "metric3dv2.vit.large", "metric3dv2.vit.giant2")},
|
|
**{m: "1x3x544x1216" for m in ("metric3d.v1.convnext.tiny", "metric3d.v1.convnext.large")},
|
|
**{m: "1x3x462x616" for m in ("unidepth.v1.convnext.l", "unidepth.v1.vit.l")},
|
|
**{m: "1x3x518x518" for m in ("unidepth.v2.vit.s", "unidepth.v2.vit.b", "unidepth.v2.vit.l")},
|
|
"depthpro": "1x3x1536x1536",
|
|
}
|
|
# Additive (batch 11): families whose official network emits a spatial size that
|
|
# differs from its input spec (Metric3D v2 RAFT decoder upsamples the 129x129
|
|
# quarter-resolution field x4 -> 516). Empty for every pre-existing family, so
|
|
# the frozen restore shape check below is unchanged for them.
|
|
FAMILY_OUTPUT_SPECS = {
|
|
"metric3dv2.vit.small": "1x1x516x516",
|
|
"metric3dv2.vit.large": "1x1x516x516",
|
|
"metric3dv2.vit.giant2": "1x1x516x516",
|
|
}
|
|
|
|
|
|
def inputs(t1, item: dict, input_spec: str, model: str) -> np.ndarray:
|
|
if model not in SUPPORTED_MODELS:
|
|
raise ValueError(f"no verified MDE input recipe for {model}")
|
|
return get(model).preprocess(t1.image(item), input_spec)
|
|
|
|
|
|
def ground_truth(t1, item: dict) -> tuple[np.ndarray, np.ndarray]:
|
|
if t1.manifest.get("dataset") != "nyuv2" or item.get("depth_unit") != "m":
|
|
raise ValueError("expected NYUv2 original metre-depth labels")
|
|
if item.get("depth_format") != "npy_float32_m":
|
|
raise ValueError("depth labels must explicitly declare npy_float32_m")
|
|
with t1.tar.extractfile(item["depth"]) as stream:
|
|
depth = np.load(io.BytesIO(stream.read()), allow_pickle=False)
|
|
if depth.dtype != np.float32 or depth.shape != (480, 640) or t1.image(item).size != (640, 480):
|
|
raise ValueError("expected aligned original NYUv2 RGB/depth at 640x480")
|
|
mask = np.zeros(depth.shape, dtype=bool)
|
|
y1, x1, y2, x2 = PROTOCOL["crop_yxyx"]
|
|
mask[y1:y2, x1:x2] = True
|
|
mask &= np.isfinite(depth) & (depth > PROTOCOL["min_depth_m"]) & (depth < PROTOCOL["max_depth_m"])
|
|
if not mask.any():
|
|
raise ValueError("no valid NYUv2 depth pixels")
|
|
return depth, mask
|
|
|
|
|
|
def restore_prediction(outputs, original_hw: tuple[int, int], model: str | None = None) -> np.ndarray:
|
|
import torch
|
|
import torch.nn.functional as F
|
|
|
|
if len(outputs) != 1:
|
|
raise ValueError("expected one relative inverse-depth output")
|
|
prediction = np.asarray(outputs[0], dtype=np.float32)
|
|
# Frozen Depth-Anything shape check stays (1, 518, 518); additive families
|
|
# declare their own pinned static resolution (FAMILY_OUTPUT_SPECS only
|
|
# covers batch-11 families whose official output size differs from the
|
|
# input spec, so existing families keep the exact input-spec shape check).
|
|
family_spec = FAMILY_INPUT_SPECS.get(model) if model is not None else None
|
|
output_spec = FAMILY_OUTPUT_SPECS.get(model) if model is not None else None
|
|
expected_shape = (1, *(int(v) for v in ((output_spec or family_spec or INPUT_SPEC).split("x")[2:])))
|
|
if prediction.shape != expected_shape or not np.isfinite(prediction).all():
|
|
raise ValueError(f"invalid relative inverse-depth output: {prediction.shape}")
|
|
if np.any(prediction < 0):
|
|
raise ValueError("official Depth Anything forward must return nonnegative values")
|
|
if model is not None and model not in SUPPORTED_MODELS:
|
|
raise ValueError(f"no verified MDE restore recipe for {model}")
|
|
align_corners = getattr(get(model), "RESTORE_ALIGN_CORNERS", True) if model else True
|
|
mode = getattr(get(model), "RESTORE_INTERPOLATION", "bilinear") if model else "bilinear"
|
|
# Preserve each author's interpolation; the two-argument V2 API stays True.
|
|
return F.interpolate(torch.from_numpy(prediction[:, None]), size=original_hw,
|
|
mode=mode, align_corners=align_corners)[0, 0].numpy()
|
|
|
|
|
|
def aligned_metrics(prediction: np.ndarray, depth_m: np.ndarray, mask: np.ndarray) -> dict:
|
|
if prediction.shape != depth_m.shape or mask.shape != depth_m.shape or not mask.any():
|
|
raise ValueError("prediction/GT/mask mismatch or no valid pixels")
|
|
if not np.isfinite(prediction).all() or not np.isfinite(depth_m[mask]).all() or np.any(depth_m[mask] <= 0):
|
|
raise ValueError("non-finite prediction or invalid metric GT")
|
|
x = prediction[mask].astype(np.float64)
|
|
gt = depth_m[mask].astype(np.float64)
|
|
inverse_gt = 1.0 / gt
|
|
dx = x - x.mean()
|
|
variance_sum = float(dx @ dx)
|
|
if variance_sum <= np.finfo(np.float64).eps * max(1.0, float(x @ x)):
|
|
raise ValueError("relative-depth alignment is singular (constant/degenerate prediction)")
|
|
scale = float(dx @ (inverse_gt - inverse_gt.mean()) / variance_sum)
|
|
shift = float(inverse_gt.mean() - scale * x.mean())
|
|
# Unconstrained affine least squares has a signed solution. Positive
|
|
# definiteness applies to the normal matrix, not the fitted coefficient.
|
|
if not np.isfinite([scale, shift]).all():
|
|
raise ValueError(f"relative inverse-depth alignment is nonfinite: scale={scale}, shift={shift}")
|
|
inverse = scale * x + shift
|
|
near, far = PROTOCOL["min_depth_m"], PROTOCOL["max_depth_m"]
|
|
clipped = np.clip(inverse, 1.0 / far, 1.0 / near)
|
|
aligned = 1.0 / clipped
|
|
ratio = np.maximum(aligned / gt, gt / aligned)
|
|
return {"absrel": float(np.mean(np.abs(aligned - gt) / gt)),
|
|
"rmse": float(np.sqrt(np.mean((aligned - gt) ** 2))),
|
|
"delta1": float(np.mean(ratio < 1.25) * 100),
|
|
"valid_pixels": int(mask.sum()), "scale": scale, "shift": shift,
|
|
"clipped_inverse_pixels": int(np.count_nonzero(clipped != inverse))}
|
|
|
|
|
|
def evaluate(runner, t1, input_spec: str, model: str) -> tuple[dict, list[dict]]:
|
|
if input_spec != INPUT_SPEC and input_spec != FAMILY_INPUT_SPECS.get(model):
|
|
raise ValueError("MDE evaluation input recipe mismatch")
|
|
if t1.manifest.get("evaluation_protocol") != PROTOCOL:
|
|
raise ValueError("T1 must freeze the explicit relative-depth evaluation protocol")
|
|
from .common import groups, run_batched
|
|
samples = []
|
|
for items in groups(runner, t1.items):
|
|
for item, out in zip(items, run_batched(runner, [inputs(t1, item, input_spec, model) for item in items]), strict=True):
|
|
depth, mask = ground_truth(t1, item)
|
|
prediction = restore_prediction(out, depth.shape, model)
|
|
samples.append({"id": item["id"], **aligned_metrics(prediction, depth, mask)})
|
|
if not samples:
|
|
raise ValueError("empty depth evaluation")
|
|
metrics = {key: float(np.mean([item[key] for item in samples])) for key in ("absrel", "rmse", "delta1")}
|
|
metrics.update(n=len(samples), valid_pixels=sum(item["valid_pixels"] for item in samples))
|
|
return metrics, samples
|
|
|
|
|
|
def run_fp32(runner, t1, input_spec: str, out: dict, resize: int = 0) -> None:
|
|
out["metrics"], out["_depth_samples"] = evaluate(runner, t1, input_spec, out["model"])
|
|
out["depth_protocol"] = PROTOCOL
|
|
|
|
|
|
def run_quant(runner, t1, input_spec: str, baseline: dict, out: dict, resize: int = 0) -> None:
|
|
if baseline.get("model") != out["model"] or baseline.get("depth_protocol") != PROTOCOL:
|
|
raise ValueError("MDE baseline model/protocol mismatch")
|
|
run_fp32(runner, t1, input_spec, out, resize)
|
|
if [s["id"] for s in baseline.get("_depth_samples", [])] != [s["id"] for s in out["_depth_samples"]]:
|
|
raise ValueError("MDE baseline and measured sample identities differ")
|
|
fp, measured = baseline["metrics"], out["metrics"]
|
|
alignment_samples = [
|
|
{"id": quant["id"], "fp32_scale": reference["scale"], "fp32_shift": reference["shift"],
|
|
"quant_scale": quant["scale"], "quant_shift": quant["shift"]}
|
|
for reference, quant in zip(baseline["_depth_samples"], out["_depth_samples"])
|
|
]
|
|
out["G3"] = {"metric": "absrel", "fp32": fp["absrel"], "quant": measured["absrel"],
|
|
"tier": "t1", "n": measured["n"], "se": None,
|
|
"also": {"rmse": measured["rmse"], "fp32_rmse": fp["rmse"],
|
|
"delta1": measured["delta1"], "fp32_delta1": fp["delta1"]},
|
|
"extra": {"scale_alignment": {"status": "measured", "protocol": PROTOCOL,
|
|
"n": measured["n"], "space": "inverse_depth",
|
|
"samples": alignment_samples},
|
|
"delta1": {"status": "measured", "unit": "percent",
|
|
"fp32": fp["delta1"], "quant": measured["delta1"],
|
|
"delta_pp": measured["delta1"] - fp["delta1"]}}}
|
|
# Trusted eval.run attaches execution provenance (board/sim); this module does not.
|