"""Probe the fourteen batch-11 mde checkpoints on the NYUv2 T1. Same admission evidence and verbatim judgement criteria as probe_da_metric_adapters.py (the batch-7 metric Depth Anything probe): strict official checkpoint load, real-data FP32 forward, ONNX export plus checker, onnxruntime parity gates cosine >= .99999 and relative_max_abs <= .001 on real images, per-image alignment audit under the frozen nyuv2-relative-inverse-ls-v1 protocol (the affine least squares carries a signed scale, so metric-metre predictions are aligned, never trusted as metres) and a 128-image train-split calibration package keyed to each family recipe. Sources are the pinned batch-11 upstream trees under ~/bench-cache/sources/mde-batch11/. Two export toolchains are pinned per family (adapter EXPORTER): TorchScript for ZoeDepth/Metric3D, dynamo for UniDepth/DepthPro (the TorchScript LayerNormalization decomposition diverges from eager beyond the frozen gate in the official Nystroem/FOV paths; both toolchains face the same frozen gates). Run with the batch-11 overlay venv (CPU only, no installs): ~/bench-cache/venvs/mde-batch11/bin/python scripts/probe_mde_batch11_adapters.py \ --model zoedepth.n --t1-manifest data/t1/nyuv2/n100-s20260914/manifest.json \ --weights-dir ~/bench-cache/weights/mde-batch11 \ --mat ~/depth-probe-20260917-dav2/downloads/nyu_depth_v2_labeled.mat \ --splits ~/depth-probe-20260917-dav2/downloads/splits.mat \ --out ~/depth-probe-20260919-mde-b11 --n 3 """ from __future__ import annotations import argparse import hashlib import importlib import json import os from pathlib import Path import sys import time import traceback sys.path.insert(0, str(Path(__file__).resolve().parents[1])) MODELS = ["zoedepth.n", "zoedepth.k", "zoedepth.nk", "metric3dv2.vit.small", "metric3dv2.vit.large", "metric3dv2.vit.giant2", "metric3d.v1.convnext.tiny", "metric3d.v1.convnext.large", "unidepth.v1.convnext.l", "unidepth.v1.vit.l", "unidepth.v2.vit.s", "unidepth.v2.vit.b", "unidepth.v2.vit.l", "depthpro"] def digest(path: Path) -> str: h = hashlib.sha256() with path.open("rb") as stream: for block in iter(lambda: stream.read(1 << 20), b""): h.update(block) return h.hexdigest() def save(path: Path, value: dict): path.parent.mkdir(parents=True, exist_ok=True) temp = path.with_suffix(path.suffix + ".tmp") temp.write_text(json.dumps(value, indent=2, allow_nan=False)) temp.replace(path) def open_eval_set(t1_manifest: Path) -> dict: from harness.continuation import load_eval_set dataset, evaluation = load_eval_set(t1_manifest) if dataset != "nyuv2" or evaluation.get("n") != 100: raise ValueError(f"expected the frozen nyuv2 n100 T1, got {dataset}/{evaluation.get('n')}") manifest = json.loads(t1_manifest.read_text()) evaluation = dict(evaluation) evaluation["eval_set_id"] = manifest["eval_set_id"] evaluation["content_sha"] = manifest["content_sha256"] evaluation["manifest"] = str(t1_manifest) return evaluation def expected_input_shape(model: str) -> tuple[int, ...]: from harness.eval.mde import FAMILY_INPUT_SPECS return tuple(int(v) for v in FAMILY_INPUT_SPECS[model].split("x")) def expected_output_shape(model: str) -> tuple[int, ...]: from harness.eval import mde spec = mde.FAMILY_OUTPUT_SPECS.get(model) or mde.FAMILY_INPUT_SPECS[model] return (1, *(int(v) for v in spec.split("x")[2:])) def calibration(mat: Path, splits: Path, evaluation: dict, models: list[str], out: Path) -> None: """Deterministic 128-image train-split tensors for each family recipe.""" import h5py import numpy as np import random from PIL import Image from scipy.io import loadmat from harness.adapters import calibration_dir, get, model_spec from harness.registry import model_row ids = loadmat(splits) train, test = [sorted(int(v) for v in ids[key].ravel()) for key in ("trainNdxs", "testNdxs")] if len(train) != 795 or len(test) != 654: raise ValueError("unexpected NYUv2 official split") manifest = json.loads(Path(evaluation["manifest"]).read_text()) raw_sha, split_sha = digest(mat), digest(splits) if (manifest["source"]["raw_sha256"] != raw_sha or manifest["source"]["split_sha256"] != split_sha): raise ValueError("raw NYUv2 material differs from the frozen T1 source") calib_ids = sorted(set(manifest["source"]["calibration_train_ids"])) if len(calib_ids) != 128 or not set(calib_ids) <= set(train): raise ValueError("calibration ids must be 128 official train images") if calib_ids != sorted(random.Random(manifest["source"]["seed"]).sample(train, 128)): raise ValueError("calibration ids are not the recorded seed sample") test_ids = set(manifest["source"]["test_ids"]) with h5py.File(mat, "r") as raw: if raw["images"].shape != (1449, 3, 640, 480): raise ValueError("unexpected NYU original HDF5 dimensions") def read_image(index: int): rgb = np.asarray(raw["images"][index - 1]).transpose(2, 1, 0) if rgb.dtype != np.uint8 or rgb.shape != (480, 640, 3): raise ValueError("invalid NYU RGB") return Image.fromarray(rgb) for model in models: row = model_spec(model_row(model)) recipe = get(model).PREPROCESS target = calibration_dir(row) if (target / "manifest.json").exists(): cm = json.loads((target / "manifest.json").read_text()) files = sorted((target / "npy_f32").glob("*.npy")) h = hashlib.sha256() for p in files: h.update(p.read_bytes()) if (cm.get("count") != 128 or len(files) != 128 or cm.get("preprocess") != recipe or cm.get("sha256") != h.hexdigest() or cm.get("t1_sha256") != digest(Path(evaluation["path"]))): raise ValueError(f"existing calibration differs; refusing to overwrite {target}") print(f"{model}: calibration verified at {target}", flush=True) continue if target.exists(): raise FileExistsError(f"partial calibration; refusing to overwrite {target}") temporary = target.with_name(target.name + f".partial-{os.getpid()}") (temporary / "npy_f32").mkdir(parents=True, exist_ok=False) items, h = [], hashlib.sha256() for index in calib_ids: image = read_image(index) if index in test_ids: raise ValueError("calibration/test split overlap") x = get(model).preprocess(image, row["input_spec"]) if x.shape != expected_input_shape(model) or not np.isfinite(x).all(): raise ValueError("invalid real calibration tensor") path = temporary / "npy_f32" / f"{len(items):05d}.npy" np.save(path, x) h.update(path.read_bytes()) items.append({"id": f"nyuv2/train/{index:04d}", "official_index": index, "rgb_sha256": hashlib.sha256(np.asarray(image).tobytes()).hexdigest(), "npy": path.name, "npy_sha256": digest(path)}) save(temporary / "manifest.json", {"task": "mde", "input_spec": row["input_spec"], "count": 128, "seed": manifest["source"]["seed"], "source": "official-labeled-train", "dataset": "nyuv2", "items": items, "files": [item["id"] for item in items], "sha256": h.hexdigest(), "preprocess": recipe, "npy_dir": "npy_f32", "layout": "NCHW", "dtype": "float32", "test_content_overlap": 0, "t1_sha256": digest(Path(evaluation["path"])), "raw_source": {"raw_mat": str(mat), "raw_sha256": raw_sha, "split_sha256": split_sha}}) temporary.rename(target) print(f"{model}: calibration written at {target}", flush=True) def _official_preprocess_reference(model: str, image): """Independent per-family re-derivation of the official preprocessing on a real image.""" import numpy as np if model.startswith("zoedepth."): import torch import torch.nn.functional as F from torchvision.transforms import Normalize rgb = torch.from_numpy(np.asarray(image.convert("RGB"), dtype=np.float32) / 255.0).permute(2, 0, 1)[None] # official PrepForMidas on a 480x640 input at img_size (384,512), minimal keep-aspect rgb = F.interpolate(rgb, size=(384, 512), mode="bilinear", align_corners=True) return Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])(rgb).numpy() if model.startswith(("metric3d.", "metric3dv2.")): import cv2 from harness.adapters.metric3d_common import deployment_spec variant = {m: v for m, v in (("metric3dv2.vit.small", "v2.vit.small"), ("metric3dv2.vit.large", "v2.vit.large"), ("metric3dv2.vit.giant2", "v2.vit.giant2"), ("metric3d.v1.convnext.tiny", "v1.convnext.tiny"), ("metric3d.v1.convnext.large", "v1.convnext.large"))}[model] spec = deployment_spec(variant) rgb = np.asarray(image.convert("RGB")) scale = spec["scale"] rgb = cv2.resize(rgb, (int(640 * scale), int(480 * scale)), interpolation=cv2.INTER_LINEAR) pt, pb, pl, pr = spec["pad"] rgb = cv2.copyMakeBorder(rgb, pt, pb, pl, pr, cv2.BORDER_CONSTANT, value=[123.675, 116.28, 103.53]) rgb = rgb.astype(np.float32).transpose(2, 0, 1)[None] mean = np.array([123.675, 116.28, 103.53], dtype=np.float32)[:, None, None] std = np.array([58.395, 57.12, 57.375], dtype=np.float32)[:, None, None] return (rgb - mean) / std if model.startswith("unidepth.v1."): import torch import torch.nn.functional as F from torchvision.transforms import functional as TF rgb = torch.from_numpy(np.asarray(image.convert("RGB"), dtype=np.float32).transpose(2, 0, 1)[None]) x = TF.normalize(rgb / 255.0, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) return F.interpolate(x, size=(462, 616), mode="bilinear", align_corners=False, antialias=True).numpy() if model.startswith("unidepth.v2."): import torch import torch.nn.functional as F from torchvision.transforms import functional as TF rgb = torch.from_numpy(np.asarray(image.convert("RGB"), dtype=np.float32).transpose(2, 0, 1)[None]) # official order (normalize, then resize); the deployment graph keeps the # official [0,255] input contract and normalizes inside, so un-normalize # the resized tensor back to the [0,255] scale (linear ops commute up to # float rounding) x = TF.normalize(rgb / 255.0, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) x = F.interpolate(x, size=(518, 518), mode="bilinear", align_corners=False, antialias=True) mean = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1) std = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1) return ((x * std + mean) * 255.0).numpy() if model == "depthpro": import torch import torch.nn.functional as F rgb = torch.from_numpy(np.asarray(image.convert("RGB"), dtype=np.float32) / 255.0).permute(2, 0, 1)[None] rgb = (rgb - 0.5) / 0.5 return F.interpolate(rgb, size=(1536, 1536), mode="bilinear", align_corners=False, antialias=True).numpy() raise ValueError(model) def verify_contract(evaluation: dict, models: list[str], out: Path) -> None: """Independent preprocessing/geometry checks on the frozen T1 package.""" import numpy as np from harness.eval.common import T1 from harness.eval import mde from harness.adapters import get t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) try: total_valid = 0 for item in t1.items: for key, expected in item["file_sha256"].items(): assert hashlib.sha256(t1.tar.extractfile(item[key]).read()).hexdigest() == expected depth, mask = mde.ground_truth(t1, item) total_valid += int(mask.sum()) image = t1.image(t1.items[0]) checks = ["all real RGB/depth member hashes", "original metre GT and fixed crop", "affine inverse-depth exact recovery", "signed LS recovers (-1,3) for depth-direction predictions", "reject constant/NaN prediction"] for model in models: adapter = get(model) reference = _official_preprocess_reference(model, image) produced = adapter.preprocess(image, adapter.INPUT_SPEC) np.testing.assert_allclose(produced, reference, rtol=1e-6, atol=1e-4) truth = np.linspace(.5, 9.5, 10000, dtype=np.float32).reshape(100, 100) relative = (1.0 / truth - .02) / .04 score = mde.aligned_metrics(relative, truth, np.ones_like(truth, dtype=bool)) assert score["absrel"] < 1e-6 and score["delta1"] == 100 signed = mde.aligned_metrics(3.0 - 1.0 / truth, truth, np.ones_like(truth, dtype=bool)) np.testing.assert_allclose([signed["scale"], signed["shift"]], [-1, 3], atol=1e-6) assert signed["absrel"] < 1e-6 and signed["delta1"] == 100 for bad in [np.ones_like(truth), np.full_like(truth, np.nan)]: try: mde.aligned_metrics(bad, truth, np.ones_like(truth, dtype=bool)) except ValueError: pass else: raise AssertionError("degenerate/nonfinite input accepted") shape = expected_output_shape(next(iter(models))) constant = np.full(shape, 2, dtype=np.float32) model0 = next(iter(models)) np.testing.assert_array_equal(mde.restore_prediction([constant], (480, 640), model0), np.full((480, 640), 2)) save(out / "contract_checks.json", {"status": "passed", "t1_sha256": digest(Path(evaluation["path"])), "decoded_real_labels": len(t1.items), "valid_pixels": total_valid, "models": models, "checks": checks + ["per-family official preprocessing equality on a real NYU image", "restore constant depth at the family output shape without vector scaling"]}) finally: t1.tar.close() def probe(model: str, evaluation: dict, weights_dir: Path, out: Path, n: int, confirm: bool, threads: int): import numpy as np import torch import torchvision import onnx import onnxruntime as ort from harness.eval.common import T1, cosine from harness.eval import EVALUATOR_VERSION, mde recipe = importlib.import_module("harness.adapters." + model.replace(".", "_")) directory = out / model directory.mkdir(parents=True, exist_ok=True) weights = weights_dir / recipe.CHECKPOINT.get("filename", recipe.CHECKPOINT["url"].rsplit("/", 1)[1]) result = {"model": model, "status": "running", "started_at": time.time(), "input_spec": recipe.INPUT_SPEC, "checkpoint": recipe.CHECKPOINT, "preprocess": recipe.PREPROCESS, "opset": recipe.OPSET, "exporter_version": recipe.EXPORTER_VERSION, "exporter": getattr(recipe, "EXPORTER", "torchscript"), "weights_path": str(weights), "t1_path": evaluation["path"], "t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation, "evaluation_protocol": mde.PROTOCOL, "runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__, "onnx": onnx.__version__, "onnxruntime": ort.__version__, "device": "cpu", "threads": threads}, "input_contract": {"status": "unverified"}, "board_tested": False, "quantization_tested": False} out_shape = expected_output_shape(model) in_shape = expected_input_shape(model) def phase(name): result["phase"] = name save(directory / "probe.json", result) print(f"{model}: {name}", flush=True) try: common = {"zoedepth": "harness/adapters/zoedepth_common.py", "metric3dv2": "harness/adapters/metric3d_common.py", "metric3d": "harness/adapters/metric3d_common.py", "unidepth": "harness/adapters/unidepth_common.py", "depthpro": "harness/adapters/depthpro.py"} family = next(f for f in common if model.startswith(f)) source_files = [common[family], "harness/adapters/" + model.replace(".", "_") + ".py", "harness/eval/mde.py"] root = Path(__file__).resolve().parents[1] result["source_sha256"] = {name: digest(root / name) for name in source_files} phase("strict_official_load") result["weights_sha256"] = digest(weights) model_object = recipe.build(weights, recipe.INPUT_SPEC) result.update(strict_load=True, parameters=sum(p.numel() for p in model_object.parameters())) t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) try: t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)} if len(t1.items) != n: raise ValueError("requested sample count differs from actual T1 subset") result["sample_ids"] = [item["id"] for item in t1.items] references = {} def native(x): key = hashlib.sha256(x.tobytes()).hexdigest() if key in references: return [references[key]] with torch.inference_mode(): value = model_object(torch.from_numpy(x)).numpy() if value.shape != out_shape or not np.isfinite(value).all(): raise ValueError(f"invalid official forward output: {value.shape} vs {out_shape}") references[key] = value print(f"{model}: FP32 {len(references)}/{n}", flush=True) return [value] phase("real_nyu_fp32_forward") baseline = {"task": "mde", "model": model, "n": n, "evaluator_version": EVALUATOR_VERSION, "t1_manifest_sha": t1.manifest["sha256"]} for item in t1.items: native(mde.inputs(t1, item, recipe.INPUT_SPEC, model)) np.savez(directory / "fp32_predictions.npz", **references) result["fp32_forward_n"] = len(references) first = mde.inputs(t1, t1.items[0], recipe.INPUT_SPEC, model) np.save(directory / "input_first.npy", first) np.save(directory / "fp32_first.npy", references[hashlib.sha256(first.tobytes()).hexdigest()]) phase("onnx_export_and_checker") path = directory / f"{model}.onnx" started = time.monotonic() with torch.no_grad(): torch.onnx.export(model_object, torch.from_numpy(first), str(path), dynamo=getattr(recipe, "EXPORTER", "torchscript") == "dynamo", opset_version=recipe.OPSET, input_names=["input"], output_names=["depth"], dynamic_axes=None) result["onnx"] = {"path": str(path), "sha256": digest(path), "size": path.stat().st_size, "export_seconds": time.monotonic() - started} onnx.checker.check_model(str(path), full_check=True) phase("real_nyu_onnxruntime_and_paired_metrics") options = ort.SessionOptions() options.intra_op_num_threads = threads options.inter_op_num_threads = 1 session = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"]) result["onnx"]["input_shape"] = session.get_inputs()[0].shape result["onnx"]["output_shape"] = session.get_outputs()[0].shape if (result["onnx"]["input_shape"] != list(in_shape) or result["onnx"]["output_shape"] != list(out_shape)): raise ValueError("ONNX static input/output shape mismatch") comparisons = [] converted_outputs = {} result["comparisons"] = comparisons def converted(x): key = hashlib.sha256(x.tobytes()).hexdigest() if key in converted_outputs: return [converted_outputs[key]] value = session.run(None, {"input": x})[0] ref = references[key] difference = np.abs(ref - value) comp = {"sample_id": result["sample_ids"][len(comparisons)], "cosine": cosine(ref, value), "max_abs": float(difference.max()), "mean_abs": float(difference.mean()), "relative_max_abs": float(difference.max() / max(float(np.abs(ref).max()), 1e-12))} comparisons.append(comp) print(json.dumps(comp), flush=True) if not np.isfinite(value).all() or comp["cosine"] < .99999 or comp["relative_max_abs"] > .001: raise ValueError(f"ONNX mismatch: {comp}") converted_outputs[key] = value return [value] for item in t1.items: converted(mde.inputs(t1, item, recipe.INPUT_SPEC, model)) np.savez(directory / "onnx_predictions.npz", **converted_outputs) result["onnxruntime_forward_n"] = len(converted_outputs) phase("real_nyu_alignment_audit") audit, failures = [], [] for item in t1.items: x = mde.inputs(t1, item, recipe.INPUT_SPEC, model) depth, mask = mde.ground_truth(t1, item) sample = {"id": item["id"]} for name, runner in (("fp32", native), ("onnxruntime", converted)): prediction = mde.restore_prediction(runner(x), depth.shape, model) try: sample[name] = {"status": "measured", **mde.aligned_metrics(prediction, depth, mask)} except ValueError as exc: sample[name] = {"status": "blocked", "error": str(exc)} failures.append({"id": item["id"], "runner": name, "error": str(exc)}) audit.append(sample) save(directory / "alignment_audit.json", {"protocol": mde.PROTOCOL, "n": n, "samples": audit, "failures": failures, "status": "blocked" if failures else "passed"}) if failures: result["evaluation_contract"] = {"status": "blocked", "protocol": mde.PROTOCOL, "failures": failures} raise ValueError(f"alignment failed for {len({s['id'] for s in failures})}/{n} real samples") mde.run_fp32(native, t1, recipe.INPUT_SPEC, baseline) result["fp32"] = baseline["metrics"] measured = {"task": "mde", "model": model} mde.run_quant(converted, t1, recipe.INPUT_SPEC, baseline, measured) paired = measured["G3"]["extra"]["scale_alignment"]["samples"] if len(paired) != n or len({sample["id"] for sample in paired}) != n: raise ValueError("incomplete/duplicate per-image alignment proof") for sample, fp, quant in zip(paired, baseline["_depth_samples"], measured["_depth_samples"]): if (sample["id"] != fp["id"] or sample["id"] != quant["id"] or sample["fp32_scale"] != fp["scale"] or sample["fp32_shift"] != fp["shift"] or sample["quant_scale"] != quant["scale"] or sample["quant_shift"] != quant["shift"] or not np.isfinite([sample[key] for key in ("fp32_scale", "fp32_shift", "quant_scale", "quant_shift")]).all()): raise ValueError("invalid/mispaired real per-image alignment proof") if (abs(baseline["metrics"]["absrel"] - measured["metrics"]["absrel"]) > .001 or abs(baseline["metrics"]["delta1"] - measured["metrics"]["delta1"]) > .1): raise ValueError("FP32/ONNX paired depth metric mismatch") checks = json.loads((out / "contract_checks.json").read_text()) if checks.get("status") != "passed" or checks.get("t1_sha256") != result["t1_sha256"]: raise ValueError("missing matching real-data and alignment contract checks") result.update(comparisons=comparisons, onnxruntime=measured["metrics"], G3_probe=measured["G3"], status="validated_fp32_export_only" if confirm else "validated_export_pending_eval_protocol", phase="complete") result["input_contract"] = {"status": "validated", "input_spec": recipe.INPUT_SPEC, "output_semantics": "nonnegative_metric_depth_m_aligned_not_trusted", "t1_sha256": result["t1_sha256"], "checks_path": str(out / "contract_checks.json"), "sample_ids": result["sample_ids"], "onnx_sha256": result["onnx"]["sha256"]} result["evaluation_contract"] = {"status": "validated" if confirm else "pending_mainline_protocol", "protocol": mde.PROTOCOL} baseline["note"] = "probe subset only, not the complete 100-image FP32 baseline" save(directory / "baseline_probe.json", baseline) save(directory / "onnx_metrics_probe.json", measured) finally: t1.tar.close() except Exception as exc: result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") (directory / "error.txt").write_text(traceback.format_exc()) result["finished_at"] = time.time() save(directory / "probe.json", result) print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True) return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model", action="append", choices=MODELS, required=True) parser.add_argument("--t1-manifest", type=Path, required=True, help="manifest.json of the existing frozen nyuv2 n100 T1 package") parser.add_argument("--weights-dir", type=Path, required=True) parser.add_argument("--mat", type=Path, required=True, help="original nyu_depth_v2_labeled.mat") parser.add_argument("--splits", type=Path, required=True, help="official splits.mat") parser.add_argument("--out", type=Path, required=True) parser.add_argument("--n", type=int, default=3) parser.add_argument("--threads", type=int, default=2) parser.add_argument("--confirm-protocol", choices=["nyuv2-relative-inverse-ls-v1"]) args = parser.parse_args() if not 1 <= args.n <= 100 or args.threads < 1: parser.error("require 1 <= n <= 100 and positive threads") os.environ["CUDA_VISIBLE_DEVICES"] = "-1" import torch torch.set_num_threads(args.threads) torch.set_num_interop_threads(1) models = list(dict.fromkeys(args.model)) evaluation = open_eval_set(args.t1_manifest) calibration(args.mat, args.splits, evaluation, models, args.out) verify_contract(evaluation, models, args.out) save(args.out / "data_preparation.json", {"t1_path": evaluation["path"], "t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation, "n": args.n}) results = [probe(model, evaluation, args.weights_dir, args.out, args.n, bool(args.confirm_protocol), args.threads) for model in models] save(args.out / "summary.json", {"results": results, "complete": True}) return int(any(result["status"] == "blocked" for result in results)) if __name__ == "__main__": raise SystemExit(main())