"""Prepare original NYUv2 data and probe the three official relative DA-V2 models. All work is private except an explicitly supplied existing --calib-root. No package install, run/queue mutation, quantization, or board pass is performed. --confirm-protocol is an explicit mainline choice, not an official-model claim. """ from __future__ import annotations import argparse import hashlib import importlib import io import json import os from pathlib import Path import random import sys import tarfile import time import traceback sys.path.insert(0, str(Path(__file__).resolve().parents[1])) def digest(path: Path) -> str: h = hashlib.sha256() with path.open("rb") as stream: for block in iter(lambda: stream.read(1 << 20), b""): h.update(block) return h.hexdigest() def save(path: Path, value: dict): path.parent.mkdir(parents=True, exist_ok=True) temp = path.with_suffix(path.suffix + ".tmp") temp.write_text(json.dumps(value, indent=2, allow_nan=False)) temp.replace(path) def data(mat: Path, splits: Path, out: Path, calib_root: Path, seed: int): import h5py import numpy as np from PIL import Image from scipy.io import loadmat from harness.adapters.depth_anything_v2_common import INPUT_SPEC, PREPROCESS, preprocess from harness.eval.mde import PROTOCOL ids = loadmat(splits) train, test = [sorted(int(v) for v in ids[key].ravel()) for key in ("trainNdxs", "testNdxs")] if len(train) != 795 or len(test) != 654 or set(train) & set(test) or sorted(train + test) != list(range(1, 1450)): raise ValueError("unexpected NYUv2 official split; require disjoint 795 train / 654 test") selected = sorted(random.Random(seed).sample(test, 100)) calib_ids = sorted(random.Random(seed).sample(train, 128)) raw_sha, split_sha = digest(mat), digest(splits) source = {"dataset": "nyuv2", "raw_mat": str(mat), "raw_sha256": raw_sha, "split_file": str(splits), "split_sha256": split_sha, "test_total": len(test), "train_total": len(train), "unit": "m", "depth_field": "depths", "matlab_indices_are_one_based": True, "test_ids": selected, "calibration_train_ids": calib_ids, "split_overlap": 0, "seed": seed} destination = out / f"t1-n100-s{seed}" destination.mkdir(parents=True, exist_ok=True) package = destination / f"{seed}.tar" recipe = hashlib.sha256(json.dumps(PREPROCESS, sort_keys=True).encode()).hexdigest()[:16] calibration = calib_root / "mde" / INPUT_SPEC / recipe if (destination / "manifest.json").exists() and (calibration / "manifest.json").exists(): old = json.loads((destination / "manifest.json").read_text()) cm = json.loads((calibration / "manifest.json").read_text()) files = sorted((calibration / "npy_f32").glob("*.npy")) h = hashlib.sha256() for p in files: h.update(p.read_bytes()) if (old["source"]["raw_sha256"] != raw_sha or old["source"]["split_sha256"] != split_sha or old.get("evaluation_protocol") != PROTOCOL or old["sha256"] != digest(package) or cm["count"] != 128 or len(files) != 128 or cm["preprocess"] != PREPROCESS or cm["sha256"] != h.hexdigest()): raise ValueError("existing depth data differs; refusing to overwrite") return package, calibration if package.exists() or calibration.exists(): raise FileExistsError("partial/existing data; choose a new private --out/calib-root") temporary = calibration.with_name(calibration.name + f".partial-{os.getpid()}") (temporary / "npy_f32").mkdir(parents=True, exist_ok=False) items, members, image_hashes = [], {}, set() calibration_items, chash = [], hashlib.sha256() with h5py.File(mat, "r") as raw: if raw["images"].shape != (1449, 3, 640, 480) or raw["depths"].shape != (1449, 640, 480): raise ValueError("unexpected NYU original HDF5 dimensions") def read_image(index): rgb = np.asarray(raw["images"][index - 1]).transpose(2, 1, 0) if rgb.dtype != np.uint8 or rgb.shape != (480, 640, 3): raise ValueError("invalid NYU RGB") return Image.fromarray(rgb), hashlib.sha256(rgb.tobytes()).hexdigest() for index in selected: image, image_sha = read_image(index) if image_sha in image_hashes: raise ValueError("duplicate test RGB content") image_hashes.add(image_sha) depth = np.ascontiguousarray(raw["depths"][index - 1].T, dtype=np.float32) if not np.isfinite(depth).all() or not np.any((depth > 0) & (depth < 10)): raise ValueError("invalid original metre depth") image_buf, depth_buf = io.BytesIO(), io.BytesIO() image.save(image_buf, format="PNG") np.save(depth_buf, depth) paths = {"file": f"images/{index:04d}.png", "depth": f"depth/{index:04d}.npy"} members[paths["file"]], members[paths["depth"]] = image_buf.getvalue(), depth_buf.getvalue() items.append({"id": f"nyuv2/test/{index:04d}", **paths, "official_index": index, "depth_unit": "m", "depth_format": "npy_float32_m", "rgb_sha256": image_sha, "file_sha256": {key: hashlib.sha256(members[value]).hexdigest() for key, value in paths.items()}}) used = set() for index in calib_ids: image, image_sha = read_image(index) if image_sha in image_hashes or image_sha in used: raise ValueError("calibration/test content overlap or duplicate calibration RGB") used.add(image_sha) path = temporary / "npy_f32" / f"{len(calibration_items):05d}.npy" x = preprocess(image) if x.shape != (1, 3, 518, 518) or not np.isfinite(x).all(): raise ValueError("invalid real calibration tensor") np.save(path, x) chash.update(path.read_bytes()) calibration_items.append({"id": f"nyuv2/train/{index:04d}", "official_index": index, "rgb_sha256": image_sha, "npy": path.name, "npy_sha256": digest(path)}) content_sha = hashlib.sha256(json.dumps({"items": items, "protocol": PROTOCOL, "source": source}, sort_keys=True).encode()).hexdigest() manifest = {"dataset": "nyuv2", "version": "test-654", "seed": seed, "n": 100, "sha256": content_sha, "items": items, "source": source, "eval_set_id": f"nyuv2-relative-n100-{content_sha[:12]}", "evaluation_protocol": PROTOCOL} members["manifest.json"] = json.dumps(manifest, sort_keys=True).encode() with tarfile.open(package.with_suffix(".tmp"), "w") as archive: for name, value in sorted(members.items()): info = tarfile.TarInfo(name) info.size = len(value) archive.addfile(info, io.BytesIO(value)) package.with_suffix(".tmp").replace(package) save(destination / "manifest.json", {key: value for key, value in manifest.items() if key != "items"} | {"sha256": digest(package), "content_sha256": content_sha, "tar": package.name}) save(temporary / "manifest.json", {"task": "mde", "input_spec": INPUT_SPEC, "count": 128, "seed": seed, "source": "official-labeled-train", "dataset": "nyuv2", "items": calibration_items, "files": [item["id"] for item in calibration_items], "sha256": chash.hexdigest(), "preprocess": PREPROCESS, "npy_dir": "npy_f32", "layout": "NCHW", "dtype": "float32", "t1_sha256": digest(package), "test_content_overlap": 0, "raw_source": source}) temporary.rename(calibration) save(out / "dataset_checks.json", source | {"status": "passed", "t1_images": 100, "t1_depth_maps": 100, "calibration_images": 128, "test_calibration_content_overlap": 0, "t1_path": str(package), "t1_sha256": digest(package), "calibration_dir": str(calibration)}) return package, calibration def verify_contract(package: Path, out: Path): import numpy as np from harness.eval.common import T1 from harness.eval import mde from harness.adapters import depth_anything_v2_common as recipe from depth_anything_v2.util.transform import Resize, NormalizeImage, PrepareForNet import cv2 t1 = T1.open(package) try: sample = t1.items[0] image = t1.image(sample) native = {"image": np.asarray(image, dtype=np.float64) / 255.0} native = Resize(518, 518, resize_target=False, keep_aspect_ratio=False, ensure_multiple_of=14, resize_method="lower_bound", image_interpolation_method=cv2.INTER_CUBIC)(native) native = NormalizeImage(mean=[.485, .456, .406], std=[.229, .224, .225])(native) native = PrepareForNet()(native)["image"][None] np.testing.assert_array_equal(recipe.preprocess(image), native) total_valid = 0 for item in t1.items: for key, expected in item["file_sha256"].items(): assert hashlib.sha256(t1.tar.extractfile(item[key]).read()).hexdigest() == expected truth, mask = mde.ground_truth(t1, item) total_valid += int(mask.sum()) # Exact affine inverse depth must recover GT; raw prediction is not metres. truth = np.linspace(.5, 9.5, 10000, dtype=np.float32).reshape(100, 100) relative = (1.0 / truth - .02) / .04 score = mde.aligned_metrics(relative, truth, np.ones_like(truth, dtype=bool)) assert score["absrel"] < 1e-6 and score["delta1"] == 100 signed = mde.aligned_metrics(3.0 - 1.0 / truth, truth, np.ones_like(truth, dtype=bool)) np.testing.assert_allclose([signed["scale"], signed["shift"]], [-1, 3], atol=1e-6) assert signed["absrel"] < 1e-6 and signed["delta1"] == 100 for bad in [np.ones_like(truth), np.full_like(truth, np.nan)]: try: mde.aligned_metrics(bad, truth, np.ones_like(truth, dtype=bool)) except ValueError: pass else: raise AssertionError("degenerate/nonfinite input accepted") constant = np.full((1, 518, 518), 2, dtype=np.float32) np.testing.assert_array_equal(mde.restore_prediction([constant], (480, 640)), np.full((480, 640), 2)) save(out / "contract_checks.json", {"status": "passed", "t1_sha256": digest(package), "decoded_real_labels": len(t1.items), "valid_pixels": total_valid, "checks": ["all real RGB/depth member hashes", "official static resize/normalization equality", "original metre GT and fixed crop", "affine inverse-depth exact recovery", "signed LS recovers (-1,3)", "reject constant/NaN prediction", "restore constant depth without vector scaling"]}) finally: t1.tar.close() def probe(model: str, package: Path, weights_dir: Path, out: Path, n: int, confirm: bool, threads: int, reuse_onnx: bool = False): import numpy as np import torch import torchvision import onnx import onnxruntime as ort from harness.eval.common import T1, cosine from harness.eval import EVALUATOR_VERSION, mde recipe = importlib.import_module("harness.adapters." + model.replace(".", "_")) directory = out / model if n == 3 else out / model / f"n{n}" directory.mkdir(parents=True, exist_ok=True) weights = weights_dir / recipe.CHECKPOINT["url"].rsplit("/", 1)[1] result = {"model": model, "status": "running", "started_at": time.time(), "input_spec": recipe.INPUT_SPEC, "checkpoint": recipe.CHECKPOINT, "preprocess": recipe.PREPROCESS, "opset": recipe.OPSET, "weights_path": str(weights), "t1_path": str(package), "t1_sha256": digest(package), "evaluation_protocol": mde.PROTOCOL, "runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__, "onnx": onnx.__version__, "onnxruntime": ort.__version__, "device": "cpu", "threads": threads}, "input_contract": {"status": "unverified"}, "board_tested": False, "quantization_tested": False} def phase(name): result["phase"] = name save(directory / "probe.json", result) print(f"{model}: {name}", flush=True) try: source_files = ["harness/adapters/depth_anything_v2_common.py", "harness/adapters/" + model.replace(".", "_") + ".py", "harness/eval/mde.py"] root = Path(__file__).resolve().parents[1] result["source_sha256"] = {name: digest(root / name) for name in source_files} phase("strict_official_load") result["weights_sha256"] = digest(weights) model_object = recipe.build(weights, recipe.INPUT_SPEC) result.update(strict_load=True, parameters=sum(p.numel() for p in model_object.parameters())) t1 = T1.open(package, result["t1_sha256"]) try: t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)} if len(t1.items) != n: raise ValueError("requested sample count differs from actual T1 subset") result["sample_ids"] = [item["id"] for item in t1.items] references = {} def native(x): key = hashlib.sha256(x.tobytes()).hexdigest() if key in references: return [references[key]] with torch.inference_mode(): value = model_object(torch.from_numpy(x)).numpy() if value.shape != (1, 518, 518) or not np.isfinite(value).all(): raise ValueError("invalid official forward output") references[key] = value print(f"{model}: FP32 {len(references)}/{n}", flush=True) return [value] phase("real_nyu_fp32_forward") baseline = {"task": "mde", "model": model, "n": n, "evaluator_version": EVALUATOR_VERSION, "t1_manifest_sha": t1.manifest["sha256"]} for item in t1.items: native(mde.inputs(t1, item, recipe.INPUT_SPEC, model)) np.savez(directory / "fp32_predictions.npz", **references) result["fp32_forward_n"] = len(references) first = mde.inputs(t1, t1.items[0], recipe.INPUT_SPEC, model) np.save(directory / "input_first.npy", first) np.save(directory / "fp32_first.npy", references[hashlib.sha256(first.tobytes()).hexdigest()]) if n == 100: xs = {f"s{i:03d}": mde.inputs(t1, item, recipe.INPUT_SPEC, model) for i, item in enumerate(t1.g2_subset(32))} np.savez(directory / "g2_inputs.npz", **xs) np.savez(directory / "g2_fp32.npz", **{ f"{key}/out0": references[hashlib.sha256(x.tobytes()).hexdigest()] for key, x in xs.items()}) phase("onnx_reuse_and_checker" if reuse_onnx else "onnx_export_and_checker") if reuse_onnx: previous_path = out / model / "probe.json" previous = json.loads(previous_path.read_text()) path = out / model / f"{model}.onnx" if (previous.get("input_contract", {}).get("status") != "validated" or not previous.get("strict_load") or previous["weights_sha256"] != result["weights_sha256"] or previous["input_spec"] != recipe.INPUT_SPEC or previous["opset"] != recipe.OPSET or any(previous["source_sha256"][name] != result["source_sha256"][name] for name in source_files[:2]) or digest(path) != previous["onnx"]["sha256"]): raise ValueError("existing ONNX identity/adapter/weights proof mismatch") result["onnx"] = {**previous["onnx"], "path": str(path), "reused_from": str(previous_path)} else: path = directory / f"{model}.onnx" started = time.monotonic() with torch.no_grad(): torch.onnx.export(model_object, torch.from_numpy(first), str(path), dynamo=False, opset_version=recipe.OPSET, input_names=["input"], output_names=["inverse_depth"], dynamic_axes=None, do_constant_folding=True) result["onnx"] = {"path": str(path), "sha256": digest(path), "size": path.stat().st_size, "export_seconds": time.monotonic() - started} onnx.checker.check_model(str(path), full_check=True) phase("real_nyu_onnxruntime_and_paired_metrics") options = ort.SessionOptions() options.intra_op_num_threads = threads options.inter_op_num_threads = 1 session = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"]) result["onnx"]["input_shape"] = session.get_inputs()[0].shape result["onnx"]["output_shape"] = session.get_outputs()[0].shape if result["onnx"]["input_shape"] != [1, 3, 518, 518] or result["onnx"]["output_shape"] != [1, 518, 518]: raise ValueError("ONNX static input/output shape mismatch") comparisons = [] converted_outputs = {} result["comparisons"] = comparisons def converted(x): key = hashlib.sha256(x.tobytes()).hexdigest() if key in converted_outputs: return [converted_outputs[key]] value = session.run(None, {"input": x})[0] ref = references[key] difference = np.abs(ref - value) comp = {"sample_id": result["sample_ids"][len(comparisons)], "cosine": cosine(ref, value), "max_abs": float(difference.max()), "mean_abs": float(difference.mean()), "relative_max_abs": float(difference.max() / max(float(np.abs(ref).max()), 1e-12))} comparisons.append(comp) print(json.dumps(comp), flush=True) if not np.isfinite(value).all() or comp["cosine"] < .99999 or comp["relative_max_abs"] > .001: raise ValueError(f"ONNX mismatch: {comp}") converted_outputs[key] = value return [value] for item in t1.items: converted(mde.inputs(t1, item, recipe.INPUT_SPEC, model)) np.savez(directory / "onnx_predictions.npz", **converted_outputs) result["onnxruntime_forward_n"] = len(converted_outputs) # Audit every real image even if one alignment fails; never omit it from a score. phase("real_nyu_alignment_audit") audit, failures = [], [] for item in t1.items: x = mde.inputs(t1, item, recipe.INPUT_SPEC, model) depth, mask = mde.ground_truth(t1, item) sample = {"id": item["id"]} for name, runner in (("fp32", native), ("onnxruntime", converted)): prediction = mde.restore_prediction(runner(x), depth.shape) try: sample[name] = {"status": "measured", **mde.aligned_metrics(prediction, depth, mask)} except ValueError as exc: sample[name] = {"status": "blocked", "error": str(exc)} failures.append({"id": item["id"], "runner": name, "error": str(exc)}) audit.append(sample) save(directory / "alignment_audit.json", {"protocol": mde.PROTOCOL, "n": n, "samples": audit, "failures": failures, "status": "blocked" if failures else "passed"}) if failures: result["evaluation_contract"] = {"status": "blocked", "protocol": mde.PROTOCOL, "failures": failures} raise ValueError(f"alignment failed for {len({s['id'] for s in failures})}/{n} real samples") mde.run_fp32(native, t1, recipe.INPUT_SPEC, baseline) result["fp32"] = baseline["metrics"] if n == 100: save(directory / "metrics.json", baseline) measured = {"task": "mde", "model": model} mde.run_quant(converted, t1, recipe.INPUT_SPEC, baseline, measured) paired = measured["G3"]["extra"]["scale_alignment"]["samples"] if len(paired) != n or len({sample["id"] for sample in paired}) != n: raise ValueError("incomplete/duplicate per-image alignment proof") for sample, fp, quant in zip(paired, baseline["_depth_samples"], measured["_depth_samples"]): if (sample["id"] != fp["id"] or sample["id"] != quant["id"] or sample["fp32_scale"] != fp["scale"] or sample["fp32_shift"] != fp["shift"] or sample["quant_scale"] != quant["scale"] or sample["quant_shift"] != quant["shift"] or not np.isfinite([sample[key] for key in ("fp32_scale", "fp32_shift", "quant_scale", "quant_shift")]).all()): raise ValueError("invalid/mispaired real per-image alignment proof") if abs(baseline["metrics"]["absrel"] - measured["metrics"]["absrel"]) > .001 or abs(baseline["metrics"]["delta1"] - measured["metrics"]["delta1"]) > .1: raise ValueError("FP32/ONNX paired depth metric mismatch") checks = json.loads((out / "contract_checks.json").read_text()) if checks.get("status") != "passed" or checks.get("t1_sha256") != result["t1_sha256"]: raise ValueError("missing matching real-data and alignment contract checks") result.update(comparisons=comparisons, onnxruntime=measured["metrics"], G3_probe=measured["G3"], status="validated_fp32_export_only" if confirm else "validated_export_pending_eval_protocol", phase="complete") result["input_contract"] = {"status": "validated", "input_spec": recipe.INPUT_SPEC, "output_semantics": "affine_invariant_inverse_depth_not_metric", "t1_sha256": result["t1_sha256"], "checks_path": str(out / "contract_checks.json"), "sample_ids": result["sample_ids"], "onnx_sha256": result["onnx"]["sha256"]} result["evaluation_contract"] = {"status": "validated" if confirm else "pending_mainline_protocol", "protocol": mde.PROTOCOL} baseline["note"] = ("complete fixed 100-image FP32 baseline; bench protocol, not paper scores" if n == 100 else "probe subset only, not the complete 100-image FP32 baseline") save(directory / "baseline_probe.json", baseline) save(directory / "onnx_metrics_probe.json", measured) finally: t1.tar.close() except Exception as exc: result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") (directory / "error.txt").write_text(traceback.format_exc()) result["finished_at"] = time.time() save(directory / "probe.json", result) print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True) return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model", action="append", choices=[f"depth.anything.v2.{v}" for v in ("small", "base", "large")], required=True) parser.add_argument("--source", type=Path, required=True) parser.add_argument("--data-reader", type=Path, help="private unpacked h5py/scipy wheels, only used to convert source MAT") parser.add_argument("--mat", type=Path, required=True) parser.add_argument("--splits", type=Path, required=True) parser.add_argument("--weights-dir", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--calib-root", type=Path) parser.add_argument("--n", type=int, default=3) parser.add_argument("--seed", type=int, default=20260914) parser.add_argument("--threads", type=int, default=2) parser.add_argument("--confirm-protocol", choices=["nyuv2-relative-inverse-ls-v1"]) parser.add_argument("--reuse-onnx", action="store_true", help="verify and reuse the original three-image ONNX proof") parser.add_argument("--prepare-only", action="store_true") args = parser.parse_args() if not 1 <= args.n <= 100 or args.threads < 1: parser.error("require 1 <= n <= 100 and positive threads") if args.reuse_onnx and args.n == 3: parser.error("reuse requires a separate sample count to preserve the original three-image proof") os.environ["CUDA_VISIBLE_DEVICES"] = "-1" os.environ["BENCH_DEPTH_ANYTHING_V2_SOURCE"] = str(args.source.resolve()) if args.data_reader: sys.path.insert(0, str(args.data_reader.resolve())) sys.path.insert(0, str(args.source.resolve())) import torch torch.set_num_threads(args.threads) torch.set_num_interop_threads(1) from harness.adapters.depth_anything_v2_common import source_root source_root() package, calibration = data(args.mat, args.splits, args.out, args.calib_root or args.out / "data/calib", args.seed) verify_contract(package, args.out) save(args.out / "data_preparation.json", {"t1_path": str(package), "t1_sha256": digest(package), "n": 100, "calibration_dir": str(calibration), "calibration_n": 128}) if args.prepare_only: return 0 results = [probe(model, package, args.weights_dir, args.out, args.n, bool(args.confirm_protocol), args.threads, args.reuse_onnx) for model in dict.fromkeys(args.model)] save(args.out / ("summary.json" if args.n == 3 else f"summary-n{args.n}.json"), {"results": results, "complete": True}) return int(any(result["status"] == "blocked" for result in results)) if __name__ == "__main__": raise SystemExit(main())