data-gen-bpu/scripts/probe_midas_adapters.py

421 lines
23 KiB
Python

"""Probe the three pinned official MiDaS models on the existing NYUv2 T1.
Same admission evidence as probe_depth_adapters.py: strict official checkpoint
load, real-data FP32 forward, ONNX export plus checker, onnxruntime parity on
real images, per-image alignment audit against the frozen nyuv2 protocol and a
128-image train-split calibration package. No installs, no run/queue mutation,
no quantization and no board pass. Static deployment uses each variant's own
midas.model_loader normalization (v3 DPT mean=std=0.5, v2.1 small ImageNet).
"""
from __future__ import annotations
import argparse
import hashlib
import importlib
import json
import os
from pathlib import Path
import sys
import time
import traceback
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
VARIANTS = {"midas.small": "small", "dpt.hybrid": "hybrid", "dpt.large": "large"}
def digest(path: Path) -> str:
h = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
def save(path: Path, value: dict):
path.parent.mkdir(parents=True, exist_ok=True)
temp = path.with_suffix(path.suffix + ".tmp")
temp.write_text(json.dumps(value, indent=2, allow_nan=False))
temp.replace(path)
def open_eval_set(t1_manifest: Path) -> dict:
from harness.continuation import load_eval_set
dataset, evaluation = load_eval_set(t1_manifest)
if dataset != "nyuv2" or evaluation.get("n") != 100:
raise ValueError(f"expected the frozen nyuv2 n100 T1, got {dataset}/{evaluation.get('n')}")
manifest = json.loads(t1_manifest.read_text())
evaluation = dict(evaluation)
evaluation["eval_set_id"] = manifest["eval_set_id"]
evaluation["content_sha"] = manifest["content_sha256"]
evaluation["manifest"] = str(t1_manifest)
return evaluation
def calibration(mat: Path, splits: Path, evaluation: dict, models: list[str], out: Path) -> None:
"""Deterministic 128-image train-split tensors for each new recipe."""
import h5py
import numpy as np
from PIL import Image
from scipy.io import loadmat
from harness import paths
from harness.adapters import calibration_dir, get, model_spec
from harness.registry import model_row
ids = loadmat(splits)
train, test = [sorted(int(v) for v in ids[key].ravel()) for key in ("trainNdxs", "testNdxs")]
if len(train) != 795 or len(test) != 654:
raise ValueError("unexpected NYUv2 official split")
manifest = json.loads(Path(evaluation["manifest"]).read_text())
raw_sha, split_sha = digest(mat), digest(splits)
if (manifest["source"]["raw_sha256"] != raw_sha or manifest["source"]["split_sha256"] != split_sha):
raise ValueError("raw NYUv2 material differs from the frozen T1 source")
calib_ids = sorted(set(manifest["source"]["calibration_train_ids"]))
if len(calib_ids) != 128 or not set(calib_ids) <= set(train):
raise ValueError("calibration ids must be 128 official train images")
import random
if calib_ids != sorted(random.Random(manifest["source"]["seed"]).sample(train, 128)):
raise ValueError("calibration ids are not the recorded seed sample")
test_ids = set(manifest["source"]["test_ids"])
with h5py.File(mat, "r") as raw:
if raw["images"].shape != (1449, 3, 640, 480):
raise ValueError("unexpected NYU original HDF5 dimensions")
def read_image(index: int):
rgb = np.asarray(raw["images"][index - 1]).transpose(2, 1, 0)
if rgb.dtype != np.uint8 or rgb.shape != (480, 640, 3):
raise ValueError("invalid NYU RGB")
return Image.fromarray(rgb)
for model in models:
row = model_spec(model_row(model))
recipe = get(model).PREPROCESS
target = calibration_dir(row)
if (target / "manifest.json").exists():
cm = json.loads((target / "manifest.json").read_text())
files = sorted((target / "npy_f32").glob("*.npy"))
h = hashlib.sha256()
for p in files:
h.update(p.read_bytes())
if (cm.get("count") != 128 or len(files) != 128 or cm.get("preprocess") != recipe
or cm.get("sha256") != h.hexdigest() or cm.get("t1_sha256") != digest(Path(evaluation["path"]))):
raise ValueError(f"existing calibration differs; refusing to overwrite {target}")
print(f"{model}: calibration verified at {target}", flush=True)
continue
if target.exists():
raise FileExistsError(f"partial calibration; refusing to overwrite {target}")
temporary = target.with_name(target.name + f".partial-{os.getpid()}")
(temporary / "npy_f32").mkdir(parents=True, exist_ok=False)
items, h = [], hashlib.sha256()
for index in calib_ids:
image = read_image(index)
if index in test_ids:
raise ValueError("calibration/test split overlap")
x = get(model).preprocess(image, row["input_spec"])
if x.shape != tuple(int(v) for v in row["input_spec"].split("x")) or not np.isfinite(x).all():
raise ValueError("invalid real calibration tensor")
path = temporary / "npy_f32" / f"{len(items):05d}.npy"
np.save(path, x)
h.update(path.read_bytes())
items.append({"id": f"nyuv2/train/{index:04d}", "official_index": index,
"rgb_sha256": hashlib.sha256(np.asarray(image).tobytes()).hexdigest(),
"npy": path.name, "npy_sha256": digest(path)})
save(temporary / "manifest.json", {"task": "mde", "input_spec": row["input_spec"],
"count": 128, "seed": manifest["source"]["seed"], "source": "official-labeled-train",
"dataset": "nyuv2", "items": items, "files": [item["id"] for item in items],
"sha256": h.hexdigest(), "preprocess": recipe, "npy_dir": "npy_f32",
"layout": "NCHW", "dtype": "float32", "test_content_overlap": 0,
"t1_sha256": digest(Path(evaluation["path"])),
"raw_source": {"raw_mat": str(mat), "raw_sha256": raw_sha, "split_sha256": split_sha}})
temporary.rename(target)
print(f"{model}: calibration written at {target}", flush=True)
def verify_contract(evaluation: dict, models: list[str], out: Path) -> None:
"""Independent geometry/preprocessing checks on the frozen T1 package."""
import numpy as np
from harness.eval.common import T1
from harness.eval import mde
from harness.adapters import get
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
try:
from harness.adapters import midas_common
root = midas_common.source_root()
if str(root) not in sys.path:
sys.path.insert(0, str(root))
import cv2 # noqa: F401 (the pinned midas transform imports cv2)
from midas.transforms import NormalizeImage, PrepareForNet, Resize
import midas.transforms as pinned_transforms
if Path(pinned_transforms.__file__).resolve() != root / "midas/transforms.py":
raise RuntimeError("a different MiDaS source is already imported")
total_valid = 0
for item in t1.items:
for key, expected in item["file_sha256"].items():
assert hashlib.sha256(t1.tar.extractfile(item[key]).read()).hexdigest() == expected
depth, mask = mde.ground_truth(t1, item)
total_valid += int(mask.sum())
image = t1.image(t1.items[0])
for model in models:
adapter = get(model)
variant = VARIANTS[model]
size = int(adapter.INPUT_SPEC.split("x")[-1])
mean, std = (([.5, .5, .5], [.5, .5, .5]) if variant != "small"
else ([.485, .456, .406], [.229, .224, .225]))
native = {"image": np.asarray(image, dtype=np.float64) / 255.0}
native = Resize(size, size, resize_target=None, keep_aspect_ratio=False, ensure_multiple_of=32,
resize_method="upper_bound" if variant == "small" else "minimal",
image_interpolation_method=cv2.INTER_CUBIC)(native)
native = NormalizeImage(mean=mean, std=std)(native)
native = PrepareForNet()(native)["image"][None]
np.testing.assert_array_equal(adapter.preprocess(image, adapter.INPUT_SPEC), native)
# Exact affine inverse depth must recover GT; raw prediction is not metres.
truth = np.linspace(.5, 9.5, 10000, dtype=np.float32).reshape(100, 100)
relative = (1.0 / truth - .02) / .04
score = mde.aligned_metrics(relative, truth, np.ones_like(truth, dtype=bool))
assert score["absrel"] < 1e-6 and score["delta1"] == 100
signed = mde.aligned_metrics(3.0 - 1.0 / truth, truth, np.ones_like(truth, dtype=bool))
np.testing.assert_allclose([signed["scale"], signed["shift"]], [-1, 3], atol=1e-6)
for bad in [np.ones_like(truth), np.full_like(truth, np.nan)]:
try:
mde.aligned_metrics(bad, truth, np.ones_like(truth, dtype=bool))
except ValueError:
pass
else:
raise AssertionError("degenerate/nonfinite input accepted")
for model in models:
size = int(get(model).INPUT_SPEC.split("x")[-1])
constant = np.full((1, size, size), 2, dtype=np.float32)
restored = mde.restore_prediction([constant], (480, 640), model)
np.testing.assert_allclose(restored, np.full((480, 640), 2), atol=1e-5)
save(out / "contract_checks.json", {"status": "passed", "t1_sha256": digest(Path(evaluation["path"])),
"decoded_real_labels": len(t1.items), "valid_pixels": total_valid,
"models": models,
"checks": ["all real RGB/depth member hashes",
"official per-variant static resize/normalization equality (model_loader statistics)",
"original metre GT and fixed crop", "affine inverse-depth exact recovery",
"signed LS recovers (-1,3)", "reject constant/NaN prediction",
"restore constant depth per family shape without vector scaling"]})
finally:
t1.tar.close()
def probe(model: str, evaluation: dict, weights_dir: Path, out: Path, n: int, confirm: bool, threads: int):
import numpy as np
import torch
import torchvision
import onnx
import onnxruntime as ort
from harness.eval.common import T1, cosine
from harness.eval import EVALUATOR_VERSION, mde
recipe = importlib.import_module("harness.adapters." + model.replace(".", "_"))
directory = out / model
directory.mkdir(parents=True, exist_ok=True)
weights = weights_dir / recipe.CHECKPOINT["url"].rsplit("/", 1)[1]
size = int(recipe.INPUT_SPEC.split("x")[-1])
result = {"model": model, "status": "running", "started_at": time.time(), "input_spec": recipe.INPUT_SPEC,
"checkpoint": recipe.CHECKPOINT, "preprocess": recipe.PREPROCESS, "opset": recipe.OPSET,
"exporter_version": recipe.EXPORTER_VERSION,
"weights_path": str(weights), "t1_path": evaluation["path"],
"t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation,
"evaluation_protocol": mde.PROTOCOL,
"runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__,
"onnx": onnx.__version__, "onnxruntime": ort.__version__, "device": "cpu",
"threads": threads},
"input_contract": {"status": "unverified"}, "board_tested": False, "quantization_tested": False}
def phase(name):
result["phase"] = name
save(directory / "probe.json", result)
print(f"{model}: {name}", flush=True)
try:
source_files = ["harness/adapters/midas_common.py",
"harness/adapters/" + model.replace(".", "_") + ".py", "harness/eval/mde.py"]
root = Path(__file__).resolve().parents[1]
result["source_sha256"] = {name: digest(root / name) for name in source_files}
phase("strict_official_load")
result["weights_sha256"] = digest(weights)
model_object = recipe.build(weights, recipe.INPUT_SPEC)
result.update(strict_load=True, parameters=sum(p.numel() for p in model_object.parameters()))
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
try:
t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)}
if len(t1.items) != n:
raise ValueError("requested sample count differs from actual T1 subset")
result["sample_ids"] = [item["id"] for item in t1.items]
references = {}
def native(x):
key = hashlib.sha256(x.tobytes()).hexdigest()
if key in references:
return [references[key]]
with torch.inference_mode():
value = model_object(torch.from_numpy(x)).numpy()
if value.shape != (1, size, size) or not np.isfinite(value).all():
raise ValueError("invalid official forward output")
references[key] = value
print(f"{model}: FP32 {len(references)}/{n}", flush=True)
return [value]
phase("real_nyu_fp32_forward")
baseline = {"task": "mde", "model": model, "n": n,
"evaluator_version": EVALUATOR_VERSION,
"t1_manifest_sha": t1.manifest["sha256"]}
for item in t1.items:
native(mde.inputs(t1, item, recipe.INPUT_SPEC, model))
np.savez(directory / "fp32_predictions.npz", **references)
result["fp32_forward_n"] = len(references)
first = mde.inputs(t1, t1.items[0], recipe.INPUT_SPEC, model)
np.save(directory / "input_first.npy", first)
np.save(directory / "fp32_first.npy", references[hashlib.sha256(first.tobytes()).hexdigest()])
phase("onnx_export_and_checker")
path = directory / f"{model}.onnx"
started = time.monotonic()
with torch.no_grad():
torch.onnx.export(model_object, torch.from_numpy(first), str(path), dynamo=False,
opset_version=recipe.OPSET, input_names=["input"],
output_names=["inverse_depth"], dynamic_axes=None,
do_constant_folding=True)
result["onnx"] = {"path": str(path), "sha256": digest(path), "size": path.stat().st_size,
"export_seconds": time.monotonic() - started}
onnx.checker.check_model(str(path), full_check=True)
phase("real_nyu_onnxruntime_and_paired_metrics")
options = ort.SessionOptions()
options.intra_op_num_threads = threads
options.inter_op_num_threads = 1
session = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"])
result["onnx"]["input_shape"] = session.get_inputs()[0].shape
result["onnx"]["output_shape"] = session.get_outputs()[0].shape
if (result["onnx"]["input_shape"] != [1, 3, size, size]
or result["onnx"]["output_shape"] != [1, size, size]):
raise ValueError("ONNX static input/output shape mismatch")
comparisons = []
converted_outputs = {}
result["comparisons"] = comparisons
def converted(x):
key = hashlib.sha256(x.tobytes()).hexdigest()
if key in converted_outputs:
return [converted_outputs[key]]
value = session.run(None, {"input": x})[0]
ref = references[key]
difference = np.abs(ref - value)
comp = {"sample_id": result["sample_ids"][len(comparisons)], "cosine": cosine(ref, value),
"max_abs": float(difference.max()), "mean_abs": float(difference.mean()),
"relative_max_abs": float(difference.max() / max(float(np.abs(ref).max()), 1e-12))}
comparisons.append(comp)
print(json.dumps(comp), flush=True)
if not np.isfinite(value).all() or comp["cosine"] < .99999 or comp["relative_max_abs"] > .001:
raise ValueError(f"ONNX mismatch: {comp}")
converted_outputs[key] = value
return [value]
for item in t1.items:
converted(mde.inputs(t1, item, recipe.INPUT_SPEC, model))
np.savez(directory / "onnx_predictions.npz", **converted_outputs)
result["onnxruntime_forward_n"] = len(converted_outputs)
phase("real_nyu_alignment_audit")
audit, failures = [], []
for item in t1.items:
x = mde.inputs(t1, item, recipe.INPUT_SPEC, model)
depth, mask = mde.ground_truth(t1, item)
sample = {"id": item["id"]}
for name, runner in (("fp32", native), ("onnxruntime", converted)):
prediction = mde.restore_prediction(runner(x), depth.shape, model)
try:
sample[name] = {"status": "measured", **mde.aligned_metrics(prediction, depth, mask)}
except ValueError as exc:
sample[name] = {"status": "blocked", "error": str(exc)}
failures.append({"id": item["id"], "runner": name, "error": str(exc)})
audit.append(sample)
save(directory / "alignment_audit.json", {"protocol": mde.PROTOCOL, "n": n,
"samples": audit, "failures": failures, "status": "blocked" if failures else "passed"})
if failures:
result["evaluation_contract"] = {"status": "blocked", "protocol": mde.PROTOCOL,
"failures": failures}
raise ValueError(f"alignment failed for {len({s['id'] for s in failures})}/{n} real samples")
mde.run_fp32(native, t1, recipe.INPUT_SPEC, baseline)
result["fp32"] = baseline["metrics"]
measured = {"task": "mde", "model": model}
mde.run_quant(converted, t1, recipe.INPUT_SPEC, baseline, measured)
paired = measured["G3"]["extra"]["scale_alignment"]["samples"]
if len(paired) != n or len({sample["id"] for sample in paired}) != n:
raise ValueError("incomplete/duplicate per-image alignment proof")
for sample, fp, quant in zip(paired, baseline["_depth_samples"], measured["_depth_samples"]):
if (sample["id"] != fp["id"] or sample["id"] != quant["id"]
or sample["fp32_scale"] != fp["scale"] or sample["fp32_shift"] != fp["shift"]
or sample["quant_scale"] != quant["scale"] or sample["quant_shift"] != quant["shift"]
or not np.isfinite([sample[key] for key in
("fp32_scale", "fp32_shift", "quant_scale", "quant_shift")]).all()):
raise ValueError("invalid/mispaired real per-image alignment proof")
if (abs(baseline["metrics"]["absrel"] - measured["metrics"]["absrel"]) > .001
or abs(baseline["metrics"]["delta1"] - measured["metrics"]["delta1"]) > .1):
raise ValueError("FP32/ONNX paired depth metric mismatch")
checks = json.loads((out / "contract_checks.json").read_text())
if checks.get("status") != "passed" or checks.get("t1_sha256") != result["t1_sha256"]:
raise ValueError("missing matching real-data and alignment contract checks")
result.update(comparisons=comparisons, onnxruntime=measured["metrics"], G3_probe=measured["G3"],
status="validated_fp32_export_only" if confirm else "validated_export_pending_eval_protocol",
phase="complete")
result["input_contract"] = {"status": "validated", "input_spec": recipe.INPUT_SPEC,
"output_semantics": "nonnegative_relative_inverse_depth_not_metric",
"t1_sha256": result["t1_sha256"], "checks_path": str(out / "contract_checks.json"),
"sample_ids": result["sample_ids"], "onnx_sha256": result["onnx"]["sha256"]}
result["evaluation_contract"] = {"status": "validated" if confirm else "pending_mainline_protocol",
"protocol": mde.PROTOCOL}
baseline["note"] = "probe subset only, not the complete 100-image FP32 baseline"
save(directory / "baseline_probe.json", baseline)
save(directory / "onnx_metrics_probe.json", measured)
finally:
t1.tar.close()
except Exception as exc:
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
(directory / "error.txt").write_text(traceback.format_exc())
result["finished_at"] = time.time()
save(directory / "probe.json", result)
print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", action="append", choices=sorted(VARIANTS), required=True)
parser.add_argument("--t1-manifest", type=Path, required=True,
help="manifest.json of the existing frozen nyuv2 n100 T1 package")
parser.add_argument("--weights-dir", type=Path, required=True)
parser.add_argument("--data-reader", type=Path,
help="private unpacked h5py/scipy wheels used only to read the source MAT")
parser.add_argument("--mat", type=Path, required=True, help="original nyu_depth_v2_labeled.mat")
parser.add_argument("--splits", type=Path, required=True, help="official splits.mat")
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--n", type=int, default=3)
parser.add_argument("--threads", type=int, default=2)
parser.add_argument("--confirm-protocol", choices=["nyuv2-relative-inverse-ls-v1"])
args = parser.parse_args()
if not 1 <= args.n <= 100 or args.threads < 1:
parser.error("require 1 <= n <= 100 and positive threads")
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import torch
torch.set_num_threads(args.threads)
torch.set_num_interop_threads(1)
if args.data_reader:
sys.path.insert(0, str(args.data_reader.resolve()))
models = list(dict.fromkeys(args.model))
evaluation = open_eval_set(args.t1_manifest)
from harness.adapters import midas_common
midas_common.source_root()
calibration(args.mat, args.splits, evaluation, models, args.out)
verify_contract(evaluation, models, args.out)
save(args.out / "data_preparation.json", {"t1_path": evaluation["path"],
"t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation, "n": args.n})
results = [probe(model, evaluation, args.weights_dir, args.out, args.n,
bool(args.confirm_protocol), args.threads) for model in models]
save(args.out / "summary.json", {"results": results, "complete": True})
return int(any(result["status"] == "blocked" for result in results))
if __name__ == "__main__":
raise SystemExit(main())