data-gen-bpu/scripts/probe_depth_adapters.py

433 lines
26 KiB
Python

"""Prepare original NYUv2 data and probe the three official relative DA-V2 models.
All work is private except an explicitly supplied existing --calib-root. No
package install, run/queue mutation, quantization, or board pass is performed.
--confirm-protocol is an explicit mainline choice, not an official-model claim.
"""
from __future__ import annotations
import argparse
import hashlib
import importlib
import io
import json
import os
from pathlib import Path
import random
import sys
import tarfile
import time
import traceback
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
def digest(path: Path) -> str:
h = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
def save(path: Path, value: dict):
path.parent.mkdir(parents=True, exist_ok=True)
temp = path.with_suffix(path.suffix + ".tmp")
temp.write_text(json.dumps(value, indent=2, allow_nan=False))
temp.replace(path)
def data(mat: Path, splits: Path, out: Path, calib_root: Path, seed: int):
import h5py
import numpy as np
from PIL import Image
from scipy.io import loadmat
from harness.adapters.depth_anything_v2_common import INPUT_SPEC, PREPROCESS, preprocess
from harness.eval.mde import PROTOCOL
ids = loadmat(splits)
train, test = [sorted(int(v) for v in ids[key].ravel()) for key in ("trainNdxs", "testNdxs")]
if len(train) != 795 or len(test) != 654 or set(train) & set(test) or sorted(train + test) != list(range(1, 1450)):
raise ValueError("unexpected NYUv2 official split; require disjoint 795 train / 654 test")
selected = sorted(random.Random(seed).sample(test, 100))
calib_ids = sorted(random.Random(seed).sample(train, 128))
raw_sha, split_sha = digest(mat), digest(splits)
source = {"dataset": "nyuv2", "raw_mat": str(mat), "raw_sha256": raw_sha,
"split_file": str(splits), "split_sha256": split_sha,
"test_total": len(test), "train_total": len(train), "unit": "m", "depth_field": "depths",
"matlab_indices_are_one_based": True, "test_ids": selected, "calibration_train_ids": calib_ids,
"split_overlap": 0, "seed": seed}
destination = out / f"t1-n100-s{seed}"
destination.mkdir(parents=True, exist_ok=True)
package = destination / f"{seed}.tar"
recipe = hashlib.sha256(json.dumps(PREPROCESS, sort_keys=True).encode()).hexdigest()[:16]
calibration = calib_root / "mde" / INPUT_SPEC / recipe
if (destination / "manifest.json").exists() and (calibration / "manifest.json").exists():
old = json.loads((destination / "manifest.json").read_text())
cm = json.loads((calibration / "manifest.json").read_text())
files = sorted((calibration / "npy_f32").glob("*.npy"))
h = hashlib.sha256()
for p in files:
h.update(p.read_bytes())
if (old["source"]["raw_sha256"] != raw_sha or old["source"]["split_sha256"] != split_sha
or old.get("evaluation_protocol") != PROTOCOL
or old["sha256"] != digest(package) or cm["count"] != 128 or len(files) != 128
or cm["preprocess"] != PREPROCESS or cm["sha256"] != h.hexdigest()):
raise ValueError("existing depth data differs; refusing to overwrite")
return package, calibration
if package.exists() or calibration.exists():
raise FileExistsError("partial/existing data; choose a new private --out/calib-root")
temporary = calibration.with_name(calibration.name + f".partial-{os.getpid()}")
(temporary / "npy_f32").mkdir(parents=True, exist_ok=False)
items, members, image_hashes = [], {}, set()
calibration_items, chash = [], hashlib.sha256()
with h5py.File(mat, "r") as raw:
if raw["images"].shape != (1449, 3, 640, 480) or raw["depths"].shape != (1449, 640, 480):
raise ValueError("unexpected NYU original HDF5 dimensions")
def read_image(index):
rgb = np.asarray(raw["images"][index - 1]).transpose(2, 1, 0)
if rgb.dtype != np.uint8 or rgb.shape != (480, 640, 3):
raise ValueError("invalid NYU RGB")
return Image.fromarray(rgb), hashlib.sha256(rgb.tobytes()).hexdigest()
for index in selected:
image, image_sha = read_image(index)
if image_sha in image_hashes:
raise ValueError("duplicate test RGB content")
image_hashes.add(image_sha)
depth = np.ascontiguousarray(raw["depths"][index - 1].T, dtype=np.float32)
if not np.isfinite(depth).all() or not np.any((depth > 0) & (depth < 10)):
raise ValueError("invalid original metre depth")
image_buf, depth_buf = io.BytesIO(), io.BytesIO()
image.save(image_buf, format="PNG")
np.save(depth_buf, depth)
paths = {"file": f"images/{index:04d}.png", "depth": f"depth/{index:04d}.npy"}
members[paths["file"]], members[paths["depth"]] = image_buf.getvalue(), depth_buf.getvalue()
items.append({"id": f"nyuv2/test/{index:04d}", **paths, "official_index": index,
"depth_unit": "m", "depth_format": "npy_float32_m", "rgb_sha256": image_sha,
"file_sha256": {key: hashlib.sha256(members[value]).hexdigest() for key, value in paths.items()}})
used = set()
for index in calib_ids:
image, image_sha = read_image(index)
if image_sha in image_hashes or image_sha in used:
raise ValueError("calibration/test content overlap or duplicate calibration RGB")
used.add(image_sha)
path = temporary / "npy_f32" / f"{len(calibration_items):05d}.npy"
x = preprocess(image)
if x.shape != (1, 3, 518, 518) or not np.isfinite(x).all():
raise ValueError("invalid real calibration tensor")
np.save(path, x)
chash.update(path.read_bytes())
calibration_items.append({"id": f"nyuv2/train/{index:04d}", "official_index": index,
"rgb_sha256": image_sha, "npy": path.name, "npy_sha256": digest(path)})
content_sha = hashlib.sha256(json.dumps({"items": items, "protocol": PROTOCOL, "source": source}, sort_keys=True).encode()).hexdigest()
manifest = {"dataset": "nyuv2", "version": "test-654", "seed": seed, "n": 100,
"sha256": content_sha, "items": items, "source": source,
"eval_set_id": f"nyuv2-relative-n100-{content_sha[:12]}", "evaluation_protocol": PROTOCOL}
members["manifest.json"] = json.dumps(manifest, sort_keys=True).encode()
with tarfile.open(package.with_suffix(".tmp"), "w") as archive:
for name, value in sorted(members.items()):
info = tarfile.TarInfo(name)
info.size = len(value)
archive.addfile(info, io.BytesIO(value))
package.with_suffix(".tmp").replace(package)
save(destination / "manifest.json", {key: value for key, value in manifest.items() if key != "items"} |
{"sha256": digest(package), "content_sha256": content_sha, "tar": package.name})
save(temporary / "manifest.json", {"task": "mde", "input_spec": INPUT_SPEC, "count": 128,
"seed": seed, "source": "official-labeled-train", "dataset": "nyuv2", "items": calibration_items,
"files": [item["id"] for item in calibration_items], "sha256": chash.hexdigest(),
"preprocess": PREPROCESS, "npy_dir": "npy_f32", "layout": "NCHW", "dtype": "float32",
"t1_sha256": digest(package), "test_content_overlap": 0, "raw_source": source})
temporary.rename(calibration)
save(out / "dataset_checks.json", source | {"status": "passed", "t1_images": 100,
"t1_depth_maps": 100, "calibration_images": 128, "test_calibration_content_overlap": 0,
"t1_path": str(package), "t1_sha256": digest(package), "calibration_dir": str(calibration)})
return package, calibration
def verify_contract(package: Path, out: Path):
import numpy as np
from harness.eval.common import T1
from harness.eval import mde
from harness.adapters import depth_anything_v2_common as recipe
from depth_anything_v2.util.transform import Resize, NormalizeImage, PrepareForNet
import cv2
t1 = T1.open(package)
try:
sample = t1.items[0]
image = t1.image(sample)
native = {"image": np.asarray(image, dtype=np.float64) / 255.0}
native = Resize(518, 518, resize_target=False, keep_aspect_ratio=False, ensure_multiple_of=14,
resize_method="lower_bound", image_interpolation_method=cv2.INTER_CUBIC)(native)
native = NormalizeImage(mean=[.485, .456, .406], std=[.229, .224, .225])(native)
native = PrepareForNet()(native)["image"][None]
np.testing.assert_array_equal(recipe.preprocess(image), native)
total_valid = 0
for item in t1.items:
for key, expected in item["file_sha256"].items():
assert hashlib.sha256(t1.tar.extractfile(item[key]).read()).hexdigest() == expected
truth, mask = mde.ground_truth(t1, item)
total_valid += int(mask.sum())
# Exact affine inverse depth must recover GT; raw prediction is not metres.
truth = np.linspace(.5, 9.5, 10000, dtype=np.float32).reshape(100, 100)
relative = (1.0 / truth - .02) / .04
score = mde.aligned_metrics(relative, truth, np.ones_like(truth, dtype=bool))
assert score["absrel"] < 1e-6 and score["delta1"] == 100
signed = mde.aligned_metrics(3.0 - 1.0 / truth, truth, np.ones_like(truth, dtype=bool))
np.testing.assert_allclose([signed["scale"], signed["shift"]], [-1, 3], atol=1e-6)
assert signed["absrel"] < 1e-6 and signed["delta1"] == 100
for bad in [np.ones_like(truth), np.full_like(truth, np.nan)]:
try:
mde.aligned_metrics(bad, truth, np.ones_like(truth, dtype=bool))
except ValueError:
pass
else:
raise AssertionError("degenerate/nonfinite input accepted")
constant = np.full((1, 518, 518), 2, dtype=np.float32)
np.testing.assert_array_equal(mde.restore_prediction([constant], (480, 640)), np.full((480, 640), 2))
save(out / "contract_checks.json", {"status": "passed", "t1_sha256": digest(package),
"decoded_real_labels": len(t1.items), "valid_pixels": total_valid,
"checks": ["all real RGB/depth member hashes", "official static resize/normalization equality",
"original metre GT and fixed crop", "affine inverse-depth exact recovery",
"signed LS recovers (-1,3)", "reject constant/NaN prediction",
"restore constant depth without vector scaling"]})
finally:
t1.tar.close()
def probe(model: str, package: Path, weights_dir: Path, out: Path, n: int, confirm: bool, threads: int,
reuse_onnx: bool = False):
import numpy as np
import torch
import torchvision
import onnx
import onnxruntime as ort
from harness.eval.common import T1, cosine
from harness.eval import EVALUATOR_VERSION, mde
recipe = importlib.import_module("harness.adapters." + model.replace(".", "_"))
directory = out / model if n == 3 else out / model / f"n{n}"
directory.mkdir(parents=True, exist_ok=True)
weights = weights_dir / recipe.CHECKPOINT["url"].rsplit("/", 1)[1]
result = {"model": model, "status": "running", "started_at": time.time(), "input_spec": recipe.INPUT_SPEC,
"checkpoint": recipe.CHECKPOINT, "preprocess": recipe.PREPROCESS, "opset": recipe.OPSET,
"weights_path": str(weights), "t1_path": str(package),
"t1_sha256": digest(package), "evaluation_protocol": mde.PROTOCOL,
"runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__,
"onnx": onnx.__version__, "onnxruntime": ort.__version__, "device": "cpu", "threads": threads},
"input_contract": {"status": "unverified"}, "board_tested": False, "quantization_tested": False}
def phase(name):
result["phase"] = name
save(directory / "probe.json", result)
print(f"{model}: {name}", flush=True)
try:
source_files = ["harness/adapters/depth_anything_v2_common.py",
"harness/adapters/" + model.replace(".", "_") + ".py", "harness/eval/mde.py"]
root = Path(__file__).resolve().parents[1]
result["source_sha256"] = {name: digest(root / name) for name in source_files}
phase("strict_official_load")
result["weights_sha256"] = digest(weights)
model_object = recipe.build(weights, recipe.INPUT_SPEC)
result.update(strict_load=True, parameters=sum(p.numel() for p in model_object.parameters()))
t1 = T1.open(package, result["t1_sha256"])
try:
t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)}
if len(t1.items) != n:
raise ValueError("requested sample count differs from actual T1 subset")
result["sample_ids"] = [item["id"] for item in t1.items]
references = {}
def native(x):
key = hashlib.sha256(x.tobytes()).hexdigest()
if key in references:
return [references[key]]
with torch.inference_mode():
value = model_object(torch.from_numpy(x)).numpy()
if value.shape != (1, 518, 518) or not np.isfinite(value).all():
raise ValueError("invalid official forward output")
references[key] = value
print(f"{model}: FP32 {len(references)}/{n}", flush=True)
return [value]
phase("real_nyu_fp32_forward")
baseline = {"task": "mde", "model": model, "n": n,
"evaluator_version": EVALUATOR_VERSION,
"t1_manifest_sha": t1.manifest["sha256"]}
for item in t1.items:
native(mde.inputs(t1, item, recipe.INPUT_SPEC, model))
np.savez(directory / "fp32_predictions.npz", **references)
result["fp32_forward_n"] = len(references)
first = mde.inputs(t1, t1.items[0], recipe.INPUT_SPEC, model)
np.save(directory / "input_first.npy", first)
np.save(directory / "fp32_first.npy", references[hashlib.sha256(first.tobytes()).hexdigest()])
if n == 100:
xs = {f"s{i:03d}": mde.inputs(t1, item, recipe.INPUT_SPEC, model)
for i, item in enumerate(t1.g2_subset(32))}
np.savez(directory / "g2_inputs.npz", **xs)
np.savez(directory / "g2_fp32.npz", **{
f"{key}/out0": references[hashlib.sha256(x.tobytes()).hexdigest()]
for key, x in xs.items()})
phase("onnx_reuse_and_checker" if reuse_onnx else "onnx_export_and_checker")
if reuse_onnx:
previous_path = out / model / "probe.json"
previous = json.loads(previous_path.read_text())
path = out / model / f"{model}.onnx"
if (previous.get("input_contract", {}).get("status") != "validated"
or not previous.get("strict_load")
or previous["weights_sha256"] != result["weights_sha256"]
or previous["input_spec"] != recipe.INPUT_SPEC or previous["opset"] != recipe.OPSET
or any(previous["source_sha256"][name] != result["source_sha256"][name]
for name in source_files[:2])
or digest(path) != previous["onnx"]["sha256"]):
raise ValueError("existing ONNX identity/adapter/weights proof mismatch")
result["onnx"] = {**previous["onnx"], "path": str(path), "reused_from": str(previous_path)}
else:
path = directory / f"{model}.onnx"
started = time.monotonic()
with torch.no_grad():
torch.onnx.export(model_object, torch.from_numpy(first), str(path), dynamo=False,
opset_version=recipe.OPSET, input_names=["input"], output_names=["inverse_depth"],
dynamic_axes=None, do_constant_folding=True)
result["onnx"] = {"path": str(path), "sha256": digest(path), "size": path.stat().st_size,
"export_seconds": time.monotonic() - started}
onnx.checker.check_model(str(path), full_check=True)
phase("real_nyu_onnxruntime_and_paired_metrics")
options = ort.SessionOptions()
options.intra_op_num_threads = threads
options.inter_op_num_threads = 1
session = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"])
result["onnx"]["input_shape"] = session.get_inputs()[0].shape
result["onnx"]["output_shape"] = session.get_outputs()[0].shape
if result["onnx"]["input_shape"] != [1, 3, 518, 518] or result["onnx"]["output_shape"] != [1, 518, 518]:
raise ValueError("ONNX static input/output shape mismatch")
comparisons = []
converted_outputs = {}
result["comparisons"] = comparisons
def converted(x):
key = hashlib.sha256(x.tobytes()).hexdigest()
if key in converted_outputs:
return [converted_outputs[key]]
value = session.run(None, {"input": x})[0]
ref = references[key]
difference = np.abs(ref - value)
comp = {"sample_id": result["sample_ids"][len(comparisons)], "cosine": cosine(ref, value),
"max_abs": float(difference.max()), "mean_abs": float(difference.mean()),
"relative_max_abs": float(difference.max() / max(float(np.abs(ref).max()), 1e-12))}
comparisons.append(comp)
print(json.dumps(comp), flush=True)
if not np.isfinite(value).all() or comp["cosine"] < .99999 or comp["relative_max_abs"] > .001:
raise ValueError(f"ONNX mismatch: {comp}")
converted_outputs[key] = value
return [value]
for item in t1.items:
converted(mde.inputs(t1, item, recipe.INPUT_SPEC, model))
np.savez(directory / "onnx_predictions.npz", **converted_outputs)
result["onnxruntime_forward_n"] = len(converted_outputs)
# Audit every real image even if one alignment fails; never omit it from a score.
phase("real_nyu_alignment_audit")
audit, failures = [], []
for item in t1.items:
x = mde.inputs(t1, item, recipe.INPUT_SPEC, model)
depth, mask = mde.ground_truth(t1, item)
sample = {"id": item["id"]}
for name, runner in (("fp32", native), ("onnxruntime", converted)):
prediction = mde.restore_prediction(runner(x), depth.shape)
try:
sample[name] = {"status": "measured", **mde.aligned_metrics(prediction, depth, mask)}
except ValueError as exc:
sample[name] = {"status": "blocked", "error": str(exc)}
failures.append({"id": item["id"], "runner": name, "error": str(exc)})
audit.append(sample)
save(directory / "alignment_audit.json", {"protocol": mde.PROTOCOL, "n": n,
"samples": audit, "failures": failures, "status": "blocked" if failures else "passed"})
if failures:
result["evaluation_contract"] = {"status": "blocked", "protocol": mde.PROTOCOL,
"failures": failures}
raise ValueError(f"alignment failed for {len({s['id'] for s in failures})}/{n} real samples")
mde.run_fp32(native, t1, recipe.INPUT_SPEC, baseline)
result["fp32"] = baseline["metrics"]
if n == 100:
save(directory / "metrics.json", baseline)
measured = {"task": "mde", "model": model}
mde.run_quant(converted, t1, recipe.INPUT_SPEC, baseline, measured)
paired = measured["G3"]["extra"]["scale_alignment"]["samples"]
if len(paired) != n or len({sample["id"] for sample in paired}) != n:
raise ValueError("incomplete/duplicate per-image alignment proof")
for sample, fp, quant in zip(paired, baseline["_depth_samples"], measured["_depth_samples"]):
if (sample["id"] != fp["id"] or sample["id"] != quant["id"]
or sample["fp32_scale"] != fp["scale"] or sample["fp32_shift"] != fp["shift"]
or sample["quant_scale"] != quant["scale"] or sample["quant_shift"] != quant["shift"]
or not np.isfinite([sample[key] for key in ("fp32_scale", "fp32_shift", "quant_scale", "quant_shift")]).all()):
raise ValueError("invalid/mispaired real per-image alignment proof")
if abs(baseline["metrics"]["absrel"] - measured["metrics"]["absrel"]) > .001 or abs(baseline["metrics"]["delta1"] - measured["metrics"]["delta1"]) > .1:
raise ValueError("FP32/ONNX paired depth metric mismatch")
checks = json.loads((out / "contract_checks.json").read_text())
if checks.get("status") != "passed" or checks.get("t1_sha256") != result["t1_sha256"]:
raise ValueError("missing matching real-data and alignment contract checks")
result.update(comparisons=comparisons, onnxruntime=measured["metrics"], G3_probe=measured["G3"],
status="validated_fp32_export_only" if confirm else "validated_export_pending_eval_protocol", phase="complete")
result["input_contract"] = {"status": "validated", "input_spec": recipe.INPUT_SPEC,
"output_semantics": "affine_invariant_inverse_depth_not_metric", "t1_sha256": result["t1_sha256"],
"checks_path": str(out / "contract_checks.json"), "sample_ids": result["sample_ids"],
"onnx_sha256": result["onnx"]["sha256"]}
result["evaluation_contract"] = {"status": "validated" if confirm else "pending_mainline_protocol",
"protocol": mde.PROTOCOL}
baseline["note"] = ("complete fixed 100-image FP32 baseline; bench protocol, not paper scores"
if n == 100 else "probe subset only, not the complete 100-image FP32 baseline")
save(directory / "baseline_probe.json", baseline)
save(directory / "onnx_metrics_probe.json", measured)
finally:
t1.tar.close()
except Exception as exc:
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
(directory / "error.txt").write_text(traceback.format_exc())
result["finished_at"] = time.time()
save(directory / "probe.json", result)
print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", action="append", choices=[f"depth.anything.v2.{v}" for v in ("small", "base", "large")], required=True)
parser.add_argument("--source", type=Path, required=True)
parser.add_argument("--data-reader", type=Path, help="private unpacked h5py/scipy wheels, only used to convert source MAT")
parser.add_argument("--mat", type=Path, required=True)
parser.add_argument("--splits", type=Path, required=True)
parser.add_argument("--weights-dir", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--calib-root", type=Path)
parser.add_argument("--n", type=int, default=3)
parser.add_argument("--seed", type=int, default=20260914)
parser.add_argument("--threads", type=int, default=2)
parser.add_argument("--confirm-protocol", choices=["nyuv2-relative-inverse-ls-v1"])
parser.add_argument("--reuse-onnx", action="store_true", help="verify and reuse the original three-image ONNX proof")
parser.add_argument("--prepare-only", action="store_true")
args = parser.parse_args()
if not 1 <= args.n <= 100 or args.threads < 1:
parser.error("require 1 <= n <= 100 and positive threads")
if args.reuse_onnx and args.n == 3:
parser.error("reuse requires a separate sample count to preserve the original three-image proof")
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
os.environ["BENCH_DEPTH_ANYTHING_V2_SOURCE"] = str(args.source.resolve())
if args.data_reader:
sys.path.insert(0, str(args.data_reader.resolve()))
sys.path.insert(0, str(args.source.resolve()))
import torch
torch.set_num_threads(args.threads)
torch.set_num_interop_threads(1)
from harness.adapters.depth_anything_v2_common import source_root
source_root()
package, calibration = data(args.mat, args.splits, args.out, args.calib_root or args.out / "data/calib", args.seed)
verify_contract(package, args.out)
save(args.out / "data_preparation.json", {"t1_path": str(package), "t1_sha256": digest(package),
"n": 100, "calibration_dir": str(calibration), "calibration_n": 128})
if args.prepare_only:
return 0
results = [probe(model, package, args.weights_dir, args.out, args.n, bool(args.confirm_protocol), args.threads, args.reuse_onnx)
for model in dict.fromkeys(args.model)]
save(args.out / ("summary.json" if args.n == 3 else f"summary-n{args.n}.json"), {"results": results, "complete": True})
return int(any(result["status"] == "blocked" for result in results))
if __name__ == "__main__":
raise SystemExit(main())