"""Private batch-9 flow-family checkpoint/real KITTI pair/ONNX probe. Reuses the raft probe judgement verbatim: strict official checkpoint load, real temporal pairs from the frozen kitti2015 T1 package, ONNX export with checker, and per-sample FP32-vs-ONNX gates cosine >= .9999, mean_abs <= .005, max_abs <= .5, epe2d_difference <= .01, fl_all_difference_pp <= .1. It never touches run state; the finished proof is mirrored to cache/adapter-probes. Run with the interpreter the family adapter pins (RUNTIME_PYTHON; default bench venv). Example (CPU only, no installs): ~/toolchains/bench-venv/bin/python scripts/probe_flow_adapters.py \ --model gmflow --model unimatch.flow \ --t1-manifest /home/ming01.wang/bench-data/t1/kitti2015/n100-s20260914/manifest.json \ --kitti-root /mnt/data/qi.xiong/Dataset/StereoDataset/KITTI/data_scene_flow/kitti_2015 \ --weights-dir ~/bench-cache/weights/flow-batch9 --out ~/flow-probe-20260919 --n 3 """ from __future__ import annotations import argparse import hashlib import importlib import json import os from pathlib import Path import random import shutil import sys import tarfile import time import traceback import urllib.request sys.path.insert(0, str(Path(__file__).resolve().parents[1])) MODELS = ["gmflow", "unimatch.flow", "gma", "sea.raft", "flow1d", "memflow", "flowformer", "flowformerplusplus", "pwc.net", "liteflownet2", "flownet2"] def sha256(path: Path) -> str: digest = hashlib.sha256() with Path(path).open("rb") as stream: for block in iter(lambda: stream.read(1 << 20), b""): digest.update(block) return digest.hexdigest() def write_json(path: Path, value: dict) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") temporary.write_text(json.dumps(value, indent=2, allow_nan=False)) temporary.replace(path) def open_eval_set(t1_manifest: Path) -> dict: from harness.continuation import load_eval_set dataset, evaluation = load_eval_set(t1_manifest) evaluation["eval_set_id"] = json.loads(t1_manifest.read_text())["eval_set_id"] return evaluation def stage_weights(adapter, weights_dir: Path) -> Path: checkpoint = adapter.CHECKPOINT name = checkpoint.get("filename") or checkpoint["url"].rsplit("/", 1)[1] destination = weights_dir / name if destination.is_file() and sha256(destination) == checkpoint["sha256"]: return destination destination = destination.with_suffix(destination.suffix + ".part") destination.unlink(missing_ok=True) with urllib.request.urlopen(checkpoint["url"], timeout=300) as remote, destination.open("wb") as target: while block := remote.read(1 << 20): target.write(block) if sha256(destination) != checkpoint["sha256"]: raise ValueError(f"downloaded {checkpoint['url']} does not match the pinned sha256") destination.replace(destination.with_suffix("")) return destination.with_suffix("") def prepare_calibration(adapter, kitti_root: Path, evaluation: dict, calib_root: Path, seed: int, n: int = 128) -> Path: """Distinct testing pairs through the family recipe; no labels needed.""" import numpy as np from PIL import Image import io recipe = adapter.PREPROCESS digest16 = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest()[:16] destination = calib_root / "flow" / adapter.INPUT_SPEC / digest16 manifest_path = destination / "manifest.json" if manifest_path.exists(): manifest = json.loads(manifest_path.read_text()) verify = hashlib.sha256() files = sorted((destination / "npy_f32").glob("*.npy")) for path in files: verify.update(path.read_bytes()) if (manifest.get("count") != n or len(files) != n or manifest.get("seed") != seed or manifest.get("t1_sha256") != evaluation["manifest_sha"] or manifest.get("preprocess") != recipe or manifest.get("sha256") != verify.hexdigest()): raise ValueError(f"existing calibration differs; refusing to overwrite {destination}") return destination if destination.exists(): raise FileExistsError(f"partial/existing calibration; refusing to overwrite {destination}") with tarfile.open(evaluation["path"]) as tar: t1 = json.load(tar.extractfile("manifest.json")) excluded = {item["file_sha256"][key] for item in t1["items"] for key in ("file", "file2")} image_dir = kitti_root / "testing" / "image_2" candidates = [p for p in sorted(image_dir.glob("*_10.png")) if p.with_name(p.name.replace("_10.png", "_11.png")).is_file()] random.Random(seed).shuffle(candidates) if len(candidates) < n: raise ValueError(f"only {len(candidates)} unique testing pairs available; need {n}") temporary = destination.with_name(destination.name + f".partial-{os.getpid()}") npy_dir = temporary / "npy_f32" npy_dir.mkdir(parents=True, exist_ok=False) digest, items, seen = hashlib.sha256(), [], set() for first in candidates: second = first.with_name(first.name.replace("_10.png", "_11.png")) raw = [first.read_bytes(), second.read_bytes()] hashes = tuple(hashlib.sha256(data).hexdigest() for data in raw) if any(h in excluded for h in hashes) or hashes in seen: continue seen.add(hashes) images = [Image.open(io.BytesIO(data)).convert("RGB") for data in raw] packed = adapter.preprocess(*images, adapter.INPUT_SPEC) if packed.shape != (1, 6, 384, 1024) or not np.isfinite(packed).all(): raise ValueError("invalid real calibration pair") path = npy_dir / f"{len(items):05d}.npy" np.save(path, packed) digest.update(path.read_bytes()) items.append({"id": f"kitti2015/testing/{first.stem.removesuffix('_10')}", "file": str(first), "file2": str(second), "file_sha256": list(hashes), "npy": path.name, "npy_sha256": sha256(path)}) if len(items) == n: break if len(items) != n: raise ValueError(f"only {len(items)} unique, nonoverlapping calibration pairs; need {n}") write_json(temporary / "manifest.json", {"task": "flow", "input_spec": adapter.INPUT_SPEC, "count": n, "seed": seed, "source": "testing-unlabeled", "dataset": "kitti2015", "sha256": digest.hexdigest(), "files": [it["file"] + "|" + it["file2"] for it in items], "items": items, "t1_sha256": evaluation["manifest_sha"], "preprocess": recipe, "t1_frame_content_overlap": 0, "unique_pair_content_count": len(seen), "layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"}) temporary.rename(destination) return destination def verify_contract(adapter, evaluation: dict, kitti_root: Path, out: Path, model: str) -> Path: """Geometry/protocol checks independent of model accuracy (raft口径).""" import numpy as np import torch from PIL import Image from torchvision.datasets._optical_flow import _read_16bits_png_with_flow_and_valid_mask from torchvision.transforms import functional as F from harness.adapters.raft_common import input_shape, preprocess as raft_preprocess from harness.eval import flow from harness.eval.common import T1 t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) try: item = t1.items[0] truth, valid = flow.ground_truth(t1, item) official, official_valid = _read_16bits_png_with_flow_and_valid_mask( str(kitti_root / "training" / item["flow"])) np.testing.assert_array_equal(truth, official) np.testing.assert_array_equal(valid, official_valid) frames = (t1.image(item), t1.image({"file": item["file2"]})) packed = flow.inputs(t1, item, adapter.INPUT_SPEC, model) # Same bilinear resize geometry as the raft recipe; only the value scale differs. raft_reference = raft_preprocess(*frames, adapter.INPUT_SPEC) scale = np.float32(127.5) np.testing.assert_allclose(packed, (raft_reference + 1.0) * scale, rtol=0, atol=2e-3) # Anisotropic scaling must change u and v independently, not only resize. const = np.empty((1, 2, 384, 1024), dtype=np.float32) const[:, 0], const[:, 1] = 8.0, -4.0 restored = flow.restore_flow([const], adapter.INPUT_SPEC, (192, 2048)) np.testing.assert_array_equal(restored[0], np.full((192, 2048), 16.0)) np.testing.assert_array_equal(restored[1], np.full((192, 2048), -2.0)) pred = np.zeros((2, 2, 2), dtype=np.float32) pred[0] = [[3, 4], [100, 100]] mask = np.array([[True, True], [False, False]]) metrics = flow.sample_metrics(pred, np.zeros_like(pred), mask) assert metrics["epe2d"] == 3.5 and metrics["fl_all"] == 50 and metrics["valid_pixels"] == 2 truth_fixture = np.full((2, 1, 1), 0, dtype=np.float32) truth_fixture[0] = 100 pred_fixture = truth_fixture.copy() pred_fixture[0] += 4 assert flow.sample_metrics(pred_fixture, truth_fixture, np.ones((1, 1), dtype=bool))["fl_all"] == 0 try: input_shape("1x3x384x1024") except ValueError: pass else: raise AssertionError("legacy single-frame specification was accepted") pred[0, 0, 0] = np.nan try: flow.sample_metrics(pred, np.zeros_like(pred), mask) except ValueError: pass else: raise AssertionError("non-finite prediction was accepted") checks_path = out / f"contract_checks-{model.replace('.', '_')}.json" write_json(checks_path, {"status": "passed", "model": model, "t1_sha256": evaluation["manifest_sha"], "real_sample_id": item["id"], "real_image_shape": list(truth.shape[-2:]), "real_valid_pixels": int(valid.sum()), "checks": ["real_uint16_decode_matches_torchvision", "family_preprocess_matches_official_resize_and_scale", "anisotropic_flow_resize_and_vector_scale", "RGB_temporal_channel_order", "valid_mask_excludes_invalid_pixels", "strict_outlier_threshold_3px_and_5pct", "reject_legacy_single_image_input", "reject_nonfinite_prediction"]}) return checks_path finally: t1.tar.close() def probe(model: str, evaluation: dict, weights_dir: Path, out: Path, n: int, threads: int, kitti_root: Path) -> dict: import numpy as np import onnx import onnxruntime as ort import torch from harness.adapters import get as get_adapter from harness.eval import flow from harness.eval.common import T1, cosine adapter = get_adapter(model) directory = out / model directory.mkdir(parents=True, exist_ok=True) result = {"model": model, "status": "running", "started_at": time.time(), "input_spec": adapter.INPUT_SPEC, "preprocess": adapter.PREPROCESS, "opset": adapter.OPSET, "checkpoint": adapter.CHECKPOINT, "runtime": {"torch": torch.__version__, "onnx": onnx.__version__, "onnxruntime": ort.__version__, "device": "cpu", "threads": threads, "executable": str(Path(sys.executable).resolve())}, "runtime_python": getattr(adapter, "RUNTIME_PYTHON", None), "t1_path": evaluation["path"], "t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation, "board_tested": False, "quantization_tested": False, "input_contract": {"status": "unverified", "input_spec": adapter.INPUT_SPEC}} def phase(name): result["phase"] = name write_json(directory / "probe.json", result) print(f"{model}: {name}", flush=True) try: pinned = getattr(adapter, "RUNTIME_PYTHON", None) if pinned and Path(sys.executable).resolve() != Path(pinned).resolve(): raise ValueError(f"{model} must run under its pinned runtime {pinned}") phase("stage_official_checkpoint") weights = stage_weights(adapter, weights_dir) result["weights_path"] = str(weights) result["weights_sha256"] = sha256(weights) phase("verify_input_contract") checks_path = verify_contract(adapter, evaluation, kitti_root, out, model) phase("strict_load") built = adapter.build(weights, adapter.INPUT_SPEC) result["strict_load"] = True result["parameters"] = sum(p.numel() for p in built.parameters()) t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) try: if not 0 < n <= len(t1.items): raise ValueError("--n must be positive and <= T1 n") result["prepared_t1_n"] = len(t1.items) t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)} result["sample_ids"] = [item["id"] for item in t1.items] phase("real_pair_forward") first = flow.inputs(t1, t1.items[0], adapter.INPUT_SPEC, model) np.save(directory / "input_first_pair.npy", first) start = time.monotonic() with torch.inference_mode(): reference = built(torch.from_numpy(first)).numpy() if reference.shape != (1, 2, 384, 1024) or not np.isfinite(reference).all(): raise ValueError(f"invalid {model} forward output") result["first_forward_seconds"] = time.monotonic() - start np.save(directory / "fp32_first_flow.npy", reference) phase("onnx_export_and_check") onnx_path = directory / f"{model}.onnx" start = time.monotonic() export_kwargs = dict(opset_version=adapter.OPSET, input_names=["input"], output_names=["flow"], dynamic_axes=None, do_constant_folding=True) if tuple(int(v) for v in torch.__version__.split(".")[:2]) >= (2, 5): export_kwargs["dynamo"] = False with torch.no_grad(): torch.onnx.export(built, torch.from_numpy(first), str(onnx_path), **export_kwargs) onnx.checker.check_model(str(onnx_path), full_check=True) graph = onnx.load(str(onnx_path)) from collections import Counter result["onnx"] = {"path": str(onnx_path), "sha256": sha256(onnx_path), "size_bytes": onnx_path.stat().st_size, "operators": dict(Counter(node.op_type for node in graph.graph.node)), "export_and_check_seconds": time.monotonic() - start, "input_shapes": [[v.dim_value for v in t.type.tensor_type.shape.dim] for t in graph.graph.input], "output_shapes": [[v.dim_value for v in t.type.tensor_type.shape.dim] for t in graph.graph.output]} del graph phase("onnxruntime_real_pairs_and_ground_truth") opts = ort.SessionOptions() opts.intra_op_num_threads = threads opts.inter_op_num_threads = 1 session = ort.InferenceSession(str(onnx_path), sess_options=opts, providers=["CPUExecutionProvider"]) comparisons, fp32_samples, ort_samples = [], [], [] for item in t1.items: x = flow.inputs(t1, item, adapter.INPUT_SPEC, model) with torch.inference_mode(): native = built(torch.from_numpy(x)).numpy() exported = session.run(None, {"input": x})[0] truth, valid = flow.ground_truth(t1, item) a = flow.sample_metrics(flow.restore_flow([native], adapter.INPUT_SPEC, truth.shape[-2:]), truth, valid) b = flow.sample_metrics(flow.restore_flow([exported], adapter.INPUT_SPEC, truth.shape[-2:]), truth, valid) diff = np.abs(native - exported) comparison = {"id": item["id"], "cosine": cosine(native, exported), "max_abs": float(diff.max()), "mean_abs": float(diff.mean()), "epe2d_difference": abs(a["epe2d"] - b["epe2d"]), "fl_all_difference_pp": abs(a["fl_all"] - b["fl_all"])} comparisons.append(comparison) fp32_samples.append({"id": item["id"], **a}) ort_samples.append({"id": item["id"], **b}) print(json.dumps(comparison), flush=True) if (comparison["cosine"] < .9999 or comparison["mean_abs"] > .005 or comparison["max_abs"] > .5 or comparison["epe2d_difference"] > .01 or comparison["fl_all_difference_pp"] > .1): raise ValueError(f"FP32 export numerical mismatch: {comparison}") result.update(status="validated_fp32_export_only", phase="complete", comparisons=comparisons, fp32=flow.aggregate(fp32_samples), onnxruntime=flow.aggregate(ort_samples), fp32_samples=fp32_samples, onnxruntime_samples=ort_samples) checks = json.loads(checks_path.read_text()) if checks["status"] != "passed" or checks["t1_sha256"] != evaluation["manifest_sha"]: raise ValueError("missing matching real data/geometry contract checks") result["input_contract"] = {"status": "validated", "input_spec": adapter.INPUT_SPEC, "packing": "RGB(t) channels 0:3; RGB(t+1) channels 3:6; NCHW float32; official scale per recipe", "output": "one N,2,H,W tensor (u,v), final update; input-image pixel displacements", "ground_truth": "KITTI2015 flow_occ uint16 RGB, (RG-32768)/64, B=valid", "evaluation_coordinates": "original image; bilinear resize plus independent u/v scaling", "sample_ids": result["sample_ids"], "t1_sha256": evaluation["manifest_sha"], "checks_path": str(checks_path), "onnx_sha256": result["onnx"]["sha256"], "opset": adapter.OPSET} write_json(directory / "baseline_probe.json", {"task": "flow", "model": model, "t1_manifest_sha": evaluation["manifest_sha"], "probe_subset_ids": result["sample_ids"], "metrics": result["fp32"], "_flow_samples": fp32_samples, "note": "probe subset only; not a completed T1 baseline"}) finally: t1.tar.close() except Exception as exc: result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") (directory / "error.txt").write_text(traceback.format_exc()) result["finished_at"] = time.time() write_json(directory / "probe.json", result) print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True) return result def mirror(model: str, out: Path) -> None: """Copy the finished proof into the shared adapter-probes cache location.""" from harness import paths source = out / model / "probe.json" shared = paths.CACHE / "adapter-probes" / model / "probe.json" shared.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(source, shared) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model", action="append", choices=MODELS, required=True) parser.add_argument("--t1-manifest", type=Path, required=True, help="manifest.json of the existing frozen kitti2015 n100 T1 package") parser.add_argument("--kitti-root", type=Path, required=True, help="directory containing training/ and testing/ (calibration pairs)") parser.add_argument("--weights-dir", type=Path, required=True) parser.add_argument("--out", type=Path, required=True, help="new private probe directory") parser.add_argument("--n", type=int, default=3) parser.add_argument("--threads", type=int, default=2) parser.add_argument("--seed", type=int, default=20260914) parser.add_argument("--calib-root", type=Path, help="existing data/calib root; defaults to private --out/data/calib") args = parser.parse_args() if not 1 <= args.n <= 100 or args.threads < 1: parser.error("require 1 <= n <= 100 and positive threads") if len(set(args.model)) != 1: parser.error("probe exactly one family per invocation: upstream packages share module " "names (utils/core) across families and must not be imported together") os.environ["CUDA_VISIBLE_DEVICES"] = "-1" import torch torch.set_num_threads(args.threads) torch.set_num_interop_threads(1) args.out.mkdir(parents=True, exist_ok=True) evaluation = open_eval_set(args.t1_manifest) models = list(dict.fromkeys(args.model)) from harness.adapters import get as get_adapter adapters = {m: get_adapter(m) for m in models} calib_root = args.calib_root or args.out / "data" / "calib" calibration = {} for model, adapter in adapters.items(): calib = prepare_calibration(adapter, args.kitti_root, evaluation, calib_root, args.seed) calibration[model] = str(calib) print(f"{model}: calibration {calib}", flush=True) write_json(args.out / "data_preparation.json", {"t1": str(evaluation["path"]), "t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation, "n": args.n, "calibration": calibration}) results = [probe(model, evaluation, args.weights_dir, args.out, args.n, args.threads, args.kitti_root) for model in models] for model, result in zip(models, results): if result["status"] == "validated_fp32_export_only": mirror(model, args.out) write_json(args.out / "summary.json", {"results": results, "complete": True}) return int(any(result["status"] != "validated_fp32_export_only" for result in results)) if __name__ == "__main__": raise SystemExit(main())