421 lines
22 KiB
Python
421 lines
22 KiB
Python
"""Private batch-9 flow-family checkpoint/real KITTI pair/ONNX probe.
|
|
|
|
Reuses the raft probe judgement verbatim: strict official checkpoint load, real
|
|
temporal pairs from the frozen kitti2015 T1 package, ONNX export with checker,
|
|
and per-sample FP32-vs-ONNX gates cosine >= .9999, mean_abs <= .005,
|
|
max_abs <= .5, epe2d_difference <= .01, fl_all_difference_pp <= .1. It never
|
|
touches run state; the finished proof is mirrored to cache/adapter-probes.
|
|
|
|
Run with the interpreter the family adapter pins (RUNTIME_PYTHON; default bench
|
|
venv). Example (CPU only, no installs):
|
|
~/toolchains/bench-venv/bin/python scripts/probe_flow_adapters.py \
|
|
--model gmflow --model unimatch.flow \
|
|
--t1-manifest /home/ming01.wang/bench-data/t1/kitti2015/n100-s20260914/manifest.json \
|
|
--kitti-root /mnt/data/qi.xiong/Dataset/StereoDataset/KITTI/data_scene_flow/kitti_2015 \
|
|
--weights-dir ~/bench-cache/weights/flow-batch9 --out ~/flow-probe-20260919 --n 3
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import importlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import shutil
|
|
import sys
|
|
import tarfile
|
|
import time
|
|
import traceback
|
|
import urllib.request
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
MODELS = ["gmflow", "unimatch.flow", "gma", "sea.raft", "flow1d", "memflow",
|
|
"flowformer", "flowformerplusplus", "pwc.net", "liteflownet2", "flownet2"]
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with Path(path).open("rb") as stream:
|
|
for block in iter(lambda: stream.read(1 << 20), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def write_json(path: Path, value: dict) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_suffix(path.suffix + ".tmp")
|
|
temporary.write_text(json.dumps(value, indent=2, allow_nan=False))
|
|
temporary.replace(path)
|
|
|
|
|
|
def open_eval_set(t1_manifest: Path) -> dict:
|
|
from harness.continuation import load_eval_set
|
|
|
|
dataset, evaluation = load_eval_set(t1_manifest)
|
|
evaluation["eval_set_id"] = json.loads(t1_manifest.read_text())["eval_set_id"]
|
|
return evaluation
|
|
|
|
|
|
def stage_weights(adapter, weights_dir: Path) -> Path:
|
|
checkpoint = adapter.CHECKPOINT
|
|
name = checkpoint.get("filename") or checkpoint["url"].rsplit("/", 1)[1]
|
|
destination = weights_dir / name
|
|
if destination.is_file() and sha256(destination) == checkpoint["sha256"]:
|
|
return destination
|
|
destination = destination.with_suffix(destination.suffix + ".part")
|
|
destination.unlink(missing_ok=True)
|
|
with urllib.request.urlopen(checkpoint["url"], timeout=300) as remote, destination.open("wb") as target:
|
|
while block := remote.read(1 << 20):
|
|
target.write(block)
|
|
if sha256(destination) != checkpoint["sha256"]:
|
|
raise ValueError(f"downloaded {checkpoint['url']} does not match the pinned sha256")
|
|
destination.replace(destination.with_suffix(""))
|
|
return destination.with_suffix("")
|
|
|
|
|
|
def prepare_calibration(adapter, kitti_root: Path, evaluation: dict, calib_root: Path,
|
|
seed: int, n: int = 128) -> Path:
|
|
"""Distinct testing pairs through the family recipe; no labels needed."""
|
|
import numpy as np
|
|
from PIL import Image
|
|
import io
|
|
|
|
recipe = adapter.PREPROCESS
|
|
digest16 = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest()[:16]
|
|
destination = calib_root / "flow" / adapter.INPUT_SPEC / digest16
|
|
manifest_path = destination / "manifest.json"
|
|
if manifest_path.exists():
|
|
manifest = json.loads(manifest_path.read_text())
|
|
verify = hashlib.sha256()
|
|
files = sorted((destination / "npy_f32").glob("*.npy"))
|
|
for path in files:
|
|
verify.update(path.read_bytes())
|
|
if (manifest.get("count") != n or len(files) != n or manifest.get("seed") != seed
|
|
or manifest.get("t1_sha256") != evaluation["manifest_sha"]
|
|
or manifest.get("preprocess") != recipe
|
|
or manifest.get("sha256") != verify.hexdigest()):
|
|
raise ValueError(f"existing calibration differs; refusing to overwrite {destination}")
|
|
return destination
|
|
if destination.exists():
|
|
raise FileExistsError(f"partial/existing calibration; refusing to overwrite {destination}")
|
|
|
|
with tarfile.open(evaluation["path"]) as tar:
|
|
t1 = json.load(tar.extractfile("manifest.json"))
|
|
excluded = {item["file_sha256"][key] for item in t1["items"] for key in ("file", "file2")}
|
|
image_dir = kitti_root / "testing" / "image_2"
|
|
candidates = [p for p in sorted(image_dir.glob("*_10.png"))
|
|
if p.with_name(p.name.replace("_10.png", "_11.png")).is_file()]
|
|
random.Random(seed).shuffle(candidates)
|
|
if len(candidates) < n:
|
|
raise ValueError(f"only {len(candidates)} unique testing pairs available; need {n}")
|
|
temporary = destination.with_name(destination.name + f".partial-{os.getpid()}")
|
|
npy_dir = temporary / "npy_f32"
|
|
npy_dir.mkdir(parents=True, exist_ok=False)
|
|
digest, items, seen = hashlib.sha256(), [], set()
|
|
for first in candidates:
|
|
second = first.with_name(first.name.replace("_10.png", "_11.png"))
|
|
raw = [first.read_bytes(), second.read_bytes()]
|
|
hashes = tuple(hashlib.sha256(data).hexdigest() for data in raw)
|
|
if any(h in excluded for h in hashes) or hashes in seen:
|
|
continue
|
|
seen.add(hashes)
|
|
images = [Image.open(io.BytesIO(data)).convert("RGB") for data in raw]
|
|
packed = adapter.preprocess(*images, adapter.INPUT_SPEC)
|
|
if packed.shape != (1, 6, 384, 1024) or not np.isfinite(packed).all():
|
|
raise ValueError("invalid real calibration pair")
|
|
path = npy_dir / f"{len(items):05d}.npy"
|
|
np.save(path, packed)
|
|
digest.update(path.read_bytes())
|
|
items.append({"id": f"kitti2015/testing/{first.stem.removesuffix('_10')}",
|
|
"file": str(first), "file2": str(second), "file_sha256": list(hashes),
|
|
"npy": path.name, "npy_sha256": sha256(path)})
|
|
if len(items) == n:
|
|
break
|
|
if len(items) != n:
|
|
raise ValueError(f"only {len(items)} unique, nonoverlapping calibration pairs; need {n}")
|
|
write_json(temporary / "manifest.json", {"task": "flow", "input_spec": adapter.INPUT_SPEC,
|
|
"count": n, "seed": seed, "source": "testing-unlabeled", "dataset": "kitti2015",
|
|
"sha256": digest.hexdigest(), "files": [it["file"] + "|" + it["file2"] for it in items],
|
|
"items": items, "t1_sha256": evaluation["manifest_sha"], "preprocess": recipe,
|
|
"t1_frame_content_overlap": 0, "unique_pair_content_count": len(seen),
|
|
"layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"})
|
|
temporary.rename(destination)
|
|
return destination
|
|
|
|
|
|
def verify_contract(adapter, evaluation: dict, kitti_root: Path, out: Path, model: str) -> Path:
|
|
"""Geometry/protocol checks independent of model accuracy (raft口径)."""
|
|
import numpy as np
|
|
import torch
|
|
from PIL import Image
|
|
from torchvision.datasets._optical_flow import _read_16bits_png_with_flow_and_valid_mask
|
|
from torchvision.transforms import functional as F
|
|
|
|
from harness.adapters.raft_common import input_shape, preprocess as raft_preprocess
|
|
from harness.eval import flow
|
|
from harness.eval.common import T1
|
|
|
|
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
|
|
try:
|
|
item = t1.items[0]
|
|
truth, valid = flow.ground_truth(t1, item)
|
|
official, official_valid = _read_16bits_png_with_flow_and_valid_mask(
|
|
str(kitti_root / "training" / item["flow"]))
|
|
np.testing.assert_array_equal(truth, official)
|
|
np.testing.assert_array_equal(valid, official_valid)
|
|
frames = (t1.image(item), t1.image({"file": item["file2"]}))
|
|
packed = flow.inputs(t1, item, adapter.INPUT_SPEC, model)
|
|
# Same bilinear resize geometry as the raft recipe; only the value scale differs.
|
|
raft_reference = raft_preprocess(*frames, adapter.INPUT_SPEC)
|
|
scale = np.float32(127.5)
|
|
np.testing.assert_allclose(packed, (raft_reference + 1.0) * scale, rtol=0, atol=2e-3)
|
|
# Anisotropic scaling must change u and v independently, not only resize.
|
|
const = np.empty((1, 2, 384, 1024), dtype=np.float32)
|
|
const[:, 0], const[:, 1] = 8.0, -4.0
|
|
restored = flow.restore_flow([const], adapter.INPUT_SPEC, (192, 2048))
|
|
np.testing.assert_array_equal(restored[0], np.full((192, 2048), 16.0))
|
|
np.testing.assert_array_equal(restored[1], np.full((192, 2048), -2.0))
|
|
pred = np.zeros((2, 2, 2), dtype=np.float32)
|
|
pred[0] = [[3, 4], [100, 100]]
|
|
mask = np.array([[True, True], [False, False]])
|
|
metrics = flow.sample_metrics(pred, np.zeros_like(pred), mask)
|
|
assert metrics["epe2d"] == 3.5 and metrics["fl_all"] == 50 and metrics["valid_pixels"] == 2
|
|
truth_fixture = np.full((2, 1, 1), 0, dtype=np.float32)
|
|
truth_fixture[0] = 100
|
|
pred_fixture = truth_fixture.copy()
|
|
pred_fixture[0] += 4
|
|
assert flow.sample_metrics(pred_fixture, truth_fixture, np.ones((1, 1), dtype=bool))["fl_all"] == 0
|
|
try:
|
|
input_shape("1x3x384x1024")
|
|
except ValueError:
|
|
pass
|
|
else:
|
|
raise AssertionError("legacy single-frame specification was accepted")
|
|
pred[0, 0, 0] = np.nan
|
|
try:
|
|
flow.sample_metrics(pred, np.zeros_like(pred), mask)
|
|
except ValueError:
|
|
pass
|
|
else:
|
|
raise AssertionError("non-finite prediction was accepted")
|
|
checks_path = out / f"contract_checks-{model.replace('.', '_')}.json"
|
|
write_json(checks_path, {"status": "passed", "model": model,
|
|
"t1_sha256": evaluation["manifest_sha"], "real_sample_id": item["id"],
|
|
"real_image_shape": list(truth.shape[-2:]), "real_valid_pixels": int(valid.sum()),
|
|
"checks": ["real_uint16_decode_matches_torchvision",
|
|
"family_preprocess_matches_official_resize_and_scale",
|
|
"anisotropic_flow_resize_and_vector_scale",
|
|
"RGB_temporal_channel_order",
|
|
"valid_mask_excludes_invalid_pixels",
|
|
"strict_outlier_threshold_3px_and_5pct",
|
|
"reject_legacy_single_image_input",
|
|
"reject_nonfinite_prediction"]})
|
|
return checks_path
|
|
finally:
|
|
t1.tar.close()
|
|
|
|
|
|
def probe(model: str, evaluation: dict, weights_dir: Path, out: Path, n: int, threads: int,
|
|
kitti_root: Path) -> dict:
|
|
import numpy as np
|
|
import onnx
|
|
import onnxruntime as ort
|
|
import torch
|
|
|
|
from harness.adapters import get as get_adapter
|
|
from harness.eval import flow
|
|
from harness.eval.common import T1, cosine
|
|
|
|
adapter = get_adapter(model)
|
|
directory = out / model
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
result = {"model": model, "status": "running", "started_at": time.time(),
|
|
"input_spec": adapter.INPUT_SPEC, "preprocess": adapter.PREPROCESS,
|
|
"opset": adapter.OPSET, "checkpoint": adapter.CHECKPOINT,
|
|
"runtime": {"torch": torch.__version__, "onnx": onnx.__version__,
|
|
"onnxruntime": ort.__version__, "device": "cpu", "threads": threads,
|
|
"executable": str(Path(sys.executable).resolve())},
|
|
"runtime_python": getattr(adapter, "RUNTIME_PYTHON", None),
|
|
"t1_path": evaluation["path"], "t1_sha256": evaluation["manifest_sha"],
|
|
"eval_set": evaluation,
|
|
"board_tested": False, "quantization_tested": False,
|
|
"input_contract": {"status": "unverified", "input_spec": adapter.INPUT_SPEC}}
|
|
|
|
def phase(name):
|
|
result["phase"] = name
|
|
write_json(directory / "probe.json", result)
|
|
print(f"{model}: {name}", flush=True)
|
|
|
|
try:
|
|
pinned = getattr(adapter, "RUNTIME_PYTHON", None)
|
|
if pinned and Path(sys.executable).resolve() != Path(pinned).resolve():
|
|
raise ValueError(f"{model} must run under its pinned runtime {pinned}")
|
|
phase("stage_official_checkpoint")
|
|
weights = stage_weights(adapter, weights_dir)
|
|
result["weights_path"] = str(weights)
|
|
result["weights_sha256"] = sha256(weights)
|
|
phase("verify_input_contract")
|
|
checks_path = verify_contract(adapter, evaluation, kitti_root, out, model)
|
|
phase("strict_load")
|
|
built = adapter.build(weights, adapter.INPUT_SPEC)
|
|
result["strict_load"] = True
|
|
result["parameters"] = sum(p.numel() for p in built.parameters())
|
|
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
|
|
try:
|
|
if not 0 < n <= len(t1.items):
|
|
raise ValueError("--n must be positive and <= T1 n")
|
|
result["prepared_t1_n"] = len(t1.items)
|
|
t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)}
|
|
result["sample_ids"] = [item["id"] for item in t1.items]
|
|
phase("real_pair_forward")
|
|
first = flow.inputs(t1, t1.items[0], adapter.INPUT_SPEC, model)
|
|
np.save(directory / "input_first_pair.npy", first)
|
|
start = time.monotonic()
|
|
with torch.inference_mode():
|
|
reference = built(torch.from_numpy(first)).numpy()
|
|
if reference.shape != (1, 2, 384, 1024) or not np.isfinite(reference).all():
|
|
raise ValueError(f"invalid {model} forward output")
|
|
result["first_forward_seconds"] = time.monotonic() - start
|
|
np.save(directory / "fp32_first_flow.npy", reference)
|
|
phase("onnx_export_and_check")
|
|
onnx_path = directory / f"{model}.onnx"
|
|
start = time.monotonic()
|
|
export_kwargs = dict(opset_version=adapter.OPSET, input_names=["input"],
|
|
output_names=["flow"], dynamic_axes=None,
|
|
do_constant_folding=True)
|
|
if tuple(int(v) for v in torch.__version__.split(".")[:2]) >= (2, 5):
|
|
export_kwargs["dynamo"] = False
|
|
with torch.no_grad():
|
|
torch.onnx.export(built, torch.from_numpy(first), str(onnx_path), **export_kwargs)
|
|
onnx.checker.check_model(str(onnx_path), full_check=True)
|
|
graph = onnx.load(str(onnx_path))
|
|
from collections import Counter
|
|
result["onnx"] = {"path": str(onnx_path), "sha256": sha256(onnx_path),
|
|
"size_bytes": onnx_path.stat().st_size,
|
|
"operators": dict(Counter(node.op_type for node in graph.graph.node)),
|
|
"export_and_check_seconds": time.monotonic() - start,
|
|
"input_shapes": [[v.dim_value for v in t.type.tensor_type.shape.dim]
|
|
for t in graph.graph.input],
|
|
"output_shapes": [[v.dim_value for v in t.type.tensor_type.shape.dim]
|
|
for t in graph.graph.output]}
|
|
del graph
|
|
phase("onnxruntime_real_pairs_and_ground_truth")
|
|
opts = ort.SessionOptions()
|
|
opts.intra_op_num_threads = threads
|
|
opts.inter_op_num_threads = 1
|
|
session = ort.InferenceSession(str(onnx_path), sess_options=opts,
|
|
providers=["CPUExecutionProvider"])
|
|
comparisons, fp32_samples, ort_samples = [], [], []
|
|
for item in t1.items:
|
|
x = flow.inputs(t1, item, adapter.INPUT_SPEC, model)
|
|
with torch.inference_mode():
|
|
native = built(torch.from_numpy(x)).numpy()
|
|
exported = session.run(None, {"input": x})[0]
|
|
truth, valid = flow.ground_truth(t1, item)
|
|
a = flow.sample_metrics(flow.restore_flow([native], adapter.INPUT_SPEC, truth.shape[-2:]), truth, valid)
|
|
b = flow.sample_metrics(flow.restore_flow([exported], adapter.INPUT_SPEC, truth.shape[-2:]), truth, valid)
|
|
diff = np.abs(native - exported)
|
|
comparison = {"id": item["id"], "cosine": cosine(native, exported),
|
|
"max_abs": float(diff.max()), "mean_abs": float(diff.mean()),
|
|
"epe2d_difference": abs(a["epe2d"] - b["epe2d"]),
|
|
"fl_all_difference_pp": abs(a["fl_all"] - b["fl_all"])}
|
|
comparisons.append(comparison)
|
|
fp32_samples.append({"id": item["id"], **a})
|
|
ort_samples.append({"id": item["id"], **b})
|
|
print(json.dumps(comparison), flush=True)
|
|
if (comparison["cosine"] < .9999 or comparison["mean_abs"] > .005
|
|
or comparison["max_abs"] > .5 or comparison["epe2d_difference"] > .01
|
|
or comparison["fl_all_difference_pp"] > .1):
|
|
raise ValueError(f"FP32 export numerical mismatch: {comparison}")
|
|
result.update(status="validated_fp32_export_only", phase="complete", comparisons=comparisons,
|
|
fp32=flow.aggregate(fp32_samples), onnxruntime=flow.aggregate(ort_samples),
|
|
fp32_samples=fp32_samples, onnxruntime_samples=ort_samples)
|
|
checks = json.loads(checks_path.read_text())
|
|
if checks["status"] != "passed" or checks["t1_sha256"] != evaluation["manifest_sha"]:
|
|
raise ValueError("missing matching real data/geometry contract checks")
|
|
result["input_contract"] = {"status": "validated", "input_spec": adapter.INPUT_SPEC,
|
|
"packing": "RGB(t) channels 0:3; RGB(t+1) channels 3:6; NCHW float32; official scale per recipe",
|
|
"output": "one N,2,H,W tensor (u,v), final update; input-image pixel displacements",
|
|
"ground_truth": "KITTI2015 flow_occ uint16 RGB, (RG-32768)/64, B=valid",
|
|
"evaluation_coordinates": "original image; bilinear resize plus independent u/v scaling",
|
|
"sample_ids": result["sample_ids"], "t1_sha256": evaluation["manifest_sha"],
|
|
"checks_path": str(checks_path),
|
|
"onnx_sha256": result["onnx"]["sha256"], "opset": adapter.OPSET}
|
|
write_json(directory / "baseline_probe.json", {"task": "flow", "model": model,
|
|
"t1_manifest_sha": evaluation["manifest_sha"], "probe_subset_ids": result["sample_ids"],
|
|
"metrics": result["fp32"], "_flow_samples": fp32_samples,
|
|
"note": "probe subset only; not a completed T1 baseline"})
|
|
finally:
|
|
t1.tar.close()
|
|
except Exception as exc:
|
|
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
|
|
(directory / "error.txt").write_text(traceback.format_exc())
|
|
result["finished_at"] = time.time()
|
|
write_json(directory / "probe.json", result)
|
|
print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True)
|
|
return result
|
|
|
|
|
|
def mirror(model: str, out: Path) -> None:
|
|
"""Copy the finished proof into the shared adapter-probes cache location."""
|
|
from harness import paths
|
|
|
|
source = out / model / "probe.json"
|
|
shared = paths.CACHE / "adapter-probes" / model / "probe.json"
|
|
shared.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(source, shared)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--model", action="append", choices=MODELS, required=True)
|
|
parser.add_argument("--t1-manifest", type=Path, required=True,
|
|
help="manifest.json of the existing frozen kitti2015 n100 T1 package")
|
|
parser.add_argument("--kitti-root", type=Path, required=True,
|
|
help="directory containing training/ and testing/ (calibration pairs)")
|
|
parser.add_argument("--weights-dir", type=Path, required=True)
|
|
parser.add_argument("--out", type=Path, required=True, help="new private probe directory")
|
|
parser.add_argument("--n", type=int, default=3)
|
|
parser.add_argument("--threads", type=int, default=2)
|
|
parser.add_argument("--seed", type=int, default=20260914)
|
|
parser.add_argument("--calib-root", type=Path, help="existing data/calib root; defaults to private --out/data/calib")
|
|
args = parser.parse_args()
|
|
if not 1 <= args.n <= 100 or args.threads < 1:
|
|
parser.error("require 1 <= n <= 100 and positive threads")
|
|
if len(set(args.model)) != 1:
|
|
parser.error("probe exactly one family per invocation: upstream packages share module "
|
|
"names (utils/core) across families and must not be imported together")
|
|
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
|
import torch
|
|
torch.set_num_threads(args.threads)
|
|
torch.set_num_interop_threads(1)
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
evaluation = open_eval_set(args.t1_manifest)
|
|
models = list(dict.fromkeys(args.model))
|
|
from harness.adapters import get as get_adapter
|
|
adapters = {m: get_adapter(m) for m in models}
|
|
calib_root = args.calib_root or args.out / "data" / "calib"
|
|
calibration = {}
|
|
for model, adapter in adapters.items():
|
|
calib = prepare_calibration(adapter, args.kitti_root, evaluation, calib_root, args.seed)
|
|
calibration[model] = str(calib)
|
|
print(f"{model}: calibration {calib}", flush=True)
|
|
write_json(args.out / "data_preparation.json", {"t1": str(evaluation["path"]),
|
|
"t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation, "n": args.n,
|
|
"calibration": calibration})
|
|
results = [probe(model, evaluation, args.weights_dir, args.out, args.n, args.threads,
|
|
args.kitti_root)
|
|
for model in models]
|
|
for model, result in zip(models, results):
|
|
if result["status"] == "validated_fp32_export_only":
|
|
mirror(model, args.out)
|
|
write_json(args.out / "summary.json", {"results": results, "complete": True})
|
|
return int(any(result["status"] != "validated_fp32_export_only" for result in results))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|