data-gen-bpu/scripts/probe_flow_adapters.py

421 lines
22 KiB
Python

"""Private batch-9 flow-family checkpoint/real KITTI pair/ONNX probe.
Reuses the raft probe judgement verbatim: strict official checkpoint load, real
temporal pairs from the frozen kitti2015 T1 package, ONNX export with checker,
and per-sample FP32-vs-ONNX gates cosine >= .9999, mean_abs <= .005,
max_abs <= .5, epe2d_difference <= .01, fl_all_difference_pp <= .1. It never
touches run state; the finished proof is mirrored to cache/adapter-probes.
Run with the interpreter the family adapter pins (RUNTIME_PYTHON; default bench
venv). Example (CPU only, no installs):
~/toolchains/bench-venv/bin/python scripts/probe_flow_adapters.py \
--model gmflow --model unimatch.flow \
--t1-manifest /home/ming01.wang/bench-data/t1/kitti2015/n100-s20260914/manifest.json \
--kitti-root /mnt/data/qi.xiong/Dataset/StereoDataset/KITTI/data_scene_flow/kitti_2015 \
--weights-dir ~/bench-cache/weights/flow-batch9 --out ~/flow-probe-20260919 --n 3
"""
from __future__ import annotations
import argparse
import hashlib
import importlib
import json
import os
from pathlib import Path
import random
import shutil
import sys
import tarfile
import time
import traceback
import urllib.request
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
MODELS = ["gmflow", "unimatch.flow", "gma", "sea.raft", "flow1d", "memflow",
"flowformer", "flowformerplusplus", "pwc.net", "liteflownet2", "flownet2"]
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for block in iter(lambda: stream.read(1 << 20), b""):
digest.update(block)
return digest.hexdigest()
def write_json(path: Path, value: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(value, indent=2, allow_nan=False))
temporary.replace(path)
def open_eval_set(t1_manifest: Path) -> dict:
from harness.continuation import load_eval_set
dataset, evaluation = load_eval_set(t1_manifest)
evaluation["eval_set_id"] = json.loads(t1_manifest.read_text())["eval_set_id"]
return evaluation
def stage_weights(adapter, weights_dir: Path) -> Path:
checkpoint = adapter.CHECKPOINT
name = checkpoint.get("filename") or checkpoint["url"].rsplit("/", 1)[1]
destination = weights_dir / name
if destination.is_file() and sha256(destination) == checkpoint["sha256"]:
return destination
destination = destination.with_suffix(destination.suffix + ".part")
destination.unlink(missing_ok=True)
with urllib.request.urlopen(checkpoint["url"], timeout=300) as remote, destination.open("wb") as target:
while block := remote.read(1 << 20):
target.write(block)
if sha256(destination) != checkpoint["sha256"]:
raise ValueError(f"downloaded {checkpoint['url']} does not match the pinned sha256")
destination.replace(destination.with_suffix(""))
return destination.with_suffix("")
def prepare_calibration(adapter, kitti_root: Path, evaluation: dict, calib_root: Path,
seed: int, n: int = 128) -> Path:
"""Distinct testing pairs through the family recipe; no labels needed."""
import numpy as np
from PIL import Image
import io
recipe = adapter.PREPROCESS
digest16 = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest()[:16]
destination = calib_root / "flow" / adapter.INPUT_SPEC / digest16
manifest_path = destination / "manifest.json"
if manifest_path.exists():
manifest = json.loads(manifest_path.read_text())
verify = hashlib.sha256()
files = sorted((destination / "npy_f32").glob("*.npy"))
for path in files:
verify.update(path.read_bytes())
if (manifest.get("count") != n or len(files) != n or manifest.get("seed") != seed
or manifest.get("t1_sha256") != evaluation["manifest_sha"]
or manifest.get("preprocess") != recipe
or manifest.get("sha256") != verify.hexdigest()):
raise ValueError(f"existing calibration differs; refusing to overwrite {destination}")
return destination
if destination.exists():
raise FileExistsError(f"partial/existing calibration; refusing to overwrite {destination}")
with tarfile.open(evaluation["path"]) as tar:
t1 = json.load(tar.extractfile("manifest.json"))
excluded = {item["file_sha256"][key] for item in t1["items"] for key in ("file", "file2")}
image_dir = kitti_root / "testing" / "image_2"
candidates = [p for p in sorted(image_dir.glob("*_10.png"))
if p.with_name(p.name.replace("_10.png", "_11.png")).is_file()]
random.Random(seed).shuffle(candidates)
if len(candidates) < n:
raise ValueError(f"only {len(candidates)} unique testing pairs available; need {n}")
temporary = destination.with_name(destination.name + f".partial-{os.getpid()}")
npy_dir = temporary / "npy_f32"
npy_dir.mkdir(parents=True, exist_ok=False)
digest, items, seen = hashlib.sha256(), [], set()
for first in candidates:
second = first.with_name(first.name.replace("_10.png", "_11.png"))
raw = [first.read_bytes(), second.read_bytes()]
hashes = tuple(hashlib.sha256(data).hexdigest() for data in raw)
if any(h in excluded for h in hashes) or hashes in seen:
continue
seen.add(hashes)
images = [Image.open(io.BytesIO(data)).convert("RGB") for data in raw]
packed = adapter.preprocess(*images, adapter.INPUT_SPEC)
if packed.shape != (1, 6, 384, 1024) or not np.isfinite(packed).all():
raise ValueError("invalid real calibration pair")
path = npy_dir / f"{len(items):05d}.npy"
np.save(path, packed)
digest.update(path.read_bytes())
items.append({"id": f"kitti2015/testing/{first.stem.removesuffix('_10')}",
"file": str(first), "file2": str(second), "file_sha256": list(hashes),
"npy": path.name, "npy_sha256": sha256(path)})
if len(items) == n:
break
if len(items) != n:
raise ValueError(f"only {len(items)} unique, nonoverlapping calibration pairs; need {n}")
write_json(temporary / "manifest.json", {"task": "flow", "input_spec": adapter.INPUT_SPEC,
"count": n, "seed": seed, "source": "testing-unlabeled", "dataset": "kitti2015",
"sha256": digest.hexdigest(), "files": [it["file"] + "|" + it["file2"] for it in items],
"items": items, "t1_sha256": evaluation["manifest_sha"], "preprocess": recipe,
"t1_frame_content_overlap": 0, "unique_pair_content_count": len(seen),
"layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"})
temporary.rename(destination)
return destination
def verify_contract(adapter, evaluation: dict, kitti_root: Path, out: Path, model: str) -> Path:
"""Geometry/protocol checks independent of model accuracy (raft口径)."""
import numpy as np
import torch
from PIL import Image
from torchvision.datasets._optical_flow import _read_16bits_png_with_flow_and_valid_mask
from torchvision.transforms import functional as F
from harness.adapters.raft_common import input_shape, preprocess as raft_preprocess
from harness.eval import flow
from harness.eval.common import T1
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
try:
item = t1.items[0]
truth, valid = flow.ground_truth(t1, item)
official, official_valid = _read_16bits_png_with_flow_and_valid_mask(
str(kitti_root / "training" / item["flow"]))
np.testing.assert_array_equal(truth, official)
np.testing.assert_array_equal(valid, official_valid)
frames = (t1.image(item), t1.image({"file": item["file2"]}))
packed = flow.inputs(t1, item, adapter.INPUT_SPEC, model)
# Same bilinear resize geometry as the raft recipe; only the value scale differs.
raft_reference = raft_preprocess(*frames, adapter.INPUT_SPEC)
scale = np.float32(127.5)
np.testing.assert_allclose(packed, (raft_reference + 1.0) * scale, rtol=0, atol=2e-3)
# Anisotropic scaling must change u and v independently, not only resize.
const = np.empty((1, 2, 384, 1024), dtype=np.float32)
const[:, 0], const[:, 1] = 8.0, -4.0
restored = flow.restore_flow([const], adapter.INPUT_SPEC, (192, 2048))
np.testing.assert_array_equal(restored[0], np.full((192, 2048), 16.0))
np.testing.assert_array_equal(restored[1], np.full((192, 2048), -2.0))
pred = np.zeros((2, 2, 2), dtype=np.float32)
pred[0] = [[3, 4], [100, 100]]
mask = np.array([[True, True], [False, False]])
metrics = flow.sample_metrics(pred, np.zeros_like(pred), mask)
assert metrics["epe2d"] == 3.5 and metrics["fl_all"] == 50 and metrics["valid_pixels"] == 2
truth_fixture = np.full((2, 1, 1), 0, dtype=np.float32)
truth_fixture[0] = 100
pred_fixture = truth_fixture.copy()
pred_fixture[0] += 4
assert flow.sample_metrics(pred_fixture, truth_fixture, np.ones((1, 1), dtype=bool))["fl_all"] == 0
try:
input_shape("1x3x384x1024")
except ValueError:
pass
else:
raise AssertionError("legacy single-frame specification was accepted")
pred[0, 0, 0] = np.nan
try:
flow.sample_metrics(pred, np.zeros_like(pred), mask)
except ValueError:
pass
else:
raise AssertionError("non-finite prediction was accepted")
checks_path = out / f"contract_checks-{model.replace('.', '_')}.json"
write_json(checks_path, {"status": "passed", "model": model,
"t1_sha256": evaluation["manifest_sha"], "real_sample_id": item["id"],
"real_image_shape": list(truth.shape[-2:]), "real_valid_pixels": int(valid.sum()),
"checks": ["real_uint16_decode_matches_torchvision",
"family_preprocess_matches_official_resize_and_scale",
"anisotropic_flow_resize_and_vector_scale",
"RGB_temporal_channel_order",
"valid_mask_excludes_invalid_pixels",
"strict_outlier_threshold_3px_and_5pct",
"reject_legacy_single_image_input",
"reject_nonfinite_prediction"]})
return checks_path
finally:
t1.tar.close()
def probe(model: str, evaluation: dict, weights_dir: Path, out: Path, n: int, threads: int,
kitti_root: Path) -> dict:
import numpy as np
import onnx
import onnxruntime as ort
import torch
from harness.adapters import get as get_adapter
from harness.eval import flow
from harness.eval.common import T1, cosine
adapter = get_adapter(model)
directory = out / model
directory.mkdir(parents=True, exist_ok=True)
result = {"model": model, "status": "running", "started_at": time.time(),
"input_spec": adapter.INPUT_SPEC, "preprocess": adapter.PREPROCESS,
"opset": adapter.OPSET, "checkpoint": adapter.CHECKPOINT,
"runtime": {"torch": torch.__version__, "onnx": onnx.__version__,
"onnxruntime": ort.__version__, "device": "cpu", "threads": threads,
"executable": str(Path(sys.executable).resolve())},
"runtime_python": getattr(adapter, "RUNTIME_PYTHON", None),
"t1_path": evaluation["path"], "t1_sha256": evaluation["manifest_sha"],
"eval_set": evaluation,
"board_tested": False, "quantization_tested": False,
"input_contract": {"status": "unverified", "input_spec": adapter.INPUT_SPEC}}
def phase(name):
result["phase"] = name
write_json(directory / "probe.json", result)
print(f"{model}: {name}", flush=True)
try:
pinned = getattr(adapter, "RUNTIME_PYTHON", None)
if pinned and Path(sys.executable).resolve() != Path(pinned).resolve():
raise ValueError(f"{model} must run under its pinned runtime {pinned}")
phase("stage_official_checkpoint")
weights = stage_weights(adapter, weights_dir)
result["weights_path"] = str(weights)
result["weights_sha256"] = sha256(weights)
phase("verify_input_contract")
checks_path = verify_contract(adapter, evaluation, kitti_root, out, model)
phase("strict_load")
built = adapter.build(weights, adapter.INPUT_SPEC)
result["strict_load"] = True
result["parameters"] = sum(p.numel() for p in built.parameters())
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
try:
if not 0 < n <= len(t1.items):
raise ValueError("--n must be positive and <= T1 n")
result["prepared_t1_n"] = len(t1.items)
t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)}
result["sample_ids"] = [item["id"] for item in t1.items]
phase("real_pair_forward")
first = flow.inputs(t1, t1.items[0], adapter.INPUT_SPEC, model)
np.save(directory / "input_first_pair.npy", first)
start = time.monotonic()
with torch.inference_mode():
reference = built(torch.from_numpy(first)).numpy()
if reference.shape != (1, 2, 384, 1024) or not np.isfinite(reference).all():
raise ValueError(f"invalid {model} forward output")
result["first_forward_seconds"] = time.monotonic() - start
np.save(directory / "fp32_first_flow.npy", reference)
phase("onnx_export_and_check")
onnx_path = directory / f"{model}.onnx"
start = time.monotonic()
export_kwargs = dict(opset_version=adapter.OPSET, input_names=["input"],
output_names=["flow"], dynamic_axes=None,
do_constant_folding=True)
if tuple(int(v) for v in torch.__version__.split(".")[:2]) >= (2, 5):
export_kwargs["dynamo"] = False
with torch.no_grad():
torch.onnx.export(built, torch.from_numpy(first), str(onnx_path), **export_kwargs)
onnx.checker.check_model(str(onnx_path), full_check=True)
graph = onnx.load(str(onnx_path))
from collections import Counter
result["onnx"] = {"path": str(onnx_path), "sha256": sha256(onnx_path),
"size_bytes": onnx_path.stat().st_size,
"operators": dict(Counter(node.op_type for node in graph.graph.node)),
"export_and_check_seconds": time.monotonic() - start,
"input_shapes": [[v.dim_value for v in t.type.tensor_type.shape.dim]
for t in graph.graph.input],
"output_shapes": [[v.dim_value for v in t.type.tensor_type.shape.dim]
for t in graph.graph.output]}
del graph
phase("onnxruntime_real_pairs_and_ground_truth")
opts = ort.SessionOptions()
opts.intra_op_num_threads = threads
opts.inter_op_num_threads = 1
session = ort.InferenceSession(str(onnx_path), sess_options=opts,
providers=["CPUExecutionProvider"])
comparisons, fp32_samples, ort_samples = [], [], []
for item in t1.items:
x = flow.inputs(t1, item, adapter.INPUT_SPEC, model)
with torch.inference_mode():
native = built(torch.from_numpy(x)).numpy()
exported = session.run(None, {"input": x})[0]
truth, valid = flow.ground_truth(t1, item)
a = flow.sample_metrics(flow.restore_flow([native], adapter.INPUT_SPEC, truth.shape[-2:]), truth, valid)
b = flow.sample_metrics(flow.restore_flow([exported], adapter.INPUT_SPEC, truth.shape[-2:]), truth, valid)
diff = np.abs(native - exported)
comparison = {"id": item["id"], "cosine": cosine(native, exported),
"max_abs": float(diff.max()), "mean_abs": float(diff.mean()),
"epe2d_difference": abs(a["epe2d"] - b["epe2d"]),
"fl_all_difference_pp": abs(a["fl_all"] - b["fl_all"])}
comparisons.append(comparison)
fp32_samples.append({"id": item["id"], **a})
ort_samples.append({"id": item["id"], **b})
print(json.dumps(comparison), flush=True)
if (comparison["cosine"] < .9999 or comparison["mean_abs"] > .005
or comparison["max_abs"] > .5 or comparison["epe2d_difference"] > .01
or comparison["fl_all_difference_pp"] > .1):
raise ValueError(f"FP32 export numerical mismatch: {comparison}")
result.update(status="validated_fp32_export_only", phase="complete", comparisons=comparisons,
fp32=flow.aggregate(fp32_samples), onnxruntime=flow.aggregate(ort_samples),
fp32_samples=fp32_samples, onnxruntime_samples=ort_samples)
checks = json.loads(checks_path.read_text())
if checks["status"] != "passed" or checks["t1_sha256"] != evaluation["manifest_sha"]:
raise ValueError("missing matching real data/geometry contract checks")
result["input_contract"] = {"status": "validated", "input_spec": adapter.INPUT_SPEC,
"packing": "RGB(t) channels 0:3; RGB(t+1) channels 3:6; NCHW float32; official scale per recipe",
"output": "one N,2,H,W tensor (u,v), final update; input-image pixel displacements",
"ground_truth": "KITTI2015 flow_occ uint16 RGB, (RG-32768)/64, B=valid",
"evaluation_coordinates": "original image; bilinear resize plus independent u/v scaling",
"sample_ids": result["sample_ids"], "t1_sha256": evaluation["manifest_sha"],
"checks_path": str(checks_path),
"onnx_sha256": result["onnx"]["sha256"], "opset": adapter.OPSET}
write_json(directory / "baseline_probe.json", {"task": "flow", "model": model,
"t1_manifest_sha": evaluation["manifest_sha"], "probe_subset_ids": result["sample_ids"],
"metrics": result["fp32"], "_flow_samples": fp32_samples,
"note": "probe subset only; not a completed T1 baseline"})
finally:
t1.tar.close()
except Exception as exc:
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
(directory / "error.txt").write_text(traceback.format_exc())
result["finished_at"] = time.time()
write_json(directory / "probe.json", result)
print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True)
return result
def mirror(model: str, out: Path) -> None:
"""Copy the finished proof into the shared adapter-probes cache location."""
from harness import paths
source = out / model / "probe.json"
shared = paths.CACHE / "adapter-probes" / model / "probe.json"
shared.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, shared)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", action="append", choices=MODELS, required=True)
parser.add_argument("--t1-manifest", type=Path, required=True,
help="manifest.json of the existing frozen kitti2015 n100 T1 package")
parser.add_argument("--kitti-root", type=Path, required=True,
help="directory containing training/ and testing/ (calibration pairs)")
parser.add_argument("--weights-dir", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True, help="new private probe directory")
parser.add_argument("--n", type=int, default=3)
parser.add_argument("--threads", type=int, default=2)
parser.add_argument("--seed", type=int, default=20260914)
parser.add_argument("--calib-root", type=Path, help="existing data/calib root; defaults to private --out/data/calib")
args = parser.parse_args()
if not 1 <= args.n <= 100 or args.threads < 1:
parser.error("require 1 <= n <= 100 and positive threads")
if len(set(args.model)) != 1:
parser.error("probe exactly one family per invocation: upstream packages share module "
"names (utils/core) across families and must not be imported together")
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import torch
torch.set_num_threads(args.threads)
torch.set_num_interop_threads(1)
args.out.mkdir(parents=True, exist_ok=True)
evaluation = open_eval_set(args.t1_manifest)
models = list(dict.fromkeys(args.model))
from harness.adapters import get as get_adapter
adapters = {m: get_adapter(m) for m in models}
calib_root = args.calib_root or args.out / "data" / "calib"
calibration = {}
for model, adapter in adapters.items():
calib = prepare_calibration(adapter, args.kitti_root, evaluation, calib_root, args.seed)
calibration[model] = str(calib)
print(f"{model}: calibration {calib}", flush=True)
write_json(args.out / "data_preparation.json", {"t1": str(evaluation["path"]),
"t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation, "n": args.n,
"calibration": calibration})
results = [probe(model, evaluation, args.weights_dir, args.out, args.n, args.threads,
args.kitti_root)
for model in models]
for model, result in zip(models, results):
if result["status"] == "validated_fp32_export_only":
mirror(model, args.out)
write_json(args.out / "summary.json", {"results": results, "complete": True})
return int(any(result["status"] != "validated_fp32_export_only" for result in results))
if __name__ == "__main__":
raise SystemExit(main())