data-gen-bpu/scripts/probe_raft_adapter.py

401 lines
22 KiB
Python

"""Private RAFT checkpoint/real KITTI pair/ONNX probe; never touches run state.
Example (CPU only, no installs):
python scripts/probe_raft_adapter.py --model raft.small --model raft.large \
--kitti-root /data/kitti_2015 --out /private/raft-probe --t1-n 100 --n 3
The resulting T1 contains 100 *pairs*, but the probe scores only --n pairs.
It is not a completed 100-pair FP32 baseline or board/quantization measurement.
"""
from __future__ import annotations
import argparse
import hashlib
import importlib
import io
import json
import os
from pathlib import Path
import random
import sys
import tarfile
import time
import traceback
import urllib.request
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
def sha256(path: Path) -> str:
h = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
def write_json(path: Path, value: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temp = path.with_suffix(path.suffix + ".tmp")
temp.write_text(json.dumps(value, indent=2, allow_nan=False))
temp.replace(path)
def prepare_t1(root: Path, out: Path, n: int, seed: int) -> Path:
"""Package complete temporal pairs and labels, never arbitrary PNG files."""
training = root / "training"
audit = {"root": str(root.resolve()), "split": "training", "dataset": "kitti2015"}
complete, incomplete = [], []
for first in sorted((training / "image_2").glob("*_10.png")):
sid = first.stem.removesuffix("_10")
rel = {"file": f"image_2/{sid}_10.png", "file2": f"image_2/{sid}_11.png",
"flow": f"flow_occ/{sid}_10.png"}
missing = [v for v in rel.values() if not (training / v).is_file()]
if missing:
incomplete.append({"id": sid, "missing": missing})
else:
complete.append({"id": sid, **rel, "flow_format": "kitti_png16"})
audit.update(complete_pairs=len(complete), incomplete_pairs=incomplete,
image10_count=len(list((training / "image_2").glob("*_10.png"))),
image11_count=len(list((training / "image_2").glob("*_11.png"))),
flow_occ_count=len(list((training / "flow_occ").glob("*_10.png"))),
flow_noc_count=len(list((training / "flow_noc").glob("*_10.png"))))
write_json(out / "dataset_audit.json", audit)
if not 0 < n <= len(complete) or incomplete:
raise ValueError(f"cannot build {n} complete KITTI pairs: {audit}")
items = sorted(random.Random(seed).sample(complete, n), key=lambda it: it["id"])
destination = out / f"t1-n{n}-s{seed}"
destination.mkdir(parents=True, exist_ok=True)
tar_path = destination / f"{seed}.tar"
manifest_path = destination / "manifest.json"
if manifest_path.exists():
previous = json.loads(manifest_path.read_text())
if previous.get("source_root") != str(root.resolve()) or previous.get("n") != n or sha256(tar_path) != previous["sha256"]:
raise ValueError("existing private T1 differs; choose another --out")
return tar_path
blobs = {}
for item in items:
item["file_sha256"] = {}
for field in ("file", "file2", "flow"):
rel = item[field]
blobs[rel] = (training / rel).read_bytes()
item["file_sha256"][field] = hashlib.sha256(blobs[rel]).hexdigest()
content_sha = hashlib.sha256(json.dumps(items, sort_keys=True).encode()).hexdigest()
man = {"dataset": "kitti2015", "version": "data_scene_flow", "task": "flow", "split": "training",
"seed": seed, "n": n, "sha256": content_sha, "items": items,
"eval_set_id": f"kitti2015-flow-n{n}-{content_sha[:12]}", "source_root": str(root.resolve())}
blobs["manifest.json"] = json.dumps(man, sort_keys=True).encode()
with tarfile.open(tar_path.with_suffix(".tar.tmp"), "w") as tar:
for name, data in sorted(blobs.items()):
info = tarfile.TarInfo(name)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
tar_path.with_suffix(".tar.tmp").replace(tar_path)
write_json(manifest_path, {k: v for k, v in man.items() if k != "items"} | {
"content_sha256": content_sha, "sha256": sha256(tar_path), "tar": tar_path.name})
return tar_path
def verify_contract(t1_path: Path, root: Path, out: Path) -> None:
"""Check geometry independently of model accuracy; compare real PNG decoding."""
import numpy as np
import torch
from PIL import Image
from torchvision.datasets._optical_flow import _read_16bits_png_with_flow_and_valid_mask
from torchvision.models.optical_flow import Raft_Small_Weights
from torchvision.transforms import functional as F
from harness.adapters.raft_common import INPUT_SPEC, preprocess, input_shape
from harness.eval.common import T1
from harness.eval import flow
t1 = T1.open(t1_path)
try:
item = t1.items[0]
truth, valid = flow.ground_truth(t1, item)
official, official_valid = _read_16bits_png_with_flow_and_valid_mask(str(root / "training" / item["flow"]))
np.testing.assert_array_equal(truth, official)
np.testing.assert_array_equal(valid, official_valid)
first, second = t1.image(item), t1.image({"file": item["file2"]})
x1, x2 = [F.resize(F.convert_image_dtype(F.pil_to_tensor(im), torch.float32), [384, 1024], antialias=False)
for im in (first, second)]
x1, x2 = Raft_Small_Weights.C_T_V2.transforms()(x1, x2)
packed = flow.inputs(t1, item, INPUT_SPEC, "raft.small")
np.testing.assert_array_equal(packed, torch.cat([x1, x2])[None].numpy())
# Anisotropic scaling must change u and v independently, not only resize.
const = np.empty((1, 2, 384, 1024), dtype=np.float32)
const[:, 0], const[:, 1] = 8.0, -4.0
restored = flow.restore_flow([const], INPUT_SPEC, (192, 2048))
np.testing.assert_array_equal(restored[0], np.full((192, 2048), 16.0))
np.testing.assert_array_equal(restored[1], np.full((192, 2048), -2.0))
pixels = np.array([[[10, 30, 50]]], dtype=np.uint8)
x = preprocess(Image.fromarray(pixels), Image.fromarray(255 - pixels), "1x6x128x128")
np.testing.assert_allclose(x[0, :, 0, 0], np.r_[pixels.flatten(), (255 - pixels).flatten()] / 127.5 - 1, atol=1e-7)
pred = np.zeros((2, 2, 2), dtype=np.float32)
pred[0] = [[3, 4], [100, 100]]
mask = np.array([[True, True], [False, False]])
metrics = flow.sample_metrics(pred, np.zeros_like(pred), mask)
assert metrics["epe2d"] == 3.5 and metrics["fl_all"] == 50 and metrics["valid_pixels"] == 2
truth_fixture = np.full((2, 1, 1), 0, dtype=np.float32)
truth_fixture[0] = 100
pred_fixture = truth_fixture.copy()
pred_fixture[0] += 4
assert flow.sample_metrics(pred_fixture, truth_fixture, np.ones((1, 1), dtype=bool))["fl_all"] == 0
try:
input_shape("1x3x384x1024")
except ValueError:
pass
else:
raise AssertionError("legacy single-frame specification was accepted")
pred[0, 0, 0] = np.nan
try:
flow.sample_metrics(pred, np.zeros_like(pred), mask)
except ValueError:
pass
else:
raise AssertionError("non-finite prediction was accepted")
write_json(out / "contract_checks.json", {"status": "passed", "t1_sha256": sha256(t1_path),
"real_sample_id": item["id"],
"real_image_shape": list(truth.shape[-2:]), "real_valid_pixels": int(valid.sum()),
"checks": ["real_uint16_decode_matches_torchvision", "real_preprocess_matches_official_transform",
"anisotropic_flow_resize_and_vector_scale", "RGB_temporal_channel_order",
"valid_mask_excludes_invalid_pixels", "strict_outlier_threshold_3px_and_5pct",
"reject_legacy_single_image_input", "reject_nonfinite_prediction"]})
finally:
t1.tar.close()
def prepare_calibration(root: Path, t1_path: Path, calib_root: Path, seed: int, n: int = 128) -> Path:
"""Use distinct testing pairs; no labels are needed for calibration."""
import numpy as np
from PIL import Image
from harness.adapters.raft_common import INPUT_SPEC, PREPROCESS, preprocess
recipe = hashlib.sha256(json.dumps(PREPROCESS, sort_keys=True).encode()).hexdigest()[:16]
destination = calib_root / "flow" / INPUT_SPEC / recipe
manifest_path = destination / "manifest.json"
if manifest_path.exists():
manifest = json.loads(manifest_path.read_text())
digest = hashlib.sha256()
files = sorted((destination / "npy_f32").glob("*.npy"))
for path in files:
digest.update(path.read_bytes())
if (manifest.get("count") != n or len(files) != n or manifest.get("seed") != seed
or manifest.get("t1_sha256") != sha256(t1_path) or manifest.get("preprocess") != PREPROCESS
or manifest.get("sha256") != digest.hexdigest()):
raise ValueError(f"existing calibration differs; refusing to overwrite {destination}")
return destination
if destination.exists():
raise FileExistsError(f"partial/existing calibration; refusing to overwrite {destination}")
with tarfile.open(t1_path) as tar:
t1 = json.load(tar.extractfile("manifest.json"))
excluded = {it["file_sha256"][key] for it in t1["items"] for key in ("file", "file2")}
image_dir = root / "testing" / "image_2"
candidates = [p for p in sorted(image_dir.glob("*_10.png"))
if p.with_name(p.name.replace("_10.png", "_11.png")).is_file()]
random.Random(seed).shuffle(candidates)
if len(candidates) < n:
raise ValueError(f"only {len(candidates)} unique testing pairs available; need {n}")
# Build alongside the final path and publish only after all real pairs pass.
temporary = destination.with_name(destination.name + f".partial-{os.getpid()}")
npy_dir = temporary / "npy_f32"
npy_dir.mkdir(parents=True, exist_ok=False)
digest, items, seen = hashlib.sha256(), [], set()
for first in candidates:
second = first.with_name(first.name.replace("_10.png", "_11.png"))
raw = [first.read_bytes(), second.read_bytes()]
hashes = tuple(hashlib.sha256(data).hexdigest() for data in raw)
if any(h in excluded for h in hashes) or hashes in seen:
continue
seen.add(hashes)
images = [Image.open(io.BytesIO(data)).convert("RGB") for data in raw]
packed = preprocess(*images, INPUT_SPEC)
if packed.shape != (1, 6, 384, 1024) or not np.isfinite(packed).all():
raise ValueError("invalid real calibration pair")
path = npy_dir / f"{len(items):05d}.npy"
np.save(path, packed)
digest.update(path.read_bytes())
items.append({"id": f"kitti2015/testing/{first.stem.removesuffix('_10')}",
"file": str(first), "file2": str(second), "file_sha256": list(hashes),
"npy": path.name, "npy_sha256": sha256(path)})
if len(items) == n:
break
if len(items) != n:
raise ValueError(f"only {len(items)} unique, nonoverlapping calibration pairs; need {n}")
write_json(temporary / "manifest.json", {"task": "flow", "input_spec": INPUT_SPEC,
"count": n, "seed": seed, "source": "testing-unlabeled", "dataset": "kitti2015",
"sha256": digest.hexdigest(), "files": [it["file"] + "|" + it["file2"] for it in items],
"items": items, "t1_sha256": sha256(t1_path), "preprocess": PREPROCESS,
"t1_frame_content_overlap": 0, "unique_pair_content_count": len(seen),
"layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"})
temporary.rename(destination)
return destination
def probe(model: str, t1_path: Path, out: Path, n: int, threads: int) -> dict:
import numpy as np
import onnx
import onnxruntime as ort
import torch
import torchvision
from harness.eval.common import T1, cosine
from harness.eval import flow
adapter = importlib.import_module("harness.adapters." + model.replace(".", "_"))
directory = out / model
directory.mkdir(parents=True, exist_ok=True)
result = {"model": model, "status": "running", "started_at": time.time(),
"input_spec": adapter.INPUT_SPEC, "preprocess": adapter.PREPROCESS,
"opset": adapter.OPSET, "num_flow_updates": 12, "checkpoint": adapter.CHECKPOINT,
"runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__,
"onnx": onnx.__version__, "onnxruntime": ort.__version__,
"device": "cpu", "threads": threads},
"t1_path": str(t1_path), "t1_sha256": sha256(t1_path),
"board_tested": False, "quantization_tested": False,
"input_contract": {"status": "unverified", "input_spec": adapter.INPUT_SPEC}}
def phase(name):
result["phase"] = name
write_json(directory / "probe.json", result)
print(f"{model}: {name}", flush=True)
try:
phase("download_official_checkpoint")
weights = directory / adapter.CHECKPOINT["url"].rsplit("/", 1)[1]
if not weights.exists():
with urllib.request.urlopen(adapter.CHECKPOINT["url"], timeout=90) as remote, weights.with_suffix(".part").open("wb") as target:
while block := remote.read(1 << 20):
target.write(block)
weights.with_suffix(".part").replace(weights)
result["weights_path"] = str(weights)
result["weights_sha256"] = sha256(weights)
phase("strict_load")
m = adapter.build(weights, adapter.INPUT_SPEC)
result["strict_load"] = True
result["parameters"] = sum(p.numel() for p in m.parameters())
t1 = T1.open(t1_path, result["t1_sha256"])
try:
if not 0 < n <= len(t1.items):
raise ValueError("--n must be positive and <= --t1-n")
result["prepared_t1_n"] = len(t1.items)
t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)}
result["sample_ids"] = [it["id"] for it in t1.items]
phase("real_pair_forward")
first = flow.inputs(t1, t1.items[0], adapter.INPUT_SPEC, model)
np.save(directory / "input_first_pair.npy", first)
start = time.monotonic()
with torch.inference_mode():
reference = m(torch.from_numpy(first)).numpy()
native = m.core(torch.from_numpy(first[:, :3]), torch.from_numpy(first[:, 3:6]), num_flow_updates=12)[-1].numpy()
if reference.shape != (1, 2, 384, 1024) or not np.isfinite(reference).all():
raise ValueError("invalid RAFT forward output")
np.testing.assert_array_equal(reference, native)
result["native_pair_wrapper_max_abs"] = float(np.abs(reference - native).max())
result["first_and_native_forward_seconds"] = time.monotonic() - start
np.save(directory / "fp32_first_flow.npy", reference)
phase("onnx_export_and_check")
onnx_path = directory / f"{model}.onnx"
start = time.monotonic()
with torch.no_grad():
torch.onnx.export(m, torch.from_numpy(first), str(onnx_path), dynamo=False,
opset_version=adapter.OPSET, input_names=["input"], output_names=["flow"],
dynamic_axes=None, do_constant_folding=True)
onnx.checker.check_model(str(onnx_path), full_check=True)
graph = onnx.load(onnx_path)
from collections import Counter
result["onnx"] = {"path": str(onnx_path), "sha256": sha256(onnx_path),
"size_bytes": onnx_path.stat().st_size,
"operators": dict(Counter(node.op_type for node in graph.graph.node)),
"export_and_check_seconds": time.monotonic() - start,
"input_shapes": [[v.dim_value for v in t.type.tensor_type.shape.dim] for t in graph.graph.input],
"output_shapes": [[v.dim_value for v in t.type.tensor_type.shape.dim] for t in graph.graph.output]}
del graph
phase("onnxruntime_real_pairs_and_ground_truth")
opts = ort.SessionOptions()
opts.intra_op_num_threads = threads
opts.inter_op_num_threads = 1
session = ort.InferenceSession(str(onnx_path), sess_options=opts, providers=["CPUExecutionProvider"])
comparisons, fp32_samples, ort_samples = [], [], []
for item in t1.items:
x = flow.inputs(t1, item, adapter.INPUT_SPEC, model)
with torch.inference_mode():
native = m(torch.from_numpy(x)).numpy()
exported = session.run(None, {"input": x})[0]
truth, valid = flow.ground_truth(t1, item)
a = flow.sample_metrics(flow.restore_flow([native], adapter.INPUT_SPEC, truth.shape[-2:]), truth, valid)
b = flow.sample_metrics(flow.restore_flow([exported], adapter.INPUT_SPEC, truth.shape[-2:]), truth, valid)
diff = np.abs(native - exported)
comparison = {"id": item["id"], "cosine": cosine(native, exported),
"max_abs": float(diff.max()), "mean_abs": float(diff.mean()),
"epe2d_difference": abs(a["epe2d"] - b["epe2d"]),
"fl_all_difference_pp": abs(a["fl_all"] - b["fl_all"])}
comparisons.append(comparison)
fp32_samples.append({"id": item["id"], **a})
ort_samples.append({"id": item["id"], **b})
print(json.dumps(comparison), flush=True)
if (comparison["cosine"] < .9999 or comparison["mean_abs"] > .005
or comparison["max_abs"] > .5 or comparison["epe2d_difference"] > .01
or comparison["fl_all_difference_pp"] > .1):
raise ValueError(f"FP32 export numerical mismatch: {comparison}")
result.update(status="validated_fp32_export_only", phase="complete", comparisons=comparisons,
fp32=flow.aggregate(fp32_samples), onnxruntime=flow.aggregate(ort_samples),
fp32_samples=fp32_samples, onnxruntime_samples=ort_samples)
checks = json.loads((out / "contract_checks.json").read_text())
if checks["status"] != "passed" or checks["t1_sha256"] != result["t1_sha256"]:
raise ValueError("missing matching real data/geometry contract checks")
result["input_contract"] = {"status": "validated", "input_spec": adapter.INPUT_SPEC,
"packing": "RGB(t) channels 0:3; RGB(t+1) channels 3:6; NCHW float32 [-1,1]",
"output": "one N,2,H,W tensor (u,v), 12th update; input-image pixel displacements",
"ground_truth": "KITTI2015 flow_occ uint16 RGB, (RG-32768)/64, B=valid",
"evaluation_coordinates": "original image; bilinear resize plus independent u/v scaling",
"sample_ids": result["sample_ids"], "t1_sha256": result["t1_sha256"],
"checks_path": str(out / "contract_checks.json"),
"onnx_sha256": result["onnx"]["sha256"], "opset": adapter.OPSET}
write_json(directory / "baseline_probe.json", {"task": "flow", "model": model,
"t1_manifest_sha": t1.manifest["sha256"], "probe_subset_ids": result["sample_ids"],
"metrics": result["fp32"], "_flow_samples": fp32_samples,
"note": "probe subset only; not a completed T1 baseline"})
finally:
t1.tar.close()
except Exception as exc:
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
(directory / "error.txt").write_text(traceback.format_exc())
result["finished_at"] = time.time()
write_json(directory / "probe.json", result)
print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True)
return result
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", action="append", choices=["raft.small", "raft.large"], required=True)
parser.add_argument("--kitti-root", type=Path, required=True, help="directory containing training/image_2 and training/flow_occ")
parser.add_argument("--out", type=Path, required=True, help="new private probe directory, outside existing run/cache state")
parser.add_argument("--n", type=int, default=3)
parser.add_argument("--t1-n", type=int, default=100)
parser.add_argument("--seed", type=int, default=20260914)
parser.add_argument("--threads", type=int, default=2)
parser.add_argument("--calib-root", type=Path, help="existing data/calib root; defaults to private --out/data/calib")
parser.add_argument("--prepare-only", action="store_true", help="prepare real T1/calibration without model inference")
args = parser.parse_args()
if args.threads < 1:
parser.error("--threads must be positive")
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import torch
torch.set_num_threads(args.threads)
torch.set_num_interop_threads(1)
args.out.mkdir(parents=True, exist_ok=True)
t1_path = prepare_t1(args.kitti_root, args.out, args.t1_n, args.seed)
verify_contract(t1_path, args.kitti_root, args.out)
calib = prepare_calibration(args.kitti_root, t1_path, args.calib_root or args.out / "data" / "calib", args.seed)
write_json(args.out / "data_preparation.json", {"t1": str(t1_path), "t1_n": args.t1_n,
"t1_sha256": sha256(t1_path), "calibration_dir": str(calib),
"calibration_manifest": str(calib / "manifest.json"), "calibration_count": 128})
if args.prepare_only:
return 0
results = [probe(model, t1_path, args.out, args.n, args.threads) for model in dict.fromkeys(args.model)]
write_json(args.out / "summary.json", {"results": results, "complete": True})
return int(any(r["status"] != "validated_fp32_export_only" for r in results))
if __name__ == "__main__":
raise SystemExit(main())