data-gen-bpu/scripts/probe_ssdlite_adapter.py

253 lines
14 KiB
Python

"""Validate the exact SSDLite adapter on the existing fixed COCO n100 package.
This writes probe artifacts only: no jobs, collectors, registry edits, or installs.
Use --n 3 for a smoke check before the full --n 100 validation. A successful
adapter probe is not admission proof until --verify-shared-hooks also succeeds.
"""
from __future__ import annotations
import argparse
from collections import Counter
from dataclasses import replace
import hashlib
import json
import os
from pathlib import Path
import sys
import time
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from harness.adapters import ssdlite320_mobilenetv3_large as adapter
MODEL = "ssdlite320.mobilenetv3.large"
INPUT_SPEC = adapter.MODEL_SPEC["input_spec"]
def sha(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def detections(image_id, prediction, cat_map=None):
boxes, scores, labels = prediction
return [{"image_id": image_id, "category_id": cat_map[str(int(label))] if cat_map else int(label), "score": float(score),
"bbox": [float(b[0]), float(b[1]), float(b[2] - b[0]), float(b[3] - b[1])]}
for b, score, label in zip(boxes, scores, labels)]
def calibration(source_manifest, output, eval_ids):
"""Reuse the existing 128 COCO train selections, not the old YOLO tensors."""
import numpy as np
from PIL import Image
source = json.loads(Path(source_manifest).read_text())
files = [Path(p) for p in source["files"]]
if source.get("dataset") != "coco2017" or not source.get("source", "").startswith("train"):
raise ValueError("calibration must come from the real COCO train split")
if len(files) != 128 or len(set(files)) != 128 or any(not p.is_file() for p in files):
raise ValueError("128 distinct existing calibration source images are required")
if any(int(p.stem) in eval_ids for p in files):
raise ValueError("calibration and evaluation image IDs overlap")
output.mkdir(parents=True, exist_ok=True)
tensors = output / "npy_f32"; tensors.mkdir(exist_ok=True)
digest = hashlib.sha256()
for i, path in enumerate(files):
with Image.open(path) as image:
x = adapter.preprocess(image, INPUT_SPEC)
if x.shape != (1, 3, 320, 320) or x.dtype != np.float32 or not np.isfinite(x).all():
raise ValueError(f"invalid calibration tensor: {path}")
destination = tensors / f"{i:05d}.npy"
if destination.exists():
np.testing.assert_array_equal(np.load(destination), x)
else:
np.save(destination, x)
digest.update(destination.read_bytes())
result = {"task": "det", "input_spec": INPUT_SPEC, "count": 128, "seed": source["seed"],
"source": source["source"], "dataset": "coco2017", "sha256": digest.hexdigest(),
"files": [str(p) for p in files], "preprocess": adapter.PREPROCESS,
"layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32",
"source_manifest": str(source_manifest), "source_manifest_sha256": sha(source_manifest)}
(output / "manifest.json").write_text(json.dumps(result, indent=2))
return {"path": str(output), "count": 128, "sha256": result["sha256"], "overlap_with_t1": 0}
def probe(args):
import numpy as np
import onnx
import onnxruntime as ort
import torch
import torchvision
from torchvision.transforms.functional import pil_to_tensor
from harness.continuation import load_eval_set
from harness.eval import EVALUATOR_VERSION
from harness.eval.common import T1, cosine
from harness.eval.det import coco_eval
output = Path(args.out); output.mkdir(parents=True, exist_ok=True)
proof = output / "probe.json"
if proof.exists():
(output / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes())
result = {"model": MODEL, "task": "det", "status": "running", "started_at": time.time(),
"checkpoint": adapter.CHECKPOINT, "preprocess": adapter.PREPROCESS,
"runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__,
"onnx": onnx.__version__, "onnxruntime": ort.__version__},
"shared_hooks_verified": False}
proof.write_text(json.dumps(result, indent=2))
t1 = None
try:
dataset, evaluation = load_eval_set(args.t1)
if dataset != "coco2017" or evaluation["n"] != 100:
raise ValueError("use the existing fixed COCO n100 evaluation package")
result["eval_set"] = evaluation
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
cat_map = t1.manifest["cat_map"]
subset = replace(t1, manifest={**t1.manifest, "items": t1.items[:args.n], "n": args.n})
model = adapter.build(Path(args.weights), INPUT_SPEC)
reference = adapter.official_model(Path(args.weights), adapter.CHECKPOINT["sha256"])
def forward(x):
with torch.inference_mode():
return [model(torch.from_numpy(x)).numpy()]
onnx_path = output / f"{MODEL}.onnx"
with torch.no_grad():
torch.onnx.export(model, torch.zeros(1, 3, 320, 320), str(onnx_path), dynamo=False,
opset_version=adapter.OPSET, input_names=["input"], output_names=["detections"],
do_constant_folding=True, dynamic_axes=None)
graph = onnx.load(str(onnx_path)); onnx.checker.check_model(graph, full_check=True)
shapes = {v.name: [d.dim_value for d in v.type.tensor_type.shape.dim]
for v in list(graph.graph.input) + list(graph.graph.output)}
if shapes != {"input": [1, 3, 320, 320], "detections": [1, 3234, 95]}:
raise ValueError(f"unexpected ONNX shapes: {shapes}")
ops = dict(Counter(n.op_type for n in graph.graph.node))
if set(ops) & {"NonMaxSuppression", "TopK", "NonZero", "Loop", "If"}:
raise ValueError(f"dynamic postprocessing left in ONNX: {ops}")
options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1
session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"])
def exported(x):
return session.run(None, {"input": x})
fp_dets, onnx_dets, official_dets, comparisons = [], [], [], []
for i, item in enumerate(subset.items):
image = subset.image(item)
x = adapter.preprocess(image, INPUT_SPEC)
image_tensor = pil_to_tensor(image).float() / 255
with torch.inference_mode():
expected_x = reference.transform([image_tensor])[0].tensors.numpy()
expected = reference([image_tensor])[0]
np.testing.assert_array_equal(x, expected_x)
raw, converted = forward(x)[0], exported(x)[0]
if raw.shape != (1, 3234, 95) or not np.isfinite(raw).all() or not np.isfinite(converted).all():
raise ValueError("invalid raw output")
# Validate boxes and probabilities separately; boxes must not hide a bad classifier.
np.testing.assert_allclose(raw[:, :, :4], converted[:, :, :4], rtol=2e-4, atol=2e-3)
np.testing.assert_allclose(raw[:, :, 4:], converted[:, :, 4:], rtol=3e-4, atol=3e-5)
predicted = adapter.decode([raw], image.size)
official = tuple(expected[key].numpy() for key in ("boxes", "scores", "labels"))
mapped_labels = np.asarray([cat_map[str(int(label))] for label in predicted[2]])
np.testing.assert_array_equal(mapped_labels, official[2])
np.testing.assert_allclose(predicted[0], official[0], rtol=1e-6, atol=1e-4)
np.testing.assert_allclose(predicted[1], official[1], rtol=1e-6, atol=1e-7)
transformed = adapter.decode([converted], image.size)
fp_dets.extend(detections(item["id"], predicted, cat_map))
official_dets.extend(detections(item["id"], official))
onnx_dets.extend(detections(item["id"], transformed, cat_map))
comparisons.append({"image_id": item["id"], "original_size": list(image.size),
"boxes_max_abs": float(np.max(np.abs(raw[:, :, :4] - converted[:, :, :4]))),
"scores_max_abs": float(np.max(np.abs(raw[:, :, 4:] - converted[:, :, 4:]))),
"boxes_cosine": cosine(raw[:, :, :4], converted[:, :, :4]),
"scores_cosine": cosine(raw[:, :, 4:], converted[:, :, 4:]),
"official_predictions": len(official[1]), "official_match": True})
if (i + 1) % 25 == 0 or i == 0:
print(f"validated {i + 1}/{args.n}", flush=True)
metrics = {}
for key, ds in (("fp32", fp_dets), ("onnx", onnx_dets), ("official", official_dets)):
(output / f"{key}-predictions.json").write_text(json.dumps(ds))
metrics[key] = coco_eval(subset, ds)
if metrics["fp32"]["map_50_95"] <= 0 or abs(metrics["fp32"]["map_50_95"] - metrics["onnx"]["map_50_95"]) > .02:
raise ValueError(f"FP32/ONNX AP mismatch: {metrics}")
result.update(**metrics, comparisons=comparisons, sample_ids=[it["id"] for it in subset.items],
weights_path=str(Path(args.weights).resolve()), weights_sha256=sha(args.weights),
onnx_path=str(onnx_path), onnx_sha256=sha(onnx_path), shapes=shapes, onnx_ops=ops)
if args.n == 100:
if not args.calib_manifest:
raise ValueError("full validation requires --calib-manifest for 128 real COCO train images")
result["calibration"] = calibration(args.calib_manifest, output / "calibration", {it["id"] for it in t1.items})
baseline = {"task": "det", "model": MODEL, "evaluator_version": EVALUATOR_VERSION,
"t1_manifest_sha": t1.manifest["sha256"], "n": 100, "metrics": metrics["fp32"]}
baseline_dir = output / "baseline"; baseline_dir.mkdir(exist_ok=True)
xs = {f"s{i:03d}": adapter.preprocess(t1.image(it), INPUT_SPEC) for i, it in enumerate(t1.g2_subset(32))}
np.savez(baseline_dir / "g2_inputs.npz", **xs)
np.savez(baseline_dir / "g2_fp32.npz", **{f"{key}/out0": forward(x)[0] for key, x in xs.items()})
(baseline_dir / "metrics.json").write_text(json.dumps(baseline, indent=2))
result["baseline_path"] = str(baseline_dir)
if args.verify_shared_hooks:
from harness.adapters import calibration_dir
from harness.eval import det
from harness.eval.run import g2_inputs
from harness.registry import model_row
row = model_row(MODEL)
if (row["source"] != "custom" or row["input_spec"] != INPUT_SPEC
or row.get("checkpoint") != adapter.CHECKPOINT or row.get("preprocess") != adapter.PREPROCESS):
raise ValueError("shared model recipe differs from the validated adapter")
check = {**baseline}; det.run_fp32(forward, t1, INPUT_SPEC, check)
if check["metrics"] != baseline["metrics"]:
raise ValueError("shared detection FP32 hook differs from adapter predictions")
quant = {"model": MODEL}; det.run_quant(exported, t1, INPUT_SPEC, baseline, quant)
if quant["metrics"] != metrics["onnx"]:
raise ValueError("shared detection quant hook differs from adapter predictions")
shared_xs = g2_inputs("det", t1, INPUT_SPEC, 32, model=MODEL)
for key in xs:
np.testing.assert_array_equal(xs[key], shared_xs[key])
shared_calib = calibration_dir(row)
shared_manifest = json.loads((shared_calib / "manifest.json").read_text())
if (shared_manifest.get("preprocess") != adapter.PREPROCESS or shared_manifest.get("count") != 128
or shared_manifest.get("sha256") != result["calibration"]["sha256"]):
raise ValueError("shared calibration recipe or tensor identity differs from this probe")
digest = hashlib.sha256()
tensors = sorted((shared_calib / shared_manifest["npy_dir"]).glob("*.npy"))
if len(tensors) != 128:
raise ValueError("shared calibration requires exactly 128 actual tensors")
for path in tensors:
digest.update(path.read_bytes())
if digest.hexdigest() != shared_manifest["sha256"]:
raise ValueError("shared calibration files do not match their recorded digest")
result["calibration"]["shared_path"] = str(shared_calib)
result["shared_source_sha256"] = {
str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p)
for p in (Path(det.__file__), Path(sys.modules[g2_inputs.__module__].__file__),
Path(__file__).with_name("make_calib.py"))}
result["shared_hooks_verified"] = True
result["status"] = "validated_fp32_export_only" if args.verify_shared_hooks else "validated_adapter_only"
else:
result["status"] = "validated_smoke_only"
except Exception as exc:
import traceback
traceback.print_exc()
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
finally:
if t1 is not None:
t1.tar.close()
result["finished_at"] = time.time()
result["source_sha256"] = {p.name: sha(p) for p in (Path(__file__), Path(adapter.__file__),
Path(adapter.__file__).with_name("ssdlite_common.py"))}
proof.write_text(json.dumps(result, indent=2))
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
print("ALL DONE", flush=True)
return result
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--weights", required=True)
parser.add_argument("--t1", required=True)
parser.add_argument("--out", required=True)
parser.add_argument("--n", type=int, choices=(3, 100), default=100)
parser.add_argument("--calib-manifest")
parser.add_argument("--verify-shared-hooks", action="store_true")
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import torch
torch.set_num_threads(2)
raise SystemExit(probe(args)["status"] == "blocked")