282 lines
16 KiB
Python
282 lines
16 KiB
Python
"""Validate SSD300/YOLOX adapters on the existing fixed COCO n100 package.
|
|
|
|
This writes probe artifacts only: no jobs, collectors, registry edits, or installs.
|
|
Use --n 3 for a smoke check before the full --n 100 validation. A successful
|
|
adapter probe is not admission proof until --verify-shared-hooks also succeeds.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter
|
|
from dataclasses import replace
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
import importlib
|
|
|
|
|
|
|
|
|
|
def sha(path):
|
|
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
|
|
|
|
def detections(image_id, prediction, cat_map=None):
|
|
boxes, scores, labels = prediction
|
|
return [{"image_id": image_id, "category_id": cat_map[str(int(label))] if cat_map else int(label), "score": float(score),
|
|
"bbox": [float(b[0]), float(b[1]), float(b[2] - b[0]), float(b[3] - b[1])]}
|
|
for b, score, label in zip(boxes, scores, labels)]
|
|
|
|
|
|
def calibration(source_manifest, output, eval_ids):
|
|
"""Validate the 128 real train inputs, reusing an identical shared recipe."""
|
|
import numpy as np
|
|
from PIL import Image
|
|
|
|
source = json.loads(Path(source_manifest).read_text())
|
|
files = [Path(p) for p in source["files"]]
|
|
if source.get("dataset") != "coco2017" or not source.get("source", "").startswith("train"):
|
|
raise ValueError("calibration must come from the real COCO train split")
|
|
if len(files) != 128 or len(set(files)) != 128 or any(not p.is_file() for p in files):
|
|
raise ValueError("128 distinct existing calibration source images are required")
|
|
if any(int(p.stem) in eval_ids for p in files):
|
|
raise ValueError("calibration and evaluation image IDs overlap")
|
|
from harness.adapters import calibration_dir
|
|
from harness.registry import model_row
|
|
shared = calibration_dir(model_row(MODEL))
|
|
existing = shared / "manifest.json"
|
|
reused = existing.is_file()
|
|
if reused:
|
|
saved = json.loads(existing.read_text())
|
|
if (saved.get("preprocess") != adapter.PREPROCESS or saved.get("input_spec") != INPUT_SPEC
|
|
or saved.get("count") != 128 or saved.get("files") != source["files"]
|
|
or saved.get("seed") != source["seed"] or saved.get("source") != source["source"]
|
|
or saved.get("npy_dir") != "npy_f32"):
|
|
raise ValueError("existing shared calibration differs from the requested official recipe")
|
|
output = shared
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
tensors = output / "npy_f32"; tensors.mkdir(exist_ok=True)
|
|
digest = hashlib.sha256()
|
|
for i, path in enumerate(files):
|
|
with Image.open(path) as image:
|
|
x = adapter.preprocess(image, INPUT_SPEC)
|
|
if x.shape != tuple(map(int, INPUT_SPEC.split("x"))) or x.dtype != np.float32 or not np.isfinite(x).all():
|
|
raise ValueError(f"invalid calibration tensor: {path}")
|
|
destination = tensors / f"{i:05d}.npy"
|
|
if reused and not destination.is_file():
|
|
raise ValueError(f"shared calibration tensor missing: {destination}")
|
|
if destination.exists():
|
|
np.testing.assert_array_equal(np.load(destination), x)
|
|
else:
|
|
np.save(destination, x)
|
|
digest.update(destination.read_bytes())
|
|
result = {"task": "det", "input_spec": INPUT_SPEC, "count": 128, "seed": source["seed"],
|
|
"source": source["source"], "dataset": "coco2017", "sha256": digest.hexdigest(),
|
|
"files": [str(p) for p in files], "preprocess": adapter.PREPROCESS,
|
|
"layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32",
|
|
"source_manifest": str(source_manifest), "source_manifest_sha256": sha(source_manifest)}
|
|
if reused:
|
|
if saved["sha256"] != result["sha256"] or len(list(tensors.glob("*.npy"))) != 128:
|
|
raise ValueError("shared calibration content digest/count mismatch")
|
|
else:
|
|
(output / "manifest.json").write_text(json.dumps(result, indent=2))
|
|
return {"path": str(output), "count": 128, "sha256": result["sha256"], "overlap_with_t1": 0, "reused": reused}
|
|
|
|
|
|
def probe(args):
|
|
import numpy as np
|
|
import onnx
|
|
import onnxruntime as ort
|
|
import torch
|
|
import torchvision
|
|
from harness.continuation import load_eval_set
|
|
from harness.eval import EVALUATOR_VERSION
|
|
from harness.eval.common import T1, cosine
|
|
from harness.eval.det import coco_eval
|
|
|
|
output = Path(args.out); output.mkdir(parents=True, exist_ok=True)
|
|
proof = output / "probe.json"
|
|
if proof.exists():
|
|
(output / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes())
|
|
result = {"model": MODEL, "task": "det", "status": "running", "started_at": time.time(),
|
|
"checkpoint": adapter.CHECKPOINT, "preprocess": adapter.PREPROCESS,
|
|
"runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__,
|
|
"onnx": onnx.__version__, "onnxruntime": ort.__version__,
|
|
"cpu_threads": torch.get_num_threads()},
|
|
"shared_hooks_verified": False}
|
|
proof.write_text(json.dumps(result, indent=2))
|
|
t1 = None
|
|
try:
|
|
dataset, evaluation = load_eval_set(args.t1)
|
|
if dataset != "coco2017" or evaluation["n"] != 100:
|
|
raise ValueError("use the existing fixed COCO n100 evaluation package")
|
|
result["eval_set"] = evaluation
|
|
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
|
|
cat_map = t1.manifest["cat_map"]
|
|
subset = replace(t1, manifest={**t1.manifest, "items": t1.items[:args.n], "n": args.n})
|
|
model = adapter.build(Path(args.weights), INPUT_SPEC)
|
|
reference = adapter.reference_model(Path(args.weights))
|
|
|
|
def forward(x):
|
|
with torch.inference_mode():
|
|
return [model(torch.from_numpy(x)).numpy()]
|
|
|
|
onnx_path = output / f"{MODEL}.onnx"
|
|
with torch.no_grad():
|
|
torch.onnx.export(model, torch.zeros(*map(int, INPUT_SPEC.split("x"))), str(onnx_path), dynamo=False,
|
|
opset_version=adapter.OPSET, input_names=["input"], output_names=["detections"],
|
|
do_constant_folding=True, dynamic_axes=None)
|
|
graph = onnx.load(str(onnx_path)); onnx.checker.check_model(graph, full_check=True)
|
|
shapes = {v.name: [d.dim_value for d in v.type.tensor_type.shape.dim]
|
|
for v in list(graph.graph.input) + list(graph.graph.output)}
|
|
if shapes != {"input": list(map(int, INPUT_SPEC.split("x"))), "detections": list(adapter.OUTPUT_SHAPE)}:
|
|
raise ValueError(f"unexpected ONNX shapes: {shapes}")
|
|
ops = dict(Counter(n.op_type for n in graph.graph.node))
|
|
if set(ops) & {"NonMaxSuppression", "TopK", "NonZero", "Loop", "If"}:
|
|
raise ValueError(f"dynamic postprocessing left in ONNX: {ops}")
|
|
options = ort.SessionOptions(); options.intra_op_num_threads = torch.get_num_threads(); options.inter_op_num_threads = 1
|
|
session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"])
|
|
|
|
def exported(x):
|
|
return session.run(None, {"input": x})
|
|
|
|
fp_dets, onnx_dets, official_dets, comparisons = [], [], [], []
|
|
for i, item in enumerate(subset.items):
|
|
image = subset.image(item)
|
|
x = adapter.preprocess(image, INPUT_SPEC)
|
|
expected_x, official = adapter.reference_predict(reference, image)
|
|
np.testing.assert_array_equal(x, expected_x)
|
|
raw, converted = forward(x)[0], exported(x)[0]
|
|
if raw.shape != adapter.OUTPUT_SHAPE or not np.isfinite(raw).all() or not np.isfinite(converted).all():
|
|
raise ValueError("invalid raw output")
|
|
# Background YOLOX regressions are unconstrained by detection loss.
|
|
# Check every raw coordinate by cosine, and retain strict pointwise
|
|
# checks on the union of candidates either runtime could keep.
|
|
box_cosine = cosine(raw[:, :, :4], converted[:, :, :4])
|
|
if box_cosine < .99999:
|
|
raise ValueError(f"raw box tensor diverged: cosine={box_cosine}")
|
|
candidates = np.ones(raw.shape[:2], dtype=bool)
|
|
if MODEL.startswith("yolox."):
|
|
candidates = ((raw[:, :, 4] * raw[:, :, 5:].max(2) >= .01)
|
|
| (converted[:, :, 4] * converted[:, :, 5:].max(2) >= .01))
|
|
np.testing.assert_allclose(raw[:, :, :4][candidates], converted[:, :, :4][candidates], rtol=2e-4, atol=2e-3)
|
|
np.testing.assert_allclose(raw[:, :, 4:], converted[:, :, 4:], rtol=3e-4, atol=3e-5)
|
|
predicted = adapter.decode([raw], image.size)
|
|
np.testing.assert_array_equal(predicted[2], official[2])
|
|
np.testing.assert_allclose(predicted[0], official[0], rtol=1e-6, atol=1e-4)
|
|
np.testing.assert_allclose(predicted[1], official[1], rtol=1e-6, atol=1e-7)
|
|
transformed = adapter.decode([converted], image.size)
|
|
fp_dets.extend(detections(item["id"], predicted, cat_map))
|
|
official_dets.extend(detections(item["id"], official, cat_map))
|
|
onnx_dets.extend(detections(item["id"], transformed, cat_map))
|
|
comparisons.append({"image_id": item["id"], "original_size": list(image.size),
|
|
"boxes_max_abs": float(np.max(np.abs(raw[:, :, :4] - converted[:, :, :4]))),
|
|
"candidate_boxes_max_abs": float(np.max(np.abs(raw[:, :, :4][candidates] - converted[:, :, :4][candidates]), initial=0)),
|
|
"candidate_count": int(candidates.sum()),
|
|
"scores_max_abs": float(np.max(np.abs(raw[:, :, 4:] - converted[:, :, 4:]))),
|
|
"boxes_cosine": cosine(raw[:, :, :4], converted[:, :, :4]),
|
|
"scores_cosine": cosine(raw[:, :, 4:], converted[:, :, 4:]),
|
|
"official_predictions": len(official[1]), "official_match": True})
|
|
if (i + 1) % 25 == 0 or i == 0:
|
|
print(f"validated {i + 1}/{args.n}", flush=True)
|
|
metrics = {}
|
|
for key, ds in (("fp32", fp_dets), ("onnx", onnx_dets), ("official", official_dets)):
|
|
(output / f"{key}-predictions.json").write_text(json.dumps(ds))
|
|
metrics[key] = coco_eval(subset, ds)
|
|
if metrics["fp32"]["map_50_95"] <= 0 or abs(metrics["fp32"]["map_50_95"] - metrics["onnx"]["map_50_95"]) > .02:
|
|
raise ValueError(f"FP32/ONNX AP mismatch: {metrics}")
|
|
result.update(**metrics, comparisons=comparisons, sample_ids=[it["id"] for it in subset.items],
|
|
weights_path=str(Path(args.weights).resolve()), weights_sha256=sha(args.weights),
|
|
onnx_path=str(onnx_path), onnx_sha256=sha(onnx_path), shapes=shapes, onnx_ops=ops)
|
|
if args.n == 100:
|
|
if not args.calib_manifest:
|
|
raise ValueError("full validation requires --calib-manifest for 128 real COCO train images")
|
|
result["calibration"] = calibration(args.calib_manifest, output / "calibration", {it["id"] for it in t1.items})
|
|
baseline = {"task": "det", "model": MODEL, "evaluator_version": EVALUATOR_VERSION,
|
|
"t1_manifest_sha": t1.manifest["sha256"], "n": 100, "metrics": metrics["fp32"]}
|
|
baseline_dir = output / "baseline"; baseline_dir.mkdir(exist_ok=True)
|
|
xs = {f"s{i:03d}": adapter.preprocess(t1.image(it), INPUT_SPEC) for i, it in enumerate(t1.g2_subset(32))}
|
|
np.savez(baseline_dir / "g2_inputs.npz", **xs)
|
|
np.savez(baseline_dir / "g2_fp32.npz", **{f"{key}/out0": forward(x)[0] for key, x in xs.items()})
|
|
(baseline_dir / "metrics.json").write_text(json.dumps(baseline, indent=2))
|
|
result["baseline_path"] = str(baseline_dir)
|
|
if args.verify_shared_hooks:
|
|
from harness.adapters import calibration_dir
|
|
from harness.eval import det
|
|
from harness.eval.run import g2_inputs
|
|
from harness.registry import model_row
|
|
row = model_row(MODEL)
|
|
if (row["source"] != "custom" or row["input_spec"] != INPUT_SPEC
|
|
or row.get("checkpoint") != adapter.CHECKPOINT or row.get("preprocess") != adapter.PREPROCESS):
|
|
raise ValueError("shared model recipe differs from the validated adapter")
|
|
check = {**baseline}; det.run_fp32(forward, t1, INPUT_SPEC, check)
|
|
if check["metrics"] != baseline["metrics"]:
|
|
raise ValueError("shared detection FP32 hook differs from adapter predictions")
|
|
quant = {"model": MODEL}; det.run_quant(exported, t1, INPUT_SPEC, baseline, quant)
|
|
if quant["metrics"] != metrics["onnx"]:
|
|
raise ValueError("shared detection quant hook differs from adapter predictions")
|
|
shared_xs = g2_inputs("det", t1, INPUT_SPEC, 32, model=MODEL)
|
|
for key in xs:
|
|
np.testing.assert_array_equal(xs[key], shared_xs[key])
|
|
shared_calib = calibration_dir(row)
|
|
shared_manifest = json.loads((shared_calib / "manifest.json").read_text())
|
|
if (shared_manifest.get("preprocess") != adapter.PREPROCESS or shared_manifest.get("count") != 128
|
|
or shared_manifest.get("sha256") != result["calibration"]["sha256"]):
|
|
raise ValueError("shared calibration recipe or tensor identity differs from this probe")
|
|
digest = hashlib.sha256()
|
|
tensors = sorted((shared_calib / shared_manifest["npy_dir"]).glob("*.npy"))
|
|
if len(tensors) != 128:
|
|
raise ValueError("shared calibration requires exactly 128 actual tensors")
|
|
for path in tensors:
|
|
digest.update(path.read_bytes())
|
|
if digest.hexdigest() != shared_manifest["sha256"]:
|
|
raise ValueError("shared calibration files do not match their recorded digest")
|
|
result["calibration"]["shared_path"] = str(shared_calib)
|
|
result["shared_source_sha256"] = {
|
|
str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p)
|
|
for p in (Path(det.__file__), Path(sys.modules[g2_inputs.__module__].__file__),
|
|
Path(__file__).with_name("make_calib.py"))}
|
|
result["shared_hooks_verified"] = True
|
|
result["status"] = "validated_fp32_export_only" if args.verify_shared_hooks else "validated_adapter_only"
|
|
else:
|
|
result["status"] = "validated_smoke_only"
|
|
except Exception as exc:
|
|
import traceback
|
|
traceback.print_exc()
|
|
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
|
|
finally:
|
|
if t1 is not None:
|
|
t1.tar.close()
|
|
result["finished_at"] = time.time()
|
|
common = "yolox_common.py" if MODEL.startswith("yolox.") else "ssdlite_common.py"
|
|
result["source_sha256"] = {str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p)
|
|
for p in (Path(__file__), Path(adapter.__file__), Path(adapter.__file__).with_name(common))}
|
|
proof.write_text(json.dumps(result, indent=2))
|
|
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
|
|
print("ALL DONE", flush=True)
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--model", required=True, choices=("ssd300.vgg16", "yolox.tiny", "yolox.s", "yolox.m", "yolox.l", "yolox.x"))
|
|
parser.add_argument("--weights", required=True)
|
|
parser.add_argument("--t1", required=True)
|
|
parser.add_argument("--out", required=True)
|
|
parser.add_argument("--n", type=int, choices=(3, 100), default=100)
|
|
parser.add_argument("--calib-manifest")
|
|
parser.add_argument("--verify-shared-hooks", action="store_true")
|
|
args = parser.parse_args()
|
|
MODEL = args.model
|
|
adapter = importlib.import_module("harness.adapters." + MODEL.replace(".", "_"))
|
|
INPUT_SPEC = adapter.MODEL_SPEC["input_spec"]
|
|
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
|
import torch
|
|
torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", "2")))
|
|
raise SystemExit(probe(args)["status"] == "blocked")
|