data-gen-bpu/scripts/probe_mmdet_dense_adapters.py

376 lines
21 KiB
Python

"""Validate exact MMDetection dense detectors in private output directories.
Run each variant with the same COCO n100 manifest and train128 recipe.
All generated files stay under --out; existing official weights may be reused.
"""
from __future__ import annotations
import argparse
from copy import deepcopy
import hashlib
import json
import os
from pathlib import Path
import random
import shutil
import sys
import time
import traceback
import urllib.request
import zipfile
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from harness.adapters import get
from harness.adapters.openmmlab import get_recipe
from harness.adapters.mmdet_dense_common import RUNTIME_PYTHON, geometry, output_names, MODELS
def sha256(path):
h = hashlib.sha256()
with Path(path).open("rb") as f:
for part in iter(lambda: f.read(1 << 20), b""):
h.update(part)
return h.hexdigest()
def save(path, value):
Path(path).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
def weights_for(model, args):
info = get(model).CHECKPOINT
path = Path(args.weights_dir or Path(args.out) / "checkpoints") / info["filename"]
if not path.is_file():
if args.weights_dir:
raise FileNotFoundError(path)
path.parent.mkdir(parents=True, exist_ok=True)
partial = path.with_suffix(".partial")
with urllib.request.urlopen(info["url"], timeout=120) as response, partial.open("wb") as f:
headers = dict(response.headers)
final_url = response.url
shutil.copyfileobj(response, f)
partial.replace(path)
save(path.with_suffix(".download.json"), {
"model": model, "url": info["url"], "final_url": final_url,
"headers": headers, "bytes": path.stat().st_size, "sha256": sha256(path),
})
digest = sha256(path)
if info.get("sha256") and digest != info["sha256"]:
raise ValueError(f"{model}: checkpoint hash mismatch")
return path, digest
def official_input(network, pipeline, image, image_id):
import numpy as np
data = pipeline(dict(img=np.asarray(image)[:, :, ::-1].copy(), img_id=image_id))
batch = network.data_preprocessor({
"inputs": [data["inputs"]], "data_samples": [data["data_samples"]],
}, training=False)
return batch["inputs"].numpy(), batch["data_samples"][0].metainfo
def check_input(adapter, network, pipeline, image, image_id):
import numpy as np
x = adapter.preprocess(image, adapter.INPUT_SPEC)
expected, meta = official_input(network, pipeline, image, image_id)
_, scales, img_shape = geometry(image.size)
error = float(np.max(np.abs(x - expected)))
if (x.shape != (1, 3, 640, 640) or error != 0 or tuple(meta["img_shape"]) != img_shape
or tuple(meta["pad_shape"]) != (640, 640)
or not np.allclose(meta["scale_factor"], scales, rtol=0, atol=1e-8)):
raise ValueError(f"Preprocessing differs from official pipeline: id={image_id}, error={error}, meta={meta}")
return x, {"input_max_abs": error, "original_size": list(image.size),
"scale_factor": list(meta["scale_factor"]), "img_shape": list(meta["img_shape"])}
def calibration(args, adapter, network, pipeline, t1):
import numpy as np
from PIL import Image
files = sorted(Path(args.train_dir).glob("*.jpg"))
if len(files) < 128:
raise ValueError(f"Need 128 actual training images, found {len(files)}")
random.Random(20260914).shuffle(files)
files = sorted(files[:128])
with zipfile.ZipFile(args.annotations_zip) as archive:
with archive.open("annotations/captions_train2017.json") as f:
annotation = json.load(f)
official_train = {im["file_name"]: im for im in annotation["images"]}
train_ids = {int(p.stem) for p in files}
if train_ids & {it["id"] for it in t1.items}:
raise ValueError("Train calibration overlaps evaluation IDs")
directory = Path(args.out) / adapter.MODEL_ID / "calibration"
npy_dir = directory / "npy_f32"
npy_dir.mkdir(parents=True, exist_ok=True)
digest = hashlib.sha256()
records = []
for i, path in enumerate(files):
item = official_train.get(path.name)
if not item or item["id"] != int(path.stem):
raise ValueError(f"Not in official COCO train2017 annotations: {path}")
with Image.open(path) as opened:
image = opened.convert("RGB")
if image.size != (item["width"], item["height"]):
raise ValueError(f"Training image geometry differs from annotation: {path}")
x, comparison = check_input(adapter, network, pipeline, image, item["id"])
target = npy_dir / f"{i:05d}.npy"
if target.exists():
if not np.array_equal(np.load(target), x):
raise ValueError(f"Existing calibration changed: {target}")
else:
np.save(target, x)
digest.update(target.read_bytes())
records.append({"source": str(path), "image_id": item["id"],
"image_sha256": sha256(path), "file": target.name,
"npy_sha256": sha256(target), **comparison})
if len(list(npy_dir.glob("*.npy"))) != 128:
raise ValueError("Calibration directory contains unexpected NPY files")
manifest = {"task": "det", "dataset": "coco2017", "input_spec": adapter.INPUT_SPEC,
"count": 128, "seed": 20260914, "source": "train-dir",
"sha256": digest.hexdigest(), "files": [str(p) for p in files],
"preprocess": adapter.PREPROCESS, "layout": "NCHW", "dtype": "float32",
"npy_dir": "npy_f32", "train_dir": args.train_dir,
"annotations_zip": args.annotations_zip,
"annotation_member": "annotations/captions_train2017.json",
"official_input_exact_matches": 128, "eval_overlap": [], "items": records}
save(directory / "manifest.json", manifest)
return {"path": str(directory), "sha256": manifest["sha256"], "count": 128,
"official_input_exact_matches": 128, "eval_overlap": [],
"recipe_hash": hashlib.sha256(json.dumps(adapter.PREPROCESS, sort_keys=True).encode()).hexdigest()}
def probe(model_id, args):
import numpy as np
import onnx
import onnxruntime as ort
import torch
from mmcv.transforms import Compose
from mmdet.apis import inference_detector
from mmdet.datasets import CocoDataset
from mmdet.utils import get_test_pipeline_cfg
from harness.eval import EVALUATOR_VERSION
from harness.eval.common import T1, cosine
from harness.eval.det import coco_eval
adapter = get(model_id)
directory = Path(args.out) / model_id
directory.mkdir(parents=True, exist_ok=True)
proof = directory / "probe.json"
if proof.exists():
proof.rename(directory / f"probe-history-{time.time_ns()}.json")
bench = Path(__file__).resolve().parents[1]
sources = ["harness/adapters/mmdet_dense_common.py", "harness/adapters/openmmlab.py",
"harness/adapters/__init__.py", f"harness/adapters/{model_id.replace('.', '_')}.py",
"harness/eval/det.py", "harness/eval/common.py", "harness/eval/run.py",
"scripts/make_calib.py", "scripts/probe_mmdet_dense_adapters.py"]
if model_id.startswith(("vfnet.", "reppoints.")):
sources.append("harness/adapters/vfnet_deformconv.py")
result = {"model": model_id, "task": "det", "status": "running", "started_at": time.time(),
"runtime_python": sys.executable, "checkpoint": adapter.CHECKPOINT,
"preprocess": adapter.PREPROCESS, "source": get_recipe(model_id)["source"],
"source_sha256": {p: sha256(bench / p) for p in sources},
"recipe_snapshot": get_recipe(model_id), "sample_count": args.n,
"runtime": {"torch": torch.__version__, "onnx": onnx.__version__,
"onnxruntime": ort.__version__, "cpu_threads": args.threads},
"scope": "FP32 input/export validation only; no quantization, board, admission or Agent run"}
save(proof, result)
t1 = None
try:
weights, digest = weights_for(model_id, args)
network = adapter.build(weights, adapter.INPUT_SPEC).eval()
pipeline_cfg = deepcopy(get_test_pipeline_cfg(network.cfg))
pipeline_cfg[0]["type"] = "mmdet.LoadImageFromNDArray"
resizes = [step for step in pipeline_cfg if step["type"] == "Resize"]
if len(resizes) != 1 or tuple(resizes[0]["scale"]) != (1333, 800) or not resizes[0]["keep_ratio"]:
raise ValueError("Original MMDetection test geometry changed")
# Explicit CSV deployment-size override, not paper/native-scale accuracy.
resizes[0]["scale"] = (640, 640)
network.upstream.data_preprocessor.pad_size_divisor = 640
result["deployment_geometry"] = {
"upstream_resize": [1333, 800], "upstream_pad_divisor": 32,
"resize": [640, 640], "keep_ratio": True, "normalized_pad_value": 0,
"pad_divisor": 640, "pad_placement": "bottom_right",
"reference": "official pipeline and model under identical CSV static640 contract",
"metric_scope": "fixed COCO100 deployment score; not paper/full-COCO accuracy"}
pipeline = Compose(pipeline_cfg)
manifest_path = Path(args.t1)
manifest = json.loads(manifest_path.read_text())
if manifest["dataset"] != "coco2017" or manifest["n"] != 100:
raise ValueError("This probe requires the frozen real COCO n100 manifest")
t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"])
if len(t1.items) != 100 or t1.manifest["sha256"] != manifest["content_sha256"]:
raise ValueError("COCO T1 content identity/count mismatch")
categories = {c["id"]: c["name"] for c in json.loads(t1.tar.extractfile("annotations.json").read())["categories"]}
cat_map = t1.manifest["cat_map"]
if [categories[cat_map[str(i)]] for i in range(80)] != list(CocoDataset.METAINFO["classes"]):
raise ValueError("Official COCO class order differs from T1 category mapping")
evaluation = {"path": str(manifest_path.parent / manifest["tar"]),
"manifest_sha": manifest["sha256"], "content_sha": manifest["content_sha256"],
"eval_set_id": manifest["eval_set_id"], "n": 100}
result.update(weights_path=str(weights), weights_sha256=digest, strict_load=True,
eval_set=evaluation, sample_ids=[it["id"] for it in t1.items[:args.n]])
save(proof, result)
export_path = directory / "model.onnx"
first = adapter.preprocess(t1.image(t1.items[0]), adapter.INPUT_SPEC)
names = output_names(model_id)
with torch.inference_mode():
torch.onnx.export(network, torch.from_numpy(first), str(export_path), opset_version=11,
input_names=["images"], output_names=names)
onnx.checker.check_model(str(export_path))
graph = onnx.load(str(export_path))
forbidden = {"NonMaxSuppression", "TopK", "NonZero", "Loop", "If"}
ops = sorted({node.op_type for node in graph.graph.node})
if forbidden.intersection(ops):
raise ValueError(f"Dynamic postprocessing leaked into raw graph: {ops}")
result["graph"] = {"input": [d.dim_value for d in graph.graph.input[0].type.tensor_type.shape.dim],
"output_names": names, "opset": 11, "ops": ops,
"postprocessing_in_graph": False}
if result["graph"]["input"] != [1, 3, 640, 640]:
raise ValueError("ONNX input is not static CSV geometry")
del graph
options = ort.SessionOptions()
options.intra_op_num_threads = args.threads
options.inter_op_num_threads = 1
session = ort.InferenceSession(str(export_path), sess_options=options, providers=["CPUExecutionProvider"])
predictions = {key: [] for key in ("official", "fp32", "onnx")}
comparisons = []
g2_keys = {it["id"]: f"s{i:03d}" for i, it in enumerate(t1.g2_subset(32))}
g2_x, g2_y = {}, {}
def add_predictions(key, image_id, values):
boxes, scores, labels = values
for box, score, label in zip(boxes, scores, labels):
predictions[key].append({"image_id": image_id, "category_id": cat_map[str(int(label))],
"score": float(score), "bbox": [float(box[0]), float(box[1]),
float(box[2] - box[0]), float(box[3] - box[1])]})
for index, item in enumerate(t1.items[:args.n]):
image = t1.image(item)
x, comparison = check_input(adapter, network.upstream, pipeline, image, item["id"])
with torch.inference_mode():
raw = [value.numpy() for value in network(torch.from_numpy(x))]
official = inference_detector(network.upstream, np.asarray(image)[:, :, ::-1].copy(),
test_pipeline=pipeline).pred_instances
reference = (official.bboxes.numpy(), official.scores.numpy(), official.labels.numpy())
decoded = adapter.decode(raw, image.size)
if not np.array_equal(reference[2], decoded[2]):
raise ValueError(f"Official/adapter selected classes differ at image {item['id']}")
box_error = float(np.max(np.abs(reference[0] - decoded[0]), initial=0))
score_error = float(np.max(np.abs(reference[1] - decoded[1]), initial=0))
if box_error > 1e-4 or score_error > 1e-6:
raise ValueError(f"Official/adapter detections differ: image={item['id']} boxes={box_error} scores={score_error}")
exported = session.run(None, {session.get_inputs()[0].name: x})
head_comparisons = []
for name, a, b in zip(names, raw, exported):
if a.shape != b.shape or not np.isfinite(a).all() or not np.isfinite(b).all():
raise ValueError(f"ONNX head shape/nonfinite output: {name}")
similarity = cosine(a, b)
head_comparisons.append({"output": name, "shape": list(a.shape), "cosine": similarity,
"max_abs": float(np.max(np.abs(a - b)))})
if similarity < .9999:
raise ValueError(f"FP32/ONNX {name} cosine below .9999: {similarity}")
add_predictions("official", item["id"], reference)
add_predictions("fp32", item["id"], decoded)
add_predictions("onnx", item["id"], adapter.decode(exported, image.size))
comparisons.append({"image_id": item["id"], **comparison, "official_box_max_abs": box_error,
"official_score_max_abs": score_error, "detections": len(decoded[1]),
"onnx_heads": head_comparisons})
if item["id"] in g2_keys:
key = g2_keys[item["id"]]
g2_x[key] = x
g2_y.update({f"{key}/out{i}": value for i, value in enumerate(raw)})
if (index + 1) % 10 == 0:
print(json.dumps({"model": model_id, "n100_progress": index + 1}), flush=True)
save(directory / "comparisons.json", comparisons)
metrics = {}
save(directory / "comparisons.json", comparisons)
eval_t1 = T1({**t1.manifest, "items": t1.items[:args.n]}, t1.tar)
for key, values in predictions.items():
save(directory / f"{key}-predictions.json", values)
# pycocotools.loadRes adds area/id/iscrowd/segmentation in place.
# Keep the saved prediction contract intact for the shared hook check.
metrics[key] = coco_eval(eval_t1, deepcopy(values))
save(directory / "metrics-all.json", metrics)
if metrics["fp32"]["map_50_95"] <= 0 or metrics["official"] != metrics["fp32"]:
raise ValueError(f"Official/adapter COCO evaluation differs: {metrics}")
if any(abs(metrics["fp32"][key] - metrics["onnx"][key]) > .1 for key in ("map_50_95", "ap50")):
raise ValueError(f"FP32/ONNX COCO metrics differ by over .1 percentage points: {metrics}")
baseline_dir = directory / "baseline"
baseline_dir.mkdir(exist_ok=True)
baseline = {"task": "det", "model": model_id, "evaluator_version": EVALUATOR_VERSION,
"t1_manifest_sha": t1.manifest["sha256"], "n": args.n, "metrics": metrics["fp32"]}
save(baseline_dir / "metrics.json", baseline)
np.savez(baseline_dir / "g2_inputs.npz", **g2_x)
np.savez(baseline_dir / "g2_fp32.npz", **g2_y)
if args.n == 100:
from harness.eval.run import g2_inputs
shared_g2 = g2_inputs("det", t1, adapter.INPUT_SPEC, 32, model=model_id)
if len(g2_x) != 32 or len(g2_y) != 32 * len(names) or g2_x.keys() != shared_g2.keys():
raise ValueError("Missing real G2 inputs/outputs")
for key in g2_x:
np.testing.assert_array_equal(g2_x[key], shared_g2[key])
result["g2"] = {"count": 32, "outputs_per_sample": len(names),
"shared_input_hook_exact": True,
"inputs_sha256": sha256(baseline_dir / "g2_inputs.npz"),
"outputs_sha256": sha256(baseline_dir / "g2_fp32.npz")}
result.update(official=metrics["official"], fp32=metrics["fp32"], onnx=metrics["onnx"],
baseline_path=str(baseline_dir), onnx_path=str(export_path), onnx_sha256=sha256(export_path),
comparisons_path=str(directory / "comparisons.json"),
official_exact_input_count=args.n, official_predict_matches=args.n,
onnx_min_cosine=min(h["cosine"] for c in comparisons for h in c["onnx_heads"]),
official_box_max_abs=max(c["official_box_max_abs"] for c in comparisons),
official_score_max_abs=max(c["official_score_max_abs"] for c in comparisons))
save(proof, result)
if args.n == 100:
result["calibration"] = calibration(args, adapter, network.upstream, pipeline, t1)
# Exercise the unchanged shared det prediction route on real inputs.
# Run the last real image again through ONNX and the shared decoder hook.
from harness.eval.det import predict
one = T1({**t1.manifest, "items": [t1.items[-1]]}, t1.tar)
actual = predict(lambda value: session.run(None, {session.get_inputs()[0].name: value}),
one, adapter.INPUT_SPEC, model=model_id)
expected = [p for p in predictions["onnx"] if p["image_id"] == t1.items[-1]["id"]]
if actual != expected:
raise ValueError("Shared det.predict differs from validated ONNX route")
result["shared_det_hook_verified"] = True
result["status"] = "validated_fp32_export_only" if args.n == 100 else "smoke_passed"
except Exception as exc:
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc())
finally:
if t1:
t1.tar.close()
result["finished_at"] = time.time()
save(proof, result)
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", choices=MODELS, action="append", required=True)
parser.add_argument("--out", required=True)
parser.add_argument("--weights-dir")
parser.add_argument("--download-only", action="store_true")
parser.add_argument("--t1", default="/home/ming01.wang/bench-data/t1/coco2017/n100-s20260914/manifest.json")
parser.add_argument("--train-dir", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/train2017")
parser.add_argument("--annotations-zip", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/annotations_trainval2017.zip")
parser.add_argument("--threads", type=int, default=2)
parser.add_argument("--n", type=int, choices=[3, 100], default=100)
args = parser.parse_args()
if Path(sys.executable).absolute() != Path(RUNTIME_PYTHON):
os.execv(RUNTIME_PYTHON, [RUNTIME_PYTHON, str(Path(__file__).resolve()), *sys.argv[1:]])
Path(args.out).mkdir(parents=True, exist_ok=True)
if args.download_only:
for model in args.model:
path, digest = weights_for(model, args)
print(json.dumps({"model": model, "path": str(path), "sha256": digest}), flush=True)
return 0
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import torch
torch.set_num_threads(args.threads)
torch.set_num_interop_threads(1)
return int(any([probe(model, args)["status"] not in ("validated_fp32_export_only", "smoke_passed") for model in args.model]))
if __name__ == "__main__":
raise SystemExit(main())