data-gen-bpu/scripts/probe_rtmdet_adapters.py

313 lines
16 KiB
Python

"""Validate RTMDet inputs in an isolated runtime, without admission or cache writes.
Run each variant with the same COCO n100 manifest and train128 recipe.
All generated files stay under --out; existing official weights may be reused.
"""
from __future__ import annotations
import argparse
from copy import deepcopy
import hashlib
import json
import os
from pathlib import Path
import random
import shutil
import sys
import time
import traceback
import urllib.request
import zipfile
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from harness.adapters import get
from harness.adapters.openmmlab import get_recipe
from harness.adapters.rtmdet_common import RUNTIME_PYTHON, geometry
def sha256(path):
h = hashlib.sha256()
with Path(path).open("rb") as f:
for part in iter(lambda: f.read(1 << 20), b""):
h.update(part)
return h.hexdigest()
def save(path, value):
Path(path).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
def weights_for(model, args):
info = get(model).CHECKPOINT
path = Path(args.weights_dir or Path(args.out) / "checkpoints") / info["filename"]
if not path.is_file():
if args.weights_dir:
raise FileNotFoundError(path)
path.parent.mkdir(parents=True, exist_ok=True)
partial = path.with_suffix(".partial")
with urllib.request.urlopen(info["url"], timeout=120) as response, partial.open("wb") as f:
headers = dict(response.headers)
final_url = response.url
shutil.copyfileobj(response, f)
partial.replace(path)
save(path.with_suffix(".download.json"), {
"model": model, "url": info["url"], "final_url": final_url,
"headers": headers, "bytes": path.stat().st_size, "sha256": sha256(path),
})
digest = sha256(path)
if info.get("sha256") and digest != info["sha256"]:
raise ValueError(f"{model}: checkpoint hash mismatch")
return path, digest
def official_input(network, pipeline, image, image_id):
import numpy as np
data = pipeline(dict(img=np.asarray(image)[:, :, ::-1].copy(), img_id=image_id))
batch = network.data_preprocessor({
"inputs": [data["inputs"]], "data_samples": [data["data_samples"]],
}, training=False)
return batch["inputs"].numpy(), batch["data_samples"][0].metainfo
def check_input(adapter, network, pipeline, image, image_id):
import numpy as np
x = adapter.preprocess(image, adapter.INPUT_SPEC)
expected, meta = official_input(network, pipeline, image, image_id)
_, scales, padded_shape = geometry(image.size)
error = float(np.max(np.abs(x - expected)))
if error != 0 or tuple(meta["img_shape"]) != padded_shape or not np.allclose(meta["scale_factor"], scales, rtol=0, atol=1e-8):
raise ValueError(f"Preprocessing differs from official pipeline: id={image_id}, error={error}, meta={meta}")
return x, {"input_max_abs": error, "original_size": list(image.size),
"scale_factor": list(meta["scale_factor"]), "img_shape": list(meta["img_shape"])}
def calibration(args, adapter, network, pipeline, t1):
import numpy as np
from PIL import Image
files = sorted(Path(args.train_dir).glob("*.jpg"))
if len(files) < 128:
raise ValueError(f"Need 128 actual training images, found {len(files)}")
random.Random(20260914).shuffle(files)
files = sorted(files[:128])
with zipfile.ZipFile(args.annotations_zip) as archive:
with archive.open("annotations/captions_train2017.json") as f:
annotation = json.load(f)
official_train = {im["file_name"]: im for im in annotation["images"]}
train_ids = {int(p.stem) for p in files}
if train_ids & {it["id"] for it in t1.items}:
raise ValueError("Train calibration overlaps evaluation IDs")
directory = Path(args.out) / "calibration"
npy_dir = directory / "npy_f32"
npy_dir.mkdir(parents=True, exist_ok=True)
digest = hashlib.sha256()
records = []
for i, path in enumerate(files):
item = official_train.get(path.name)
if not item or item["id"] != int(path.stem):
raise ValueError(f"Not in official COCO train2017 annotations: {path}")
with Image.open(path) as opened:
image = opened.convert("RGB")
if image.size != (item["width"], item["height"]):
raise ValueError(f"Training image geometry differs from annotation: {path}")
x, comparison = check_input(adapter, network, pipeline, image, item["id"])
target = npy_dir / f"{i:05d}.npy"
if target.exists():
if not np.array_equal(np.load(target), x):
raise ValueError(f"Existing calibration changed: {target}")
else:
np.save(target, x)
digest.update(target.read_bytes())
records.append({"source": str(path), "image_id": item["id"],
"image_sha256": sha256(path), "file": target.name,
"npy_sha256": sha256(target), **comparison})
if len(list(npy_dir.glob("*.npy"))) != 128:
raise ValueError("Calibration directory contains unexpected NPY files")
manifest = {"task": "det", "dataset": "coco2017", "input_spec": adapter.INPUT_SPEC,
"count": 128, "seed": 20260914, "source": "train-dir",
"sha256": digest.hexdigest(), "files": [str(p) for p in files],
"preprocess": adapter.PREPROCESS, "layout": "NCHW", "dtype": "float32",
"npy_dir": "npy_f32", "train_dir": args.train_dir,
"annotations_zip": args.annotations_zip,
"annotation_member": "annotations/captions_train2017.json",
"official_input_exact_matches": 128, "eval_overlap": [], "items": records}
save(directory / "manifest.json", manifest)
return {"path": str(directory), "sha256": manifest["sha256"], "count": 128,
"official_input_exact_matches": 128, "eval_overlap": [],
"recipe_hash": hashlib.sha256(json.dumps(adapter.PREPROCESS, sort_keys=True).encode()).hexdigest()}
def probe(model_id, args):
import numpy as np
import onnx
import onnxruntime as ort
import torch
from mmcv.transforms import Compose
from mmdet.apis import inference_detector
from mmdet.datasets import CocoDataset
from mmdet.utils import get_test_pipeline_cfg
from harness.eval import EVALUATOR_VERSION
from harness.eval.common import T1, cosine
from harness.eval.det import coco_eval
adapter = get(model_id)
directory = Path(args.out) / model_id
directory.mkdir(parents=True, exist_ok=True)
proof = directory / "probe.json"
if proof.exists():
proof.rename(directory / f"probe-history-{time.time_ns()}.json")
result = {"model": model_id, "task": "det", "status": "running", "started_at": time.time(),
"runtime_python": sys.executable, "checkpoint": adapter.CHECKPOINT,
"preprocess": adapter.PREPROCESS, "source": get_recipe(model_id)["source"],
"scope": "FP32 input/export validation only; no quantization, board, admission or Agent run"}
save(proof, result)
t1 = None
try:
weights, digest = weights_for(model_id, args)
network = adapter.build(weights, adapter.INPUT_SPEC).eval()
pipeline_cfg = deepcopy(get_test_pipeline_cfg(network.cfg))
pipeline_cfg[0]["type"] = "mmdet.LoadImageFromNDArray"
pipeline = Compose(pipeline_cfg)
manifest_path = Path(args.t1)
manifest = json.loads(manifest_path.read_text())
if manifest["dataset"] != "coco2017" or manifest["n"] != 100:
raise ValueError("This probe requires the frozen real COCO n100 manifest")
t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"])
if len(t1.items) != 100 or t1.manifest["sha256"] != manifest["content_sha256"]:
raise ValueError("COCO T1 content identity/count mismatch")
categories = {c["id"]: c["name"] for c in json.loads(t1.tar.extractfile("annotations.json").read())["categories"]}
cat_map = t1.manifest["cat_map"]
if [categories[cat_map[str(i)]] for i in range(80)] != list(CocoDataset.METAINFO["classes"]):
raise ValueError("Official COCO class order differs from T1 category mapping")
evaluation = {"path": str(manifest_path.parent / manifest["tar"]),
"manifest_sha": manifest["sha256"], "content_sha": manifest["content_sha256"],
"eval_set_id": manifest["eval_set_id"], "n": 100}
result.update(weights_path=str(weights), weights_sha256=digest, strict_load=True,
eval_set=evaluation, sample_ids=[it["id"] for it in t1.items])
save(proof, result)
export_path = directory / "model.onnx"
first = adapter.preprocess(t1.image(t1.items[0]), adapter.INPUT_SPEC)
output_names = [f"cls_s{s}" for s in (8, 16, 32)] + [f"dist_s{s}" for s in (8, 16, 32)]
with torch.inference_mode():
torch.onnx.export(network, torch.from_numpy(first), str(export_path), opset_version=11,
input_names=["images"], output_names=output_names)
onnx.checker.check_model(str(export_path))
options = ort.SessionOptions()
options.intra_op_num_threads = args.threads
options.inter_op_num_threads = 1
session = ort.InferenceSession(str(export_path), sess_options=options, providers=["CPUExecutionProvider"])
predictions = {key: [] for key in ("official", "fp32", "onnx")}
comparisons = []
g2_keys = {it["id"]: f"s{i:03d}" for i, it in enumerate(t1.g2_subset(32))}
g2_x, g2_y = {}, {}
def add_predictions(key, image_id, values):
boxes, scores, labels = values
for box, score, label in zip(boxes, scores, labels):
predictions[key].append({"image_id": image_id, "category_id": cat_map[str(int(label))],
"score": float(score), "bbox": [float(box[0]), float(box[1]),
float(box[2] - box[0]), float(box[3] - box[1])]})
for index, item in enumerate(t1.items):
image = t1.image(item)
x, comparison = check_input(adapter, network.upstream, pipeline, image, item["id"])
with torch.inference_mode():
raw = [value.numpy() for value in network(torch.from_numpy(x))]
official = inference_detector(network.upstream, np.asarray(image)[:, :, ::-1].copy(),
test_pipeline=pipeline).pred_instances
reference = (official.bboxes.numpy(), official.scores.numpy(), official.labels.numpy())
decoded = adapter.decode(raw, image.size)
if not np.array_equal(reference[2], decoded[2]):
raise ValueError(f"Official/adapter selected classes differ at image {item['id']}")
box_error = float(np.max(np.abs(reference[0] - decoded[0]), initial=0))
score_error = float(np.max(np.abs(reference[1] - decoded[1]), initial=0))
if box_error > 1e-4 or score_error > 1e-6:
raise ValueError(f"Official/adapter detections differ: image={item['id']} boxes={box_error} scores={score_error}")
exported = session.run(None, {session.get_inputs()[0].name: x})
head_comparisons = []
for name, a, b in zip(output_names, raw, exported):
if a.shape != b.shape or not np.isfinite(a).all() or not np.isfinite(b).all():
raise ValueError(f"ONNX head shape/nonfinite output: {name}")
similarity = cosine(a, b)
head_comparisons.append({"output": name, "shape": list(a.shape), "cosine": similarity,
"max_abs": float(np.max(np.abs(a - b)))})
if similarity < .9999:
raise ValueError(f"FP32/ONNX {name} cosine below .9999: {similarity}")
add_predictions("official", item["id"], reference)
add_predictions("fp32", item["id"], decoded)
add_predictions("onnx", item["id"], adapter.decode(exported, image.size))
comparisons.append({"image_id": item["id"], **comparison, "official_box_max_abs": box_error,
"official_score_max_abs": score_error, "detections": len(decoded[1]),
"onnx_heads": head_comparisons})
if item["id"] in g2_keys:
key = g2_keys[item["id"]]
g2_x[key] = x
g2_y.update({f"{key}/out{i}": value for i, value in enumerate(raw)})
if (index + 1) % 10 == 0:
print(json.dumps({"model": model_id, "n100_progress": index + 1}), flush=True)
save(directory / "comparisons.json", comparisons)
metrics = {}
for key, values in predictions.items():
save(directory / f"{key}-predictions.json", values)
metrics[key] = coco_eval(t1, values)
save(directory / "metrics-all.json", metrics)
if metrics["fp32"]["map_50_95"] <= 0 or metrics["official"] != metrics["fp32"]:
raise ValueError(f"Official/adapter COCO evaluation differs: {metrics}")
if any(abs(metrics["fp32"][key] - metrics["onnx"][key]) > .1 for key in ("map_50_95", "ap50")):
raise ValueError(f"FP32/ONNX COCO metrics differ by over .1 percentage points: {metrics}")
baseline_dir = directory / "baseline"
baseline_dir.mkdir(exist_ok=True)
baseline = {"task": "det", "model": model_id, "evaluator_version": EVALUATOR_VERSION,
"t1_manifest_sha": t1.manifest["sha256"], "n": 100, "metrics": metrics["fp32"]}
save(baseline_dir / "metrics.json", baseline)
np.savez(baseline_dir / "g2_inputs.npz", **g2_x)
np.savez(baseline_dir / "g2_fp32.npz", **g2_y)
result.update(official=metrics["official"], fp32=metrics["fp32"], onnx=metrics["onnx"],
baseline_path=str(baseline_dir), onnx_path=str(export_path), onnx_sha256=sha256(export_path),
comparisons_path=str(directory / "comparisons.json"),
official_exact_input_count=100, official_predict_matches=100,
onnx_min_cosine=min(h["cosine"] for c in comparisons for h in c["onnx_heads"]),
official_box_max_abs=max(c["official_box_max_abs"] for c in comparisons),
official_score_max_abs=max(c["official_score_max_abs"] for c in comparisons))
save(proof, result)
result["calibration"] = calibration(args, adapter, network.upstream, pipeline, t1)
result["status"] = "validated_fp32_export_only"
except Exception as exc:
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc())
finally:
if t1:
t1.tar.close()
result["finished_at"] = time.time()
save(proof, result)
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", choices=["rtmdet.tiny", "rtmdet.s", "rtmdet.m", "rtmdet.l", "rtmdet.x"], action="append", required=True)
parser.add_argument("--out", required=True)
parser.add_argument("--weights-dir")
parser.add_argument("--download-only", action="store_true")
parser.add_argument("--t1", default="/home/ming01.wang/bench-data/t1/coco2017/n100-s20260914/manifest.json")
parser.add_argument("--train-dir", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/train2017")
parser.add_argument("--annotations-zip", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/annotations_trainval2017.zip")
parser.add_argument("--threads", type=int, default=2)
args = parser.parse_args()
if Path(sys.executable).absolute() != Path(RUNTIME_PYTHON):
os.execv(RUNTIME_PYTHON, [RUNTIME_PYTHON, str(Path(__file__).resolve()), *sys.argv[1:]])
Path(args.out).mkdir(parents=True, exist_ok=True)
if args.download_only:
for model in args.model:
path, digest = weights_for(model, args)
print(json.dumps({"model": model, "path": str(path), "sha256": digest}), flush=True)
return 0
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import torch
torch.set_num_threads(args.threads)
torch.set_num_interop_threads(1)
return int(any([probe(model, args)["status"] != "validated_fp32_export_only" for model in args.model]))
if __name__ == "__main__":
raise SystemExit(main())