376 lines
21 KiB
Python
376 lines
21 KiB
Python
"""Validate exact MMDetection dense detectors in private output directories.
|
|
|
|
Run each variant with the same COCO n100 manifest and train128 recipe.
|
|
All generated files stay under --out; existing official weights may be reused.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from copy import deepcopy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import urllib.request
|
|
import zipfile
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from harness.adapters import get
|
|
from harness.adapters.openmmlab import get_recipe
|
|
from harness.adapters.mmdet_dense_common import RUNTIME_PYTHON, geometry, output_names, MODELS
|
|
|
|
|
|
def sha256(path):
|
|
h = hashlib.sha256()
|
|
with Path(path).open("rb") as f:
|
|
for part in iter(lambda: f.read(1 << 20), b""):
|
|
h.update(part)
|
|
return h.hexdigest()
|
|
|
|
|
|
def save(path, value):
|
|
Path(path).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
|
|
|
|
|
|
def weights_for(model, args):
|
|
info = get(model).CHECKPOINT
|
|
path = Path(args.weights_dir or Path(args.out) / "checkpoints") / info["filename"]
|
|
if not path.is_file():
|
|
if args.weights_dir:
|
|
raise FileNotFoundError(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
partial = path.with_suffix(".partial")
|
|
with urllib.request.urlopen(info["url"], timeout=120) as response, partial.open("wb") as f:
|
|
headers = dict(response.headers)
|
|
final_url = response.url
|
|
shutil.copyfileobj(response, f)
|
|
partial.replace(path)
|
|
save(path.with_suffix(".download.json"), {
|
|
"model": model, "url": info["url"], "final_url": final_url,
|
|
"headers": headers, "bytes": path.stat().st_size, "sha256": sha256(path),
|
|
})
|
|
digest = sha256(path)
|
|
if info.get("sha256") and digest != info["sha256"]:
|
|
raise ValueError(f"{model}: checkpoint hash mismatch")
|
|
return path, digest
|
|
|
|
|
|
def official_input(network, pipeline, image, image_id):
|
|
import numpy as np
|
|
data = pipeline(dict(img=np.asarray(image)[:, :, ::-1].copy(), img_id=image_id))
|
|
batch = network.data_preprocessor({
|
|
"inputs": [data["inputs"]], "data_samples": [data["data_samples"]],
|
|
}, training=False)
|
|
return batch["inputs"].numpy(), batch["data_samples"][0].metainfo
|
|
|
|
|
|
def check_input(adapter, network, pipeline, image, image_id):
|
|
import numpy as np
|
|
x = adapter.preprocess(image, adapter.INPUT_SPEC)
|
|
expected, meta = official_input(network, pipeline, image, image_id)
|
|
_, scales, img_shape = geometry(image.size)
|
|
error = float(np.max(np.abs(x - expected)))
|
|
if (x.shape != (1, 3, 640, 640) or error != 0 or tuple(meta["img_shape"]) != img_shape
|
|
or tuple(meta["pad_shape"]) != (640, 640)
|
|
or not np.allclose(meta["scale_factor"], scales, rtol=0, atol=1e-8)):
|
|
raise ValueError(f"Preprocessing differs from official pipeline: id={image_id}, error={error}, meta={meta}")
|
|
return x, {"input_max_abs": error, "original_size": list(image.size),
|
|
"scale_factor": list(meta["scale_factor"]), "img_shape": list(meta["img_shape"])}
|
|
|
|
|
|
def calibration(args, adapter, network, pipeline, t1):
|
|
import numpy as np
|
|
from PIL import Image
|
|
files = sorted(Path(args.train_dir).glob("*.jpg"))
|
|
if len(files) < 128:
|
|
raise ValueError(f"Need 128 actual training images, found {len(files)}")
|
|
random.Random(20260914).shuffle(files)
|
|
files = sorted(files[:128])
|
|
with zipfile.ZipFile(args.annotations_zip) as archive:
|
|
with archive.open("annotations/captions_train2017.json") as f:
|
|
annotation = json.load(f)
|
|
official_train = {im["file_name"]: im for im in annotation["images"]}
|
|
train_ids = {int(p.stem) for p in files}
|
|
if train_ids & {it["id"] for it in t1.items}:
|
|
raise ValueError("Train calibration overlaps evaluation IDs")
|
|
directory = Path(args.out) / adapter.MODEL_ID / "calibration"
|
|
npy_dir = directory / "npy_f32"
|
|
npy_dir.mkdir(parents=True, exist_ok=True)
|
|
digest = hashlib.sha256()
|
|
records = []
|
|
for i, path in enumerate(files):
|
|
item = official_train.get(path.name)
|
|
if not item or item["id"] != int(path.stem):
|
|
raise ValueError(f"Not in official COCO train2017 annotations: {path}")
|
|
with Image.open(path) as opened:
|
|
image = opened.convert("RGB")
|
|
if image.size != (item["width"], item["height"]):
|
|
raise ValueError(f"Training image geometry differs from annotation: {path}")
|
|
x, comparison = check_input(adapter, network, pipeline, image, item["id"])
|
|
target = npy_dir / f"{i:05d}.npy"
|
|
if target.exists():
|
|
if not np.array_equal(np.load(target), x):
|
|
raise ValueError(f"Existing calibration changed: {target}")
|
|
else:
|
|
np.save(target, x)
|
|
digest.update(target.read_bytes())
|
|
records.append({"source": str(path), "image_id": item["id"],
|
|
"image_sha256": sha256(path), "file": target.name,
|
|
"npy_sha256": sha256(target), **comparison})
|
|
if len(list(npy_dir.glob("*.npy"))) != 128:
|
|
raise ValueError("Calibration directory contains unexpected NPY files")
|
|
manifest = {"task": "det", "dataset": "coco2017", "input_spec": adapter.INPUT_SPEC,
|
|
"count": 128, "seed": 20260914, "source": "train-dir",
|
|
"sha256": digest.hexdigest(), "files": [str(p) for p in files],
|
|
"preprocess": adapter.PREPROCESS, "layout": "NCHW", "dtype": "float32",
|
|
"npy_dir": "npy_f32", "train_dir": args.train_dir,
|
|
"annotations_zip": args.annotations_zip,
|
|
"annotation_member": "annotations/captions_train2017.json",
|
|
"official_input_exact_matches": 128, "eval_overlap": [], "items": records}
|
|
save(directory / "manifest.json", manifest)
|
|
return {"path": str(directory), "sha256": manifest["sha256"], "count": 128,
|
|
"official_input_exact_matches": 128, "eval_overlap": [],
|
|
"recipe_hash": hashlib.sha256(json.dumps(adapter.PREPROCESS, sort_keys=True).encode()).hexdigest()}
|
|
|
|
|
|
def probe(model_id, args):
|
|
import numpy as np
|
|
import onnx
|
|
import onnxruntime as ort
|
|
import torch
|
|
from mmcv.transforms import Compose
|
|
from mmdet.apis import inference_detector
|
|
from mmdet.datasets import CocoDataset
|
|
from mmdet.utils import get_test_pipeline_cfg
|
|
from harness.eval import EVALUATOR_VERSION
|
|
from harness.eval.common import T1, cosine
|
|
from harness.eval.det import coco_eval
|
|
|
|
adapter = get(model_id)
|
|
directory = Path(args.out) / model_id
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
proof = directory / "probe.json"
|
|
if proof.exists():
|
|
proof.rename(directory / f"probe-history-{time.time_ns()}.json")
|
|
bench = Path(__file__).resolve().parents[1]
|
|
sources = ["harness/adapters/mmdet_dense_common.py", "harness/adapters/openmmlab.py",
|
|
"harness/adapters/__init__.py", f"harness/adapters/{model_id.replace('.', '_')}.py",
|
|
"harness/eval/det.py", "harness/eval/common.py", "harness/eval/run.py",
|
|
"scripts/make_calib.py", "scripts/probe_mmdet_dense_adapters.py"]
|
|
if model_id.startswith(("vfnet.", "reppoints.")):
|
|
sources.append("harness/adapters/vfnet_deformconv.py")
|
|
result = {"model": model_id, "task": "det", "status": "running", "started_at": time.time(),
|
|
"runtime_python": sys.executable, "checkpoint": adapter.CHECKPOINT,
|
|
"preprocess": adapter.PREPROCESS, "source": get_recipe(model_id)["source"],
|
|
"source_sha256": {p: sha256(bench / p) for p in sources},
|
|
"recipe_snapshot": get_recipe(model_id), "sample_count": args.n,
|
|
"runtime": {"torch": torch.__version__, "onnx": onnx.__version__,
|
|
"onnxruntime": ort.__version__, "cpu_threads": args.threads},
|
|
"scope": "FP32 input/export validation only; no quantization, board, admission or Agent run"}
|
|
save(proof, result)
|
|
t1 = None
|
|
try:
|
|
weights, digest = weights_for(model_id, args)
|
|
network = adapter.build(weights, adapter.INPUT_SPEC).eval()
|
|
pipeline_cfg = deepcopy(get_test_pipeline_cfg(network.cfg))
|
|
pipeline_cfg[0]["type"] = "mmdet.LoadImageFromNDArray"
|
|
resizes = [step for step in pipeline_cfg if step["type"] == "Resize"]
|
|
if len(resizes) != 1 or tuple(resizes[0]["scale"]) != (1333, 800) or not resizes[0]["keep_ratio"]:
|
|
raise ValueError("Original MMDetection test geometry changed")
|
|
# Explicit CSV deployment-size override, not paper/native-scale accuracy.
|
|
resizes[0]["scale"] = (640, 640)
|
|
network.upstream.data_preprocessor.pad_size_divisor = 640
|
|
result["deployment_geometry"] = {
|
|
"upstream_resize": [1333, 800], "upstream_pad_divisor": 32,
|
|
"resize": [640, 640], "keep_ratio": True, "normalized_pad_value": 0,
|
|
"pad_divisor": 640, "pad_placement": "bottom_right",
|
|
"reference": "official pipeline and model under identical CSV static640 contract",
|
|
"metric_scope": "fixed COCO100 deployment score; not paper/full-COCO accuracy"}
|
|
pipeline = Compose(pipeline_cfg)
|
|
manifest_path = Path(args.t1)
|
|
manifest = json.loads(manifest_path.read_text())
|
|
if manifest["dataset"] != "coco2017" or manifest["n"] != 100:
|
|
raise ValueError("This probe requires the frozen real COCO n100 manifest")
|
|
t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"])
|
|
if len(t1.items) != 100 or t1.manifest["sha256"] != manifest["content_sha256"]:
|
|
raise ValueError("COCO T1 content identity/count mismatch")
|
|
categories = {c["id"]: c["name"] for c in json.loads(t1.tar.extractfile("annotations.json").read())["categories"]}
|
|
cat_map = t1.manifest["cat_map"]
|
|
if [categories[cat_map[str(i)]] for i in range(80)] != list(CocoDataset.METAINFO["classes"]):
|
|
raise ValueError("Official COCO class order differs from T1 category mapping")
|
|
evaluation = {"path": str(manifest_path.parent / manifest["tar"]),
|
|
"manifest_sha": manifest["sha256"], "content_sha": manifest["content_sha256"],
|
|
"eval_set_id": manifest["eval_set_id"], "n": 100}
|
|
result.update(weights_path=str(weights), weights_sha256=digest, strict_load=True,
|
|
eval_set=evaluation, sample_ids=[it["id"] for it in t1.items[:args.n]])
|
|
save(proof, result)
|
|
export_path = directory / "model.onnx"
|
|
first = adapter.preprocess(t1.image(t1.items[0]), adapter.INPUT_SPEC)
|
|
names = output_names(model_id)
|
|
with torch.inference_mode():
|
|
torch.onnx.export(network, torch.from_numpy(first), str(export_path), opset_version=11,
|
|
input_names=["images"], output_names=names)
|
|
onnx.checker.check_model(str(export_path))
|
|
graph = onnx.load(str(export_path))
|
|
forbidden = {"NonMaxSuppression", "TopK", "NonZero", "Loop", "If"}
|
|
ops = sorted({node.op_type for node in graph.graph.node})
|
|
if forbidden.intersection(ops):
|
|
raise ValueError(f"Dynamic postprocessing leaked into raw graph: {ops}")
|
|
result["graph"] = {"input": [d.dim_value for d in graph.graph.input[0].type.tensor_type.shape.dim],
|
|
"output_names": names, "opset": 11, "ops": ops,
|
|
"postprocessing_in_graph": False}
|
|
if result["graph"]["input"] != [1, 3, 640, 640]:
|
|
raise ValueError("ONNX input is not static CSV geometry")
|
|
del graph
|
|
options = ort.SessionOptions()
|
|
options.intra_op_num_threads = args.threads
|
|
options.inter_op_num_threads = 1
|
|
session = ort.InferenceSession(str(export_path), sess_options=options, providers=["CPUExecutionProvider"])
|
|
predictions = {key: [] for key in ("official", "fp32", "onnx")}
|
|
comparisons = []
|
|
g2_keys = {it["id"]: f"s{i:03d}" for i, it in enumerate(t1.g2_subset(32))}
|
|
g2_x, g2_y = {}, {}
|
|
|
|
def add_predictions(key, image_id, values):
|
|
boxes, scores, labels = values
|
|
for box, score, label in zip(boxes, scores, labels):
|
|
predictions[key].append({"image_id": image_id, "category_id": cat_map[str(int(label))],
|
|
"score": float(score), "bbox": [float(box[0]), float(box[1]),
|
|
float(box[2] - box[0]), float(box[3] - box[1])]})
|
|
|
|
for index, item in enumerate(t1.items[:args.n]):
|
|
image = t1.image(item)
|
|
x, comparison = check_input(adapter, network.upstream, pipeline, image, item["id"])
|
|
with torch.inference_mode():
|
|
raw = [value.numpy() for value in network(torch.from_numpy(x))]
|
|
official = inference_detector(network.upstream, np.asarray(image)[:, :, ::-1].copy(),
|
|
test_pipeline=pipeline).pred_instances
|
|
reference = (official.bboxes.numpy(), official.scores.numpy(), official.labels.numpy())
|
|
decoded = adapter.decode(raw, image.size)
|
|
if not np.array_equal(reference[2], decoded[2]):
|
|
raise ValueError(f"Official/adapter selected classes differ at image {item['id']}")
|
|
box_error = float(np.max(np.abs(reference[0] - decoded[0]), initial=0))
|
|
score_error = float(np.max(np.abs(reference[1] - decoded[1]), initial=0))
|
|
if box_error > 1e-4 or score_error > 1e-6:
|
|
raise ValueError(f"Official/adapter detections differ: image={item['id']} boxes={box_error} scores={score_error}")
|
|
exported = session.run(None, {session.get_inputs()[0].name: x})
|
|
head_comparisons = []
|
|
for name, a, b in zip(names, raw, exported):
|
|
if a.shape != b.shape or not np.isfinite(a).all() or not np.isfinite(b).all():
|
|
raise ValueError(f"ONNX head shape/nonfinite output: {name}")
|
|
similarity = cosine(a, b)
|
|
head_comparisons.append({"output": name, "shape": list(a.shape), "cosine": similarity,
|
|
"max_abs": float(np.max(np.abs(a - b)))})
|
|
if similarity < .9999:
|
|
raise ValueError(f"FP32/ONNX {name} cosine below .9999: {similarity}")
|
|
add_predictions("official", item["id"], reference)
|
|
add_predictions("fp32", item["id"], decoded)
|
|
add_predictions("onnx", item["id"], adapter.decode(exported, image.size))
|
|
comparisons.append({"image_id": item["id"], **comparison, "official_box_max_abs": box_error,
|
|
"official_score_max_abs": score_error, "detections": len(decoded[1]),
|
|
"onnx_heads": head_comparisons})
|
|
if item["id"] in g2_keys:
|
|
key = g2_keys[item["id"]]
|
|
g2_x[key] = x
|
|
g2_y.update({f"{key}/out{i}": value for i, value in enumerate(raw)})
|
|
if (index + 1) % 10 == 0:
|
|
print(json.dumps({"model": model_id, "n100_progress": index + 1}), flush=True)
|
|
save(directory / "comparisons.json", comparisons)
|
|
metrics = {}
|
|
save(directory / "comparisons.json", comparisons)
|
|
eval_t1 = T1({**t1.manifest, "items": t1.items[:args.n]}, t1.tar)
|
|
for key, values in predictions.items():
|
|
save(directory / f"{key}-predictions.json", values)
|
|
# pycocotools.loadRes adds area/id/iscrowd/segmentation in place.
|
|
# Keep the saved prediction contract intact for the shared hook check.
|
|
metrics[key] = coco_eval(eval_t1, deepcopy(values))
|
|
save(directory / "metrics-all.json", metrics)
|
|
if metrics["fp32"]["map_50_95"] <= 0 or metrics["official"] != metrics["fp32"]:
|
|
raise ValueError(f"Official/adapter COCO evaluation differs: {metrics}")
|
|
if any(abs(metrics["fp32"][key] - metrics["onnx"][key]) > .1 for key in ("map_50_95", "ap50")):
|
|
raise ValueError(f"FP32/ONNX COCO metrics differ by over .1 percentage points: {metrics}")
|
|
baseline_dir = directory / "baseline"
|
|
baseline_dir.mkdir(exist_ok=True)
|
|
baseline = {"task": "det", "model": model_id, "evaluator_version": EVALUATOR_VERSION,
|
|
"t1_manifest_sha": t1.manifest["sha256"], "n": args.n, "metrics": metrics["fp32"]}
|
|
save(baseline_dir / "metrics.json", baseline)
|
|
np.savez(baseline_dir / "g2_inputs.npz", **g2_x)
|
|
np.savez(baseline_dir / "g2_fp32.npz", **g2_y)
|
|
if args.n == 100:
|
|
from harness.eval.run import g2_inputs
|
|
shared_g2 = g2_inputs("det", t1, adapter.INPUT_SPEC, 32, model=model_id)
|
|
if len(g2_x) != 32 or len(g2_y) != 32 * len(names) or g2_x.keys() != shared_g2.keys():
|
|
raise ValueError("Missing real G2 inputs/outputs")
|
|
for key in g2_x:
|
|
np.testing.assert_array_equal(g2_x[key], shared_g2[key])
|
|
result["g2"] = {"count": 32, "outputs_per_sample": len(names),
|
|
"shared_input_hook_exact": True,
|
|
"inputs_sha256": sha256(baseline_dir / "g2_inputs.npz"),
|
|
"outputs_sha256": sha256(baseline_dir / "g2_fp32.npz")}
|
|
result.update(official=metrics["official"], fp32=metrics["fp32"], onnx=metrics["onnx"],
|
|
baseline_path=str(baseline_dir), onnx_path=str(export_path), onnx_sha256=sha256(export_path),
|
|
comparisons_path=str(directory / "comparisons.json"),
|
|
official_exact_input_count=args.n, official_predict_matches=args.n,
|
|
onnx_min_cosine=min(h["cosine"] for c in comparisons for h in c["onnx_heads"]),
|
|
official_box_max_abs=max(c["official_box_max_abs"] for c in comparisons),
|
|
official_score_max_abs=max(c["official_score_max_abs"] for c in comparisons))
|
|
save(proof, result)
|
|
if args.n == 100:
|
|
result["calibration"] = calibration(args, adapter, network.upstream, pipeline, t1)
|
|
# Exercise the unchanged shared det prediction route on real inputs.
|
|
# Run the last real image again through ONNX and the shared decoder hook.
|
|
from harness.eval.det import predict
|
|
one = T1({**t1.manifest, "items": [t1.items[-1]]}, t1.tar)
|
|
actual = predict(lambda value: session.run(None, {session.get_inputs()[0].name: value}),
|
|
one, adapter.INPUT_SPEC, model=model_id)
|
|
expected = [p for p in predictions["onnx"] if p["image_id"] == t1.items[-1]["id"]]
|
|
if actual != expected:
|
|
raise ValueError("Shared det.predict differs from validated ONNX route")
|
|
result["shared_det_hook_verified"] = True
|
|
result["status"] = "validated_fp32_export_only" if args.n == 100 else "smoke_passed"
|
|
except Exception as exc:
|
|
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc())
|
|
finally:
|
|
if t1:
|
|
t1.tar.close()
|
|
result["finished_at"] = time.time()
|
|
save(proof, result)
|
|
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--model", choices=MODELS, action="append", required=True)
|
|
parser.add_argument("--out", required=True)
|
|
parser.add_argument("--weights-dir")
|
|
parser.add_argument("--download-only", action="store_true")
|
|
parser.add_argument("--t1", default="/home/ming01.wang/bench-data/t1/coco2017/n100-s20260914/manifest.json")
|
|
parser.add_argument("--train-dir", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/train2017")
|
|
parser.add_argument("--annotations-zip", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/annotations_trainval2017.zip")
|
|
parser.add_argument("--threads", type=int, default=2)
|
|
parser.add_argument("--n", type=int, choices=[3, 100], default=100)
|
|
args = parser.parse_args()
|
|
if Path(sys.executable).absolute() != Path(RUNTIME_PYTHON):
|
|
os.execv(RUNTIME_PYTHON, [RUNTIME_PYTHON, str(Path(__file__).resolve()), *sys.argv[1:]])
|
|
Path(args.out).mkdir(parents=True, exist_ok=True)
|
|
if args.download_only:
|
|
for model in args.model:
|
|
path, digest = weights_for(model, args)
|
|
print(json.dumps({"model": model, "path": str(path), "sha256": digest}), flush=True)
|
|
return 0
|
|
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
|
import torch
|
|
torch.set_num_threads(args.threads)
|
|
torch.set_num_interop_threads(1)
|
|
return int(any([probe(model, args)["status"] not in ("validated_fp32_export_only", "smoke_passed") for model in args.model]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|