"""Real batch-8 topdown heatmap validation; fixed COCO100, official crops. Run in the pinned OpenMMLab runtime. Validates SimpleBaseline/LiteHRNet standard MSRA heatmaps and the MSPN multi-unit Megvii heatmaps against the unmodified official pipeline (bitwise input, sub-0.01px coordinates), then records FP32/ONNX COCO100 baselines and the <=2px export-shift evidence. """ from __future__ import annotations import argparse import hashlib import json import os import subprocess import sys import time import traceback from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) MODELS = ("simplebaseline.r50", "simplebaseline.r101", "simplebaseline.r152", "litehrnet.18", "litehrnet.30", "mspn.r50") COMMONS = {"mspn.r50": "harness/adapters/mspn_common.py"} COMMONS_DEFAULT = "harness/adapters/topdown_pose_batch8.py" def sha(path): digest = hashlib.sha256() with Path(path).open("rb") as stream: for block in iter(lambda: stream.read(1 << 20), b""): digest.update(block) return digest.hexdigest() def save(path, value): Path(path).write_text(json.dumps(value, indent=2) + "\n") def check_shared_g2(mid, t1, inputs): import numpy as np from harness.eval import run from harness.registry import model_row shared = run.g2_inputs("pose", t1, model_row(mid)["input_spec"], 32, model=mid) if list(shared) != list(inputs) or len(shared) != 32: raise ValueError("Shared G2 keys/count differ from saved probe inputs") for key in inputs: if not np.array_equal(shared[key], inputs[key]): raise ValueError(f"Shared G2 input differs from probe input: {key}") return {"count": 32, "elementwise_identical": True, "shared_run_sha256": sha(run.__file__)} def installed_calibration(mid: str) -> dict: """Verify the already-installed shared calibration bytes for this recipe.""" from harness import paths from harness.adapters import get recipe = get(mid).PREPROCESS digest = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest()[:16] directory = paths.DATA / "calib" / "pose" / recipe["input_spec"] / digest manifest_path = directory / "manifest.json" if not manifest_path.is_file(): raise ValueError(f"Shared calibration missing: {directory}") manifest = json.loads(manifest_path.read_text()) files = sorted((directory / "npy_f32").glob("*.npy")) digest_all = hashlib.sha256() for path in files: digest_all.update(path.read_bytes()) if (manifest.get("preprocess") != recipe or len(files) != 128 or manifest.get("count") != 128 or manifest.get("sha256") != digest_all.hexdigest() or len({x["image_id"] for x in manifest["crops"]}) != 128): raise ValueError(f"Installed calibration disagrees with the recipe: {directory}") return {"path": str(directory), "sha256": manifest["sha256"], "count": 128, "recipe_digest": digest, "source": manifest.get("selection_manifest") or manifest_path} def probe(mid: str, manifest: str, weights_dir: Path, output: Path, calib: dict): import numpy as np import onnx import onnxruntime as ort import torch from mmengine.dataset import Compose, pseudo_collate from mmpose.apis.inference import inference_topdown from harness import registry, scheduler from harness.adapters import get from harness.adapters.openmmlab import get_recipe from harness.adapters.rtmpose_common import crop, boxes_for, diagnostic_box, BBOX_FILE, BBOX_SHA256 from harness.continuation import load_eval_set from harness.eval import EVALUATOR_VERSION, coco_extra from harness.eval.common import T1, TorchRunner, cosine from harness.tools.export_onnx import exporter_version if mid == "mspn.r50": from harness.adapters import mspn_common from harness.adapters.mspn_common import output_shapes def components(_model, _cache=mspn_common.components): return _cache() else: from harness.adapters.topdown_pose_batch8 import components, output_shapes directory = output / mid directory.mkdir(parents=True, exist_ok=True) proof = directory / "probe.json" if proof.exists(): (directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes()) result = {"model": mid, "task": "pose", "status": "running", "started_at": time.time()} save(proof, result) try: adapter, row = get(mid), registry.model_row(mid) dataset, evaluation = load_eval_set(manifest) if dataset != "coco2017" or row["dataset"] != dataset: raise ValueError("Batch-8 topdown models require the fixed COCO2017 T1") weights = weights_dir / row["checkpoint"]["filename"] if sha(weights) != row["checkpoint"]["sha256"]: raise ValueError("Checkpoint bytes are not pinned in the shared catalog") result.update(checkpoint=row["checkpoint"], preprocess=row["preprocess"], eval_set=evaluation, weights_path=str(weights), weights_sha256=sha(weights), calibration=calib, bbox_file=str(BBOX_FILE), bbox_sha256=BBOX_SHA256, official_recipe=get_recipe(mid), implementation_sha256={str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p) for p in (Path(__file__).resolve(), Path(adapter.__file__), Path(__file__).resolve().parents[1] / COMMONS.get(mid, COMMONS_DEFAULT), Path(__file__).resolve().parents[1] / "harness/adapters/rtmpose_common.py", Path(__file__).resolve().parents[1] / "harness/adapters/openmmlab.py", Path(coco_extra.__file__))}) with scheduler.acquire("cpu"): runner = TorchRunner(mid, row["source"], weights, row["input_spec"]) result["strict_load"] = True onnx_path = directory / "model.onnx" meta_path = directory / "meta.json" version = exporter_version(mid) previous = json.loads(meta_path.read_text()) if meta_path.exists() else {} if not (onnx_path.exists() and previous.get("exporter_version") == version and previous.get("weights_sha256") == sha(weights) and previous.get("onnx_sha256") == sha(onnx_path)): with (directory / "export.log").open("a") as log: subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid, "--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"], "--out", str(onnx_path)], check=True, stdout=log, stderr=log) save(meta_path, {"model": mid, "exporter_version": version, "input_spec": row["input_spec"], "weights_sha256": sha(weights), "onnx_sha256": sha(onnx_path)}) onnx.checker.check_model(str(onnx_path)) options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1 session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"]) def exported(x): return session.run(None, {session.get_inputs()[0].name: x}) t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) try: if len(t1.items) != 100: raise ValueError("This admission probe requires exactly the fixed 100 COCO images") eligible = [i for i, item in enumerate(t1.items) if diagnostic_box(t1, item) is not None] if len(eligible) < 32: raise ValueError("Fewer than 32 images have genuine detector boxes") indices = sorted(np.random.default_rng(20260914).permutation(eligible)[:32].tolist()) native = runner.m.upstream native.cfg = runner.m.cfg native.dataset_meta = components(mid)[2] pipeline = Compose(native.cfg.test_dataloader.dataset.pipeline) inputs, outputs, comparisons, references = {}, {}, [], [] shapes = output_shapes() if mid == "mspn.r50" else output_shapes(mid) for i, index in enumerate(indices): item = t1.items[index] image = t1.image(item) box = diagnostic_box(t1, item) x, matrix, center, scale = crop(image, row["input_spec"], box["bbox_xyxy"]) key = f"s{i:03d}" inputs[key] = x a, b = runner(x), exported(x) if len(a) != len(shapes) or len(b) != len(shapes): raise ValueError(f"Unexpected official head tensor count {len(a)}/{len(b)} vs {len(shapes)}") for j, (aa, bb) in enumerate(zip(a, b)): if aa.shape != bb.shape or not np.isfinite(aa).all() or not np.isfinite(bb).all(): raise ValueError("Invalid FP32/ONNX output") score = cosine(aa, bb) comparisons.append({"sample": key, "image_id": item["id"], "output": j, "cosine": score, "max_abs": float(np.abs(aa-bb).max())}) if score < .9999: raise ValueError(f"Heatmap parity failed: {score}") outputs[f"{key}/out{j}"] = aa bgr = np.asarray(image)[:, :, ::-1].copy() original = {"img": bgr, "bbox": box["bbox_xyxy"][None].copy(), "bbox_score": np.ones(1, np.float32), **native.dataset_meta} packed = pipeline(original) with torch.no_grad(): official_input = native.data_preprocessor(pseudo_collate([packed]), training=False)["inputs"].cpu().numpy() input_delta = float(np.abs(x - official_input).max()) ref = inference_topdown(native, bgr, bboxes=box["bbox_xyxy"][None].copy(), bbox_format="xyxy")[0].pred_instances points = adapter.decode(a, runner(np.ascontiguousarray(x[..., ::-1])), center, scale, row["input_spec"]) coordinate_delta = float(np.abs(points[:, :2] - ref.keypoints[0]).max()) score_delta = float(np.abs(points[:, 2] - ref.keypoint_scores[0]).max()) references.append({"sample": key, "image_id": item["id"], "source_index": box["source_index"], "affine": matrix.tolist(), "input_max_abs": input_delta, "coordinate_max_abs_px": coordinate_delta, "score_max_abs": score_delta}) if input_delta > 1e-6 or coordinate_delta > .01 or score_delta > 1e-4: raise ValueError(f"Official pipeline mismatch: {references[-1]}") result.update(comparisons=comparisons, official_reference=references, no_bbox_image_ids=[it["id"] for it in t1.items if not boxes_for(t1, it)], g2_selection="seed20260914 sample32 eligible detector images; sorted T1 order") save(proof, result) print(f"{mid}: strict load/export and 32 official-reference/G2 comparisons passed", flush=True) baseline = {"task": "pose", "model": mid, "evaluator_version": EVALUATOR_VERSION, "t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)} converted = dict(baseline) coco_extra.run_fp32(runner, t1, row["input_spec"], baseline) save(directory / "metrics.json", baseline) print(f"{mid}: FP32 COCO100 {baseline['metrics']}", flush=True) coco_extra.run_quant(exported, t1, row["input_spec"], baseline, converted) converted["execution"] = {"source": "onnxruntime_cpu", "purpose": "export_parity_only"} save(directory / "eval.json", converted) shift = converted["pose_shift_evidence"] delta = abs(baseline["metrics"]["oks_ap"] - converted["metrics"]["oks_ap"]) if (baseline["metrics"]["oks_ap"] <= 0 or delta > .1 or shift["status"] != "measured" or shift["keypoint_mean_shift_px"] > 2): raise ValueError(f"COCO100/shift parity failed: delta={delta}, shift={coco_extra.pose_shift_summary(shift)}") np.savez(directory / "g2_inputs.npz", **inputs) np.savez(directory / "g2_fp32.npz", **outputs) result["shared_g2"] = check_shared_g2(mid, t1, inputs) evidence_path = directory / "pose-export-evidence.json" save(evidence_path, shift) result.update(status="validated_fp32_export_only", fp32=baseline["metrics"], onnx=converted["metrics"], pose_export_shift={**coco_extra.pose_shift_summary(shift), "evidence": str(evidence_path)}, pose_export_evidence={"path": str(evidence_path), "sha256": sha(evidence_path)}, onnx_path=str(onnx_path), onnx_sha256=sha(onnx_path), baseline_path=str(directory), sample_ids=[it["id"] for it in t1.items]) finally: t1.tar.close() except Exception as exc: result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc()) result["finished_at"] = time.time() save(proof, result) print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model", action="append", required=True, choices=list(MODELS)) parser.add_argument("--t1", required=True) parser.add_argument("--weights-dir", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) args = parser.parse_args() if len(args.model) > 3 or len(set(args.model)) != len(args.model): parser.error("at most three distinct models per batch") os.environ["CUDA_VISIBLE_DEVICES"] = "-1" import torch torch.set_num_threads(2) args.out.mkdir(parents=True, exist_ok=True) results = [] for mid in args.model: calib = installed_calibration(mid) print(json.dumps({"model": mid, "calibration": calib}), flush=True) results.append(probe(mid, args.t1, args.weights_dir, args.out, calib)) return int(any(r["status"] != "validated_fp32_export_only" for r in results)) if __name__ == "__main__": raise SystemExit(main())