"""Real RTMO admission probes: official 640x640 fit resize, COCO100, 32 G2. Run in the pinned OpenMMLab runtime. RTMO is a one-stage whole-image model: the official BottomupResize geometry is replayed exactly, decoding follows RTMOHead.predict (DCC coordinate classification), and parity is measured against unmodified inference_bottomup. Also renders the 128-image whole-image calibration pack for the shared recipe digest. """ from __future__ import annotations import argparse import hashlib import io import json import os import random import subprocess import sys import time import traceback import zipfile from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) MODELS = ("rtmo.s", "rtmo.m", "rtmo.l") def sha(path): digest = hashlib.sha256() with Path(path).open("rb") as stream: for block in iter(lambda: stream.read(1 << 20), b""): digest.update(block) return digest.hexdigest() def save(path, value): Path(path).write_text(json.dumps(value, indent=2) + "\n") def check_shared_g2(mid, t1, inputs): import numpy as np from harness.eval import run from harness.registry import model_row shared = run.g2_inputs("pose", t1, model_row(mid)["input_spec"], 32, model=mid) if list(shared) != list(inputs) or len(shared) != 32: raise ValueError("Shared G2 keys/count differ from saved probe inputs") for key in inputs: if not np.array_equal(shared[key], inputs[key]): raise ValueError(f"Shared G2 input differs from probe input: {key}") return {"count": 32, "elementwise_identical": True, "shared_run_sha256": sha(run.__file__)} def calibration(root: Path, output: Path, model: str): """Render the recorded train128 images with the official whole-image fit resize.""" import numpy as np from PIL import Image from harness.adapters import get from harness.adapters.rtmo_common import prepare recipe = get(model).PREPROCESS input_spec = recipe["input_spec"] shape = tuple(int(v) for v in input_spec.split("x")) recipe_sha = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest() directory = output / "calib" / recipe_sha[:16] manifest_path = directory / "manifest.json" if manifest_path.exists(): manifest = json.loads(manifest_path.read_text()) files = sorted((directory / "npy_f32").glob("*.npy")) digest = hashlib.sha256() for path in files: digest.update(path.read_bytes()) if (len(files) != 128 or manifest["preprocess"] != recipe or manifest["sha256"] != digest.hexdigest() or len({x["image_id"] for x in manifest["images"]}) != 128): raise ValueError(f"Existing calibration differs: {directory}") return {"path": str(directory), "sha256": manifest["sha256"], "count": 128} if directory.exists(): raise ValueError(f"Incomplete calibration requires inspection: {directory}") # Same deterministic person-image identities as the verified top-down packs. prior_path = Path("/mnt/data/ming01.wang/model-compile-hbdk/bench/data/calib/pose/1x3x256x192/e28f899732c4ebf2/manifest.json") prior = json.loads(prior_path.read_text()) if len(prior["crops"]) != 128 or len({r["image_id"] for r in prior["crops"]}) != 128: raise ValueError("Expected verified train128 person identities") member = "annotations/person_keypoints_train2017.json" with zipfile.ZipFile(root / "annotations_trainval2017.zip") as archive: annotation_bytes = archive.read(member) annotation_sha = hashlib.sha256(annotation_bytes).hexdigest() annotations = json.loads(annotation_bytes) del annotation_bytes names = {im["id"]: f"train2017/{im['file_name']}" for im in annotations["images"]} by_image = {} for ann in annotations["annotations"]: box = ann["bbox"] if (ann["category_id"] == 1 and not ann.get("iscrowd", 0) and ann.get("num_keypoints", 0) > 0 and min(box[2:]) > 0): by_image.setdefault(ann["image_id"], []).append(ann) digest, records, sources = hashlib.sha256(), [], [] prior_images = {r["image_id"]: r["image_sha256"] for r in prior["crops"]} with zipfile.ZipFile(root / "train2017.zip") as archive: available = set(archive.namelist()) selected = sorted(prior_images) if len(selected) != 128 or any(names[i] not in available or i not in by_image for i in selected): raise ValueError("Verified train128 identities are not present in the train zip") (directory / "npy_f32").mkdir(parents=True) for index, image_id in enumerate(selected): name = names[image_id] content = archive.read(name) if hashlib.sha256(content).hexdigest() != prior_images[image_id]: raise ValueError(f"train2017 bytes changed for image {image_id}") image = Image.open(io.BytesIO(content)).convert("RGB") x, geometry = prepare(image, input_spec) if x.shape != shape or x.dtype != np.float32 or not np.isfinite(x).all(): raise ValueError("Invalid calibration input") path = directory / "npy_f32" / f"{index:05d}.npy" np.save(path, x) digest.update(path.read_bytes()) sources.append(f"{root / 'train2017.zip'}#{name}") records.append({"image_id": image_id, "image_sha256": hashlib.sha256(content).hexdigest(), "npy_sha256": sha(path), "input_center": geometry["input_center"], "input_scale": geometry["input_scale"]}) manifest = {"task": "pose", "dataset": "coco2017", "input_spec": input_spec, "count": 128, "seed": 20260914, "source": "train-zip-person-annotations", "sha256": digest.hexdigest(), "files": sources, "preprocess": recipe, "layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32", "annotations": f"{root / 'annotations_trainval2017.zip'}#{member}", "annotations_sha256": annotation_sha, "selection": "the verified train128 distinct images (largest noncrowd labeled person per image); whole-image official fit resize", "selection_manifest": str(prior_path), "selection_manifest_sha256": sha(prior_path), "images": records} save(manifest_path, manifest) return {"path": str(directory), "sha256": digest.hexdigest(), "count": 128} def probe(mid: str, manifest: str, weights_dir: Path, output: Path, calib: dict): import numpy as np import onnx import onnxruntime as ort import torch from mmengine.dataset import Compose, pseudo_collate from mmpose.apis.inference import inference_bottomup from harness import registry, scheduler from harness.adapters import get from harness.adapters.openmmlab import get_recipe from harness.adapters.rtmo_common import prepare, decode, output_shapes, components from harness.continuation import load_eval_set from harness.eval import EVALUATOR_VERSION, coco_extra from harness.eval.common import T1, TorchRunner, cosine from harness.tools.export_onnx import exporter_version directory = output / mid directory.mkdir(parents=True, exist_ok=True) proof = directory / "probe.json" if proof.exists(): (directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes()) result = {"model": mid, "task": "pose", "status": "running", "started_at": time.time()} save(proof, result) try: adapter, row = get(mid), registry.model_row(mid) dataset, evaluation = load_eval_set(manifest) if dataset != "coco2017" or row["dataset"] != dataset: raise ValueError("RTMO requires the fixed COCO2017 T1") weights = weights_dir / row["checkpoint"]["filename"] if sha(weights) != row["checkpoint"]["sha256"]: raise ValueError("Checkpoint bytes are not pinned in the shared catalog") result.update(checkpoint=row["checkpoint"], preprocess=row["preprocess"], eval_set=evaluation, weights_path=str(weights), weights_sha256=sha(weights), calibration=calib, official_recipe=get_recipe(mid), implementation_sha256={str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p) for p in (Path(__file__).resolve(), Path(adapter.__file__), Path(__file__).resolve().parents[1] / "harness/adapters/rtmo_common.py", Path(__file__).resolve().parents[1] / "harness/adapters/openmmlab.py", Path(coco_extra.__file__))}) with scheduler.acquire("cpu"): runner = TorchRunner(mid, row["source"], weights, row["input_spec"]) result["strict_load"] = True onnx_path = directory / "model.onnx" meta_path = directory / "meta.json" version = exporter_version(mid) previous = json.loads(meta_path.read_text()) if meta_path.exists() else {} if not (onnx_path.exists() and previous.get("exporter_version") == version and previous.get("weights_sha256") == sha(weights) and previous.get("onnx_sha256") == sha(onnx_path)): with (directory / "export.log").open("a") as log: subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid, "--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"], "--out", str(onnx_path)], check=True, stdout=log, stderr=log) save(meta_path, {"model": mid, "exporter_version": version, "input_spec": row["input_spec"], "weights_sha256": sha(weights), "onnx_sha256": sha(onnx_path)}) onnx.checker.check_model(str(onnx_path)) options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1 session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"]) def exported(x): return session.run(None, {session.get_inputs()[0].name: x}) t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) try: if len(t1.items) != 100: raise ValueError("This admission probe requires exactly the fixed 100 COCO images") indices = sorted(np.random.default_rng(20260914).permutation(len(t1.items))[:32].tolist()) native = runner.m.upstream native.cfg = runner.m.cfg from mmpose.apis.inference import dataset_meta_from_config native.dataset_meta = dataset_meta_from_config(native.cfg) pipeline = Compose(native.cfg.test_dataloader.dataset.pipeline) inputs, outputs, comparisons, references = {}, {}, [], [] n_out = len(output_shapes(row["input_spec"])) for i, index in enumerate(indices): item = t1.items[index] image = t1.image(item) x, geometry = prepare(image, row["input_spec"]) key = f"s{i:03d}" inputs[key] = x a, b = runner(x), exported(x) if len(a) != n_out or len(b) != n_out: raise ValueError(f"Unexpected official head tensor count {len(a)}/{len(b)} vs {n_out}") for j, (aa, bb) in enumerate(zip(a, b)): if aa.shape != bb.shape or not np.isfinite(aa).all() or not np.isfinite(bb).all(): raise ValueError("Invalid FP32/ONNX output") score = cosine(aa, bb) comparisons.append({"sample": key, "image_id": item["id"], "output": j, "cosine": score, "max_abs": float(np.abs(aa-bb).max())}) if score < .9999: raise ValueError(f"RTMO tensor parity failed: {score}") outputs[f"{key}/out{j}"] = aa bgr = np.asarray(image)[:, :, ::-1].copy() packed = pipeline({"img": bgr, "ori_shape": bgr.shape[:2]}) with torch.no_grad(): official_input = native.data_preprocessor(pseudo_collate([packed]), training=False)["inputs"].cpu().numpy() input_delta = float(np.abs(x - official_input).max()) ref = inference_bottomup(native, bgr)[0].pred_instances decoded = decode(runner, a, geometry, mid) if len(decoded["keypoints"]) != len(ref.keypoints): raise ValueError(f"Instance count mismatch vs official: {len(decoded['keypoints'])} != {len(ref.keypoints)}") coordinate_delta = float(np.abs(decoded["keypoints"][:, :, :2] - ref.keypoints).max()) \ if len(ref.keypoints) else 0.0 score_delta = float(np.abs(decoded["keypoint_scores"] - ref.keypoint_scores).max()) \ if len(ref.keypoint_scores) else 0.0 bbox_delta = float(np.abs(decoded["bboxes"] - ref.bboxes).max()) if len(ref.bboxes) else 0.0 box_score_delta = float(np.abs(decoded["scores"] - ref.scores).max()) if len(ref.scores) else 0.0 references.append({"sample": key, "image_id": item["id"], "input_center": geometry["input_center"], "input_scale": geometry["input_scale"], "input_max_abs": input_delta, "coordinate_max_abs_px": coordinate_delta, "score_max_abs": score_delta, "bbox_max_abs_px": bbox_delta, "bbox_score_max_abs": box_score_delta, "n_instances": int(len(ref.keypoints))}) if input_delta > 1e-6 or coordinate_delta > .01 or score_delta > 1e-4 \ or bbox_delta > .01 or box_score_delta > 1e-4: raise ValueError(f"Official pipeline mismatch: {references[-1]}") result.update(comparisons=comparisons, official_reference=references, g2_selection="seed20260914 sample32 of all T1 images; sorted T1 order") save(proof, result) print(f"{mid}: strict load/export and 32 official-reference/G2 comparisons passed", flush=True) baseline = {"task": "pose", "model": mid, "evaluator_version": EVALUATOR_VERSION, "t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)} converted = dict(baseline) coco_extra.run_fp32(runner, t1, row["input_spec"], baseline) save(directory / "metrics.json", baseline) print(f"{mid}: FP32 COCO100 {baseline['metrics']}", flush=True) coco_extra.run_quant(exported, t1, row["input_spec"], baseline, converted) converted["execution"] = {"source": "onnxruntime_cpu", "purpose": "export_parity_only"} save(directory / "eval.json", converted) shift = converted["pose_shift_evidence"] delta = abs(baseline["metrics"]["oks_ap"] - converted["metrics"]["oks_ap"]) if (baseline["metrics"]["oks_ap"] <= 0 or delta > .1 or shift["status"] != "measured" or shift["keypoint_mean_shift_px"] > 2): raise ValueError(f"COCO100/shift parity failed: delta={delta}, shift={coco_extra.pose_shift_summary(shift)}") np.savez(directory / "g2_inputs.npz", **inputs) np.savez(directory / "g2_fp32.npz", **outputs) result["shared_g2"] = check_shared_g2(mid, t1, inputs) evidence_path = directory / "pose-export-evidence.json" save(evidence_path, shift) result.update(status="validated_fp32_export_only", fp32=baseline["metrics"], onnx=converted["metrics"], pose_export_shift={**coco_extra.pose_shift_summary(shift), "evidence": str(evidence_path)}, pose_export_evidence={"path": str(evidence_path), "sha256": sha(evidence_path)}, onnx_path=str(onnx_path), onnx_sha256=sha(onnx_path), baseline_path=str(directory), sample_ids=[it["id"] for it in t1.items]) finally: t1.tar.close() except Exception as exc: result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc()) result["finished_at"] = time.time() save(proof, result) print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model", action="append", required=True, choices=list(MODELS)) parser.add_argument("--t1", required=True) parser.add_argument("--weights-dir", type=Path, required=True) parser.add_argument("--train-root", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) args = parser.parse_args() if len(args.model) > 3 or len(set(args.model)) != len(args.model): parser.error("at most three distinct models per batch") os.environ["CUDA_VISIBLE_DEVICES"] = "-1" import torch torch.set_num_threads(2) args.out.mkdir(parents=True, exist_ok=True) results = [] for mid in args.model: calib = calibration(args.train_root, args.out, mid) print(json.dumps({"model": mid, "calibration": calib}), flush=True) results.append(probe(mid, args.t1, args.weights_dir, args.out, calib)) return int(any(r["status"] != "validated_fp32_export_only" for r in results)) if __name__ == "__main__": raise SystemExit(main())