data-gen-bpu/scripts/probe_rtmo_adapters.py

315 lines
18 KiB
Python

"""Real RTMO admission probes: official 640x640 fit resize, COCO100, 32 G2.
Run in the pinned OpenMMLab runtime. RTMO is a one-stage whole-image model:
the official BottomupResize geometry is replayed exactly, decoding follows
RTMOHead.predict (DCC coordinate classification), and parity is measured
against unmodified inference_bottomup. Also renders the 128-image
whole-image calibration pack for the shared recipe digest.
"""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import os
import random
import subprocess
import sys
import time
import traceback
import zipfile
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
MODELS = ("rtmo.s", "rtmo.m", "rtmo.l")
def sha(path):
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for block in iter(lambda: stream.read(1 << 20), b""):
digest.update(block)
return digest.hexdigest()
def save(path, value):
Path(path).write_text(json.dumps(value, indent=2) + "\n")
def check_shared_g2(mid, t1, inputs):
import numpy as np
from harness.eval import run
from harness.registry import model_row
shared = run.g2_inputs("pose", t1, model_row(mid)["input_spec"], 32, model=mid)
if list(shared) != list(inputs) or len(shared) != 32:
raise ValueError("Shared G2 keys/count differ from saved probe inputs")
for key in inputs:
if not np.array_equal(shared[key], inputs[key]):
raise ValueError(f"Shared G2 input differs from probe input: {key}")
return {"count": 32, "elementwise_identical": True,
"shared_run_sha256": sha(run.__file__)}
def calibration(root: Path, output: Path, model: str):
"""Render the recorded train128 images with the official whole-image fit resize."""
import numpy as np
from PIL import Image
from harness.adapters import get
from harness.adapters.rtmo_common import prepare
recipe = get(model).PREPROCESS
input_spec = recipe["input_spec"]
shape = tuple(int(v) for v in input_spec.split("x"))
recipe_sha = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest()
directory = output / "calib" / recipe_sha[:16]
manifest_path = directory / "manifest.json"
if manifest_path.exists():
manifest = json.loads(manifest_path.read_text())
files = sorted((directory / "npy_f32").glob("*.npy"))
digest = hashlib.sha256()
for path in files:
digest.update(path.read_bytes())
if (len(files) != 128 or manifest["preprocess"] != recipe
or manifest["sha256"] != digest.hexdigest()
or len({x["image_id"] for x in manifest["images"]}) != 128):
raise ValueError(f"Existing calibration differs: {directory}")
return {"path": str(directory), "sha256": manifest["sha256"], "count": 128}
if directory.exists():
raise ValueError(f"Incomplete calibration requires inspection: {directory}")
# Same deterministic person-image identities as the verified top-down packs.
prior_path = Path("/mnt/data/ming01.wang/model-compile-hbdk/bench/data/calib/pose/1x3x256x192/e28f899732c4ebf2/manifest.json")
prior = json.loads(prior_path.read_text())
if len(prior["crops"]) != 128 or len({r["image_id"] for r in prior["crops"]}) != 128:
raise ValueError("Expected verified train128 person identities")
member = "annotations/person_keypoints_train2017.json"
with zipfile.ZipFile(root / "annotations_trainval2017.zip") as archive:
annotation_bytes = archive.read(member)
annotation_sha = hashlib.sha256(annotation_bytes).hexdigest()
annotations = json.loads(annotation_bytes)
del annotation_bytes
names = {im["id"]: f"train2017/{im['file_name']}" for im in annotations["images"]}
by_image = {}
for ann in annotations["annotations"]:
box = ann["bbox"]
if (ann["category_id"] == 1 and not ann.get("iscrowd", 0)
and ann.get("num_keypoints", 0) > 0 and min(box[2:]) > 0):
by_image.setdefault(ann["image_id"], []).append(ann)
digest, records, sources = hashlib.sha256(), [], []
prior_images = {r["image_id"]: r["image_sha256"] for r in prior["crops"]}
with zipfile.ZipFile(root / "train2017.zip") as archive:
available = set(archive.namelist())
selected = sorted(prior_images)
if len(selected) != 128 or any(names[i] not in available or i not in by_image for i in selected):
raise ValueError("Verified train128 identities are not present in the train zip")
(directory / "npy_f32").mkdir(parents=True)
for index, image_id in enumerate(selected):
name = names[image_id]
content = archive.read(name)
if hashlib.sha256(content).hexdigest() != prior_images[image_id]:
raise ValueError(f"train2017 bytes changed for image {image_id}")
image = Image.open(io.BytesIO(content)).convert("RGB")
x, geometry = prepare(image, input_spec)
if x.shape != shape or x.dtype != np.float32 or not np.isfinite(x).all():
raise ValueError("Invalid calibration input")
path = directory / "npy_f32" / f"{index:05d}.npy"
np.save(path, x)
digest.update(path.read_bytes())
sources.append(f"{root / 'train2017.zip'}#{name}")
records.append({"image_id": image_id, "image_sha256": hashlib.sha256(content).hexdigest(),
"npy_sha256": sha(path), "input_center": geometry["input_center"],
"input_scale": geometry["input_scale"]})
manifest = {"task": "pose", "dataset": "coco2017", "input_spec": input_spec,
"count": 128, "seed": 20260914, "source": "train-zip-person-annotations",
"sha256": digest.hexdigest(), "files": sources, "preprocess": recipe,
"layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32",
"annotations": f"{root / 'annotations_trainval2017.zip'}#{member}",
"annotations_sha256": annotation_sha,
"selection": "the verified train128 distinct images (largest noncrowd labeled person per image); whole-image official fit resize",
"selection_manifest": str(prior_path), "selection_manifest_sha256": sha(prior_path),
"images": records}
save(manifest_path, manifest)
return {"path": str(directory), "sha256": digest.hexdigest(), "count": 128}
def probe(mid: str, manifest: str, weights_dir: Path, output: Path, calib: dict):
import numpy as np
import onnx
import onnxruntime as ort
import torch
from mmengine.dataset import Compose, pseudo_collate
from mmpose.apis.inference import inference_bottomup
from harness import registry, scheduler
from harness.adapters import get
from harness.adapters.openmmlab import get_recipe
from harness.adapters.rtmo_common import prepare, decode, output_shapes, components
from harness.continuation import load_eval_set
from harness.eval import EVALUATOR_VERSION, coco_extra
from harness.eval.common import T1, TorchRunner, cosine
from harness.tools.export_onnx import exporter_version
directory = output / mid
directory.mkdir(parents=True, exist_ok=True)
proof = directory / "probe.json"
if proof.exists():
(directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes())
result = {"model": mid, "task": "pose", "status": "running", "started_at": time.time()}
save(proof, result)
try:
adapter, row = get(mid), registry.model_row(mid)
dataset, evaluation = load_eval_set(manifest)
if dataset != "coco2017" or row["dataset"] != dataset:
raise ValueError("RTMO requires the fixed COCO2017 T1")
weights = weights_dir / row["checkpoint"]["filename"]
if sha(weights) != row["checkpoint"]["sha256"]:
raise ValueError("Checkpoint bytes are not pinned in the shared catalog")
result.update(checkpoint=row["checkpoint"], preprocess=row["preprocess"], eval_set=evaluation,
weights_path=str(weights), weights_sha256=sha(weights), calibration=calib,
official_recipe=get_recipe(mid),
implementation_sha256={str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p)
for p in (Path(__file__).resolve(), Path(adapter.__file__),
Path(__file__).resolve().parents[1] / "harness/adapters/rtmo_common.py",
Path(__file__).resolve().parents[1] / "harness/adapters/openmmlab.py",
Path(coco_extra.__file__))})
with scheduler.acquire("cpu"):
runner = TorchRunner(mid, row["source"], weights, row["input_spec"])
result["strict_load"] = True
onnx_path = directory / "model.onnx"
meta_path = directory / "meta.json"
version = exporter_version(mid)
previous = json.loads(meta_path.read_text()) if meta_path.exists() else {}
if not (onnx_path.exists() and previous.get("exporter_version") == version
and previous.get("weights_sha256") == sha(weights)
and previous.get("onnx_sha256") == sha(onnx_path)):
with (directory / "export.log").open("a") as log:
subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid,
"--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"],
"--out", str(onnx_path)], check=True, stdout=log, stderr=log)
save(meta_path, {"model": mid, "exporter_version": version, "input_spec": row["input_spec"],
"weights_sha256": sha(weights), "onnx_sha256": sha(onnx_path)})
onnx.checker.check_model(str(onnx_path))
options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1
session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"])
def exported(x):
return session.run(None, {session.get_inputs()[0].name: x})
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
try:
if len(t1.items) != 100:
raise ValueError("This admission probe requires exactly the fixed 100 COCO images")
indices = sorted(np.random.default_rng(20260914).permutation(len(t1.items))[:32].tolist())
native = runner.m.upstream
native.cfg = runner.m.cfg
from mmpose.apis.inference import dataset_meta_from_config
native.dataset_meta = dataset_meta_from_config(native.cfg)
pipeline = Compose(native.cfg.test_dataloader.dataset.pipeline)
inputs, outputs, comparisons, references = {}, {}, [], []
n_out = len(output_shapes(row["input_spec"]))
for i, index in enumerate(indices):
item = t1.items[index]
image = t1.image(item)
x, geometry = prepare(image, row["input_spec"])
key = f"s{i:03d}"
inputs[key] = x
a, b = runner(x), exported(x)
if len(a) != n_out or len(b) != n_out:
raise ValueError(f"Unexpected official head tensor count {len(a)}/{len(b)} vs {n_out}")
for j, (aa, bb) in enumerate(zip(a, b)):
if aa.shape != bb.shape or not np.isfinite(aa).all() or not np.isfinite(bb).all():
raise ValueError("Invalid FP32/ONNX output")
score = cosine(aa, bb)
comparisons.append({"sample": key, "image_id": item["id"], "output": j,
"cosine": score, "max_abs": float(np.abs(aa-bb).max())})
if score < .9999:
raise ValueError(f"RTMO tensor parity failed: {score}")
outputs[f"{key}/out{j}"] = aa
bgr = np.asarray(image)[:, :, ::-1].copy()
packed = pipeline({"img": bgr, "ori_shape": bgr.shape[:2]})
with torch.no_grad():
official_input = native.data_preprocessor(pseudo_collate([packed]), training=False)["inputs"].cpu().numpy()
input_delta = float(np.abs(x - official_input).max())
ref = inference_bottomup(native, bgr)[0].pred_instances
decoded = decode(runner, a, geometry, mid)
if len(decoded["keypoints"]) != len(ref.keypoints):
raise ValueError(f"Instance count mismatch vs official: {len(decoded['keypoints'])} != {len(ref.keypoints)}")
coordinate_delta = float(np.abs(decoded["keypoints"][:, :, :2] - ref.keypoints).max()) \
if len(ref.keypoints) else 0.0
score_delta = float(np.abs(decoded["keypoint_scores"] - ref.keypoint_scores).max()) \
if len(ref.keypoint_scores) else 0.0
bbox_delta = float(np.abs(decoded["bboxes"] - ref.bboxes).max()) if len(ref.bboxes) else 0.0
box_score_delta = float(np.abs(decoded["scores"] - ref.scores).max()) if len(ref.scores) else 0.0
references.append({"sample": key, "image_id": item["id"],
"input_center": geometry["input_center"], "input_scale": geometry["input_scale"],
"input_max_abs": input_delta, "coordinate_max_abs_px": coordinate_delta,
"score_max_abs": score_delta, "bbox_max_abs_px": bbox_delta,
"bbox_score_max_abs": box_score_delta, "n_instances": int(len(ref.keypoints))})
if input_delta > 1e-6 or coordinate_delta > .01 or score_delta > 1e-4 \
or bbox_delta > .01 or box_score_delta > 1e-4:
raise ValueError(f"Official pipeline mismatch: {references[-1]}")
result.update(comparisons=comparisons, official_reference=references,
g2_selection="seed20260914 sample32 of all T1 images; sorted T1 order")
save(proof, result)
print(f"{mid}: strict load/export and 32 official-reference/G2 comparisons passed", flush=True)
baseline = {"task": "pose", "model": mid, "evaluator_version": EVALUATOR_VERSION,
"t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)}
converted = dict(baseline)
coco_extra.run_fp32(runner, t1, row["input_spec"], baseline)
save(directory / "metrics.json", baseline)
print(f"{mid}: FP32 COCO100 {baseline['metrics']}", flush=True)
coco_extra.run_quant(exported, t1, row["input_spec"], baseline, converted)
converted["execution"] = {"source": "onnxruntime_cpu", "purpose": "export_parity_only"}
save(directory / "eval.json", converted)
shift = converted["pose_shift_evidence"]
delta = abs(baseline["metrics"]["oks_ap"] - converted["metrics"]["oks_ap"])
if (baseline["metrics"]["oks_ap"] <= 0 or delta > .1 or shift["status"] != "measured"
or shift["keypoint_mean_shift_px"] > 2):
raise ValueError(f"COCO100/shift parity failed: delta={delta}, shift={coco_extra.pose_shift_summary(shift)}")
np.savez(directory / "g2_inputs.npz", **inputs)
np.savez(directory / "g2_fp32.npz", **outputs)
result["shared_g2"] = check_shared_g2(mid, t1, inputs)
evidence_path = directory / "pose-export-evidence.json"
save(evidence_path, shift)
result.update(status="validated_fp32_export_only", fp32=baseline["metrics"], onnx=converted["metrics"],
pose_export_shift={**coco_extra.pose_shift_summary(shift), "evidence": str(evidence_path)},
pose_export_evidence={"path": str(evidence_path), "sha256": sha(evidence_path)},
onnx_path=str(onnx_path),
onnx_sha256=sha(onnx_path), baseline_path=str(directory),
sample_ids=[it["id"] for it in t1.items])
finally:
t1.tar.close()
except Exception as exc:
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc())
result["finished_at"] = time.time()
save(proof, result)
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", action="append", required=True, choices=list(MODELS))
parser.add_argument("--t1", required=True)
parser.add_argument("--weights-dir", type=Path, required=True)
parser.add_argument("--train-root", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
if len(args.model) > 3 or len(set(args.model)) != len(args.model):
parser.error("at most three distinct models per batch")
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import torch
torch.set_num_threads(2)
args.out.mkdir(parents=True, exist_ok=True)
results = []
for mid in args.model:
calib = calibration(args.train_root, args.out, mid)
print(json.dumps({"model": mid, "calibration": calib}), flush=True)
results.append(probe(mid, args.t1, args.weights_dir, args.out, calib))
return int(any(r["status"] != "validated_fp32_export_only" for r in results))
if __name__ == "__main__":
raise SystemExit(main())