315 lines
18 KiB
Python
315 lines
18 KiB
Python
"""Real RTMO admission probes: official 640x640 fit resize, COCO100, 32 G2.
|
|
|
|
Run in the pinned OpenMMLab runtime. RTMO is a one-stage whole-image model:
|
|
the official BottomupResize geometry is replayed exactly, decoding follows
|
|
RTMOHead.predict (DCC coordinate classification), and parity is measured
|
|
against unmodified inference_bottomup. Also renders the 128-image
|
|
whole-image calibration pack for the shared recipe digest.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
MODELS = ("rtmo.s", "rtmo.m", "rtmo.l")
|
|
|
|
|
|
def sha(path):
|
|
digest = hashlib.sha256()
|
|
with Path(path).open("rb") as stream:
|
|
for block in iter(lambda: stream.read(1 << 20), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def save(path, value):
|
|
Path(path).write_text(json.dumps(value, indent=2) + "\n")
|
|
|
|
|
|
def check_shared_g2(mid, t1, inputs):
|
|
import numpy as np
|
|
from harness.eval import run
|
|
from harness.registry import model_row
|
|
|
|
shared = run.g2_inputs("pose", t1, model_row(mid)["input_spec"], 32, model=mid)
|
|
if list(shared) != list(inputs) or len(shared) != 32:
|
|
raise ValueError("Shared G2 keys/count differ from saved probe inputs")
|
|
for key in inputs:
|
|
if not np.array_equal(shared[key], inputs[key]):
|
|
raise ValueError(f"Shared G2 input differs from probe input: {key}")
|
|
return {"count": 32, "elementwise_identical": True,
|
|
"shared_run_sha256": sha(run.__file__)}
|
|
|
|
|
|
def calibration(root: Path, output: Path, model: str):
|
|
"""Render the recorded train128 images with the official whole-image fit resize."""
|
|
import numpy as np
|
|
from PIL import Image
|
|
from harness.adapters import get
|
|
from harness.adapters.rtmo_common import prepare
|
|
|
|
recipe = get(model).PREPROCESS
|
|
input_spec = recipe["input_spec"]
|
|
shape = tuple(int(v) for v in input_spec.split("x"))
|
|
recipe_sha = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest()
|
|
directory = output / "calib" / recipe_sha[:16]
|
|
manifest_path = directory / "manifest.json"
|
|
if manifest_path.exists():
|
|
manifest = json.loads(manifest_path.read_text())
|
|
files = sorted((directory / "npy_f32").glob("*.npy"))
|
|
digest = hashlib.sha256()
|
|
for path in files:
|
|
digest.update(path.read_bytes())
|
|
if (len(files) != 128 or manifest["preprocess"] != recipe
|
|
or manifest["sha256"] != digest.hexdigest()
|
|
or len({x["image_id"] for x in manifest["images"]}) != 128):
|
|
raise ValueError(f"Existing calibration differs: {directory}")
|
|
return {"path": str(directory), "sha256": manifest["sha256"], "count": 128}
|
|
if directory.exists():
|
|
raise ValueError(f"Incomplete calibration requires inspection: {directory}")
|
|
# Same deterministic person-image identities as the verified top-down packs.
|
|
prior_path = Path("/mnt/data/ming01.wang/model-compile-hbdk/bench/data/calib/pose/1x3x256x192/e28f899732c4ebf2/manifest.json")
|
|
prior = json.loads(prior_path.read_text())
|
|
if len(prior["crops"]) != 128 or len({r["image_id"] for r in prior["crops"]}) != 128:
|
|
raise ValueError("Expected verified train128 person identities")
|
|
member = "annotations/person_keypoints_train2017.json"
|
|
with zipfile.ZipFile(root / "annotations_trainval2017.zip") as archive:
|
|
annotation_bytes = archive.read(member)
|
|
annotation_sha = hashlib.sha256(annotation_bytes).hexdigest()
|
|
annotations = json.loads(annotation_bytes)
|
|
del annotation_bytes
|
|
names = {im["id"]: f"train2017/{im['file_name']}" for im in annotations["images"]}
|
|
by_image = {}
|
|
for ann in annotations["annotations"]:
|
|
box = ann["bbox"]
|
|
if (ann["category_id"] == 1 and not ann.get("iscrowd", 0)
|
|
and ann.get("num_keypoints", 0) > 0 and min(box[2:]) > 0):
|
|
by_image.setdefault(ann["image_id"], []).append(ann)
|
|
digest, records, sources = hashlib.sha256(), [], []
|
|
prior_images = {r["image_id"]: r["image_sha256"] for r in prior["crops"]}
|
|
with zipfile.ZipFile(root / "train2017.zip") as archive:
|
|
available = set(archive.namelist())
|
|
selected = sorted(prior_images)
|
|
if len(selected) != 128 or any(names[i] not in available or i not in by_image for i in selected):
|
|
raise ValueError("Verified train128 identities are not present in the train zip")
|
|
(directory / "npy_f32").mkdir(parents=True)
|
|
for index, image_id in enumerate(selected):
|
|
name = names[image_id]
|
|
content = archive.read(name)
|
|
if hashlib.sha256(content).hexdigest() != prior_images[image_id]:
|
|
raise ValueError(f"train2017 bytes changed for image {image_id}")
|
|
image = Image.open(io.BytesIO(content)).convert("RGB")
|
|
x, geometry = prepare(image, input_spec)
|
|
if x.shape != shape or x.dtype != np.float32 or not np.isfinite(x).all():
|
|
raise ValueError("Invalid calibration input")
|
|
path = directory / "npy_f32" / f"{index:05d}.npy"
|
|
np.save(path, x)
|
|
digest.update(path.read_bytes())
|
|
sources.append(f"{root / 'train2017.zip'}#{name}")
|
|
records.append({"image_id": image_id, "image_sha256": hashlib.sha256(content).hexdigest(),
|
|
"npy_sha256": sha(path), "input_center": geometry["input_center"],
|
|
"input_scale": geometry["input_scale"]})
|
|
manifest = {"task": "pose", "dataset": "coco2017", "input_spec": input_spec,
|
|
"count": 128, "seed": 20260914, "source": "train-zip-person-annotations",
|
|
"sha256": digest.hexdigest(), "files": sources, "preprocess": recipe,
|
|
"layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32",
|
|
"annotations": f"{root / 'annotations_trainval2017.zip'}#{member}",
|
|
"annotations_sha256": annotation_sha,
|
|
"selection": "the verified train128 distinct images (largest noncrowd labeled person per image); whole-image official fit resize",
|
|
"selection_manifest": str(prior_path), "selection_manifest_sha256": sha(prior_path),
|
|
"images": records}
|
|
save(manifest_path, manifest)
|
|
return {"path": str(directory), "sha256": digest.hexdigest(), "count": 128}
|
|
|
|
|
|
def probe(mid: str, manifest: str, weights_dir: Path, output: Path, calib: dict):
|
|
import numpy as np
|
|
import onnx
|
|
import onnxruntime as ort
|
|
import torch
|
|
from mmengine.dataset import Compose, pseudo_collate
|
|
from mmpose.apis.inference import inference_bottomup
|
|
from harness import registry, scheduler
|
|
from harness.adapters import get
|
|
from harness.adapters.openmmlab import get_recipe
|
|
from harness.adapters.rtmo_common import prepare, decode, output_shapes, components
|
|
from harness.continuation import load_eval_set
|
|
from harness.eval import EVALUATOR_VERSION, coco_extra
|
|
from harness.eval.common import T1, TorchRunner, cosine
|
|
from harness.tools.export_onnx import exporter_version
|
|
|
|
directory = output / mid
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
proof = directory / "probe.json"
|
|
if proof.exists():
|
|
(directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes())
|
|
result = {"model": mid, "task": "pose", "status": "running", "started_at": time.time()}
|
|
save(proof, result)
|
|
try:
|
|
adapter, row = get(mid), registry.model_row(mid)
|
|
dataset, evaluation = load_eval_set(manifest)
|
|
if dataset != "coco2017" or row["dataset"] != dataset:
|
|
raise ValueError("RTMO requires the fixed COCO2017 T1")
|
|
weights = weights_dir / row["checkpoint"]["filename"]
|
|
if sha(weights) != row["checkpoint"]["sha256"]:
|
|
raise ValueError("Checkpoint bytes are not pinned in the shared catalog")
|
|
result.update(checkpoint=row["checkpoint"], preprocess=row["preprocess"], eval_set=evaluation,
|
|
weights_path=str(weights), weights_sha256=sha(weights), calibration=calib,
|
|
official_recipe=get_recipe(mid),
|
|
implementation_sha256={str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p)
|
|
for p in (Path(__file__).resolve(), Path(adapter.__file__),
|
|
Path(__file__).resolve().parents[1] / "harness/adapters/rtmo_common.py",
|
|
Path(__file__).resolve().parents[1] / "harness/adapters/openmmlab.py",
|
|
Path(coco_extra.__file__))})
|
|
with scheduler.acquire("cpu"):
|
|
runner = TorchRunner(mid, row["source"], weights, row["input_spec"])
|
|
result["strict_load"] = True
|
|
onnx_path = directory / "model.onnx"
|
|
meta_path = directory / "meta.json"
|
|
version = exporter_version(mid)
|
|
previous = json.loads(meta_path.read_text()) if meta_path.exists() else {}
|
|
if not (onnx_path.exists() and previous.get("exporter_version") == version
|
|
and previous.get("weights_sha256") == sha(weights)
|
|
and previous.get("onnx_sha256") == sha(onnx_path)):
|
|
with (directory / "export.log").open("a") as log:
|
|
subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid,
|
|
"--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"],
|
|
"--out", str(onnx_path)], check=True, stdout=log, stderr=log)
|
|
save(meta_path, {"model": mid, "exporter_version": version, "input_spec": row["input_spec"],
|
|
"weights_sha256": sha(weights), "onnx_sha256": sha(onnx_path)})
|
|
onnx.checker.check_model(str(onnx_path))
|
|
options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1
|
|
session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"])
|
|
|
|
def exported(x):
|
|
return session.run(None, {session.get_inputs()[0].name: x})
|
|
|
|
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
|
|
try:
|
|
if len(t1.items) != 100:
|
|
raise ValueError("This admission probe requires exactly the fixed 100 COCO images")
|
|
indices = sorted(np.random.default_rng(20260914).permutation(len(t1.items))[:32].tolist())
|
|
native = runner.m.upstream
|
|
native.cfg = runner.m.cfg
|
|
from mmpose.apis.inference import dataset_meta_from_config
|
|
native.dataset_meta = dataset_meta_from_config(native.cfg)
|
|
pipeline = Compose(native.cfg.test_dataloader.dataset.pipeline)
|
|
inputs, outputs, comparisons, references = {}, {}, [], []
|
|
n_out = len(output_shapes(row["input_spec"]))
|
|
for i, index in enumerate(indices):
|
|
item = t1.items[index]
|
|
image = t1.image(item)
|
|
x, geometry = prepare(image, row["input_spec"])
|
|
key = f"s{i:03d}"
|
|
inputs[key] = x
|
|
a, b = runner(x), exported(x)
|
|
if len(a) != n_out or len(b) != n_out:
|
|
raise ValueError(f"Unexpected official head tensor count {len(a)}/{len(b)} vs {n_out}")
|
|
for j, (aa, bb) in enumerate(zip(a, b)):
|
|
if aa.shape != bb.shape or not np.isfinite(aa).all() or not np.isfinite(bb).all():
|
|
raise ValueError("Invalid FP32/ONNX output")
|
|
score = cosine(aa, bb)
|
|
comparisons.append({"sample": key, "image_id": item["id"], "output": j,
|
|
"cosine": score, "max_abs": float(np.abs(aa-bb).max())})
|
|
if score < .9999:
|
|
raise ValueError(f"RTMO tensor parity failed: {score}")
|
|
outputs[f"{key}/out{j}"] = aa
|
|
bgr = np.asarray(image)[:, :, ::-1].copy()
|
|
packed = pipeline({"img": bgr, "ori_shape": bgr.shape[:2]})
|
|
with torch.no_grad():
|
|
official_input = native.data_preprocessor(pseudo_collate([packed]), training=False)["inputs"].cpu().numpy()
|
|
input_delta = float(np.abs(x - official_input).max())
|
|
ref = inference_bottomup(native, bgr)[0].pred_instances
|
|
decoded = decode(runner, a, geometry, mid)
|
|
if len(decoded["keypoints"]) != len(ref.keypoints):
|
|
raise ValueError(f"Instance count mismatch vs official: {len(decoded['keypoints'])} != {len(ref.keypoints)}")
|
|
coordinate_delta = float(np.abs(decoded["keypoints"][:, :, :2] - ref.keypoints).max()) \
|
|
if len(ref.keypoints) else 0.0
|
|
score_delta = float(np.abs(decoded["keypoint_scores"] - ref.keypoint_scores).max()) \
|
|
if len(ref.keypoint_scores) else 0.0
|
|
bbox_delta = float(np.abs(decoded["bboxes"] - ref.bboxes).max()) if len(ref.bboxes) else 0.0
|
|
box_score_delta = float(np.abs(decoded["scores"] - ref.scores).max()) if len(ref.scores) else 0.0
|
|
references.append({"sample": key, "image_id": item["id"],
|
|
"input_center": geometry["input_center"], "input_scale": geometry["input_scale"],
|
|
"input_max_abs": input_delta, "coordinate_max_abs_px": coordinate_delta,
|
|
"score_max_abs": score_delta, "bbox_max_abs_px": bbox_delta,
|
|
"bbox_score_max_abs": box_score_delta, "n_instances": int(len(ref.keypoints))})
|
|
if input_delta > 1e-6 or coordinate_delta > .01 or score_delta > 1e-4 \
|
|
or bbox_delta > .01 or box_score_delta > 1e-4:
|
|
raise ValueError(f"Official pipeline mismatch: {references[-1]}")
|
|
result.update(comparisons=comparisons, official_reference=references,
|
|
g2_selection="seed20260914 sample32 of all T1 images; sorted T1 order")
|
|
save(proof, result)
|
|
print(f"{mid}: strict load/export and 32 official-reference/G2 comparisons passed", flush=True)
|
|
baseline = {"task": "pose", "model": mid, "evaluator_version": EVALUATOR_VERSION,
|
|
"t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)}
|
|
converted = dict(baseline)
|
|
coco_extra.run_fp32(runner, t1, row["input_spec"], baseline)
|
|
save(directory / "metrics.json", baseline)
|
|
print(f"{mid}: FP32 COCO100 {baseline['metrics']}", flush=True)
|
|
coco_extra.run_quant(exported, t1, row["input_spec"], baseline, converted)
|
|
converted["execution"] = {"source": "onnxruntime_cpu", "purpose": "export_parity_only"}
|
|
save(directory / "eval.json", converted)
|
|
shift = converted["pose_shift_evidence"]
|
|
delta = abs(baseline["metrics"]["oks_ap"] - converted["metrics"]["oks_ap"])
|
|
if (baseline["metrics"]["oks_ap"] <= 0 or delta > .1 or shift["status"] != "measured"
|
|
or shift["keypoint_mean_shift_px"] > 2):
|
|
raise ValueError(f"COCO100/shift parity failed: delta={delta}, shift={coco_extra.pose_shift_summary(shift)}")
|
|
np.savez(directory / "g2_inputs.npz", **inputs)
|
|
np.savez(directory / "g2_fp32.npz", **outputs)
|
|
result["shared_g2"] = check_shared_g2(mid, t1, inputs)
|
|
evidence_path = directory / "pose-export-evidence.json"
|
|
save(evidence_path, shift)
|
|
result.update(status="validated_fp32_export_only", fp32=baseline["metrics"], onnx=converted["metrics"],
|
|
pose_export_shift={**coco_extra.pose_shift_summary(shift), "evidence": str(evidence_path)},
|
|
pose_export_evidence={"path": str(evidence_path), "sha256": sha(evidence_path)},
|
|
onnx_path=str(onnx_path),
|
|
onnx_sha256=sha(onnx_path), baseline_path=str(directory),
|
|
sample_ids=[it["id"] for it in t1.items])
|
|
finally:
|
|
t1.tar.close()
|
|
except Exception as exc:
|
|
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc())
|
|
result["finished_at"] = time.time()
|
|
save(proof, result)
|
|
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--model", action="append", required=True, choices=list(MODELS))
|
|
parser.add_argument("--t1", required=True)
|
|
parser.add_argument("--weights-dir", type=Path, required=True)
|
|
parser.add_argument("--train-root", type=Path, required=True)
|
|
parser.add_argument("--out", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if len(args.model) > 3 or len(set(args.model)) != len(args.model):
|
|
parser.error("at most three distinct models per batch")
|
|
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
|
import torch
|
|
torch.set_num_threads(2)
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
results = []
|
|
for mid in args.model:
|
|
calib = calibration(args.train_root, args.out, mid)
|
|
print(json.dumps({"model": mid, "calibration": calib}), flush=True)
|
|
results.append(probe(mid, args.t1, args.weights_dir, args.out, calib))
|
|
return int(any(r["status"] != "validated_fp32_export_only" for r in results))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|