data-gen-bpu/scripts/probe_pose_batch8.py

264 lines
15 KiB
Python

"""Real batch-8 topdown heatmap validation; fixed COCO100, official crops.
Run in the pinned OpenMMLab runtime. Validates SimpleBaseline/LiteHRNet
standard MSRA heatmaps and the MSPN multi-unit Megvii heatmaps against the
unmodified official pipeline (bitwise input, sub-0.01px coordinates), then
records FP32/ONNX COCO100 baselines and the <=2px export-shift evidence.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import subprocess
import sys
import time
import traceback
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
MODELS = ("simplebaseline.r50", "simplebaseline.r101", "simplebaseline.r152",
"litehrnet.18", "litehrnet.30", "mspn.r50")
COMMONS = {"mspn.r50": "harness/adapters/mspn_common.py"}
COMMONS_DEFAULT = "harness/adapters/topdown_pose_batch8.py"
def sha(path):
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for block in iter(lambda: stream.read(1 << 20), b""):
digest.update(block)
return digest.hexdigest()
def save(path, value):
Path(path).write_text(json.dumps(value, indent=2) + "\n")
def check_shared_g2(mid, t1, inputs):
import numpy as np
from harness.eval import run
from harness.registry import model_row
shared = run.g2_inputs("pose", t1, model_row(mid)["input_spec"], 32, model=mid)
if list(shared) != list(inputs) or len(shared) != 32:
raise ValueError("Shared G2 keys/count differ from saved probe inputs")
for key in inputs:
if not np.array_equal(shared[key], inputs[key]):
raise ValueError(f"Shared G2 input differs from probe input: {key}")
return {"count": 32, "elementwise_identical": True,
"shared_run_sha256": sha(run.__file__)}
def installed_calibration(mid: str) -> dict:
"""Verify the already-installed shared calibration bytes for this recipe."""
from harness import paths
from harness.adapters import get
recipe = get(mid).PREPROCESS
digest = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest()[:16]
directory = paths.DATA / "calib" / "pose" / recipe["input_spec"] / digest
manifest_path = directory / "manifest.json"
if not manifest_path.is_file():
raise ValueError(f"Shared calibration missing: {directory}")
manifest = json.loads(manifest_path.read_text())
files = sorted((directory / "npy_f32").glob("*.npy"))
digest_all = hashlib.sha256()
for path in files:
digest_all.update(path.read_bytes())
if (manifest.get("preprocess") != recipe or len(files) != 128
or manifest.get("count") != 128 or manifest.get("sha256") != digest_all.hexdigest()
or len({x["image_id"] for x in manifest["crops"]}) != 128):
raise ValueError(f"Installed calibration disagrees with the recipe: {directory}")
return {"path": str(directory), "sha256": manifest["sha256"], "count": 128,
"recipe_digest": digest, "source": manifest.get("selection_manifest") or manifest_path}
def probe(mid: str, manifest: str, weights_dir: Path, output: Path, calib: dict):
import numpy as np
import onnx
import onnxruntime as ort
import torch
from mmengine.dataset import Compose, pseudo_collate
from mmpose.apis.inference import inference_topdown
from harness import registry, scheduler
from harness.adapters import get
from harness.adapters.openmmlab import get_recipe
from harness.adapters.rtmpose_common import crop, boxes_for, diagnostic_box, BBOX_FILE, BBOX_SHA256
from harness.continuation import load_eval_set
from harness.eval import EVALUATOR_VERSION, coco_extra
from harness.eval.common import T1, TorchRunner, cosine
from harness.tools.export_onnx import exporter_version
if mid == "mspn.r50":
from harness.adapters import mspn_common
from harness.adapters.mspn_common import output_shapes
def components(_model, _cache=mspn_common.components):
return _cache()
else:
from harness.adapters.topdown_pose_batch8 import components, output_shapes
directory = output / mid
directory.mkdir(parents=True, exist_ok=True)
proof = directory / "probe.json"
if proof.exists():
(directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes())
result = {"model": mid, "task": "pose", "status": "running", "started_at": time.time()}
save(proof, result)
try:
adapter, row = get(mid), registry.model_row(mid)
dataset, evaluation = load_eval_set(manifest)
if dataset != "coco2017" or row["dataset"] != dataset:
raise ValueError("Batch-8 topdown models require the fixed COCO2017 T1")
weights = weights_dir / row["checkpoint"]["filename"]
if sha(weights) != row["checkpoint"]["sha256"]:
raise ValueError("Checkpoint bytes are not pinned in the shared catalog")
result.update(checkpoint=row["checkpoint"], preprocess=row["preprocess"], eval_set=evaluation,
weights_path=str(weights), weights_sha256=sha(weights), calibration=calib,
bbox_file=str(BBOX_FILE), bbox_sha256=BBOX_SHA256,
official_recipe=get_recipe(mid),
implementation_sha256={str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p)
for p in (Path(__file__).resolve(), Path(adapter.__file__),
Path(__file__).resolve().parents[1] / COMMONS.get(mid, COMMONS_DEFAULT),
Path(__file__).resolve().parents[1] / "harness/adapters/rtmpose_common.py",
Path(__file__).resolve().parents[1] / "harness/adapters/openmmlab.py",
Path(coco_extra.__file__))})
with scheduler.acquire("cpu"):
runner = TorchRunner(mid, row["source"], weights, row["input_spec"])
result["strict_load"] = True
onnx_path = directory / "model.onnx"
meta_path = directory / "meta.json"
version = exporter_version(mid)
previous = json.loads(meta_path.read_text()) if meta_path.exists() else {}
if not (onnx_path.exists() and previous.get("exporter_version") == version
and previous.get("weights_sha256") == sha(weights)
and previous.get("onnx_sha256") == sha(onnx_path)):
with (directory / "export.log").open("a") as log:
subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid,
"--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"],
"--out", str(onnx_path)], check=True, stdout=log, stderr=log)
save(meta_path, {"model": mid, "exporter_version": version, "input_spec": row["input_spec"],
"weights_sha256": sha(weights), "onnx_sha256": sha(onnx_path)})
onnx.checker.check_model(str(onnx_path))
options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1
session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"])
def exported(x):
return session.run(None, {session.get_inputs()[0].name: x})
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
try:
if len(t1.items) != 100:
raise ValueError("This admission probe requires exactly the fixed 100 COCO images")
eligible = [i for i, item in enumerate(t1.items) if diagnostic_box(t1, item) is not None]
if len(eligible) < 32:
raise ValueError("Fewer than 32 images have genuine detector boxes")
indices = sorted(np.random.default_rng(20260914).permutation(eligible)[:32].tolist())
native = runner.m.upstream
native.cfg = runner.m.cfg
native.dataset_meta = components(mid)[2]
pipeline = Compose(native.cfg.test_dataloader.dataset.pipeline)
inputs, outputs, comparisons, references = {}, {}, [], []
shapes = output_shapes() if mid == "mspn.r50" else output_shapes(mid)
for i, index in enumerate(indices):
item = t1.items[index]
image = t1.image(item)
box = diagnostic_box(t1, item)
x, matrix, center, scale = crop(image, row["input_spec"], box["bbox_xyxy"])
key = f"s{i:03d}"
inputs[key] = x
a, b = runner(x), exported(x)
if len(a) != len(shapes) or len(b) != len(shapes):
raise ValueError(f"Unexpected official head tensor count {len(a)}/{len(b)} vs {len(shapes)}")
for j, (aa, bb) in enumerate(zip(a, b)):
if aa.shape != bb.shape or not np.isfinite(aa).all() or not np.isfinite(bb).all():
raise ValueError("Invalid FP32/ONNX output")
score = cosine(aa, bb)
comparisons.append({"sample": key, "image_id": item["id"], "output": j,
"cosine": score, "max_abs": float(np.abs(aa-bb).max())})
if score < .9999:
raise ValueError(f"Heatmap parity failed: {score}")
outputs[f"{key}/out{j}"] = aa
bgr = np.asarray(image)[:, :, ::-1].copy()
original = {"img": bgr, "bbox": box["bbox_xyxy"][None].copy(),
"bbox_score": np.ones(1, np.float32), **native.dataset_meta}
packed = pipeline(original)
with torch.no_grad():
official_input = native.data_preprocessor(pseudo_collate([packed]), training=False)["inputs"].cpu().numpy()
input_delta = float(np.abs(x - official_input).max())
ref = inference_topdown(native, bgr, bboxes=box["bbox_xyxy"][None].copy(), bbox_format="xyxy")[0].pred_instances
points = adapter.decode(a, runner(np.ascontiguousarray(x[..., ::-1])), center, scale, row["input_spec"])
coordinate_delta = float(np.abs(points[:, :2] - ref.keypoints[0]).max())
score_delta = float(np.abs(points[:, 2] - ref.keypoint_scores[0]).max())
references.append({"sample": key, "image_id": item["id"], "source_index": box["source_index"],
"affine": matrix.tolist(), "input_max_abs": input_delta,
"coordinate_max_abs_px": coordinate_delta, "score_max_abs": score_delta})
if input_delta > 1e-6 or coordinate_delta > .01 or score_delta > 1e-4:
raise ValueError(f"Official pipeline mismatch: {references[-1]}")
result.update(comparisons=comparisons, official_reference=references,
no_bbox_image_ids=[it["id"] for it in t1.items if not boxes_for(t1, it)],
g2_selection="seed20260914 sample32 eligible detector images; sorted T1 order")
save(proof, result)
print(f"{mid}: strict load/export and 32 official-reference/G2 comparisons passed", flush=True)
baseline = {"task": "pose", "model": mid, "evaluator_version": EVALUATOR_VERSION,
"t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)}
converted = dict(baseline)
coco_extra.run_fp32(runner, t1, row["input_spec"], baseline)
save(directory / "metrics.json", baseline)
print(f"{mid}: FP32 COCO100 {baseline['metrics']}", flush=True)
coco_extra.run_quant(exported, t1, row["input_spec"], baseline, converted)
converted["execution"] = {"source": "onnxruntime_cpu", "purpose": "export_parity_only"}
save(directory / "eval.json", converted)
shift = converted["pose_shift_evidence"]
delta = abs(baseline["metrics"]["oks_ap"] - converted["metrics"]["oks_ap"])
if (baseline["metrics"]["oks_ap"] <= 0 or delta > .1 or shift["status"] != "measured"
or shift["keypoint_mean_shift_px"] > 2):
raise ValueError(f"COCO100/shift parity failed: delta={delta}, shift={coco_extra.pose_shift_summary(shift)}")
np.savez(directory / "g2_inputs.npz", **inputs)
np.savez(directory / "g2_fp32.npz", **outputs)
result["shared_g2"] = check_shared_g2(mid, t1, inputs)
evidence_path = directory / "pose-export-evidence.json"
save(evidence_path, shift)
result.update(status="validated_fp32_export_only", fp32=baseline["metrics"], onnx=converted["metrics"],
pose_export_shift={**coco_extra.pose_shift_summary(shift), "evidence": str(evidence_path)},
pose_export_evidence={"path": str(evidence_path), "sha256": sha(evidence_path)},
onnx_path=str(onnx_path),
onnx_sha256=sha(onnx_path), baseline_path=str(directory),
sample_ids=[it["id"] for it in t1.items])
finally:
t1.tar.close()
except Exception as exc:
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc())
result["finished_at"] = time.time()
save(proof, result)
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", action="append", required=True, choices=list(MODELS))
parser.add_argument("--t1", required=True)
parser.add_argument("--weights-dir", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
if len(args.model) > 3 or len(set(args.model)) != len(args.model):
parser.error("at most three distinct models per batch")
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import torch
torch.set_num_threads(2)
args.out.mkdir(parents=True, exist_ok=True)
results = []
for mid in args.model:
calib = installed_calibration(mid)
print(json.dumps({"model": mid, "calibration": calib}), flush=True)
results.append(probe(mid, args.t1, args.weights_dir, args.out, calib))
return int(any(r["status"] != "validated_fp32_export_only" for r in results))
if __name__ == "__main__":
raise SystemExit(main())