238 lines
15 KiB
Python
238 lines
15 KiB
Python
"""Real official HRNet/SimCC validation; private outputs, fixed COCO100."""
|
|
from __future__ import annotations
|
|
import argparse, json, os, subprocess, sys, time, traceback
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from probe_rtmpose_adapters import check_shared_g2, save, sha
|
|
|
|
|
|
def calibration(root, output, model):
|
|
"""Render the same recorded train128 people with canonical official normalization."""
|
|
import hashlib, io, zipfile
|
|
import numpy as np
|
|
from PIL import Image
|
|
from harness.adapters import get
|
|
from harness.adapters.topdown_pose_common import crop
|
|
|
|
recipe = get(model).PREPROCESS
|
|
recipe_sha = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest()
|
|
directory = output / "calib" / recipe_sha[:16]
|
|
manifest_path = directory / "manifest.json"
|
|
if not manifest_path.exists():
|
|
if directory.exists():
|
|
raise ValueError(f"Incomplete calibration requires inspection: {directory}")
|
|
prior_path = Path("/home/ming01.wang/outputs/pose/verified/calib/e28f899732c4ebf2/manifest.json")
|
|
prior = json.loads(prior_path.read_text())
|
|
if len(prior["crops"]) != 128 or len({r["image_id"] for r in prior["crops"]}) != 128:
|
|
raise ValueError("Expected verified train128 person identities")
|
|
with zipfile.ZipFile(root / "annotations_trainval2017.zip") as archive:
|
|
annotations = archive.read("annotations/person_keypoints_train2017.json")
|
|
assert hashlib.sha256(annotations).hexdigest() == prior["annotations_sha256"]
|
|
(directory / "npy_f32").mkdir(parents=True)
|
|
records, digest = [], hashlib.sha256()
|
|
with zipfile.ZipFile(root / "train2017.zip") as archive:
|
|
for i, (source, record) in enumerate(zip(prior["files"], prior["crops"])):
|
|
content = archive.read(source.split("#", 1)[1])
|
|
assert hashlib.sha256(content).hexdigest() == record["image_sha256"]
|
|
x, affine, _, _ = crop(Image.open(io.BytesIO(content)).convert("RGB"), recipe["input_spec"], record["bbox_xyxy"])
|
|
assert x.shape == (1, 3, 256, 192) and x.dtype == np.float32 and np.isfinite(x).all()
|
|
path = directory / "npy_f32" / f"{i:05d}.npy"
|
|
np.save(path, x); digest.update(path.read_bytes())
|
|
records.append({**record, "affine": affine.tolist(), "npy_sha256": sha(path)})
|
|
save(manifest_path, {**prior, "preprocess": recipe, "crops": records, "sha256": digest.hexdigest(),
|
|
"selection_manifest": str(prior_path), "selection_manifest_sha256": sha(prior_path)})
|
|
manifest = json.loads(manifest_path.read_text())
|
|
files = sorted((directory / "npy_f32").glob("*.npy"))
|
|
digest = hashlib.sha256()
|
|
for path in files:
|
|
digest.update(path.read_bytes())
|
|
if len(files) != 128 or manifest["preprocess"] != recipe or digest.hexdigest() != manifest["sha256"]:
|
|
raise ValueError("Calibration bytes do not match recipe")
|
|
return {"path": str(directory), "sha256": digest.hexdigest(), "count": 128}
|
|
|
|
|
|
def probe(mid: str, manifest: str, weights_dir: Path, output: Path, calib: dict):
|
|
import numpy as np
|
|
import onnx
|
|
import onnxruntime as ort
|
|
import torch
|
|
from mmengine.dataset import Compose, pseudo_collate
|
|
from mmpose.apis.inference import inference_topdown
|
|
from harness import registry, scheduler
|
|
from harness.adapters import get
|
|
from harness.adapters.openmmlab import get_recipe
|
|
from harness.adapters.topdown_pose_common import (crop, components, diagnostic_box,
|
|
boxes_for, BBOX_FILE, BBOX_SHA256)
|
|
from harness.continuation import load_eval_set
|
|
from harness.eval import EVALUATOR_VERSION, coco_extra
|
|
from harness.eval.common import T1, TorchRunner, cosine
|
|
from harness.tools.export_onnx import exporter_version
|
|
|
|
directory = output / mid
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
proof = directory / "probe.json"
|
|
if proof.exists():
|
|
(directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes())
|
|
result = {"model": mid, "task": "pose", "status": "running", "started_at": time.time()}
|
|
save(proof, result)
|
|
try:
|
|
adapter, row = get(mid), registry.model_row(mid)
|
|
dataset, evaluation = load_eval_set(manifest)
|
|
if dataset != "coco2017" or row["dataset"] != dataset:
|
|
raise ValueError("RTMPose requires the fixed COCO2017 T1")
|
|
weights = weights_dir / row["checkpoint"]["filename"]
|
|
if sha(weights) != row["checkpoint"]["sha256"]:
|
|
raise ValueError("Checkpoint bytes are not pinned in the shared catalog")
|
|
audit_path = output.parent / "canonical-audit.json"
|
|
audit = json.loads(audit_path.read_text())
|
|
normal = next(r for r in audit["models"] if r["model"] == mid)
|
|
assert normal["checkpoint_sha256"] == sha(weights)
|
|
assert normal["crop_vs_stock_official_preprocessor_bitwise_equal"]
|
|
result.update(checkpoint=row["checkpoint"], preprocess=row["preprocess"], eval_set=evaluation,
|
|
weights_path=str(weights), weights_sha256=sha(weights), calibration=calib,
|
|
bbox_file=str(BBOX_FILE), bbox_sha256=BBOX_SHA256,
|
|
official_recipe=get_recipe(mid),
|
|
canonical_audit={"path": str(audit_path), "sha256": sha(audit_path)},
|
|
official_reference_scope=audit["reference_scope"],
|
|
implementation_sha256={str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p)
|
|
for p in (Path(__file__).resolve(), Path(adapter.__file__),
|
|
Path(__file__).resolve().parents[1] / "harness/adapters/topdown_pose_common.py",
|
|
Path(__file__).resolve().parents[1] / "harness/adapters/openmmlab.py",
|
|
Path(__file__).resolve().parents[1] / "harness/adapters/rtmpose_common.py",
|
|
Path(coco_extra.__file__))})
|
|
with scheduler.acquire("cpu"):
|
|
runner = TorchRunner(mid, row["source"], weights, row["input_spec"])
|
|
result["strict_load"] = True
|
|
onnx_path = directory / "model.onnx"
|
|
meta_path = directory / "meta.json"
|
|
version = exporter_version(mid)
|
|
previous = json.loads(meta_path.read_text()) if meta_path.exists() else {}
|
|
if not (onnx_path.exists() and previous.get("exporter_version") == version
|
|
and previous.get("weights_sha256") == sha(weights)
|
|
and previous.get("onnx_sha256") == sha(onnx_path)):
|
|
with (directory / "export.log").open("a") as log:
|
|
subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid,
|
|
"--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"],
|
|
"--out", str(onnx_path)], check=True, stdout=log, stderr=log)
|
|
save(meta_path, {"model": mid, "exporter_version": version, "input_spec": row["input_spec"],
|
|
"weights_sha256": sha(weights), "onnx_sha256": sha(onnx_path)})
|
|
onnx.checker.check_model(str(onnx_path))
|
|
options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1
|
|
session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"])
|
|
|
|
def exported(x):
|
|
return session.run(None, {session.get_inputs()[0].name: x})
|
|
|
|
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
|
|
try:
|
|
if len(t1.items) != 100:
|
|
raise ValueError("This admission probe requires exactly the fixed 100 COCO images")
|
|
eligible = [i for i, item in enumerate(t1.items) if diagnostic_box(t1, item) is not None]
|
|
if len(eligible) < 32:
|
|
raise ValueError("Fewer than 32 images have genuine detector boxes")
|
|
indices = sorted(np.random.default_rng(20260914).permutation(eligible)[:32].tolist())
|
|
native = runner.m.upstream
|
|
native.cfg = runner.m.cfg
|
|
native.dataset_meta = components(mid)[2]
|
|
pipeline = Compose(native.cfg.test_dataloader.dataset.pipeline)
|
|
inputs, outputs, comparisons, references = {}, {}, [], []
|
|
for i, index in enumerate(indices):
|
|
item = t1.items[index]
|
|
image = t1.image(item)
|
|
box = diagnostic_box(t1, item)
|
|
x, matrix, center, scale = crop(image, row["input_spec"], box["bbox_xyxy"])
|
|
key = f"s{i:03d}"
|
|
inputs[key] = x
|
|
a, b = runner(x), exported(x)
|
|
if len(a) != len(b) or len(a) != (1 if mid.startswith("hrnet.") else 2):
|
|
raise ValueError("Unexpected official head tensor count")
|
|
for j, (aa, bb) in enumerate(zip(a, b)):
|
|
if aa.shape != bb.shape or not np.isfinite(aa).all() or not np.isfinite(bb).all():
|
|
raise ValueError("Invalid FP32/ONNX output")
|
|
score = cosine(aa, bb)
|
|
comparisons.append({"sample": key, "image_id": item["id"], "output": j,
|
|
"cosine": score, "max_abs": float(np.abs(aa-bb).max())})
|
|
if score < .9999:
|
|
raise ValueError(f"SimCC parity failed: {score}")
|
|
outputs[f"{key}/out{j}"] = aa
|
|
bgr = np.asarray(image)[:, :, ::-1].copy()
|
|
original = {"img": bgr, "bbox": box["bbox_xyxy"][None].copy(),
|
|
"bbox_score": np.ones(1, np.float32), **native.dataset_meta}
|
|
packed = pipeline(original)
|
|
with torch.no_grad():
|
|
official_input = native.data_preprocessor(pseudo_collate([packed]), training=False)["inputs"].cpu().numpy()
|
|
input_delta = float(np.abs(x - official_input).max())
|
|
ref = inference_topdown(native, bgr, bboxes=box["bbox_xyxy"][None].copy(), bbox_format="xyxy")[0].pred_instances
|
|
points = adapter.decode(a, runner(np.ascontiguousarray(x[..., ::-1])), center, scale, row["input_spec"])
|
|
coordinate_delta = float(np.abs(points[:, :2] - ref.keypoints[0]).max())
|
|
score_delta = float(np.abs(points[:, 2] - ref.keypoint_scores[0]).max())
|
|
references.append({"sample": key, "image_id": item["id"], "source_index": box["source_index"],
|
|
"affine": matrix.tolist(), "input_max_abs": input_delta,
|
|
"coordinate_max_abs_px": coordinate_delta, "score_max_abs": score_delta})
|
|
if input_delta > 1e-6 or coordinate_delta > .01 or score_delta > 1e-4:
|
|
raise ValueError(f"Official pipeline mismatch: {references[-1]}")
|
|
result.update(comparisons=comparisons, official_reference=references,
|
|
no_bbox_image_ids=[it["id"] for it in t1.items if not boxes_for(t1, it)],
|
|
g2_selection="seed20260914 sample32 eligible detector images; sorted T1 order")
|
|
save(proof, result)
|
|
print(f"{mid}: strict load/export and 32 official-reference/G2 comparisons passed", flush=True)
|
|
baseline = {"task": "pose", "model": mid, "evaluator_version": EVALUATOR_VERSION,
|
|
"t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)}
|
|
converted = dict(baseline)
|
|
coco_extra.run_fp32(runner, t1, row["input_spec"], baseline)
|
|
save(directory / "metrics.json", baseline)
|
|
print(f"{mid}: FP32 COCO100 {baseline['metrics']}", flush=True)
|
|
coco_extra.run_quant(exported, t1, row["input_spec"], baseline, converted)
|
|
converted["execution"] = {"source": "onnxruntime_cpu", "purpose": "export_parity_only"}
|
|
save(directory / "eval.json", converted)
|
|
shift = converted["pose_shift_evidence"]
|
|
delta = abs(baseline["metrics"]["oks_ap"] - converted["metrics"]["oks_ap"])
|
|
if (baseline["metrics"]["oks_ap"] <= 0 or delta > .1 or shift["status"] != "measured"
|
|
or shift["keypoint_mean_shift_px"] > 2):
|
|
raise ValueError(f"COCO100/shift parity failed: delta={delta}, shift={coco_extra.pose_shift_summary(shift)}")
|
|
np.savez(directory / "g2_inputs.npz", **inputs)
|
|
np.savez(directory / "g2_fp32.npz", **outputs)
|
|
result["shared_g2"] = check_shared_g2(mid, t1, inputs)
|
|
evidence_path = directory / "pose-export-evidence.json"
|
|
save(evidence_path, shift)
|
|
result.update(status="validated_fp32_export_only", fp32=baseline["metrics"], onnx=converted["metrics"],
|
|
pose_export_shift={**coco_extra.pose_shift_summary(shift), "evidence": str(evidence_path)},
|
|
pose_export_evidence={"path": str(evidence_path), "sha256": sha(evidence_path)},
|
|
onnx_path=str(onnx_path),
|
|
onnx_sha256=sha(onnx_path), baseline_path=str(directory),
|
|
sample_ids=[it["id"] for it in t1.items])
|
|
finally:
|
|
t1.tar.close()
|
|
except Exception as exc:
|
|
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc())
|
|
result["finished_at"] = time.time()
|
|
save(proof, result)
|
|
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--model", action="append", required=True, choices=["hrnet.w32", "hrnet.w48.2", "simcc.resnet50"])
|
|
parser.add_argument("--t1", required=True)
|
|
parser.add_argument("--weights-dir", type=Path, required=True)
|
|
parser.add_argument("--train-root", type=Path, required=True)
|
|
parser.add_argument("--out", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if len(args.model) > 3 or len(set(args.model)) != len(args.model):
|
|
parser.error("at most three distinct models per batch")
|
|
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
|
import torch
|
|
torch.set_num_threads(2)
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
results = []
|
|
for mid in args.model:
|
|
calib = calibration(args.train_root, args.out, mid)
|
|
print(json.dumps({"model": mid, "calibration": calib}), flush=True)
|
|
results.append(probe(mid, args.t1, args.weights_dir, args.out, calib))
|
|
return int(any(r["status"] != "validated_fp32_export_only" for r in results))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|