data-gen-bpu/scripts/probe_coco_adapters.py

120 lines
6.5 KiB
Python

"""Real checkpoint -> ONNX -> paired FP32 inference/COCO score, before queue admission.
Uses the existing fetch/export/evaluation code and resource lock. Does not label a
model ready, insert tasks, change packages, or claim quantization/board completion.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from harness import paths, registry, scheduler
def probe(row, n):
import numpy as np
import onnxruntime as ort
from harness.eval.common import T1, TorchRunner, cosine
from harness.eval.run import g2_inputs
from harness.eval.coco_extra import predict, evaluate, run_fp32, run_quant
mid, task = row["model_id"], row["task"]
directory = paths.CACHE / "adapter-probes" / mid
directory.mkdir(parents=True, exist_ok=True)
onnx = directory / (mid + ".onnx")
result = {"model": mid, "task": task, "started_at": time.time(), "status": "running"}
try:
with scheduler.acquire("cpu"), (directory / "probe.log").open("a") as log:
subprocess.run([sys.executable, "-m", "harness.tools.fetch_weights", "--model", mid,
"--uri", row["weights"], "--out-dir", str(directory)], check=True, stdout=log, stderr=log)
weights = directory / row["weights"].split("/")[-1]
subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid,
"--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"],
"--out", str(onnx)], check=True, stdout=log, stderr=log)
torch_runner = TorchRunner(mid, row["source"], weights, row["input_spec"])
opts = ort.SessionOptions()
opts.intra_op_num_threads = 2
opts.inter_op_num_threads = 1
session = ort.InferenceSession(str(onnx), sess_options=opts, providers=["CPUExecutionProvider"])
def exported(x):
return session.run(None, {session.get_inputs()[0].name: x})
manifest_path = paths.DATA / "t1" / row["dataset"] / "manifest.json"
manifest = json.loads(manifest_path.read_text())
t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"])
t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)}
try:
comparisons = []
for sample, x in g2_inputs(task, t1, row["input_spec"], n).items():
raw, converted = torch_runner(x), exported(x)
# TorchRunner can expose extra training/debug tensors. Match the
# official export outputs by shape; require an unambiguous match.
for y in converted:
matches = [v for v in raw if v.shape == y.shape]
if len(matches) != 1:
raise ValueError(f"ambiguous output mapping: {y.shape}, {[v.shape for v in raw]}")
cos = cosine(matches[0], y)
comparisons.append({"sample": sample, "shape": list(y.shape), "cosine": cos,
"max_abs": float(np.abs(matches[0] - y).max())})
if cos < .9999:
raise ValueError(f"export numeric mismatch: {cos}")
if task == "pose":
baseline = {"task": task, "model": mid}
converted = {"task": task, "model": mid}
run_fp32(torch_runner, t1, row["input_spec"], baseline)
run_quant(exported, t1, row["input_spec"], baseline, converted)
a, b = baseline["metrics"], converted["metrics"]
shift = converted["G3"]["extra"]["keypoint_shift"]
# This compares two FP32 implementations; it is never a
# source=board measurement or a quantization gate pass.
if shift["status"] != "measured" or shift["keypoint_mean_shift_px"] > 2:
raise ValueError(f"export pose geometry mismatch: {shift.get('reason', shift['keypoint_mean_shift_px'])}")
evidence_path = directory / f"pose-export-evidence-{time.time_ns()}.json"
evidence_path.write_text(json.dumps(converted["pose_shift_evidence"], indent=2))
result["pose_export_shift"] = {**shift, "evidence": str(evidence_path)}
result["pose_export_evidence"] = {
"path": str(evidence_path), "sha256": hashlib.sha256(evidence_path.read_bytes()).hexdigest()}
else:
a = evaluate(t1, predict(torch_runner, t1, row["input_spec"], task), task)
b = evaluate(t1, predict(exported, t1, row["input_spec"], task), task)
metric = "mask_ap" if task == "iseg" else "oks_ap"
if a[metric] <= 0 or abs(a[metric] - b[metric]) > .1:
raise ValueError(f"export task metric mismatch or zero baseline: {a} / {b}")
result.update(status="validated_fp32_export_only", comparisons=comparisons, fp32=a, onnx=b,
sample_ids=[it["id"] for it in t1.items], t1_sha256=manifest["sha256"],
weights_sha256=hashlib.sha256(weights.read_bytes()).hexdigest(),
onnx_sha256=hashlib.sha256(onnx.read_bytes()).hexdigest())
finally:
t1.tar.close()
except Exception as exc:
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
result["finished_at"] = time.time()
previous = directory / "probe.json"
if previous.exists():
import shutil
shutil.copy2(previous, directory / f"probe-history-{time.time_ns()}.json")
previous.write_text(json.dumps(result, indent=2))
print(json.dumps(result), flush=True)
return result
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--model", action="append", required=True)
ap.add_argument("--n", type=int, default=8)
args = ap.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import torch
torch.set_num_threads(2)
failures = 0
for mid in args.model:
row = registry.model_row(mid)
if row["source"] != "ultralytics" or row["task"] not in ("iseg", "pose") or mid.startswith("fastsam."):
raise SystemExit(f"unsupported probe scope: {mid}")
failures += probe(row, args.n)["status"] == "blocked"
sys.exit(bool(failures))