"""Real checkpoint -> ONNX -> paired FP32 inference/COCO score, before queue admission. Uses the existing fetch/export/evaluation code and resource lock. Does not label a model ready, insert tasks, change packages, or claim quantization/board completion. """ from __future__ import annotations import argparse import hashlib import json import os from pathlib import Path import subprocess import sys import time sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from harness import paths, registry, scheduler def probe(row, n): import numpy as np import onnxruntime as ort from harness.eval.common import T1, TorchRunner, cosine from harness.eval.run import g2_inputs from harness.eval.coco_extra import predict, evaluate, run_fp32, run_quant mid, task = row["model_id"], row["task"] directory = paths.CACHE / "adapter-probes" / mid directory.mkdir(parents=True, exist_ok=True) onnx = directory / (mid + ".onnx") result = {"model": mid, "task": task, "started_at": time.time(), "status": "running"} try: with scheduler.acquire("cpu"), (directory / "probe.log").open("a") as log: subprocess.run([sys.executable, "-m", "harness.tools.fetch_weights", "--model", mid, "--uri", row["weights"], "--out-dir", str(directory)], check=True, stdout=log, stderr=log) weights = directory / row["weights"].split("/")[-1] subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid, "--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"], "--out", str(onnx)], check=True, stdout=log, stderr=log) torch_runner = TorchRunner(mid, row["source"], weights, row["input_spec"]) opts = ort.SessionOptions() opts.intra_op_num_threads = 2 opts.inter_op_num_threads = 1 session = ort.InferenceSession(str(onnx), sess_options=opts, providers=["CPUExecutionProvider"]) def exported(x): return session.run(None, {session.get_inputs()[0].name: x}) manifest_path = paths.DATA / "t1" / row["dataset"] / "manifest.json" manifest = json.loads(manifest_path.read_text()) t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"]) t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)} try: comparisons = [] for sample, x in g2_inputs(task, t1, row["input_spec"], n).items(): raw, converted = torch_runner(x), exported(x) # TorchRunner can expose extra training/debug tensors. Match the # official export outputs by shape; require an unambiguous match. for y in converted: matches = [v for v in raw if v.shape == y.shape] if len(matches) != 1: raise ValueError(f"ambiguous output mapping: {y.shape}, {[v.shape for v in raw]}") cos = cosine(matches[0], y) comparisons.append({"sample": sample, "shape": list(y.shape), "cosine": cos, "max_abs": float(np.abs(matches[0] - y).max())}) if cos < .9999: raise ValueError(f"export numeric mismatch: {cos}") if task == "pose": baseline = {"task": task, "model": mid} converted = {"task": task, "model": mid} run_fp32(torch_runner, t1, row["input_spec"], baseline) run_quant(exported, t1, row["input_spec"], baseline, converted) a, b = baseline["metrics"], converted["metrics"] shift = converted["G3"]["extra"]["keypoint_shift"] # This compares two FP32 implementations; it is never a # source=board measurement or a quantization gate pass. if shift["status"] != "measured" or shift["keypoint_mean_shift_px"] > 2: raise ValueError(f"export pose geometry mismatch: {shift.get('reason', shift['keypoint_mean_shift_px'])}") evidence_path = directory / f"pose-export-evidence-{time.time_ns()}.json" evidence_path.write_text(json.dumps(converted["pose_shift_evidence"], indent=2)) result["pose_export_shift"] = {**shift, "evidence": str(evidence_path)} result["pose_export_evidence"] = { "path": str(evidence_path), "sha256": hashlib.sha256(evidence_path.read_bytes()).hexdigest()} else: a = evaluate(t1, predict(torch_runner, t1, row["input_spec"], task), task) b = evaluate(t1, predict(exported, t1, row["input_spec"], task), task) metric = "mask_ap" if task == "iseg" else "oks_ap" if a[metric] <= 0 or abs(a[metric] - b[metric]) > .1: raise ValueError(f"export task metric mismatch or zero baseline: {a} / {b}") result.update(status="validated_fp32_export_only", comparisons=comparisons, fp32=a, onnx=b, sample_ids=[it["id"] for it in t1.items], t1_sha256=manifest["sha256"], weights_sha256=hashlib.sha256(weights.read_bytes()).hexdigest(), onnx_sha256=hashlib.sha256(onnx.read_bytes()).hexdigest()) finally: t1.tar.close() except Exception as exc: result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") result["finished_at"] = time.time() previous = directory / "probe.json" if previous.exists(): import shutil shutil.copy2(previous, directory / f"probe-history-{time.time_ns()}.json") previous.write_text(json.dumps(result, indent=2)) print(json.dumps(result), flush=True) return result if __name__ == "__main__": ap = argparse.ArgumentParser() ap.add_argument("--model", action="append", required=True) ap.add_argument("--n", type=int, default=8) args = ap.parse_args() os.environ["CUDA_VISIBLE_DEVICES"] = "-1" import torch torch.set_num_threads(2) failures = 0 for mid in args.model: row = registry.model_row(mid) if row["source"] != "ultralytics" or row["task"] not in ("iseg", "pose") or mid.startswith("fastsam."): raise SystemExit(f"unsupported probe scope: {mid}") failures += probe(row, args.n)["status"] == "blocked" sys.exit(bool(failures))