120 lines
6.5 KiB
Python
120 lines
6.5 KiB
Python
"""Real checkpoint -> ONNX -> paired FP32 inference/COCO score, before queue admission.
|
|
|
|
Uses the existing fetch/export/evaluation code and resource lock. Does not label a
|
|
model ready, insert tasks, change packages, or claim quantization/board completion.
|
|
"""
|
|
from __future__ import annotations
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from harness import paths, registry, scheduler
|
|
|
|
|
|
def probe(row, n):
|
|
import numpy as np
|
|
import onnxruntime as ort
|
|
from harness.eval.common import T1, TorchRunner, cosine
|
|
from harness.eval.run import g2_inputs
|
|
from harness.eval.coco_extra import predict, evaluate, run_fp32, run_quant
|
|
mid, task = row["model_id"], row["task"]
|
|
directory = paths.CACHE / "adapter-probes" / mid
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
onnx = directory / (mid + ".onnx")
|
|
result = {"model": mid, "task": task, "started_at": time.time(), "status": "running"}
|
|
try:
|
|
with scheduler.acquire("cpu"), (directory / "probe.log").open("a") as log:
|
|
subprocess.run([sys.executable, "-m", "harness.tools.fetch_weights", "--model", mid,
|
|
"--uri", row["weights"], "--out-dir", str(directory)], check=True, stdout=log, stderr=log)
|
|
weights = directory / row["weights"].split("/")[-1]
|
|
subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid,
|
|
"--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"],
|
|
"--out", str(onnx)], check=True, stdout=log, stderr=log)
|
|
torch_runner = TorchRunner(mid, row["source"], weights, row["input_spec"])
|
|
opts = ort.SessionOptions()
|
|
opts.intra_op_num_threads = 2
|
|
opts.inter_op_num_threads = 1
|
|
session = ort.InferenceSession(str(onnx), sess_options=opts, providers=["CPUExecutionProvider"])
|
|
def exported(x):
|
|
return session.run(None, {session.get_inputs()[0].name: x})
|
|
manifest_path = paths.DATA / "t1" / row["dataset"] / "manifest.json"
|
|
manifest = json.loads(manifest_path.read_text())
|
|
t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"])
|
|
t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)}
|
|
try:
|
|
comparisons = []
|
|
for sample, x in g2_inputs(task, t1, row["input_spec"], n).items():
|
|
raw, converted = torch_runner(x), exported(x)
|
|
# TorchRunner can expose extra training/debug tensors. Match the
|
|
# official export outputs by shape; require an unambiguous match.
|
|
for y in converted:
|
|
matches = [v for v in raw if v.shape == y.shape]
|
|
if len(matches) != 1:
|
|
raise ValueError(f"ambiguous output mapping: {y.shape}, {[v.shape for v in raw]}")
|
|
cos = cosine(matches[0], y)
|
|
comparisons.append({"sample": sample, "shape": list(y.shape), "cosine": cos,
|
|
"max_abs": float(np.abs(matches[0] - y).max())})
|
|
if cos < .9999:
|
|
raise ValueError(f"export numeric mismatch: {cos}")
|
|
if task == "pose":
|
|
baseline = {"task": task, "model": mid}
|
|
converted = {"task": task, "model": mid}
|
|
run_fp32(torch_runner, t1, row["input_spec"], baseline)
|
|
run_quant(exported, t1, row["input_spec"], baseline, converted)
|
|
a, b = baseline["metrics"], converted["metrics"]
|
|
shift = converted["G3"]["extra"]["keypoint_shift"]
|
|
# This compares two FP32 implementations; it is never a
|
|
# source=board measurement or a quantization gate pass.
|
|
if shift["status"] != "measured" or shift["keypoint_mean_shift_px"] > 2:
|
|
raise ValueError(f"export pose geometry mismatch: {shift.get('reason', shift['keypoint_mean_shift_px'])}")
|
|
evidence_path = directory / f"pose-export-evidence-{time.time_ns()}.json"
|
|
evidence_path.write_text(json.dumps(converted["pose_shift_evidence"], indent=2))
|
|
result["pose_export_shift"] = {**shift, "evidence": str(evidence_path)}
|
|
result["pose_export_evidence"] = {
|
|
"path": str(evidence_path), "sha256": hashlib.sha256(evidence_path.read_bytes()).hexdigest()}
|
|
else:
|
|
a = evaluate(t1, predict(torch_runner, t1, row["input_spec"], task), task)
|
|
b = evaluate(t1, predict(exported, t1, row["input_spec"], task), task)
|
|
metric = "mask_ap" if task == "iseg" else "oks_ap"
|
|
if a[metric] <= 0 or abs(a[metric] - b[metric]) > .1:
|
|
raise ValueError(f"export task metric mismatch or zero baseline: {a} / {b}")
|
|
result.update(status="validated_fp32_export_only", comparisons=comparisons, fp32=a, onnx=b,
|
|
sample_ids=[it["id"] for it in t1.items], t1_sha256=manifest["sha256"],
|
|
weights_sha256=hashlib.sha256(weights.read_bytes()).hexdigest(),
|
|
onnx_sha256=hashlib.sha256(onnx.read_bytes()).hexdigest())
|
|
finally:
|
|
t1.tar.close()
|
|
except Exception as exc:
|
|
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
|
|
result["finished_at"] = time.time()
|
|
previous = directory / "probe.json"
|
|
if previous.exists():
|
|
import shutil
|
|
shutil.copy2(previous, directory / f"probe-history-{time.time_ns()}.json")
|
|
previous.write_text(json.dumps(result, indent=2))
|
|
print(json.dumps(result), flush=True)
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--model", action="append", required=True)
|
|
ap.add_argument("--n", type=int, default=8)
|
|
args = ap.parse_args()
|
|
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
|
import torch
|
|
torch.set_num_threads(2)
|
|
failures = 0
|
|
for mid in args.model:
|
|
row = registry.model_row(mid)
|
|
if row["source"] != "ultralytics" or row["task"] not in ("iseg", "pose") or mid.startswith("fastsam."):
|
|
raise SystemExit(f"unsupported probe scope: {mid}")
|
|
failures += probe(row, args.n)["status"] == "blocked"
|
|
sys.exit(bool(failures))
|