"""Validate real classification inputs once; reuse the normal preparation cache. No quantization, board runs, task insertion, or training traces are synthesized. The subsequent Agent job reuses these weights, ONNX and measured FP32 baseline. """ from __future__ import annotations import argparse import hashlib import json import os from pathlib import Path import subprocess import sys import time from types import SimpleNamespace sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from harness import paths, registry, scheduler from harness.continuation import load_eval_set from harness.eval import EVALUATOR_VERSION def probe(mid: str, manifest: str, weights_directory: str | None = None) -> dict: import numpy as np import onnxruntime as ort from harness.eval import cls from harness.eval.common import T1, TorchRunner, cosine from harness.eval.run import g2_inputs from harness.stages.s01_fetch import Fetch, weights_file from harness.stages.s02_baseline import baseline_dir from harness.stages.s03_export import Export, onnx_path row = registry.model_row(mid) if row["task"] != "cls" or not row.get("checkpoint") or not row.get("preprocess"): raise ValueError(f"{mid}: explicit checkpoint and image recipe required") dataset, evaluation = load_eval_set(manifest) if dataset != row["dataset"]: raise ValueError("model and evaluation dataset differ") spec = registry.expand_matrix(["oracle"], [mid])[0] job = registry.build_job_json(spec, "adapter-probe", EVALUATOR_VERSION) job["data"]["t1"] = evaluation ctx = SimpleNamespace(job=job) directory = paths.CACHE / "adapter-probes" / mid directory.mkdir(parents=True, exist_ok=True) result = {"model": mid, "task": "cls", "status": "running", "started_at": time.time(), "checkpoint": row["checkpoint"], "preprocess": row["preprocess"], "eval_set": evaluation} proof = directory / "probe.json" if proof.exists(): (directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes()) proof.write_text(json.dumps(result, indent=2)) try: with scheduler.acquire("cpu"), (directory / "probe.log").open("a") as log: fetch = Fetch() if not fetch.cached(ctx): cmd = fetch.command(ctx) if weights_directory: source = Path(weights_directory) / row["checkpoint"]["filename"] cmd[cmd.index("--uri") + 1] = source.resolve().as_uri() subprocess.run(cmd, check=True, stdout=log, stderr=log) fetch.measure(ctx, directory, None) weights = weights_file(mid) actual_sha = hashlib.sha256(weights.read_bytes()).hexdigest() if actual_sha != row["checkpoint"]["sha256"]: raise ValueError("checkpoint digest differs from the pinned official recipe") runner = TorchRunner(mid, row["source"], weights, row["input_spec"]) export = Export() if not export.cached(ctx): subprocess.run(export.command(ctx), check=True, stdout=log, stderr=log) export.measure(ctx, directory, None) onnx = onnx_path(job) options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1 session = ort.InferenceSession(str(onnx), sess_options=options, providers=["CPUExecutionProvider"]) def exported(x): return session.run(None, {session.get_inputs()[0].name: x}) t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) try: xs = g2_inputs("cls", t1, row["input_spec"], job["gates"]["G2"]["n_samples"], model=mid) outputs, comparisons = {}, [] for key, x in xs.items(): a, b = runner(x)[0], exported(x)[0] if a.shape != (1, 1000) or b.shape != a.shape or not np.isfinite(a).all() or not np.isfinite(b).all(): raise ValueError(f"invalid classifier output: {a.shape}, {b.shape}") score = cosine(a, b) comparisons.append({"sample": key, "cosine": score, "max_abs": float(np.abs(a - b).max())}) if score < .9999: raise ValueError(f"ONNX/FP32 output mismatch: {score}") outputs[f"{key}/out0"] = a baseline = {"task": "cls", "model": mid, "evaluator_version": EVALUATOR_VERSION, "t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)} converted = {**baseline} cls.run_fp32(runner, t1, row["input_spec"], baseline) cls.run_fp32(exported, t1, row["input_spec"], converted) if baseline["metrics"]["top1"] <= 0 or abs(baseline["metrics"]["top1"] - converted["metrics"]["top1"]) > .1: raise ValueError(f"FP32/ONNX accuracy mismatch: {baseline['metrics']} / {converted['metrics']}") # Populate precisely the cache consumed by stage 02, after verification. cache = baseline_dir(job); cache.mkdir(parents=True, exist_ok=True) np.savez(cache / "g2_inputs.npz", **xs) np.savez(cache / "g2_fp32.npz", **outputs) (cache / "metrics.json").write_text(json.dumps(baseline, indent=2)) result.update(status="validated_fp32_export_only", fp32=baseline["metrics"], onnx=converted["metrics"], comparisons=comparisons, weights_path=str(weights), weights_sha256=actual_sha, onnx_path=str(onnx), onnx_sha256=hashlib.sha256(onnx.read_bytes()).hexdigest(), baseline_path=str(cache), sample_ids=[it["id"] for it in t1.items]) finally: t1.tar.close() except Exception as exc: result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") result["finished_at"] = time.time() proof.write_text(json.dumps(result, indent=2)) print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) return result if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model", action="append", required=True) parser.add_argument("--t1", required=True) parser.add_argument("--weights-dir") args = parser.parse_args() os.environ["CUDA_VISIBLE_DEVICES"] = "-1" import torch torch.set_num_threads(2) failures = sum(probe(mid, args.t1, args.weights_dir)["status"] != "validated_fp32_export_only" for mid in args.model) raise SystemExit(bool(failures))