data-gen-bpu/scripts/probe_classification_adapters.py

127 lines
6.7 KiB
Python

"""Validate real classification inputs once; reuse the normal preparation cache.
No quantization, board runs, task insertion, or training traces are synthesized.
The subsequent Agent job reuses these weights, ONNX and measured FP32 baseline.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
from types import SimpleNamespace
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from harness import paths, registry, scheduler
from harness.continuation import load_eval_set
from harness.eval import EVALUATOR_VERSION
def probe(mid: str, manifest: str, weights_directory: str | None = None) -> dict:
import numpy as np
import onnxruntime as ort
from harness.eval import cls
from harness.eval.common import T1, TorchRunner, cosine
from harness.eval.run import g2_inputs
from harness.stages.s01_fetch import Fetch, weights_file
from harness.stages.s02_baseline import baseline_dir
from harness.stages.s03_export import Export, onnx_path
row = registry.model_row(mid)
if row["task"] != "cls" or not row.get("checkpoint") or not row.get("preprocess"):
raise ValueError(f"{mid}: explicit checkpoint and image recipe required")
dataset, evaluation = load_eval_set(manifest)
if dataset != row["dataset"]:
raise ValueError("model and evaluation dataset differ")
spec = registry.expand_matrix(["oracle"], [mid])[0]
job = registry.build_job_json(spec, "adapter-probe", EVALUATOR_VERSION)
job["data"]["t1"] = evaluation
ctx = SimpleNamespace(job=job)
directory = paths.CACHE / "adapter-probes" / mid
directory.mkdir(parents=True, exist_ok=True)
result = {"model": mid, "task": "cls", "status": "running", "started_at": time.time(),
"checkpoint": row["checkpoint"], "preprocess": row["preprocess"], "eval_set": evaluation}
proof = directory / "probe.json"
if proof.exists():
(directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes())
proof.write_text(json.dumps(result, indent=2))
try:
with scheduler.acquire("cpu"), (directory / "probe.log").open("a") as log:
fetch = Fetch()
if not fetch.cached(ctx):
cmd = fetch.command(ctx)
if weights_directory:
source = Path(weights_directory) / row["checkpoint"]["filename"]
cmd[cmd.index("--uri") + 1] = source.resolve().as_uri()
subprocess.run(cmd, check=True, stdout=log, stderr=log)
fetch.measure(ctx, directory, None)
weights = weights_file(mid)
actual_sha = hashlib.sha256(weights.read_bytes()).hexdigest()
if actual_sha != row["checkpoint"]["sha256"]:
raise ValueError("checkpoint digest differs from the pinned official recipe")
runner = TorchRunner(mid, row["source"], weights, row["input_spec"])
export = Export()
if not export.cached(ctx):
subprocess.run(export.command(ctx), check=True, stdout=log, stderr=log)
export.measure(ctx, directory, None)
onnx = onnx_path(job)
options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1
session = ort.InferenceSession(str(onnx), sess_options=options, providers=["CPUExecutionProvider"])
def exported(x):
return session.run(None, {session.get_inputs()[0].name: x})
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
try:
xs = g2_inputs("cls", t1, row["input_spec"], job["gates"]["G2"]["n_samples"], model=mid)
outputs, comparisons = {}, []
for key, x in xs.items():
a, b = runner(x)[0], exported(x)[0]
if a.shape != (1, 1000) or b.shape != a.shape or not np.isfinite(a).all() or not np.isfinite(b).all():
raise ValueError(f"invalid classifier output: {a.shape}, {b.shape}")
score = cosine(a, b)
comparisons.append({"sample": key, "cosine": score, "max_abs": float(np.abs(a - b).max())})
if score < .9999:
raise ValueError(f"ONNX/FP32 output mismatch: {score}")
outputs[f"{key}/out0"] = a
baseline = {"task": "cls", "model": mid, "evaluator_version": EVALUATOR_VERSION,
"t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)}
converted = {**baseline}
cls.run_fp32(runner, t1, row["input_spec"], baseline)
cls.run_fp32(exported, t1, row["input_spec"], converted)
if baseline["metrics"]["top1"] <= 0 or abs(baseline["metrics"]["top1"] - converted["metrics"]["top1"]) > .1:
raise ValueError(f"FP32/ONNX accuracy mismatch: {baseline['metrics']} / {converted['metrics']}")
# Populate precisely the cache consumed by stage 02, after verification.
cache = baseline_dir(job); cache.mkdir(parents=True, exist_ok=True)
np.savez(cache / "g2_inputs.npz", **xs)
np.savez(cache / "g2_fp32.npz", **outputs)
(cache / "metrics.json").write_text(json.dumps(baseline, indent=2))
result.update(status="validated_fp32_export_only", fp32=baseline["metrics"],
onnx=converted["metrics"], comparisons=comparisons,
weights_path=str(weights), weights_sha256=actual_sha,
onnx_path=str(onnx), onnx_sha256=hashlib.sha256(onnx.read_bytes()).hexdigest(),
baseline_path=str(cache), sample_ids=[it["id"] for it in t1.items])
finally:
t1.tar.close()
except Exception as exc:
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
result["finished_at"] = time.time()
proof.write_text(json.dumps(result, indent=2))
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
return result
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", action="append", required=True)
parser.add_argument("--t1", required=True)
parser.add_argument("--weights-dir")
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import torch
torch.set_num_threads(2)
failures = sum(probe(mid, args.t1, args.weights_dir)["status"] != "validated_fp32_export_only" for mid in args.model)
raise SystemExit(bool(failures))