127 lines
6.7 KiB
Python
127 lines
6.7 KiB
Python
"""Validate real classification inputs once; reuse the normal preparation cache.
|
|
|
|
No quantization, board runs, task insertion, or training traces are synthesized.
|
|
The subsequent Agent job reuses these weights, ONNX and measured FP32 baseline.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from types import SimpleNamespace
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from harness import paths, registry, scheduler
|
|
from harness.continuation import load_eval_set
|
|
from harness.eval import EVALUATOR_VERSION
|
|
|
|
|
|
def probe(mid: str, manifest: str, weights_directory: str | None = None) -> dict:
|
|
import numpy as np
|
|
import onnxruntime as ort
|
|
from harness.eval import cls
|
|
from harness.eval.common import T1, TorchRunner, cosine
|
|
from harness.eval.run import g2_inputs
|
|
from harness.stages.s01_fetch import Fetch, weights_file
|
|
from harness.stages.s02_baseline import baseline_dir
|
|
from harness.stages.s03_export import Export, onnx_path
|
|
|
|
row = registry.model_row(mid)
|
|
if row["task"] != "cls" or not row.get("checkpoint") or not row.get("preprocess"):
|
|
raise ValueError(f"{mid}: explicit checkpoint and image recipe required")
|
|
dataset, evaluation = load_eval_set(manifest)
|
|
if dataset != row["dataset"]:
|
|
raise ValueError("model and evaluation dataset differ")
|
|
spec = registry.expand_matrix(["oracle"], [mid])[0]
|
|
job = registry.build_job_json(spec, "adapter-probe", EVALUATOR_VERSION)
|
|
job["data"]["t1"] = evaluation
|
|
ctx = SimpleNamespace(job=job)
|
|
directory = paths.CACHE / "adapter-probes" / mid
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
result = {"model": mid, "task": "cls", "status": "running", "started_at": time.time(),
|
|
"checkpoint": row["checkpoint"], "preprocess": row["preprocess"], "eval_set": evaluation}
|
|
proof = directory / "probe.json"
|
|
if proof.exists():
|
|
(directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes())
|
|
proof.write_text(json.dumps(result, indent=2))
|
|
try:
|
|
with scheduler.acquire("cpu"), (directory / "probe.log").open("a") as log:
|
|
fetch = Fetch()
|
|
if not fetch.cached(ctx):
|
|
cmd = fetch.command(ctx)
|
|
if weights_directory:
|
|
source = Path(weights_directory) / row["checkpoint"]["filename"]
|
|
cmd[cmd.index("--uri") + 1] = source.resolve().as_uri()
|
|
subprocess.run(cmd, check=True, stdout=log, stderr=log)
|
|
fetch.measure(ctx, directory, None)
|
|
weights = weights_file(mid)
|
|
actual_sha = hashlib.sha256(weights.read_bytes()).hexdigest()
|
|
if actual_sha != row["checkpoint"]["sha256"]:
|
|
raise ValueError("checkpoint digest differs from the pinned official recipe")
|
|
runner = TorchRunner(mid, row["source"], weights, row["input_spec"])
|
|
export = Export()
|
|
if not export.cached(ctx):
|
|
subprocess.run(export.command(ctx), check=True, stdout=log, stderr=log)
|
|
export.measure(ctx, directory, None)
|
|
onnx = onnx_path(job)
|
|
options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1
|
|
session = ort.InferenceSession(str(onnx), sess_options=options, providers=["CPUExecutionProvider"])
|
|
def exported(x):
|
|
return session.run(None, {session.get_inputs()[0].name: x})
|
|
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
|
|
try:
|
|
xs = g2_inputs("cls", t1, row["input_spec"], job["gates"]["G2"]["n_samples"], model=mid)
|
|
outputs, comparisons = {}, []
|
|
for key, x in xs.items():
|
|
a, b = runner(x)[0], exported(x)[0]
|
|
if a.shape != (1, 1000) or b.shape != a.shape or not np.isfinite(a).all() or not np.isfinite(b).all():
|
|
raise ValueError(f"invalid classifier output: {a.shape}, {b.shape}")
|
|
score = cosine(a, b)
|
|
comparisons.append({"sample": key, "cosine": score, "max_abs": float(np.abs(a - b).max())})
|
|
if score < .9999:
|
|
raise ValueError(f"ONNX/FP32 output mismatch: {score}")
|
|
outputs[f"{key}/out0"] = a
|
|
baseline = {"task": "cls", "model": mid, "evaluator_version": EVALUATOR_VERSION,
|
|
"t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)}
|
|
converted = {**baseline}
|
|
cls.run_fp32(runner, t1, row["input_spec"], baseline)
|
|
cls.run_fp32(exported, t1, row["input_spec"], converted)
|
|
if baseline["metrics"]["top1"] <= 0 or abs(baseline["metrics"]["top1"] - converted["metrics"]["top1"]) > .1:
|
|
raise ValueError(f"FP32/ONNX accuracy mismatch: {baseline['metrics']} / {converted['metrics']}")
|
|
# Populate precisely the cache consumed by stage 02, after verification.
|
|
cache = baseline_dir(job); cache.mkdir(parents=True, exist_ok=True)
|
|
np.savez(cache / "g2_inputs.npz", **xs)
|
|
np.savez(cache / "g2_fp32.npz", **outputs)
|
|
(cache / "metrics.json").write_text(json.dumps(baseline, indent=2))
|
|
result.update(status="validated_fp32_export_only", fp32=baseline["metrics"],
|
|
onnx=converted["metrics"], comparisons=comparisons,
|
|
weights_path=str(weights), weights_sha256=actual_sha,
|
|
onnx_path=str(onnx), onnx_sha256=hashlib.sha256(onnx.read_bytes()).hexdigest(),
|
|
baseline_path=str(cache), sample_ids=[it["id"] for it in t1.items])
|
|
finally:
|
|
t1.tar.close()
|
|
except Exception as exc:
|
|
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
|
|
result["finished_at"] = time.time()
|
|
proof.write_text(json.dumps(result, indent=2))
|
|
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--model", action="append", required=True)
|
|
parser.add_argument("--t1", required=True)
|
|
parser.add_argument("--weights-dir")
|
|
args = parser.parse_args()
|
|
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
|
import torch
|
|
torch.set_num_threads(2)
|
|
failures = sum(probe(mid, args.t1, args.weights_dir)["status"] != "validated_fp32_export_only" for mid in args.model)
|
|
raise SystemExit(bool(failures))
|