data-gen-bpu/scripts/probe_seg_adapters.py

148 lines
8.0 KiB
Python

"""Validate real segmentation preparation through the existing stage caches.
Requires an explicit labeled T1 with at least 100 images. No quantization,
board run, task insertion, or training trace is synthesized. One dataset per
invocation; at most three models. Run B0/Cityscapes before B1+B2/ADE20K.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
from types import SimpleNamespace
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from harness import paths, registry, scheduler
from harness.continuation import load_eval_set
from harness.eval import EVALUATOR_VERSION
def probe(mid: str, manifest: str, weights_directory: str | None = None) -> dict:
import numpy as np
import onnxruntime as ort
from harness.eval import seg
from harness.eval.common import T1, TorchRunner, cosine
from harness.stages.s01_fetch import Fetch, weights_file
from harness.stages.s02_baseline import baseline_dir
from harness.stages.s03_export import Export, onnx_path
directory = paths.CACHE / "adapter-probes" / mid
directory.mkdir(parents=True, exist_ok=True)
result = {"model": mid, "task": "seg", "status": "running", "started_at": time.time()}
proof = directory / "probe.json"
if proof.exists():
(directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes())
proof.write_text(json.dumps(result, indent=2))
try:
row = registry.model_row(mid)
if row["task"] != "seg" or not row.get("checkpoint") or not row.get("preprocess"):
raise ValueError(f"{mid}: explicit segmentation checkpoint and recipe required")
dataset, evaluation = load_eval_set(manifest)
if dataset != row["dataset"]:
raise ValueError(f"model/T1 dataset mismatch: {row['dataset']} / {dataset}")
spec = registry.expand_matrix(["oracle"], [mid])[0]
job = registry.build_job_json(spec, "adapter-probe", EVALUATOR_VERSION)
job["data"]["t1"] = evaluation
ctx = SimpleNamespace(job=job)
result.update(checkpoint=row["checkpoint"], preprocess=row["preprocess"],
model_spec=row, eval_set=evaluation)
with scheduler.acquire("cpu"), (directory / "probe.log").open("a") as log:
fetch = Fetch()
if not fetch.cached(ctx):
cmd = fetch.command(ctx)
if weights_directory:
source = Path(weights_directory) / row["checkpoint"]["filename"]
cmd[cmd.index("--uri") + 1] = source.resolve().as_uri()
subprocess.run(cmd, check=True, stdout=log, stderr=log)
fetch.measure(ctx, directory, None)
weights = weights_file(mid)
actual_sha = hashlib.sha256(weights.read_bytes()).hexdigest()
if actual_sha != row["checkpoint"]["sha256"]:
raise ValueError("checkpoint digest differs from the pinned official segmentation recipe")
runner = TorchRunner(mid, row["source"], weights, row["input_spec"])
export = Export()
if not export.cached(ctx):
subprocess.run(export.command(ctx), check=True, stdout=log, stderr=log)
export.measure(ctx, directory, None)
onnx = onnx_path(job)
options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1
session = ort.InferenceSession(str(onnx), sess_options=options, providers=["CPUExecutionProvider"])
diagnostic_outputs = {}
def native(x):
cached = diagnostic_outputs.get(hashlib.sha256(x.tobytes()).digest())
return [cached[0]] if cached is not None else runner(x)
def exported(x):
cached = diagnostic_outputs.get(hashlib.sha256(x.tobytes()).digest())
return [cached[1]] if cached is not None else session.run(None, {session.get_inputs()[0].name: x})
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
try:
if len(t1.items) < 100:
raise ValueError("segmentation admission requires at least 100 real labeled T1 images")
xs = {f"s{i:03d}": seg.inputs(t1, it, row["input_spec"], mid)
for i, it in enumerate(t1.g2_subset(job["gates"]["G2"]["n_samples"]))}
outputs, comparisons = {}, []
for key, x in xs.items():
a, b = runner(x)[0], exported(x)[0]
if (a.ndim != 4 or a.shape[:2] != (1, row["checkpoint"]["classes"])
or b.shape != a.shape or not np.isfinite(a).all() or not np.isfinite(b).all()):
raise ValueError(f"invalid segmentation outputs: {a.shape}, {b.shape}")
score = cosine(a, b)
agreement = float((a.argmax(1) == b.argmax(1)).mean())
comparisons.append({"sample": key, "cosine": score,
"max_abs": float(np.abs(a-b).max()), "argmax_agreement": agreement})
if score < .9999 or agreement < .999:
raise ValueError(f"ONNX/FP32 segmentation output mismatch: cosine={score}, agreement={agreement}")
outputs[f"{key}/out0"] = a
# The same 32 images also belong to T1: reuse their measured
# outputs instead of running both models on them twice.
diagnostic_outputs[hashlib.sha256(x.tobytes()).digest()] = (a, b)
baseline = {"task": "seg", "model": mid, "evaluator_version": EVALUATOR_VERSION,
"t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)}
converted = dict(baseline)
seg.run_fp32(native, t1, row["input_spec"], baseline)
seg.run_fp32(exported, t1, row["input_spec"], converted)
delta = abs(baseline["metrics"]["miou"] - converted["metrics"]["miou"])
if baseline["metrics"]["miou"] <= 0 or delta > .01:
raise ValueError(f"FP32/ONNX mIoU mismatch: delta={delta} pp")
cache = baseline_dir(job); cache.mkdir(parents=True, exist_ok=True)
np.savez(cache / "g2_inputs.npz", **xs)
np.savez(cache / "g2_fp32.npz", **outputs)
(cache / "metrics.json").write_text(json.dumps(baseline, indent=2))
result.update(status="validated_fp32_export_only", fp32=baseline["metrics"],
onnx=converted["metrics"], comparisons=comparisons,
segmentation_protocol=baseline["seg_protocol"],
weights_path=str(weights), weights_sha256=actual_sha,
onnx_path=str(onnx), onnx_sha256=hashlib.sha256(onnx.read_bytes()).hexdigest(),
baseline_path=str(cache), sample_ids=[it["id"] for it in t1.items])
finally:
t1.tar.close()
except Exception as exc:
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
result["finished_at"] = time.time()
proof.write_text(json.dumps(result, indent=2))
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
return result
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", action="append", required=True)
parser.add_argument("--t1", required=True)
parser.add_argument("--weights-dir")
args = parser.parse_args()
if len(args.model) > 3:
parser.error("at most three segmentation models per batch")
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import torch
torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", "2")))
failures = sum(probe(mid, args.t1, args.weights_dir)["status"] != "validated_fp32_export_only" for mid in args.model)
raise SystemExit(bool(failures))