148 lines
8.0 KiB
Python
148 lines
8.0 KiB
Python
"""Validate real segmentation preparation through the existing stage caches.
|
|
|
|
Requires an explicit labeled T1 with at least 100 images. No quantization,
|
|
board run, task insertion, or training trace is synthesized. One dataset per
|
|
invocation; at most three models. Run B0/Cityscapes before B1+B2/ADE20K.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from types import SimpleNamespace
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from harness import paths, registry, scheduler
|
|
from harness.continuation import load_eval_set
|
|
from harness.eval import EVALUATOR_VERSION
|
|
|
|
|
|
def probe(mid: str, manifest: str, weights_directory: str | None = None) -> dict:
|
|
import numpy as np
|
|
import onnxruntime as ort
|
|
from harness.eval import seg
|
|
from harness.eval.common import T1, TorchRunner, cosine
|
|
from harness.stages.s01_fetch import Fetch, weights_file
|
|
from harness.stages.s02_baseline import baseline_dir
|
|
from harness.stages.s03_export import Export, onnx_path
|
|
|
|
directory = paths.CACHE / "adapter-probes" / mid
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
result = {"model": mid, "task": "seg", "status": "running", "started_at": time.time()}
|
|
proof = directory / "probe.json"
|
|
if proof.exists():
|
|
(directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes())
|
|
proof.write_text(json.dumps(result, indent=2))
|
|
try:
|
|
row = registry.model_row(mid)
|
|
if row["task"] != "seg" or not row.get("checkpoint") or not row.get("preprocess"):
|
|
raise ValueError(f"{mid}: explicit segmentation checkpoint and recipe required")
|
|
dataset, evaluation = load_eval_set(manifest)
|
|
if dataset != row["dataset"]:
|
|
raise ValueError(f"model/T1 dataset mismatch: {row['dataset']} / {dataset}")
|
|
spec = registry.expand_matrix(["oracle"], [mid])[0]
|
|
job = registry.build_job_json(spec, "adapter-probe", EVALUATOR_VERSION)
|
|
job["data"]["t1"] = evaluation
|
|
ctx = SimpleNamespace(job=job)
|
|
result.update(checkpoint=row["checkpoint"], preprocess=row["preprocess"],
|
|
model_spec=row, eval_set=evaluation)
|
|
with scheduler.acquire("cpu"), (directory / "probe.log").open("a") as log:
|
|
fetch = Fetch()
|
|
if not fetch.cached(ctx):
|
|
cmd = fetch.command(ctx)
|
|
if weights_directory:
|
|
source = Path(weights_directory) / row["checkpoint"]["filename"]
|
|
cmd[cmd.index("--uri") + 1] = source.resolve().as_uri()
|
|
subprocess.run(cmd, check=True, stdout=log, stderr=log)
|
|
fetch.measure(ctx, directory, None)
|
|
weights = weights_file(mid)
|
|
actual_sha = hashlib.sha256(weights.read_bytes()).hexdigest()
|
|
if actual_sha != row["checkpoint"]["sha256"]:
|
|
raise ValueError("checkpoint digest differs from the pinned official segmentation recipe")
|
|
runner = TorchRunner(mid, row["source"], weights, row["input_spec"])
|
|
export = Export()
|
|
if not export.cached(ctx):
|
|
subprocess.run(export.command(ctx), check=True, stdout=log, stderr=log)
|
|
export.measure(ctx, directory, None)
|
|
onnx = onnx_path(job)
|
|
options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1
|
|
session = ort.InferenceSession(str(onnx), sess_options=options, providers=["CPUExecutionProvider"])
|
|
diagnostic_outputs = {}
|
|
|
|
def native(x):
|
|
cached = diagnostic_outputs.get(hashlib.sha256(x.tobytes()).digest())
|
|
return [cached[0]] if cached is not None else runner(x)
|
|
|
|
def exported(x):
|
|
cached = diagnostic_outputs.get(hashlib.sha256(x.tobytes()).digest())
|
|
return [cached[1]] if cached is not None else session.run(None, {session.get_inputs()[0].name: x})
|
|
|
|
t1 = T1.open(evaluation["path"], evaluation["manifest_sha"])
|
|
try:
|
|
if len(t1.items) < 100:
|
|
raise ValueError("segmentation admission requires at least 100 real labeled T1 images")
|
|
xs = {f"s{i:03d}": seg.inputs(t1, it, row["input_spec"], mid)
|
|
for i, it in enumerate(t1.g2_subset(job["gates"]["G2"]["n_samples"]))}
|
|
outputs, comparisons = {}, []
|
|
for key, x in xs.items():
|
|
a, b = runner(x)[0], exported(x)[0]
|
|
if (a.ndim != 4 or a.shape[:2] != (1, row["checkpoint"]["classes"])
|
|
or b.shape != a.shape or not np.isfinite(a).all() or not np.isfinite(b).all()):
|
|
raise ValueError(f"invalid segmentation outputs: {a.shape}, {b.shape}")
|
|
score = cosine(a, b)
|
|
agreement = float((a.argmax(1) == b.argmax(1)).mean())
|
|
comparisons.append({"sample": key, "cosine": score,
|
|
"max_abs": float(np.abs(a-b).max()), "argmax_agreement": agreement})
|
|
if score < .9999 or agreement < .999:
|
|
raise ValueError(f"ONNX/FP32 segmentation output mismatch: cosine={score}, agreement={agreement}")
|
|
outputs[f"{key}/out0"] = a
|
|
# The same 32 images also belong to T1: reuse their measured
|
|
# outputs instead of running both models on them twice.
|
|
diagnostic_outputs[hashlib.sha256(x.tobytes()).digest()] = (a, b)
|
|
baseline = {"task": "seg", "model": mid, "evaluator_version": EVALUATOR_VERSION,
|
|
"t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)}
|
|
converted = dict(baseline)
|
|
seg.run_fp32(native, t1, row["input_spec"], baseline)
|
|
seg.run_fp32(exported, t1, row["input_spec"], converted)
|
|
delta = abs(baseline["metrics"]["miou"] - converted["metrics"]["miou"])
|
|
if baseline["metrics"]["miou"] <= 0 or delta > .01:
|
|
raise ValueError(f"FP32/ONNX mIoU mismatch: delta={delta} pp")
|
|
cache = baseline_dir(job); cache.mkdir(parents=True, exist_ok=True)
|
|
np.savez(cache / "g2_inputs.npz", **xs)
|
|
np.savez(cache / "g2_fp32.npz", **outputs)
|
|
(cache / "metrics.json").write_text(json.dumps(baseline, indent=2))
|
|
result.update(status="validated_fp32_export_only", fp32=baseline["metrics"],
|
|
onnx=converted["metrics"], comparisons=comparisons,
|
|
segmentation_protocol=baseline["seg_protocol"],
|
|
weights_path=str(weights), weights_sha256=actual_sha,
|
|
onnx_path=str(onnx), onnx_sha256=hashlib.sha256(onnx.read_bytes()).hexdigest(),
|
|
baseline_path=str(cache), sample_ids=[it["id"] for it in t1.items])
|
|
finally:
|
|
t1.tar.close()
|
|
except Exception as exc:
|
|
result.update(status="blocked", error=f"{type(exc).__name__}: {exc}")
|
|
result["finished_at"] = time.time()
|
|
proof.write_text(json.dumps(result, indent=2))
|
|
print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True)
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--model", action="append", required=True)
|
|
parser.add_argument("--t1", required=True)
|
|
parser.add_argument("--weights-dir")
|
|
args = parser.parse_args()
|
|
if len(args.model) > 3:
|
|
parser.error("at most three segmentation models per batch")
|
|
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
|
import torch
|
|
torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", "2")))
|
|
failures = sum(probe(mid, args.t1, args.weights_dir)["status"] != "validated_fp32_export_only" for mid in args.model)
|
|
raise SystemExit(bool(failures))
|