"""Real official/adapter/ORT + HPatches geometry probe, outside all existing runs.""" from __future__ import annotations import argparse import ast import hashlib import importlib import json import os from pathlib import Path import sys import time sys.path.insert(0, str(Path(__file__).resolve().parents[1])) def sha(path): h = hashlib.sha256() with open(path, 'rb') as f: for b in iter(lambda: f.read(1 << 20), b''): h.update(b) return h.hexdigest() def write(path, obj): path.parent.mkdir(parents=True, exist_ok=True) temp = path.with_suffix(path.suffix + '.tmp') temp.write_text(json.dumps(obj, indent=2, allow_nan=False) + '\n'); temp.replace(path) def prepare_calibration(adapter, selection, destination): import numpy as np from PIL import Image recipe_sha = hashlib.sha256(json.dumps(adapter.PREPROCESS, sort_keys=True).encode()).hexdigest()[:16] out = destination / adapter.INPUT_SPEC / recipe_sha s = json.loads(selection.read_text()); source = Path(s['source_pool']) if (out / 'manifest.json').exists(): man = json.loads((out / 'manifest.json').read_text()) assert man['count'] == 128 and man['preprocess'] == adapter.PREPROCESS and man['t1_sha256'] == s['t1_sha256'] for item in man['items']: assert sha(out / 'npy_f32' / item['npy']) == item['npy_sha256'] metadata = {'task': 'feat', 'dataset': 'hpatches', 'source': 'val-holdout', 'calib_source': 'val-holdout', 'files': [it['file'] for it in man['items']]} if any(man.get(k) != v for k, v in metadata.items()): man.update(metadata); write(out / 'manifest.json', man) return out if out.exists(): raise FileExistsError(f'refuse partial calibration overwrite: {out}') stage = out.with_name(out.name + '.partial'); (stage / 'npy_f32').mkdir(parents=True, exist_ok=False) items = []; digest = hashlib.sha256() for i, record in enumerate(s['items']): path = source / record['file']; assert sha(path) == record['sha256'] with Image.open(path) as im: x = adapter.preprocess(im, adapter.INPUT_SPEC) p = stage / 'npy_f32' / f'{i:05d}.npy'; np.save(p, x); digest.update(p.read_bytes()) items.append({**record, 'npy': p.name, 'npy_sha256': sha(p)}) write(stage / 'manifest.json', {**{k: v for k, v in s.items() if k != 'items'}, 'items': items, 'preprocess': adapter.PREPROCESS, 'count': 128, 'sha256': digest.hexdigest(), 'input_spec': adapter.INPUT_SPEC, 'npy_dir': 'npy_f32', 'layout': 'NCHW', 'dtype': 'float32', 'task': 'feat', 'dataset': 'hpatches', 'source': 'val-holdout', 'calib_source': 'val-holdout', 'files': [it['file'] for it in items]}) stage.rename(out); return out def main(): ap = argparse.ArgumentParser() ap.add_argument('--model', choices=['superpoint', 'disk', 'aliked', 'xfeat', 'd2.net', 'r2d2', 'dedode.v2', 'keynetplushardnet'], required=True) ap.add_argument('--t1', type=Path, required=True); ap.add_argument('--out', type=Path, required=True) ap.add_argument('--calibration-selection', type=Path, required=True) ap.add_argument('--threads', type=int, default=2) ap.add_argument('--stage', choices=['all', 'probe', 'evaluate'], default='all') a = ap.parse_args(); a.out.mkdir(parents=True, exist_ok=True) os.environ['CUDA_VISIBLE_DEVICES'] = '' import cv2 import numpy as np import torch import onnx import onnxruntime as ort from harness.adapters import feature_common as common from harness.eval.common import T1 from harness.eval import feat torch.set_num_threads(a.threads); torch.set_num_interop_threads(1); cv2.setNumThreads(1) torch.manual_seed(20260914) from harness.adapters import get adapter = get(a.model) weights = common.ROOT / adapter.CHECKPOINT['official_relative_path'] calib = prepare_calibration(adapter, a.calibration_selection, a.out.parent / 'calibration') print(json.dumps({'model': a.model, 'stage': 'calibration_ready', 'path': str(calib)}), flush=True) t1 = T1.open(a.t1); assert len(t1.items) == 100 graph = adapter.build(weights, adapter.INPUT_SPEC).eval() def torch_run(x): with torch.no_grad(): output = graph(torch.from_numpy(x)) return [t.detach().numpy() for t in (output if isinstance(output, tuple) else (output,))] def compare_features(left, right): # Coordinates uniquely identify sparse detections; row order is not a semantic contract. x, y = left['keypoints'], right['keypoints'] assert len(x) == len(y), (len(x), len(y)) aidx = np.lexsort((x[:, 0], x[:, 1])); bidx = np.lexsort((y[:, 0], y[:, 1])) result = {} for key in ('keypoints', 'scores', 'descriptors'): p, q = left[key][aidx], right[key][bidx] result[key + '_max_abs'] = float(np.max(abs(p - q), initial=0)) np.testing.assert_allclose(p, q, atol=2e-3, rtol=2e-3) return result x = feat.inputs(t1, t1.items[0], adapter.INPUT_SPEC, a.model) onnx_path = a.out / 'model.onnx' checks = [] if a.stage != 'evaluate': official = adapter.official_model(weights) if hasattr(adapter, 'official_model') else common.official_model(a.model, weights) for item in [t1.items[i] for i in (0, 49, 99)]: value = adapter.preprocess(t1.image({'file': item['file2']}), adapter.INPUT_SPEC) with torch.no_grad(): k, s, d = (adapter.official_features(official, torch.from_numpy(value)) if hasattr(adapter, 'official_features') else common.official_features(a.model, official, torch.from_numpy(value))) expected = {'keypoints': k.numpy(), 'scores': s.numpy(), 'descriptors': d.numpy()} actual = common.decode(a.model, torch_run(value)) checks.append({'id': item['id'], 'official_vs_adapter': compare_features(expected, actual)}) del official print(json.dumps({'model': a.model, 'stage': 'official_numeric_passed', 'checks': checks}), flush=True) with torch.no_grad(): torch.onnx.export(graph, torch.from_numpy(x), str(onnx_path), opset_version=adapter.OPSET, input_names=['input'], dynamic_axes=None, do_constant_folding=True, dynamo=False) onnx.checker.check_model(str(onnx_path)) so = ort.SessionOptions(); so.intra_op_num_threads = a.threads; so.inter_op_num_threads = 1 session = ort.InferenceSession(str(onnx_path), so, providers=['CPUExecutionProvider']) def ort_run(value): return session.run(None, {'input': value}) if a.stage != 'evaluate': for idx, item in enumerate([t1.items[i] for i in (0, 49, 99)]): value = adapter.preprocess(t1.image({'file': item['file2']}), adapter.INPUT_SPEC) eager, ort_out = torch_run(value), ort_run(value) assert all(p.shape == q.shape for p, q in zip(eager, ort_out)) checks[idx]['dense_or_sparse_output_max_abs'] = [float(np.max(abs(p-q))) for p, q in zip(eager, ort_out)] checks[idx]['adapter_vs_ort'] = compare_features(common.decode(a.model, eager), common.decode(a.model, ort_out)) static = onnx.load(str(onnx_path)) ops = sorted({node.op_type for node in static.graph.node}) output_shapes = [[dim.dim_value for dim in output.type.tensor_type.shape.dim] for output in static.graph.output] assert all(all(n > 0 for n in shape) for shape in output_shapes), output_shapes write(a.out / 'probe.json', {'status': 'official_adapter_ort_verified', 'model': a.model, 'checkpoint': adapter.CHECKPOINT, 'checkpoint_sha256': sha(weights), 'input_spec': adapter.INPUT_SPEC, 'output_shapes': output_shapes, 'onnx_sha256': sha(onnx_path), 'onnx_ops': ops, 'checks': checks, 'calibration': str(calib), 't1': str(a.t1), 't1_sha256': sha(a.t1), 'feature_protocol': feat.PROTOCOL, 'feature_protocol_sha256': feat.PROTOCOL_SHA, 'torch_version': torch.__version__, 'ort_version': ort.__version__, 'cv2_version': cv2.__version__, 'board_tested': False, 'quant_compile_tested': False, 'admitted': False}) print(json.dumps({'model': a.model, 'stage': 'onnx_probe_passed', 'shapes': output_shapes}), flush=True) if a.stage == 'probe': return for label, runner in [('fp32', torch_run), ('ort', ort_run)]: started = time.time() out = {'model': a.model, 'task': 'feat', 't1_manifest_sha': t1.manifest['sha256'], 'evaluator_version': 'feature-static-1'} feat.run_fp32(runner, t1, adapter.INPUT_SPEC, out) out['wall_seconds'] = time.time() - started write(a.out / f'{label}-metrics.json', out) print(json.dumps({'model': a.model, 'stage': label + '_100pair_done', 'metrics': out['metrics'], 'seconds': out['wall_seconds']}), flush=True) baseline = json.loads((a.out / 'fp32-metrics.json').read_text()); measured = json.loads((a.out / 'ort-metrics.json').read_text()) b = baseline['metrics'][feat.METRIC]; m = measured['metrics'][feat.METRIC] loss_pct = 100 * (b - m) / b if b > 0 else None write(a.out / 'receipt.json', {'model': a.model, 'status': 'fp32_ort_100pair_complete', 'n': 100, 'fp32': baseline['metrics'], 'ort': measured['metrics'], 'ort_relative_auc_loss_pct': loss_pct, 'within_existing_5pct_tolerance': loss_pct is not None and loss_pct <= 5, 'protocol_sha256': feat.PROTOCOL_SHA, 'calibration_count': 128, 'calibration': str(calib), 't1_sha256': sha(a.t1), 'onnx_sha256': sha(onnx_path), 'probe': str(a.out/'probe.json'), 'admitted': False, 'quant_compile_tested': False, 'board_tested': False, 'agent_trajectory_collected': False}) t1.tar.close() if loss_pct is None or loss_pct > 5: raise ValueError(f'ORT geometric AUC exceeds 5% degradation: {loss_pct}') if __name__ == '__main__': main()