162 lines
9.7 KiB
Python
162 lines
9.7 KiB
Python
"""Real official/adapter/ORT + HPatches geometry probe, outside all existing runs."""
|
|
from __future__ import annotations
|
|
import argparse
|
|
import ast
|
|
import hashlib
|
|
import importlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
|
|
def sha(path):
|
|
h = hashlib.sha256()
|
|
with open(path, 'rb') as f:
|
|
for b in iter(lambda: f.read(1 << 20), b''): h.update(b)
|
|
return h.hexdigest()
|
|
|
|
|
|
def write(path, obj):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temp = path.with_suffix(path.suffix + '.tmp')
|
|
temp.write_text(json.dumps(obj, indent=2, allow_nan=False) + '\n'); temp.replace(path)
|
|
|
|
|
|
def prepare_calibration(adapter, selection, destination):
|
|
import numpy as np
|
|
from PIL import Image
|
|
recipe_sha = hashlib.sha256(json.dumps(adapter.PREPROCESS, sort_keys=True).encode()).hexdigest()[:16]
|
|
out = destination / adapter.INPUT_SPEC / recipe_sha
|
|
s = json.loads(selection.read_text()); source = Path(s['source_pool'])
|
|
if (out / 'manifest.json').exists():
|
|
man = json.loads((out / 'manifest.json').read_text())
|
|
assert man['count'] == 128 and man['preprocess'] == adapter.PREPROCESS and man['t1_sha256'] == s['t1_sha256']
|
|
for item in man['items']: assert sha(out / 'npy_f32' / item['npy']) == item['npy_sha256']
|
|
metadata = {'task': 'feat', 'dataset': 'hpatches', 'source': 'val-holdout',
|
|
'calib_source': 'val-holdout', 'files': [it['file'] for it in man['items']]}
|
|
if any(man.get(k) != v for k, v in metadata.items()):
|
|
man.update(metadata); write(out / 'manifest.json', man)
|
|
return out
|
|
if out.exists(): raise FileExistsError(f'refuse partial calibration overwrite: {out}')
|
|
stage = out.with_name(out.name + '.partial'); (stage / 'npy_f32').mkdir(parents=True, exist_ok=False)
|
|
items = []; digest = hashlib.sha256()
|
|
for i, record in enumerate(s['items']):
|
|
path = source / record['file']; assert sha(path) == record['sha256']
|
|
with Image.open(path) as im: x = adapter.preprocess(im, adapter.INPUT_SPEC)
|
|
p = stage / 'npy_f32' / f'{i:05d}.npy'; np.save(p, x); digest.update(p.read_bytes())
|
|
items.append({**record, 'npy': p.name, 'npy_sha256': sha(p)})
|
|
write(stage / 'manifest.json', {**{k: v for k, v in s.items() if k != 'items'}, 'items': items,
|
|
'preprocess': adapter.PREPROCESS, 'count': 128, 'sha256': digest.hexdigest(), 'input_spec': adapter.INPUT_SPEC,
|
|
'npy_dir': 'npy_f32', 'layout': 'NCHW', 'dtype': 'float32', 'task': 'feat', 'dataset': 'hpatches',
|
|
'source': 'val-holdout', 'calib_source': 'val-holdout', 'files': [it['file'] for it in items]})
|
|
stage.rename(out); return out
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('--model', choices=['superpoint', 'disk', 'aliked', 'xfeat', 'd2.net', 'r2d2'], required=True)
|
|
ap.add_argument('--t1', type=Path, required=True); ap.add_argument('--out', type=Path, required=True)
|
|
ap.add_argument('--calibration-selection', type=Path, required=True)
|
|
ap.add_argument('--threads', type=int, default=2)
|
|
ap.add_argument('--stage', choices=['all', 'probe', 'evaluate'], default='all')
|
|
a = ap.parse_args(); a.out.mkdir(parents=True, exist_ok=True)
|
|
os.environ['CUDA_VISIBLE_DEVICES'] = ''
|
|
import cv2
|
|
import numpy as np
|
|
import torch
|
|
import onnx
|
|
import onnxruntime as ort
|
|
from harness.adapters import feature_common as common
|
|
from harness.eval.common import T1
|
|
from harness.eval import feat
|
|
torch.set_num_threads(a.threads); torch.set_num_interop_threads(1); cv2.setNumThreads(1)
|
|
torch.manual_seed(20260914)
|
|
from harness.adapters import get
|
|
adapter = get(a.model)
|
|
weights = common.ROOT / adapter.CHECKPOINT['official_relative_path']
|
|
calib = prepare_calibration(adapter, a.calibration_selection, a.out.parent / 'calibration')
|
|
print(json.dumps({'model': a.model, 'stage': 'calibration_ready', 'path': str(calib)}), flush=True)
|
|
t1 = T1.open(a.t1); assert len(t1.items) == 100
|
|
graph = adapter.build(weights, adapter.INPUT_SPEC).eval()
|
|
def torch_run(x):
|
|
with torch.no_grad(): output = graph(torch.from_numpy(x))
|
|
return [t.detach().numpy() for t in (output if isinstance(output, tuple) else (output,))]
|
|
def compare_features(left, right):
|
|
# Coordinates uniquely identify sparse detections; row order is not a semantic contract.
|
|
x, y = left['keypoints'], right['keypoints']
|
|
assert len(x) == len(y), (len(x), len(y))
|
|
aidx = np.lexsort((x[:, 0], x[:, 1])); bidx = np.lexsort((y[:, 0], y[:, 1]))
|
|
result = {}
|
|
for key in ('keypoints', 'scores', 'descriptors'):
|
|
p, q = left[key][aidx], right[key][bidx]
|
|
result[key + '_max_abs'] = float(np.max(abs(p - q), initial=0))
|
|
np.testing.assert_allclose(p, q, atol=2e-3, rtol=2e-3)
|
|
return result
|
|
x = feat.inputs(t1, t1.items[0], adapter.INPUT_SPEC, a.model)
|
|
onnx_path = a.out / 'model.onnx'
|
|
checks = []
|
|
if a.stage != 'evaluate':
|
|
official = adapter.official_model(weights) if hasattr(adapter, 'official_model') else common.official_model(a.model, weights)
|
|
for item in [t1.items[i] for i in (0, 49, 99)]:
|
|
value = adapter.preprocess(t1.image({'file': item['file2']}), adapter.INPUT_SPEC)
|
|
with torch.no_grad():
|
|
k, s, d = (adapter.official_features(official, torch.from_numpy(value)) if hasattr(adapter, 'official_features')
|
|
else common.official_features(a.model, official, torch.from_numpy(value)))
|
|
expected = {'keypoints': k.numpy(), 'scores': s.numpy(), 'descriptors': d.numpy()}
|
|
actual = common.decode(a.model, torch_run(value))
|
|
checks.append({'id': item['id'], 'official_vs_adapter': compare_features(expected, actual)})
|
|
del official
|
|
print(json.dumps({'model': a.model, 'stage': 'official_numeric_passed', 'checks': checks}), flush=True)
|
|
with torch.no_grad():
|
|
torch.onnx.export(graph, torch.from_numpy(x), str(onnx_path), opset_version=adapter.OPSET,
|
|
input_names=['input'], dynamic_axes=None, do_constant_folding=True, dynamo=False)
|
|
onnx.checker.check_model(str(onnx_path))
|
|
so = ort.SessionOptions(); so.intra_op_num_threads = a.threads; so.inter_op_num_threads = 1
|
|
session = ort.InferenceSession(str(onnx_path), so, providers=['CPUExecutionProvider'])
|
|
def ort_run(value): return session.run(None, {'input': value})
|
|
if a.stage != 'evaluate':
|
|
for idx, item in enumerate([t1.items[i] for i in (0, 49, 99)]):
|
|
value = adapter.preprocess(t1.image({'file': item['file2']}), adapter.INPUT_SPEC)
|
|
eager, ort_out = torch_run(value), ort_run(value)
|
|
assert all(p.shape == q.shape for p, q in zip(eager, ort_out))
|
|
checks[idx]['dense_or_sparse_output_max_abs'] = [float(np.max(abs(p-q))) for p, q in zip(eager, ort_out)]
|
|
checks[idx]['adapter_vs_ort'] = compare_features(common.decode(a.model, eager), common.decode(a.model, ort_out))
|
|
static = onnx.load(str(onnx_path))
|
|
ops = sorted({node.op_type for node in static.graph.node})
|
|
output_shapes = [[dim.dim_value for dim in output.type.tensor_type.shape.dim] for output in static.graph.output]
|
|
assert all(all(n > 0 for n in shape) for shape in output_shapes), output_shapes
|
|
write(a.out / 'probe.json', {'status': 'official_adapter_ort_verified', 'model': a.model,
|
|
'checkpoint': adapter.CHECKPOINT, 'checkpoint_sha256': sha(weights), 'input_spec': adapter.INPUT_SPEC,
|
|
'output_shapes': output_shapes, 'onnx_sha256': sha(onnx_path), 'onnx_ops': ops, 'checks': checks,
|
|
'calibration': str(calib), 't1': str(a.t1), 't1_sha256': sha(a.t1), 'feature_protocol': feat.PROTOCOL,
|
|
'feature_protocol_sha256': feat.PROTOCOL_SHA, 'torch_version': torch.__version__,
|
|
'ort_version': ort.__version__, 'cv2_version': cv2.__version__, 'board_tested': False,
|
|
'quant_compile_tested': False, 'admitted': False})
|
|
print(json.dumps({'model': a.model, 'stage': 'onnx_probe_passed', 'shapes': output_shapes}), flush=True)
|
|
if a.stage == 'probe': return
|
|
for label, runner in [('fp32', torch_run), ('ort', ort_run)]:
|
|
started = time.time()
|
|
out = {'model': a.model, 'task': 'feat', 't1_manifest_sha': t1.manifest['sha256'], 'evaluator_version': 'feature-static-1'}
|
|
feat.run_fp32(runner, t1, adapter.INPUT_SPEC, out)
|
|
out['wall_seconds'] = time.time() - started
|
|
write(a.out / f'{label}-metrics.json', out)
|
|
print(json.dumps({'model': a.model, 'stage': label + '_100pair_done', 'metrics': out['metrics'], 'seconds': out['wall_seconds']}), flush=True)
|
|
baseline = json.loads((a.out / 'fp32-metrics.json').read_text()); measured = json.loads((a.out / 'ort-metrics.json').read_text())
|
|
b = baseline['metrics'][feat.METRIC]; m = measured['metrics'][feat.METRIC]
|
|
loss_pct = 100 * (b - m) / b if b > 0 else None
|
|
write(a.out / 'receipt.json', {'model': a.model, 'status': 'fp32_ort_100pair_complete', 'n': 100,
|
|
'fp32': baseline['metrics'], 'ort': measured['metrics'], 'ort_relative_auc_loss_pct': loss_pct,
|
|
'within_existing_5pct_tolerance': loss_pct is not None and loss_pct <= 5,
|
|
'protocol_sha256': feat.PROTOCOL_SHA, 'calibration_count': 128, 'calibration': str(calib),
|
|
't1_sha256': sha(a.t1), 'onnx_sha256': sha(onnx_path), 'probe': str(a.out/'probe.json'),
|
|
'admitted': False, 'quant_compile_tested': False, 'board_tested': False, 'agent_trajectory_collected': False})
|
|
t1.tar.close()
|
|
if loss_pct is None or loss_pct > 5: raise ValueError(f'ORT geometric AUC exceeds 5% degradation: {loss_pct}')
|
|
|
|
|
|
if __name__ == '__main__': main()
|