data-gen-bpu/scripts/test_adapter_recipes.py

136 lines
7.4 KiB
Python

"""New input recipes must not alter frozen legacy inputs or evaluation sets."""
from contextlib import ExitStack
import copy
import hashlib
import io
import json
from pathlib import Path
import tarfile
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import numpy as np
from PIL import Image
from harness import adapters, db, paths, pipeline, registry
from harness.eval.common import preprocess_cls
from harness.eval import cls, det
from harness.eval.run import g2_inputs
class Recipes(unittest.TestCase):
def test_diagnostic_sampling_uses_only_real_eligible_crops(self):
t1 = SimpleNamespace(items=[{"id": i} for i in range(9)])
transform = lambda t1, item, spec: (np.full((1, 3, 2, 2), item["id"], dtype=np.float32)
if item["id"] % 2 else None)
adapter = SimpleNamespace(g2_input=transform)
with patch.object(adapters, "get", return_value=adapter):
actual = g2_inputs("pose", t1, "1x3x2x2", 3, model="crop.model")
chosen = sorted(np.random.default_rng(20260914).permutation([1, 3, 5, 7])[:3])
self.assertEqual([int(x[0, 0, 0, 0]) for x in actual.values()], chosen)
with self.assertRaisesRegex(ValueError, "only 4 valid"):
g2_inputs("pose", t1, "1x3x2x2", 5, model="crop.model")
def test_checkpoint_resolution_keeps_csv_and_old_models_identical(self):
raw = next(row for row in registry.models() if row["model_id"] == "efficientvit.b0")
before = dict(raw)
resolved = registry.model_row("efficientvit.b0")
self.assertIn("/resolve/", resolved["weights"])
self.assertEqual(raw, before)
self.assertEqual(resolved["status"], "needs_adapter")
legacy = registry.model_row("resnet18")
self.assertNotIn("preprocess", legacy)
self.assertEqual(adapters.calibration_dir(legacy), paths.DATA / "calib/cls/1x3x224x224")
self.assertNotEqual(adapters.calibration_dir(raw), adapters.calibration_dir(legacy))
self.assertEqual(adapters.calibration_dir(raw), adapters.calibration_dir(registry.model_row("efficientvit.b3")))
def test_same_recipe_reaches_fp32_quant_and_g2(self):
img = Image.new("RGB", (17, 31), (10, 20, 30))
tensor = np.arange(12, dtype=np.float32).reshape(1, 3, 2, 2)
adapter = SimpleNamespace(preprocess=lambda image, spec: tensor.copy())
t1 = SimpleNamespace(items=[{"file": "sample-0.jpg", "label": 1}], image=lambda item: img,
g2_subset=lambda n: [{"label": 1}])
seen = []
def runner(x):
seen.append(x)
return [np.array([[0., 3., 1., 2., -1.]])]
baseline, quant = {"model": "new.model"}, {"model": "new.model"}
legacy = preprocess_cls(img, size=2, resize=3)
with patch.object(adapters, "get", return_value=adapter):
cls.run_fp32(runner, t1, "1x3x2x2", baseline)
cls.run_quant(runner, t1, "1x3x2x2", baseline, quant)
diagnostic = g2_inputs("cls", t1, "1x3x2x2", 1, model="new.model")
self.assertEqual(quant["_sample_predictions"], [{"sample_id": "sample-0.jpg", "top5": [1, 3, 2, 0, 4]}])
for actual in [*seen, diagnostic["s000"]]:
np.testing.assert_array_equal(actual, tensor)
np.testing.assert_array_equal(legacy, preprocess_cls(img, size=2, resize=3, model="resnet18"))
def test_detector_adapter_owns_geometry_and_keeps_coco_category_mapping(self):
image = Image.new("RGB", (20, 40))
tensor = np.ones((1, 3, 2, 2), dtype=np.float32) * 3
original_box = np.array([[2., 4., 6., 12.]])
def decode(outputs, original_size):
self.assertEqual(original_size, (20, 40))
return original_box, np.array([.9]), np.array([0])
adapter = SimpleNamespace(preprocess=lambda image, spec: tensor.copy(), decode=decode)
t1 = SimpleNamespace(items=[{"id": 17}], image=lambda item: image,
manifest={"cat_map": {"0": 13}}, g2_subset=lambda n: [{"id": 17}])
seen = []
def runner(x):
seen.append(x)
return [np.zeros((1, 4, 1))]
with patch.object(adapters, "get", return_value=adapter):
prediction = det.predict(runner, t1, "1x3x2x2", model="custom.det")
diagnostic = g2_inputs("det", t1, "1x3x2x2", 1, model="custom.det")
self.assertEqual(prediction[0]["bbox"], [2., 4., 4., 8.])
self.assertEqual(prediction[0]["category_id"], 13)
np.testing.assert_array_equal(seen[0], diagnostic["s000"])
def test_new_batch_freezes_eval_set_and_refuses_rewrite(self):
with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack:
matrix = copy.deepcopy(registry.matrix())
matrix['modes']['e2e']['k'] = 4
stack.enter_context(patch.object(registry, 'matrix', return_value=matrix))
root = Path(tmp)
for name in ("STATE", "RUNS", "DATA"):
stack.enter_context(patch.object(paths, name, root / name.lower()))
database = root / "jobs.sqlite"
connect = db.connect
stack.enter_context(patch.object(db, "connect", side_effect=lambda: connect(database)))
stack.enter_context(patch.object(pipeline, "build_manifest", return_value={}))
stack.enter_context(patch("harness.coverage.model_blockers", return_value=[]))
rid = pipeline.new_run("adapter recipe test")
package = root / "subset.tar"
items = [{"id": "1", "file": "one.jpg", "label": 1}]
content = hashlib.sha256(json.dumps(items).encode()).hexdigest()
inner = {"dataset": "imagenet1k", "n": 1, "sha256": content, "items": items}
with tarfile.open(package, "w") as archive:
for name, payload in [("manifest.json", json.dumps(inner).encode()), ("one.jpg", b"image")]:
info = tarfile.TarInfo(name); info.size = len(payload)
archive.addfile(info, io.BytesIO(payload))
manifest = root / "manifest.json"
manifest.write_text(json.dumps({"dataset": "imagenet1k", "n": 1, "seed": 1,
"tar": package.name, "sha256": hashlib.sha256(package.read_bytes()).hexdigest(),
"content_sha256": content}))
args = dict(model_ids=["efficientvit.b0"], eval_manifests=[str(manifest)])
result = pipeline.expand(rid, ["oracle", "e2e"], **args)
self.assertEqual(result["job_ids"], 1)
job_path = next((paths.RUNS / rid).glob("*/job.json"))
job = json.loads(job_path.read_text())
self.assertEqual(job["data"]["t1"]["n"], 1)
self.assertEqual(job["rollout_plans"], [{"mode": "e2e", "policy": "teacher", "k": 4, "runner": "pi"}])
self.assertEqual(job["model_spec"]["status"], "needs_adapter")
self.assertEqual(pipeline.expand(rid, ["oracle", "e2e"], **args)["jobs_new"], 0)
original = job_path.read_bytes()
data = json.loads(manifest.read_text()); data["seed"] = 2
manifest.write_text(json.dumps(data))
with self.assertRaisesRegex(ValueError, r"frozen job content conflict \(data\)"):
pipeline.expand(rid, ["oracle", "e2e"], **args)
self.assertEqual(job_path.read_bytes(), original)
if __name__ == "__main__":
unittest.main()