136 lines
7.4 KiB
Python
136 lines
7.4 KiB
Python
"""New input recipes must not alter frozen legacy inputs or evaluation sets."""
|
|
from contextlib import ExitStack
|
|
import copy
|
|
import hashlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import tarfile
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import numpy as np
|
|
from PIL import Image
|
|
|
|
from harness import adapters, db, paths, pipeline, registry
|
|
from harness.eval.common import preprocess_cls
|
|
from harness.eval import cls, det
|
|
from harness.eval.run import g2_inputs
|
|
|
|
|
|
class Recipes(unittest.TestCase):
|
|
def test_diagnostic_sampling_uses_only_real_eligible_crops(self):
|
|
t1 = SimpleNamespace(items=[{"id": i} for i in range(9)])
|
|
transform = lambda t1, item, spec: (np.full((1, 3, 2, 2), item["id"], dtype=np.float32)
|
|
if item["id"] % 2 else None)
|
|
adapter = SimpleNamespace(g2_input=transform)
|
|
with patch.object(adapters, "get", return_value=adapter):
|
|
actual = g2_inputs("pose", t1, "1x3x2x2", 3, model="crop.model")
|
|
chosen = sorted(np.random.default_rng(20260914).permutation([1, 3, 5, 7])[:3])
|
|
self.assertEqual([int(x[0, 0, 0, 0]) for x in actual.values()], chosen)
|
|
with self.assertRaisesRegex(ValueError, "only 4 valid"):
|
|
g2_inputs("pose", t1, "1x3x2x2", 5, model="crop.model")
|
|
|
|
def test_checkpoint_resolution_keeps_csv_and_old_models_identical(self):
|
|
raw = next(row for row in registry.models() if row["model_id"] == "efficientvit.b0")
|
|
before = dict(raw)
|
|
resolved = registry.model_row("efficientvit.b0")
|
|
self.assertIn("/resolve/", resolved["weights"])
|
|
self.assertEqual(raw, before)
|
|
self.assertEqual(resolved["status"], "needs_adapter")
|
|
legacy = registry.model_row("resnet18")
|
|
self.assertNotIn("preprocess", legacy)
|
|
self.assertEqual(adapters.calibration_dir(legacy), paths.DATA / "calib/cls/1x3x224x224")
|
|
self.assertNotEqual(adapters.calibration_dir(raw), adapters.calibration_dir(legacy))
|
|
self.assertEqual(adapters.calibration_dir(raw), adapters.calibration_dir(registry.model_row("efficientvit.b3")))
|
|
|
|
def test_same_recipe_reaches_fp32_quant_and_g2(self):
|
|
img = Image.new("RGB", (17, 31), (10, 20, 30))
|
|
tensor = np.arange(12, dtype=np.float32).reshape(1, 3, 2, 2)
|
|
adapter = SimpleNamespace(preprocess=lambda image, spec: tensor.copy())
|
|
t1 = SimpleNamespace(items=[{"file": "sample-0.jpg", "label": 1}], image=lambda item: img,
|
|
g2_subset=lambda n: [{"label": 1}])
|
|
seen = []
|
|
def runner(x):
|
|
seen.append(x)
|
|
return [np.array([[0., 3., 1., 2., -1.]])]
|
|
baseline, quant = {"model": "new.model"}, {"model": "new.model"}
|
|
legacy = preprocess_cls(img, size=2, resize=3)
|
|
with patch.object(adapters, "get", return_value=adapter):
|
|
cls.run_fp32(runner, t1, "1x3x2x2", baseline)
|
|
cls.run_quant(runner, t1, "1x3x2x2", baseline, quant)
|
|
diagnostic = g2_inputs("cls", t1, "1x3x2x2", 1, model="new.model")
|
|
self.assertEqual(quant["_sample_predictions"], [{"sample_id": "sample-0.jpg", "top5": [1, 3, 2, 0, 4]}])
|
|
for actual in [*seen, diagnostic["s000"]]:
|
|
np.testing.assert_array_equal(actual, tensor)
|
|
np.testing.assert_array_equal(legacy, preprocess_cls(img, size=2, resize=3, model="resnet18"))
|
|
|
|
def test_detector_adapter_owns_geometry_and_keeps_coco_category_mapping(self):
|
|
image = Image.new("RGB", (20, 40))
|
|
tensor = np.ones((1, 3, 2, 2), dtype=np.float32) * 3
|
|
original_box = np.array([[2., 4., 6., 12.]])
|
|
def decode(outputs, original_size):
|
|
self.assertEqual(original_size, (20, 40))
|
|
return original_box, np.array([.9]), np.array([0])
|
|
adapter = SimpleNamespace(preprocess=lambda image, spec: tensor.copy(), decode=decode)
|
|
t1 = SimpleNamespace(items=[{"id": 17}], image=lambda item: image,
|
|
manifest={"cat_map": {"0": 13}}, g2_subset=lambda n: [{"id": 17}])
|
|
seen = []
|
|
def runner(x):
|
|
seen.append(x)
|
|
return [np.zeros((1, 4, 1))]
|
|
with patch.object(adapters, "get", return_value=adapter):
|
|
prediction = det.predict(runner, t1, "1x3x2x2", model="custom.det")
|
|
diagnostic = g2_inputs("det", t1, "1x3x2x2", 1, model="custom.det")
|
|
self.assertEqual(prediction[0]["bbox"], [2., 4., 4., 8.])
|
|
self.assertEqual(prediction[0]["category_id"], 13)
|
|
np.testing.assert_array_equal(seen[0], diagnostic["s000"])
|
|
|
|
def test_new_batch_freezes_eval_set_and_refuses_rewrite(self):
|
|
with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack:
|
|
matrix = copy.deepcopy(registry.matrix())
|
|
matrix['modes']['e2e']['k'] = 4
|
|
stack.enter_context(patch.object(registry, 'matrix', return_value=matrix))
|
|
root = Path(tmp)
|
|
for name in ("STATE", "RUNS", "DATA"):
|
|
stack.enter_context(patch.object(paths, name, root / name.lower()))
|
|
database = root / "jobs.sqlite"
|
|
connect = db.connect
|
|
stack.enter_context(patch.object(db, "connect", side_effect=lambda: connect(database)))
|
|
stack.enter_context(patch.object(pipeline, "build_manifest", return_value={}))
|
|
stack.enter_context(patch("harness.coverage.model_blockers", return_value=[]))
|
|
rid = pipeline.new_run("adapter recipe test")
|
|
package = root / "subset.tar"
|
|
items = [{"id": "1", "file": "one.jpg", "label": 1}]
|
|
content = hashlib.sha256(json.dumps(items).encode()).hexdigest()
|
|
inner = {"dataset": "imagenet1k", "n": 1, "sha256": content, "items": items}
|
|
with tarfile.open(package, "w") as archive:
|
|
for name, payload in [("manifest.json", json.dumps(inner).encode()), ("one.jpg", b"image")]:
|
|
info = tarfile.TarInfo(name); info.size = len(payload)
|
|
archive.addfile(info, io.BytesIO(payload))
|
|
manifest = root / "manifest.json"
|
|
manifest.write_text(json.dumps({"dataset": "imagenet1k", "n": 1, "seed": 1,
|
|
"tar": package.name, "sha256": hashlib.sha256(package.read_bytes()).hexdigest(),
|
|
"content_sha256": content}))
|
|
args = dict(model_ids=["efficientvit.b0"], eval_manifests=[str(manifest)])
|
|
result = pipeline.expand(rid, ["oracle", "e2e"], **args)
|
|
self.assertEqual(result["job_ids"], 1)
|
|
job_path = next((paths.RUNS / rid).glob("*/job.json"))
|
|
job = json.loads(job_path.read_text())
|
|
self.assertEqual(job["data"]["t1"]["n"], 1)
|
|
self.assertEqual(job["rollout_plans"], [{"mode": "e2e", "policy": "teacher", "k": 4, "runner": "pi"}])
|
|
self.assertEqual(job["model_spec"]["status"], "needs_adapter")
|
|
self.assertEqual(pipeline.expand(rid, ["oracle", "e2e"], **args)["jobs_new"], 0)
|
|
original = job_path.read_bytes()
|
|
data = json.loads(manifest.read_text()); data["seed"] = 2
|
|
manifest.write_text(json.dumps(data))
|
|
with self.assertRaisesRegex(ValueError, r"frozen job content conflict \(data\)"):
|
|
pipeline.expand(rid, ["oracle", "e2e"], **args)
|
|
self.assertEqual(job_path.read_bytes(), original)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|