data-gen-bpu/scripts/test_recipe_calibration.py

183 lines
11 KiB
Python

"""Platform training images feed only isolated preprocessing, never task labels."""
import copy
import hashlib
import io
import json
from pathlib import Path
import tarfile
import tempfile
import unittest
from unittest.mock import patch
from types import SimpleNamespace
try:
import numpy as np
from PIL import Image
except ImportError:
np = Image = None
from harness import paths, registry
from harness.stages.base import StageContext, recipe_identity, recipe_output
from harness.stages.s04_calibrate import Calibrate
from scripts.make_calib import make_recipe_calibration
@unittest.skipUnless(np is not None and Image is not None, "calibration model tests require NumPy and Pillow")
class RecipeCalibration(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name).resolve()
self.train = self.root / "training"
self.train.mkdir()
for index, value in enumerate((30, 80)):
Image.fromarray(np.full((4, 4, 3), value, dtype=np.uint8)).save(self.train / f"{index}.png")
buffer = io.BytesIO()
Image.fromarray(np.full((4, 4, 3), 255, dtype=np.uint8)).save(buffer, format="PNG")
tarpath = self.root / "t1.tar"
with tarfile.open(tarpath, "w") as archive:
manifest = {"dataset": "cityscapes", "items": [{"file": "rgb.png", "label": "forbidden-label.png"}]}
for name, data in (("manifest.json", json.dumps(manifest).encode()), ("rgb.png", buffer.getvalue())):
member = tarfile.TarInfo(name)
member.size = len(data)
archive.addfile(member, io.BytesIO(data))
self.source = {"dataset": "cityscapes", "local_root": str(self.root), "pool": str(self.root / "pool"), "seed": 4,
"dataset_config": {"sources": [], "calib": {"train_dir": "training"}}}
self.job = {"job_id": "fixture", "run_id": "run", "task": "seg", "backend": "ptq", "march": "nash-m", "qcfg": "int8",
"model_spec": {"input_spec": "1x3x4x4", "preprocess": {"resize": 4}},
"recipe": {"entrypoint": "candidate.py", "input_spec": "1x3x4x4", "preprocess": {"resize": 4}, "submission_sha256": "a" * 64,
"reference_sha256": "b" * 64},
"data": {"dataset": "cityscapes", "calib": {"path": str(self.root / "shared"), "n": 2, "recipe_source": self.source},
"t1": {"path": str(tarpath), "manifest_sha": hashlib.sha256(tarpath.read_bytes()).hexdigest()}}}
self.runtime = {"python": "configured-runtime"}
self.calls = []
def isolated_call(self, python, recipe, operation, inputs, output, **kw):
self.calls.append({"operation": operation, "inputs": inputs, "recipe": recipe, "output": output})
self.assertEqual(operation, "preprocess")
self.assertEqual(set(inputs), {"s0000", "s0001"})
self.assertTrue(all(value.dtype == np.uint8 and value.shape == (4, 4, 3) for value in inputs.values()))
self.assertTrue(all(not np.any(value == 255) for value in inputs.values()))
return {key: value.transpose(2, 0, 1)[None].astype(np.float32) for key, value in inputs.items()}
def prepare(self, name="attempt-1"):
with patch("harness.recipe_runtime.call", side_effect=self.isolated_call):
return make_recipe_calibration(self.job, self.root / name / "calibration", self.root / name / "calls",
Path("/configured/python"), self.runtime)
def test_actual_training_pixels_generate_task_local_arrays_and_identity(self):
before = {path.name: path.read_bytes() for path in self.train.iterdir()}
manifest = self.prepare()
self.assertEqual(manifest["source"], "train-dir")
self.assertEqual(manifest["count"], 2)
self.assertEqual(len(manifest["raw_rgb_sha256"]), 2)
self.assertEqual(manifest["preprocessing_identity"]["submission_sha256"], "a" * 64)
self.assertEqual(manifest["preprocessing_identity"]["runtime"], self.runtime)
arrays = sorted((self.root / "attempt-1/calibration/npy_f32").glob("*.npy"))
self.assertEqual(len(arrays), 2)
for path in arrays:
value = np.load(path, allow_pickle=False)
self.assertEqual(value.shape, (1, 3, 4, 4))
self.assertEqual(value.dtype, np.float32)
self.assertFalse((self.root / "shared").exists())
self.assertEqual(before, {path.name: path.read_bytes() for path in self.train.iterdir()})
self.assertNotIn("data", self.calls[0]["recipe"])
self.assertNotIn("t1", self.calls[0]["inputs"])
# The T1 archive deliberately has no label member: only its images were read.
def test_changed_code_identity_cannot_reuse_or_overwrite_previous_calibration(self):
first = self.prepare()
saved = (self.root / "attempt-1/calibration/manifest.json").read_bytes()
self.job["recipe"]["submission_sha256"] = "c" * 64
with self.assertRaisesRegex(ValueError, "already exists"):
self.prepare()
second = self.prepare("attempt-2")
self.assertNotEqual(first["preprocessing_identity"], second["preprocessing_identity"])
self.assertEqual(first["raw_rgb_sha256"], second["raw_rgb_sha256"])
self.assertEqual(saved, (self.root / "attempt-1/calibration/manifest.json").read_bytes())
def test_rgb_overlap_with_t1_is_rejected_before_model_code_runs(self):
Image.fromarray(np.full((4, 4, 3), 255, dtype=np.uint8)).save(self.train / "0.png", compress_level=0)
with self.assertRaisesRegex(ValueError, "overlap T1"):
self.prepare()
self.assertEqual(self.calls, [])
self.assertFalse((self.root / "attempt-1/calibration").exists())
def test_training_source_missing_fails_with_config_path_without_val_fallback(self):
self.source["dataset_config"]["calib"]["train_dir"] = "missing"
pool = self.root / "pool/val"
pool.mkdir(parents=True)
Image.fromarray(np.full((4, 4, 3), 1, dtype=np.uint8)).save(pool / "0.png")
with self.assertRaisesRegex(ValueError, r"registry/datasets.yaml datasets.cityscapes.calib.train_dir"):
self.prepare()
self.assertEqual(self.calls, [])
def test_registry_freezes_sources_only_for_recipe_jobs(self):
config = {"local_root": str(self.root), "datasets": {"cityscapes": {"version": "v1", "t1": {"seed": 1},
"calib": {"n": 2, "seed": 4, "train_dir": "training"},
"sources": [{"kind": "local_zip", "path": "images.zip", "extra": [{"path": "labels.zip"}]}]}}}
spec = registry.JobSpec("fixture", "seg", "custom", "nash-m", "ptq", "int8", "A", "e2e", overrides={"model_recipe": {}})
row = {"dataset": "cityscapes", "task": "seg", "source": "recipe", "input_spec": "1x3x4x4", "model_id": "custom"}
with patch.object(registry, "datasets", return_value=config), patch.object(paths, "DATA", self.root / "data"):
data = registry.data_snapshot(spec, row)
frozen = copy.deepcopy(data["calib"]["recipe_source"])
config["datasets"]["cityscapes"]["calib"]["train_dir"] = "changed"
self.assertEqual(data["calib"]["recipe_source"], frozen)
self.assertNotIn("extra", frozen["dataset_config"]["sources"][0])
self.assertEqual(frozen["dataset_config"]["calib"], {"train_dir": "training"})
spec.overrides = {}
self.assertNotIn("recipe_source", registry.data_snapshot(spec, row)["calib"])
def test_calibrate_stage_generates_and_pins_current_attempt_config(self):
ctx = StageContext("run", self.job, self.root, self.root / "stages", self.root / "artifacts", rollout=0,
extra={"runtime_identity": self.runtime, "recipe_stage": "04-calibrate", "compile_attempt": 3})
export = ctx.stage_root / "03-export/a00"
export.mkdir(parents=True)
graph = export / "model.onnx"
graph.write_bytes(b"fixture graph identity only")
meta = {"stage": "03-export", "attempt": 0, "state": "OK", "recipe": recipe_identity(ctx),
"recipe_outputs": {"onnx": recipe_output(graph)},
"recipe_input": {"input_spec": "1x3x4x4", "preprocess": {"resize": 4}}}
(export / "meta.json").write_text(json.dumps(meta))
sd = ctx.stage_root / "04-calibrate/a00"
sd.mkdir(parents=True)
def run(command, **kw):
self.assertEqual(command[0], "/configured/python")
self.assertTrue(command[1].endswith("scripts/make_calib.py"))
envelope = json.loads(Path(command[command.index("--recipe-job") + 1]).read_text())
make_recipe_calibration(envelope["job"], sd / "calibration", sd / "calibration_calls",
Path(command[0]), envelope["runtime_identity"])
kw["stdout"].write_text("")
kw["stderr"].write_text("")
return SimpleNamespace(state="OK")
with patch("harness.recipe_runtime.call", side_effect=self.isolated_call), \
patch.object(paths, "model_python", return_value=Path("/configured/python")), \
patch.object(paths, "config", return_value={"paths": {"work_root": str(self.root / "work")}}), \
patch.object(ctx, "watchdog_params", return_value={key: 10 for key in
("stall_s", "cpu_idle_pct", "poll_s", "hard_ceiling_s", "io_stall_s", "no_progress_ceiling_s")}), \
patch("harness.stages.s04_calibrate.watchdog.run", side_effect=run):
outcome = Calibrate().native(ctx, sd)
self.assertEqual(Calibrate.pool, "cpu")
self.assertEqual(outcome.state, "OK")
self.assertEqual(outcome.meta["calib_dir"], str(sd / "calibration"))
self.assertTrue(outcome.meta["compile_work_dir"].endswith("05-compile/a03"))
self.assertEqual(set(outcome.meta["recipe_outputs"]), {"compile_yaml", "calibration"})
self.assertFalse((self.root / "shared").exists())
receipt = {"stage": "04-calibrate", "attempt": 0, "state": "OK", "recipe": recipe_identity(ctx),
"recipe_parents": ctx.extra["recipe_parents"], **outcome.meta}
(sd / "meta.json").write_text(json.dumps(receipt))
next_dir = ctx.stage_root / "04-calibrate/a01"
next_dir.mkdir()
ctx.attempt = 1
ctx.extra["compile_attempt"] = 4
with patch("harness.stages.s04_calibrate.watchdog.run", side_effect=AssertionError("unchanged calibration regenerated")), \
patch.object(paths, "config", return_value={"paths": {"work_root": str(self.root / "work")}}):
reused = Calibrate().native(ctx, next_dir)
self.assertEqual(reused.meta["recipe_outputs"]["calibration"], outcome.meta["recipe_outputs"]["calibration"])
self.assertNotEqual(reused.meta["compile_work_dir"], outcome.meta["compile_work_dir"])
self.assertEqual(len(self.calls), 1)
if __name__ == "__main__":
unittest.main()