183 lines
11 KiB
Python
183 lines
11 KiB
Python
"""Platform training images feed only isolated preprocessing, never task labels."""
|
|
import copy
|
|
import hashlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import tarfile
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from types import SimpleNamespace
|
|
|
|
try:
|
|
import numpy as np
|
|
from PIL import Image
|
|
except ImportError:
|
|
np = Image = None
|
|
|
|
from harness import paths, registry
|
|
from harness.stages.base import StageContext, recipe_identity, recipe_output
|
|
from harness.stages.s04_calibrate import Calibrate
|
|
from scripts.make_calib import make_recipe_calibration
|
|
|
|
|
|
@unittest.skipUnless(np is not None and Image is not None, "calibration model tests require NumPy and Pillow")
|
|
class RecipeCalibration(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.root = Path(self.tmp.name).resolve()
|
|
self.train = self.root / "training"
|
|
self.train.mkdir()
|
|
for index, value in enumerate((30, 80)):
|
|
Image.fromarray(np.full((4, 4, 3), value, dtype=np.uint8)).save(self.train / f"{index}.png")
|
|
buffer = io.BytesIO()
|
|
Image.fromarray(np.full((4, 4, 3), 255, dtype=np.uint8)).save(buffer, format="PNG")
|
|
tarpath = self.root / "t1.tar"
|
|
with tarfile.open(tarpath, "w") as archive:
|
|
manifest = {"dataset": "cityscapes", "items": [{"file": "rgb.png", "label": "forbidden-label.png"}]}
|
|
for name, data in (("manifest.json", json.dumps(manifest).encode()), ("rgb.png", buffer.getvalue())):
|
|
member = tarfile.TarInfo(name)
|
|
member.size = len(data)
|
|
archive.addfile(member, io.BytesIO(data))
|
|
self.source = {"dataset": "cityscapes", "local_root": str(self.root), "pool": str(self.root / "pool"), "seed": 4,
|
|
"dataset_config": {"sources": [], "calib": {"train_dir": "training"}}}
|
|
self.job = {"job_id": "fixture", "run_id": "run", "task": "seg", "backend": "ptq", "march": "nash-m", "qcfg": "int8",
|
|
"model_spec": {"input_spec": "1x3x4x4", "preprocess": {"resize": 4}},
|
|
"recipe": {"entrypoint": "candidate.py", "input_spec": "1x3x4x4", "preprocess": {"resize": 4}, "submission_sha256": "a" * 64,
|
|
"reference_sha256": "b" * 64},
|
|
"data": {"dataset": "cityscapes", "calib": {"path": str(self.root / "shared"), "n": 2, "recipe_source": self.source},
|
|
"t1": {"path": str(tarpath), "manifest_sha": hashlib.sha256(tarpath.read_bytes()).hexdigest()}}}
|
|
self.runtime = {"python": "configured-runtime"}
|
|
self.calls = []
|
|
|
|
def isolated_call(self, python, recipe, operation, inputs, output, **kw):
|
|
self.calls.append({"operation": operation, "inputs": inputs, "recipe": recipe, "output": output})
|
|
self.assertEqual(operation, "preprocess")
|
|
self.assertEqual(set(inputs), {"s0000", "s0001"})
|
|
self.assertTrue(all(value.dtype == np.uint8 and value.shape == (4, 4, 3) for value in inputs.values()))
|
|
self.assertTrue(all(not np.any(value == 255) for value in inputs.values()))
|
|
return {key: value.transpose(2, 0, 1)[None].astype(np.float32) for key, value in inputs.items()}
|
|
|
|
def prepare(self, name="attempt-1"):
|
|
with patch("harness.recipe_runtime.call", side_effect=self.isolated_call):
|
|
return make_recipe_calibration(self.job, self.root / name / "calibration", self.root / name / "calls",
|
|
Path("/configured/python"), self.runtime)
|
|
|
|
def test_actual_training_pixels_generate_task_local_arrays_and_identity(self):
|
|
before = {path.name: path.read_bytes() for path in self.train.iterdir()}
|
|
manifest = self.prepare()
|
|
self.assertEqual(manifest["source"], "train-dir")
|
|
self.assertEqual(manifest["count"], 2)
|
|
self.assertEqual(len(manifest["raw_rgb_sha256"]), 2)
|
|
self.assertEqual(manifest["preprocessing_identity"]["submission_sha256"], "a" * 64)
|
|
self.assertEqual(manifest["preprocessing_identity"]["runtime"], self.runtime)
|
|
arrays = sorted((self.root / "attempt-1/calibration/npy_f32").glob("*.npy"))
|
|
self.assertEqual(len(arrays), 2)
|
|
for path in arrays:
|
|
value = np.load(path, allow_pickle=False)
|
|
self.assertEqual(value.shape, (1, 3, 4, 4))
|
|
self.assertEqual(value.dtype, np.float32)
|
|
self.assertFalse((self.root / "shared").exists())
|
|
self.assertEqual(before, {path.name: path.read_bytes() for path in self.train.iterdir()})
|
|
self.assertNotIn("data", self.calls[0]["recipe"])
|
|
self.assertNotIn("t1", self.calls[0]["inputs"])
|
|
# The T1 archive deliberately has no label member: only its images were read.
|
|
|
|
def test_changed_code_identity_cannot_reuse_or_overwrite_previous_calibration(self):
|
|
first = self.prepare()
|
|
saved = (self.root / "attempt-1/calibration/manifest.json").read_bytes()
|
|
self.job["recipe"]["submission_sha256"] = "c" * 64
|
|
with self.assertRaisesRegex(ValueError, "already exists"):
|
|
self.prepare()
|
|
second = self.prepare("attempt-2")
|
|
self.assertNotEqual(first["preprocessing_identity"], second["preprocessing_identity"])
|
|
self.assertEqual(first["raw_rgb_sha256"], second["raw_rgb_sha256"])
|
|
self.assertEqual(saved, (self.root / "attempt-1/calibration/manifest.json").read_bytes())
|
|
|
|
def test_rgb_overlap_with_t1_is_rejected_before_model_code_runs(self):
|
|
Image.fromarray(np.full((4, 4, 3), 255, dtype=np.uint8)).save(self.train / "0.png", compress_level=0)
|
|
with self.assertRaisesRegex(ValueError, "overlap T1"):
|
|
self.prepare()
|
|
self.assertEqual(self.calls, [])
|
|
self.assertFalse((self.root / "attempt-1/calibration").exists())
|
|
|
|
def test_training_source_missing_fails_with_config_path_without_val_fallback(self):
|
|
self.source["dataset_config"]["calib"]["train_dir"] = "missing"
|
|
pool = self.root / "pool/val"
|
|
pool.mkdir(parents=True)
|
|
Image.fromarray(np.full((4, 4, 3), 1, dtype=np.uint8)).save(pool / "0.png")
|
|
with self.assertRaisesRegex(ValueError, r"registry/datasets.yaml datasets.cityscapes.calib.train_dir"):
|
|
self.prepare()
|
|
self.assertEqual(self.calls, [])
|
|
|
|
def test_registry_freezes_sources_only_for_recipe_jobs(self):
|
|
config = {"local_root": str(self.root), "datasets": {"cityscapes": {"version": "v1", "t1": {"seed": 1},
|
|
"calib": {"n": 2, "seed": 4, "train_dir": "training"},
|
|
"sources": [{"kind": "local_zip", "path": "images.zip", "extra": [{"path": "labels.zip"}]}]}}}
|
|
spec = registry.JobSpec("fixture", "seg", "custom", "nash-m", "ptq", "int8", "A", "e2e", overrides={"model_recipe": {}})
|
|
row = {"dataset": "cityscapes", "task": "seg", "source": "recipe", "input_spec": "1x3x4x4", "model_id": "custom"}
|
|
with patch.object(registry, "datasets", return_value=config), patch.object(paths, "DATA", self.root / "data"):
|
|
data = registry.data_snapshot(spec, row)
|
|
frozen = copy.deepcopy(data["calib"]["recipe_source"])
|
|
config["datasets"]["cityscapes"]["calib"]["train_dir"] = "changed"
|
|
self.assertEqual(data["calib"]["recipe_source"], frozen)
|
|
self.assertNotIn("extra", frozen["dataset_config"]["sources"][0])
|
|
self.assertEqual(frozen["dataset_config"]["calib"], {"train_dir": "training"})
|
|
spec.overrides = {}
|
|
self.assertNotIn("recipe_source", registry.data_snapshot(spec, row)["calib"])
|
|
|
|
def test_calibrate_stage_generates_and_pins_current_attempt_config(self):
|
|
ctx = StageContext("run", self.job, self.root, self.root / "stages", self.root / "artifacts", rollout=0,
|
|
extra={"runtime_identity": self.runtime, "recipe_stage": "04-calibrate", "compile_attempt": 3})
|
|
export = ctx.stage_root / "03-export/a00"
|
|
export.mkdir(parents=True)
|
|
graph = export / "model.onnx"
|
|
graph.write_bytes(b"fixture graph identity only")
|
|
meta = {"stage": "03-export", "attempt": 0, "state": "OK", "recipe": recipe_identity(ctx),
|
|
"recipe_outputs": {"onnx": recipe_output(graph)},
|
|
"recipe_input": {"input_spec": "1x3x4x4", "preprocess": {"resize": 4}}}
|
|
(export / "meta.json").write_text(json.dumps(meta))
|
|
sd = ctx.stage_root / "04-calibrate/a00"
|
|
sd.mkdir(parents=True)
|
|
def run(command, **kw):
|
|
self.assertEqual(command[0], "/configured/python")
|
|
self.assertTrue(command[1].endswith("scripts/make_calib.py"))
|
|
envelope = json.loads(Path(command[command.index("--recipe-job") + 1]).read_text())
|
|
make_recipe_calibration(envelope["job"], sd / "calibration", sd / "calibration_calls",
|
|
Path(command[0]), envelope["runtime_identity"])
|
|
kw["stdout"].write_text("")
|
|
kw["stderr"].write_text("")
|
|
return SimpleNamespace(state="OK")
|
|
with patch("harness.recipe_runtime.call", side_effect=self.isolated_call), \
|
|
patch.object(paths, "model_python", return_value=Path("/configured/python")), \
|
|
patch.object(paths, "config", return_value={"paths": {"work_root": str(self.root / "work")}}), \
|
|
patch.object(ctx, "watchdog_params", return_value={key: 10 for key in
|
|
("stall_s", "cpu_idle_pct", "poll_s", "hard_ceiling_s", "io_stall_s", "no_progress_ceiling_s")}), \
|
|
patch("harness.stages.s04_calibrate.watchdog.run", side_effect=run):
|
|
outcome = Calibrate().native(ctx, sd)
|
|
self.assertEqual(Calibrate.pool, "cpu")
|
|
self.assertEqual(outcome.state, "OK")
|
|
self.assertEqual(outcome.meta["calib_dir"], str(sd / "calibration"))
|
|
self.assertTrue(outcome.meta["compile_work_dir"].endswith("05-compile/a03"))
|
|
self.assertEqual(set(outcome.meta["recipe_outputs"]), {"compile_yaml", "calibration"})
|
|
self.assertFalse((self.root / "shared").exists())
|
|
receipt = {"stage": "04-calibrate", "attempt": 0, "state": "OK", "recipe": recipe_identity(ctx),
|
|
"recipe_parents": ctx.extra["recipe_parents"], **outcome.meta}
|
|
(sd / "meta.json").write_text(json.dumps(receipt))
|
|
next_dir = ctx.stage_root / "04-calibrate/a01"
|
|
next_dir.mkdir()
|
|
ctx.attempt = 1
|
|
ctx.extra["compile_attempt"] = 4
|
|
with patch("harness.stages.s04_calibrate.watchdog.run", side_effect=AssertionError("unchanged calibration regenerated")), \
|
|
patch.object(paths, "config", return_value={"paths": {"work_root": str(self.root / "work")}}):
|
|
reused = Calibrate().native(ctx, next_dir)
|
|
self.assertEqual(reused.meta["recipe_outputs"]["calibration"], outcome.meta["recipe_outputs"]["calibration"])
|
|
self.assertNotEqual(reused.meta["compile_work_dir"], outcome.meta["compile_work_dir"])
|
|
self.assertEqual(len(self.calls), 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|