data-gen-bpu/scripts/test_recipe_submission.py

213 lines
10 KiB
Python

"""Task submissions preserve reference identity and cannot escape their workspace."""
import copy
import hashlib
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from harness import recipes
def sha(data):
return hashlib.sha256(data).hexdigest()
class RecipeSubmission(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.workspace = self.root / "workspace"
self.workspace.mkdir()
self.reference = b"def build(): return 'official'\n"
self.weights = b"official weights"
for name, content in {"reference.py": self.reference, "model.py": b"deployment implementation",
"weights.pt": self.weights, "graph.onnx": b"graph-v1"}.items():
(self.workspace / name).write_bytes(content)
self.job = {"model": "new-model", "model_spec": {"source": "custom", "input_spec": "1x3x4x4",
"preprocess": {"resize": 4}}, "task": "seg", "data": {"dataset": "frozen"},
"gates": {"G3": {"tol": 1}}, "march": "nash-m", "weights_uri": "official://weights",
"model_runtime": {"python": "/configured/runtime/bin/python"},
"model_recipe": {"reference": {"entrypoint": "reference.py",
"files": {"reference.py": sha(self.reference)}, "weights_sha256": sha(self.weights)}}}
self.manifest = {"schema": recipes.SCHEMA, "files": ["reference.py", "model.py", "weights.pt", "graph.onnx"],
"weights": "weights.pt", "entrypoint": "model.py", "onnx": "graph.onnx",
"input_spec": "1x3x8x8", "preprocess": {"resize": 8}}
self.save()
def save(self):
(self.workspace / "recipe.json").write_text(json.dumps(self.manifest))
def snapshot(self, name="attempt-1"):
return recipes.snapshot(self.workspace, "recipe.json", self.root / name, self.job)
def test_submission_is_independent_and_load_checks_actual_bytes(self):
frozen = self.snapshot()
self.assertEqual(recipes.load(self.root / "attempt-1/recipe.json"), frozen)
stored = json.loads((self.root / "attempt-1/recipe.json").read_text())
self.assertNotIn("root", stored)
self.assertEqual(stored["files"]["weights.pt"], sha(self.weights))
(self.workspace / "model.py").write_text("next implementation")
self.assertEqual((self.root / "attempt-1/model.py").read_text(), "deployment implementation")
changed = self.snapshot("attempt-2")
self.assertNotEqual(changed["submission_sha256"], frozen["submission_sha256"])
self.assertEqual(changed["reference_sha256"], frozen["reference_sha256"])
self.assertEqual(changed["files"]["weights.pt"], frozen["files"]["weights.pt"])
protected = self.root / "attempt-1/graph.onnx"
protected.chmod(0o600)
protected.write_bytes(b"replacement")
with self.assertRaisesRegex(ValueError, "file identity mismatch"):
recipes.load(self.root / "attempt-1/recipe.json")
def test_same_content_is_location_and_file_order_independent(self):
first = self.snapshot()
self.manifest["files"].reverse()
self.save()
second = self.snapshot("attempt-2")
self.assertEqual(first["submission_sha256"], second["submission_sha256"])
self.assertNotEqual(first["root"], second["root"])
def test_graph_iteration_changes_submission_but_not_official_reference(self):
first = self.snapshot()
(self.workspace / "graph.onnx").write_bytes(b"graph-v2 without deployment head")
second = self.snapshot("attempt-2")
self.assertNotEqual(first["submission_sha256"], second["submission_sha256"])
self.assertEqual(first["reference_sha256"], second["reference_sha256"])
self.assertEqual((self.root / "attempt-1/graph.onnx").read_bytes(), b"graph-v1")
def test_apply_preserves_task_and_reference_input_while_overriding_deployment(self):
before = copy.deepcopy(self.job)
frozen = self.snapshot()
applied = recipes.apply(self.job, frozen)
self.assertEqual(self.job, before)
self.assertEqual(applied["model_spec"], {"source": "recipe", "input_spec": "1x3x8x8", "preprocess": {"resize": 8}})
self.assertEqual(applied["recipe"]["reference_input_spec"], "1x3x4x4")
self.assertEqual(applied["recipe"]["reference_preprocess"], {"resize": 4})
for key in ("task", "data", "gates", "march", "weights_uri", "model_runtime", "model_recipe"):
self.assertEqual(applied[key], self.job[key])
applied["gates"]["G3"]["tol"] = 100
self.assertEqual(self.job["gates"]["G3"]["tol"], 1)
self.job["model_spec"]["input_spec"] = "1x3x16x16"
with self.assertRaisesRegex(ValueError, "frozen job reference"):
recipes.apply(self.job, frozen)
second = self.snapshot("attempt-2")
self.assertNotEqual(frozen["reference_sha256"], second["reference_sha256"])
def test_reference_and_weights_must_match_platform_policy(self):
for name in ("reference.py", "weights.pt"):
with self.subTest(file=name):
file = self.workspace / name
original = file.read_bytes()
file.write_bytes(b"agent replacement")
with self.assertRaisesRegex(ValueError, "reference"):
self.snapshot()
self.assertFalse((self.root / "attempt-1").exists())
self.assertEqual(list(self.root.glob(".attempt-1.*")), [])
file.write_bytes(original)
self.manifest["files"].remove("reference.py")
self.save()
frozen = self.snapshot()
self.assertNotIn("reference.py", frozen["files"])
def test_graph_only_submission_defaults_input_without_reference_sources_or_weights(self):
self.manifest = {"schema": recipes.SCHEMA, "files": ["graph.onnx"], "onnx": "graph.onnx"}
self.save()
frozen = self.snapshot()
self.assertIsNone(frozen["weights_sha256"])
self.assertEqual(frozen["input_spec"], "1x3x4x4")
self.assertEqual(frozen["preprocess"], {"resize": 4})
self.assertNotIn("entrypoint", frozen)
applied = recipes.apply(self.job, recipes.load(self.root / "attempt-1/recipe.json"))
self.assertEqual(applied["model_recipe"], self.job["model_recipe"])
self.assertEqual(applied["gates"], self.job["gates"])
def test_unconfigured_jobs_and_unknown_policy_fields_are_rejected(self):
original = self.job.pop("model_recipe")
with self.assertRaisesRegex(ValueError, "policy"):
self.snapshot()
self.job["model_recipe"] = {**original, "runtime": "agent choice"}
with self.assertRaisesRegex(ValueError, "policy"):
self.snapshot()
self.job["model_recipe"] = original
original["reference"]["extra"] = True
with self.assertRaisesRegex(ValueError, "policy"):
self.snapshot()
def test_unknown_manifest_fields_dynamic_input_and_unlisted_entries_are_rejected(self):
original = copy.deepcopy(self.manifest)
for change in ({"task": "cls"}, {"model_runtime": "agent runtime"}, {"input_spec": "Nx3x8x8"},
{"entrypoint": "not-listed.py"}, {"weights": "not-listed.pt"},
{"onnx": "not-listed.onnx"}, {"preprocess": []}):
with self.subTest(change=change):
self.manifest = {**original, **change}
self.save()
with self.assertRaises(ValueError):
self.snapshot()
def test_absolute_parent_paths_duplicates_and_reserved_files_are_rejected(self):
original = self.manifest["files"][:]
for path in ("/tmp/outside", "../outside", "nested/../outside", "nested//outside", "recipe.json", "model.py"):
with self.subTest(path=path):
self.manifest["files"] = original + [path]
self.save()
with self.assertRaises(ValueError):
self.snapshot()
def test_symlinks_hardlinks_and_nonregular_files_are_rejected(self):
outside = self.root / "outside"
outside.mkdir()
(outside / "file").write_text("outside")
for kind in ("symlink", "directory-symlink", "hardlink", "fifo"):
with self.subTest(kind=kind):
path = self.workspace / "unsafe"
if kind == "symlink":
path.symlink_to(outside / "file")
elif kind == "directory-symlink":
path.symlink_to(outside, target_is_directory=True)
elif kind == "hardlink":
os.link(outside / "file", path)
else:
os.mkfifo(path)
listed = "unsafe/file" if kind == "directory-symlink" else "unsafe"
self.manifest["files"].append(listed)
self.save()
with self.assertRaises((ValueError, OSError)):
self.snapshot()
self.assertFalse((self.root / "attempt-1").exists())
self.manifest["files"].pop()
path.unlink()
def test_budget_and_duplicate_json_keys_fail_without_publishing(self):
with patch.object(recipes, "MAX_BYTES", 8):
with self.assertRaisesRegex(ValueError, "size limit"):
self.snapshot()
self.assertFalse((self.root / "attempt-1").exists())
(self.workspace / "recipe.json").write_text('{"schema":"bench.recipe/1","schema":"bench.recipe/1"}')
with self.assertRaisesRegex(ValueError, "duplicate"):
self.snapshot()
def test_existing_submission_cannot_be_overwritten(self):
frozen = self.snapshot()
with self.assertRaisesRegex(ValueError, "already exists"):
self.snapshot()
self.assertEqual(recipes.load(self.root / "attempt-1/recipe.json"), frozen)
def test_altered_snapshot_manifest_is_rejected_before_use(self):
frozen = self.snapshot()
altered = copy.deepcopy(frozen)
altered["input_spec"] = "1x3x32x32"
with self.assertRaisesRegex(ValueError, "submission identity"):
recipes.apply(self.job, altered)
stored = self.root / "attempt-1/recipe.json"
stored.chmod(0o600)
stored.write_text(json.dumps({k: v for k, v in altered.items() if k != "root"}))
with self.assertRaisesRegex(ValueError, "submission identity"):
recipes.load(stored)
if __name__ == "__main__":
unittest.main()