213 lines
10 KiB
Python
213 lines
10 KiB
Python
"""Task submissions preserve reference identity and cannot escape their workspace."""
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import recipes
|
|
|
|
|
|
def sha(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
class RecipeSubmission(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.root = Path(self.tmp.name)
|
|
self.workspace = self.root / "workspace"
|
|
self.workspace.mkdir()
|
|
self.reference = b"def build(): return 'official'\n"
|
|
self.weights = b"official weights"
|
|
for name, content in {"reference.py": self.reference, "model.py": b"deployment implementation",
|
|
"weights.pt": self.weights, "graph.onnx": b"graph-v1"}.items():
|
|
(self.workspace / name).write_bytes(content)
|
|
self.job = {"model": "new-model", "model_spec": {"source": "custom", "input_spec": "1x3x4x4",
|
|
"preprocess": {"resize": 4}}, "task": "seg", "data": {"dataset": "frozen"},
|
|
"gates": {"G3": {"tol": 1}}, "march": "nash-m", "weights_uri": "official://weights",
|
|
"model_runtime": {"python": "/configured/runtime/bin/python"},
|
|
"model_recipe": {"reference": {"entrypoint": "reference.py",
|
|
"files": {"reference.py": sha(self.reference)}, "weights_sha256": sha(self.weights)}}}
|
|
self.manifest = {"schema": recipes.SCHEMA, "files": ["reference.py", "model.py", "weights.pt", "graph.onnx"],
|
|
"weights": "weights.pt", "entrypoint": "model.py", "onnx": "graph.onnx",
|
|
"input_spec": "1x3x8x8", "preprocess": {"resize": 8}}
|
|
self.save()
|
|
|
|
def save(self):
|
|
(self.workspace / "recipe.json").write_text(json.dumps(self.manifest))
|
|
|
|
def snapshot(self, name="attempt-1"):
|
|
return recipes.snapshot(self.workspace, "recipe.json", self.root / name, self.job)
|
|
|
|
def test_submission_is_independent_and_load_checks_actual_bytes(self):
|
|
frozen = self.snapshot()
|
|
self.assertEqual(recipes.load(self.root / "attempt-1/recipe.json"), frozen)
|
|
stored = json.loads((self.root / "attempt-1/recipe.json").read_text())
|
|
self.assertNotIn("root", stored)
|
|
self.assertEqual(stored["files"]["weights.pt"], sha(self.weights))
|
|
(self.workspace / "model.py").write_text("next implementation")
|
|
self.assertEqual((self.root / "attempt-1/model.py").read_text(), "deployment implementation")
|
|
changed = self.snapshot("attempt-2")
|
|
self.assertNotEqual(changed["submission_sha256"], frozen["submission_sha256"])
|
|
self.assertEqual(changed["reference_sha256"], frozen["reference_sha256"])
|
|
self.assertEqual(changed["files"]["weights.pt"], frozen["files"]["weights.pt"])
|
|
protected = self.root / "attempt-1/graph.onnx"
|
|
protected.chmod(0o600)
|
|
protected.write_bytes(b"replacement")
|
|
with self.assertRaisesRegex(ValueError, "file identity mismatch"):
|
|
recipes.load(self.root / "attempt-1/recipe.json")
|
|
|
|
def test_same_content_is_location_and_file_order_independent(self):
|
|
first = self.snapshot()
|
|
self.manifest["files"].reverse()
|
|
self.save()
|
|
second = self.snapshot("attempt-2")
|
|
self.assertEqual(first["submission_sha256"], second["submission_sha256"])
|
|
self.assertNotEqual(first["root"], second["root"])
|
|
|
|
def test_graph_iteration_changes_submission_but_not_official_reference(self):
|
|
first = self.snapshot()
|
|
(self.workspace / "graph.onnx").write_bytes(b"graph-v2 without deployment head")
|
|
second = self.snapshot("attempt-2")
|
|
self.assertNotEqual(first["submission_sha256"], second["submission_sha256"])
|
|
self.assertEqual(first["reference_sha256"], second["reference_sha256"])
|
|
self.assertEqual((self.root / "attempt-1/graph.onnx").read_bytes(), b"graph-v1")
|
|
|
|
def test_apply_preserves_task_and_reference_input_while_overriding_deployment(self):
|
|
before = copy.deepcopy(self.job)
|
|
frozen = self.snapshot()
|
|
applied = recipes.apply(self.job, frozen)
|
|
self.assertEqual(self.job, before)
|
|
self.assertEqual(applied["model_spec"], {"source": "recipe", "input_spec": "1x3x8x8", "preprocess": {"resize": 8}})
|
|
self.assertEqual(applied["recipe"]["reference_input_spec"], "1x3x4x4")
|
|
self.assertEqual(applied["recipe"]["reference_preprocess"], {"resize": 4})
|
|
for key in ("task", "data", "gates", "march", "weights_uri", "model_runtime", "model_recipe"):
|
|
self.assertEqual(applied[key], self.job[key])
|
|
applied["gates"]["G3"]["tol"] = 100
|
|
self.assertEqual(self.job["gates"]["G3"]["tol"], 1)
|
|
self.job["model_spec"]["input_spec"] = "1x3x16x16"
|
|
with self.assertRaisesRegex(ValueError, "frozen job reference"):
|
|
recipes.apply(self.job, frozen)
|
|
second = self.snapshot("attempt-2")
|
|
self.assertNotEqual(frozen["reference_sha256"], second["reference_sha256"])
|
|
|
|
def test_reference_and_weights_must_match_platform_policy(self):
|
|
for name in ("reference.py", "weights.pt"):
|
|
with self.subTest(file=name):
|
|
file = self.workspace / name
|
|
original = file.read_bytes()
|
|
file.write_bytes(b"agent replacement")
|
|
with self.assertRaisesRegex(ValueError, "reference"):
|
|
self.snapshot()
|
|
self.assertFalse((self.root / "attempt-1").exists())
|
|
self.assertEqual(list(self.root.glob(".attempt-1.*")), [])
|
|
file.write_bytes(original)
|
|
self.manifest["files"].remove("reference.py")
|
|
self.save()
|
|
frozen = self.snapshot()
|
|
self.assertNotIn("reference.py", frozen["files"])
|
|
|
|
def test_graph_only_submission_defaults_input_without_reference_sources_or_weights(self):
|
|
self.manifest = {"schema": recipes.SCHEMA, "files": ["graph.onnx"], "onnx": "graph.onnx"}
|
|
self.save()
|
|
frozen = self.snapshot()
|
|
self.assertIsNone(frozen["weights_sha256"])
|
|
self.assertEqual(frozen["input_spec"], "1x3x4x4")
|
|
self.assertEqual(frozen["preprocess"], {"resize": 4})
|
|
self.assertNotIn("entrypoint", frozen)
|
|
applied = recipes.apply(self.job, recipes.load(self.root / "attempt-1/recipe.json"))
|
|
self.assertEqual(applied["model_recipe"], self.job["model_recipe"])
|
|
self.assertEqual(applied["gates"], self.job["gates"])
|
|
|
|
def test_unconfigured_jobs_and_unknown_policy_fields_are_rejected(self):
|
|
original = self.job.pop("model_recipe")
|
|
with self.assertRaisesRegex(ValueError, "policy"):
|
|
self.snapshot()
|
|
self.job["model_recipe"] = {**original, "runtime": "agent choice"}
|
|
with self.assertRaisesRegex(ValueError, "policy"):
|
|
self.snapshot()
|
|
self.job["model_recipe"] = original
|
|
original["reference"]["extra"] = True
|
|
with self.assertRaisesRegex(ValueError, "policy"):
|
|
self.snapshot()
|
|
|
|
def test_unknown_manifest_fields_dynamic_input_and_unlisted_entries_are_rejected(self):
|
|
original = copy.deepcopy(self.manifest)
|
|
for change in ({"task": "cls"}, {"model_runtime": "agent runtime"}, {"input_spec": "Nx3x8x8"},
|
|
{"entrypoint": "not-listed.py"}, {"weights": "not-listed.pt"},
|
|
{"onnx": "not-listed.onnx"}, {"preprocess": []}):
|
|
with self.subTest(change=change):
|
|
self.manifest = {**original, **change}
|
|
self.save()
|
|
with self.assertRaises(ValueError):
|
|
self.snapshot()
|
|
|
|
def test_absolute_parent_paths_duplicates_and_reserved_files_are_rejected(self):
|
|
original = self.manifest["files"][:]
|
|
for path in ("/tmp/outside", "../outside", "nested/../outside", "nested//outside", "recipe.json", "model.py"):
|
|
with self.subTest(path=path):
|
|
self.manifest["files"] = original + [path]
|
|
self.save()
|
|
with self.assertRaises(ValueError):
|
|
self.snapshot()
|
|
|
|
def test_symlinks_hardlinks_and_nonregular_files_are_rejected(self):
|
|
outside = self.root / "outside"
|
|
outside.mkdir()
|
|
(outside / "file").write_text("outside")
|
|
for kind in ("symlink", "directory-symlink", "hardlink", "fifo"):
|
|
with self.subTest(kind=kind):
|
|
path = self.workspace / "unsafe"
|
|
if kind == "symlink":
|
|
path.symlink_to(outside / "file")
|
|
elif kind == "directory-symlink":
|
|
path.symlink_to(outside, target_is_directory=True)
|
|
elif kind == "hardlink":
|
|
os.link(outside / "file", path)
|
|
else:
|
|
os.mkfifo(path)
|
|
listed = "unsafe/file" if kind == "directory-symlink" else "unsafe"
|
|
self.manifest["files"].append(listed)
|
|
self.save()
|
|
with self.assertRaises((ValueError, OSError)):
|
|
self.snapshot()
|
|
self.assertFalse((self.root / "attempt-1").exists())
|
|
self.manifest["files"].pop()
|
|
path.unlink()
|
|
|
|
def test_budget_and_duplicate_json_keys_fail_without_publishing(self):
|
|
with patch.object(recipes, "MAX_BYTES", 8):
|
|
with self.assertRaisesRegex(ValueError, "size limit"):
|
|
self.snapshot()
|
|
self.assertFalse((self.root / "attempt-1").exists())
|
|
(self.workspace / "recipe.json").write_text('{"schema":"bench.recipe/1","schema":"bench.recipe/1"}')
|
|
with self.assertRaisesRegex(ValueError, "duplicate"):
|
|
self.snapshot()
|
|
|
|
def test_existing_submission_cannot_be_overwritten(self):
|
|
frozen = self.snapshot()
|
|
with self.assertRaisesRegex(ValueError, "already exists"):
|
|
self.snapshot()
|
|
self.assertEqual(recipes.load(self.root / "attempt-1/recipe.json"), frozen)
|
|
|
|
def test_altered_snapshot_manifest_is_rejected_before_use(self):
|
|
frozen = self.snapshot()
|
|
altered = copy.deepcopy(frozen)
|
|
altered["input_spec"] = "1x3x32x32"
|
|
with self.assertRaisesRegex(ValueError, "submission identity"):
|
|
recipes.apply(self.job, altered)
|
|
stored = self.root / "attempt-1/recipe.json"
|
|
stored.chmod(0o600)
|
|
stored.write_text(json.dumps({k: v for k, v in altered.items() if k != "root"}))
|
|
with self.assertRaisesRegex(ValueError, "submission identity"):
|
|
recipes.load(stored)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|