data-gen-bpu/scripts/test_recipe_runtime.py

384 lines
22 KiB
Python

"""Boundary regressions; actual namespace execution is a separate Linux check."""
import hashlib
import io
import json
import os
from pathlib import Path
import tempfile
import sys
import subprocess
import textwrap
import unittest
from unittest.mock import patch
import zipfile
import numpy as np
from harness import recipe_runtime as runtime
class RecipeBoundary(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.root = Path(self.temporary.name).resolve()
self.snapshot = self.root / "snapshot"
self.snapshot.mkdir()
(self.snapshot / "candidate.py").write_text("raise RuntimeError('must not import')")
(self.snapshot / "reference.py").write_text("import common\n")
(self.snapshot / "common.py").write_text("official = True\n")
(self.snapshot / "weights.bin").write_bytes(b"actual weights")
self.recipe = {"root": str(self.snapshot), "entrypoint": "candidate.py", "weights": "weights.bin",
"input_spec": "1x3x2x2", "reference": {"entrypoint": "reference.py", "files": {
name: hashlib.sha256((self.snapshot / name).read_bytes()).hexdigest()
for name in ("reference.py", "common.py")},
"weights_sha256": hashlib.sha256(b"actual weights").hexdigest()}}
def tearDown(self):
self.temporary.cleanup()
def test_batch_numeric_roundtrip(self):
values = {"sample/out0": np.zeros((1, 2, 3), dtype=np.float32)}
path = self.root / "result.npz"
np.savez(path, **values)
np.testing.assert_array_equal(runtime.load_arrays(path)["sample/out0"], values["sample/out0"])
def test_rejects_python_objects_nonfinite_and_unsafe_keys(self):
for values in ({"s": np.array([object()], dtype=object)}, {"s": np.array([np.nan])},
{"../../escape": np.zeros(1)}, {"s": np.array(["not numeric"])},
{"s": np.ones(1, dtype=complex)}):
with self.subTest(values=list(values)), self.assertRaises(ValueError):
runtime.arrays_checked(values)
def test_rejects_npy_header_allocation_attack_before_numpy_load(self):
header = io.BytesIO()
np.lib.format.write_array_header_1_0(header, {"descr": "<f8", "fortran_order": False,
"shape": (10**12,)})
path = self.root / "bad.npz"
with zipfile.ZipFile(path, "w") as archive:
archive.writestr("s.npy", header.getvalue())
with patch.object(np, "load", side_effect=AssertionError("unsafe parser reached")):
with self.assertRaisesRegex(ValueError, "header"):
runtime.load_arrays(path)
def test_rejects_link_and_parent_link_outputs(self):
path = self.root / "plain"
path.write_bytes(b"x")
link = self.root / "link"
link.symlink_to(path)
parent = self.root / "alias"
parent.symlink_to(self.root, target_is_directory=True)
for candidate in (link, parent / "plain"):
with self.subTest(candidate=candidate), self.assertRaises(OSError):
runtime.regular_bytes(candidate, 20)
hard = self.root / "hard"
os.link(path, hard)
with self.assertRaises(ValueError):
runtime.regular_bytes(hard, 20)
def test_request_cannot_escape_snapshot_or_use_unknown_operation(self):
for operation, changed in (("shell", {}), ("predict", {"entrypoint": "../other.py"}),
("predict", {"entrypoint": "/tmp/model.py"})):
with self.assertRaises(ValueError):
runtime._request({**self.recipe, **changed}, operation, False)
(self.snapshot / "escape").symlink_to(self.root)
with self.assertRaises(ValueError):
runtime._request(self.recipe, "predict", False)
def test_reference_only_exposes_pinned_dependencies_and_weights(self):
(self.snapshot / "torch.py").write_text("raise RuntimeError('poisoned torch')")
target = runtime._reference_root(self.snapshot, self.recipe, self.root / "reference")
self.assertEqual(sorted(p.name for p in target.iterdir()), ["common.py", "reference.py", "weights.bin"])
self.assertFalse((target / "candidate.py").exists())
self.assertFalse((target / "torch.py").exists())
def test_reference_rejects_changed_dependency_or_weights(self):
(self.snapshot / "common.py").write_text("official = False")
with self.assertRaisesRegex(ValueError, "digest"):
runtime._reference_root(self.snapshot, self.recipe, self.root / "reference")
def test_reference_rejects_changed_weight_bytes(self):
(self.snapshot / "weights.bin").write_bytes(b"changed")
with self.assertRaisesRegex(ValueError, "weight digest"):
runtime._reference_root(self.snapshot, self.recipe, self.root / "reference")
def test_reference_rejects_missing_file_pin_set(self):
self.recipe["reference"].pop("files")
with self.assertRaisesRegex(ValueError, "pinned"):
runtime._reference_root(self.snapshot, self.recipe, self.root / "reference")
def test_materials_are_checked_only_when_the_operation_uses_them(self):
minimal = {"root": str(self.snapshot), "entrypoint": "candidate.py", "input_spec": "1x3x2x2"}
for operation in ("preprocess", "decode"):
_, request = runtime._request(minimal, operation, False)
self.assertNotIn("weights", request)
with self.assertRaisesRegex(ValueError, "weights"):
runtime._request(minimal, "predict", False)
_, request = runtime._request(minimal, "export", False)
self.assertNotIn("weights", request)
(self.snapshot / "model.onnx").write_bytes(b"graph")
for operation in ("inspect_onnx", "predict_onnx", "export"):
_, request = runtime._request({"root": str(self.snapshot), "onnx": "model.onnx"}, operation, False)
self.assertNotIn("entrypoint", request)
self.assertNotIn("weights", request)
for operation in ("build", "check_onnx"):
with self.assertRaisesRegex(ValueError, "unknown"):
runtime._request(self.recipe, operation, False)
def test_reference_preprocessing_does_not_require_unused_weights(self):
self.recipe.pop("weights")
self.recipe["reference"].pop("weights_sha256")
target = runtime._reference_root(self.snapshot, self.recipe, self.root / "reference", weights=False)
self.assertEqual(sorted(p.name for p in target.iterdir()), ["common.py", "reference.py"])
with self.assertRaises(ValueError):
runtime._reference_root(self.snapshot, self.recipe, self.root / "model", weights=True)
def process(self, code, context=None, operation="preprocess", reference=False, name="out", timeout=5):
# Exercise real parent capture with an ordinary harmless child. Namespace
# protection itself is covered by RealRecipeNamespace, not this mock.
popen = subprocess.Popen
with patch.object(runtime.sys, "platform", "linux"), \
patch.object(runtime.shutil, "which", return_value="/usr/bin/unshare"), \
patch.object(runtime.subprocess, "Popen", side_effect=lambda *a, **kw: popen([sys.executable, "-c", code], **kw)):
return runtime._execute(Path(sys.executable), self.recipe, operation, {},
self.root / name, reference, timeout, context)
def test_stage_context_reuses_probe_and_copies_weights_only_once(self):
from harness import recipes
identity = {"sha256": "test-environment"}
context = {}
with patch.object(runtime, "runtime_identity", return_value=identity) as probe, \
patch.object(recipes, "_stream", wraps=recipes._stream) as copies:
for number, operation in enumerate(["preprocess", "predict", "decode"] * 3):
self.process("pass", context, operation, True, f"call-{number}")
self.assertEqual(probe.call_count, 1)
self.assertEqual(copies.call_count, 1, "reference weights must not be recopied per callback")
self.assertEqual(len(context["references"]), 2)
for path in context["references"].values():
self.assertFalse((path / "candidate.py").exists())
def test_parent_retains_full_stdout_stderr_and_prevents_fd_truncation(self):
code = "import os; print('before', flush=True); print('x'*20000, flush=True); "
code += "\ntry: os.ftruncate(1, 0)\nexcept OSError: print('cannot-truncate', flush=True)\n"
code += "os.write(2,b'failure-evidence'); raise SystemExit(7)"
with patch.object(runtime, "runtime_identity", return_value={"sha256": "environment"}):
with self.assertRaisesRegex(RuntimeError, r"failed \(7\)"):
self.process(code)
evidence = self.root / "out-process"
self.assertTrue((evidence / "stdout.log").read_text().startswith("before\n"))
self.assertGreater((evidence / "stdout.log").stat().st_size, 20000)
self.assertIn("cannot-truncate", (evidence / "stdout.log").read_text())
self.assertEqual((evidence / "stderr.log").read_text(), "failure-evidence")
record = json.loads((evidence / "process.json").read_text())
self.assertEqual(record["returncode"], 7)
self.assertFalse(record["timed_out"])
self.assertEqual(record["request"]["operation"], "preprocess")
self.assertTrue(record["command"])
def test_timeout_keeps_prior_output_and_exit_evidence(self):
with patch.object(runtime, "runtime_identity", return_value={"sha256": "environment"}):
with self.assertRaises(TimeoutError):
self.process("import time; print('before-timeout', flush=True); time.sleep(10)", timeout=1)
evidence = self.root / "out-process"
record = json.loads((evidence / "process.json").read_text())
self.assertTrue(record["timed_out"])
self.assertLess(record["returncode"], 0)
self.assertEqual((evidence / "stdout.log").read_text(), "before-timeout\n")
def test_child_cannot_escape_timeout_by_closing_its_output_streams(self):
with patch.object(runtime, "runtime_identity", return_value={"sha256": "environment"}):
with self.assertRaises(TimeoutError):
self.process("import os,time; os.close(1); os.close(2); time.sleep(10)", timeout=1)
record = json.loads((self.root / "out-process/process.json").read_text())
self.assertTrue(record["timed_out"])
self.assertLess(record["returncode"], 0)
def test_unbounded_candidate_logging_is_stopped_and_reported(self):
with patch.object(runtime, "runtime_identity", return_value={"sha256": "environment"}), \
patch.object(runtime, "MAX_GRAPH_BYTES", 1024):
with self.assertRaisesRegex(RuntimeError, "log limit"):
self.process("print('x'*20000, flush=True)")
evidence = self.root / "out-process"
self.assertEqual((evidence / "stdout.log").stat().st_size, 1024)
record = json.loads((evidence / "process.json").read_text())
self.assertTrue(record["log_limit"])
self.assertIsNotNone(record["returncode"])
def test_export_uses_one_operation_without_duplicate_inspector(self):
with patch.object(runtime, "_execute", return_value=self.root / "out") as execute, \
patch.object(runtime, "inspect_onnx", side_effect=AssertionError("duplicate inspection")):
self.assertEqual(runtime.export(Path(sys.executable), self.recipe, self.root / "out"),
self.root / "out/model.onnx")
self.assertEqual(execute.call_count, 1)
def test_nonlinux_never_runs_plain_subprocess(self):
with patch.object(runtime.sys, "platform", "darwin"), patch.object(runtime.subprocess, "Popen") as child:
with self.assertRaisesRegex(RuntimeError, "no unsafe fallback"):
runtime.call(Path("/usr/bin/python3"), self.recipe, "predict", {}, self.root / "out")
child.assert_not_called()
def test_jail_preserves_intermediate_python_symlinks(self):
from scripts.recipe_jail import interpreter_links
host = self.root / "host"
(host / "venv/bin").mkdir(parents=True)
(host / "usr/bin").mkdir(parents=True)
binary = host / "usr/bin/python3.10"
binary.write_bytes(b"executable")
alias = host / "usr/bin/python3"
alias.symlink_to("python3.10")
launcher = host / "venv/bin/python"
launcher.symlink_to(alias)
jail = self.root / "jail"
jail.mkdir()
self.assertEqual(interpreter_links(jail, str(launcher)), binary)
self.assertEqual(os.readlink(jail / str(alias).lstrip("/")), "python3.10")
self.assertEqual(os.readlink(jail / str(launcher).lstrip("/")), str(alias))
# Already mounted venv aliases are retained without mutating their inode.
before = (jail / str(launcher).lstrip("/")).lstat().st_ino
interpreter_links(jail, str(launcher))
self.assertEqual((jail / str(launcher).lstrip("/")).lstat().st_ino, before)
class RecipeBatchBudget(unittest.TestCase):
def runner(self):
from harness.eval.common import RecipeRunner
return RecipeRunner({"input_spec": "1x3x2x2", "preprocess": {}}, sys.executable, Path("unused"), "ade20k")
def test_large_static_outputs_use_bounded_batches_instead_of_fixed_sixteen(self):
runner = self.runner()
# Broadcast views express the real 150 MiB tensor shape without allocating
# multi-gigabyte test buffers. No serialization is mocked as proof of E2E.
logits = np.broadcast_to(np.zeros((), np.float32), (1, 150, 512, 512))
batches = []
def call(operation, values, **kwargs):
if operation == "predict":
batches.append(len(values))
outputs = {key + "/out0": logits for key in values}
else:
outputs = values
self.assertLess(sum(value.nbytes for value in outputs.values()), runtime.MAX_BYTES)
return outputs
with patch.object(runner, "_call", side_effect=call):
result = runner.batch([np.zeros((1, 3, 2, 2), np.float32)] * 16)
self.assertEqual(batches, [1, 3, 3, 3, 3, 3])
self.assertEqual(len(result), 16)
def test_516_samples_keep_context_and_report_actual_callback_counts(self):
runner, counts, contexts = self.runner(), {}, set()
def call(python, recipe, operation, values, output, **kwargs):
counts[operation] = counts.get(operation, 0) + 1
contexts.add(id(kwargs["context"]))
if operation == "preprocess":
return {key: value.transpose(2, 0, 1)[None].astype(np.float32) for key, value in values.items()}
if operation == "predict":
return {key + "/out0": value for key, value in values.items()}
return values
with patch.object(runtime, "call", side_effect=call):
for count in [16] * 32 + [4]:
runner.batch(runner.prepare([np.zeros((2, 2, 3), np.uint8)] * count))
# The single-sample capacity probe adds one predict/decode pair, while
# all callbacks share one environment probe and one reference copy.
self.assertEqual(counts, {"preprocess": 33, "predict": 34, "decode": 34})
self.assertEqual(len(contexts), 1)
@unittest.skipUnless(os.environ.get("BENCH_TEST_RECIPE_NAMESPACE") == "1", "requires explicit Linux namespace test")
class RealRecipeNamespace(unittest.TestCase):
"""Use the existing model Python and a disposable directory in the test checkout."""
def test_custom_export_accepts_no_weights_and_no_preprocessing(self):
with tempfile.TemporaryDirectory(prefix="recipe-custom-export-", dir=Path.cwd()) as temporary:
root = Path(temporary).resolve()
snapshot = root / "snapshot"
snapshot.mkdir()
(snapshot / "export.py").write_text(textwrap.dedent("""
def export(weights, input_spec, output):
assert weights is None
from onnx import TensorProto, helper, save
value = helper.make_tensor_value_info
graph = helper.make_graph([helper.make_node('Identity', ['input'], ['output'])], 'custom',
[value('input', TensorProto.FLOAT, [1,3,2,2])],
[value('output', TensorProto.FLOAT, [1,3,2,2])])
save(helper.make_model(graph, opset_imports=[helper.make_opsetid('',11)], ir_version=7), output)
"""))
recipe = {"root": str(snapshot), "entrypoint": "export.py", "input_spec": "1x3x2x2"}
context = {}
graph = runtime.export(Path(sys.executable), recipe, root / "export", context=context)
metadata = runtime.inspect_onnx(Path(sys.executable), {"root": str(graph.parent), "onnx": graph.name},
root / "inspect", context=context)
self.assertEqual(metadata["inputs"][0]["shape"], [1, 3, 2, 2])
def test_real_model_export_ort_reference_and_isolation(self):
import importlib.util
for name in ("torch", "onnx", "onnxruntime", "PIL"):
self.assertIsNotNone(importlib.util.find_spec(name), f"test runtime missing {name}")
with tempfile.TemporaryDirectory(prefix="recipe-namespace-", dir=Path.cwd()) as temporary:
root = Path(temporary).resolve()
snapshot = root / "snapshot"
snapshot.mkdir()
sentinel = root / "outside-secret"
sentinel.write_text("hidden from recipes")
module = textwrap.dedent('''
import os, pathlib, socket
import numpy as np
import torch
def build(weights, input_spec):
return torch.nn.Identity()
def preprocess(image, input_spec):
return np.asarray(image, dtype=np.float32).transpose(2,0,1)[None]
def decode(outputs, input_spec):
assert os.environ.get('RECIPE_TEST_SECRET') is None
assert os.environ.get('CUDA_VISIBLE_DEVICES') == ''
status = pathlib.Path('/proc/self/status').read_text()
assert 'CapEff:\\t0000000000000000' in status
assert 'NoNewPrivs:\\t1' in status
assert not pathlib.Path('/etc/shadow').exists()
assert not pathlib.Path('/dev/nvidia0').exists()
assert not pathlib.Path(OUTER_SENTINEL).exists()
assert not pathlib.Path('/proc/1/root' + OUTER_SENTINEL).exists()
assert os.environ.get('HOME') == '/tmp/home'
assert pathlib.Path('/tmp/home').is_dir()
assert not pathlib.Path(HOST_HOME, '.ssh').exists()
assert not pathlib.Path(HOST_HOME, '.aws').exists()
try:
pathlib.Path('/snapshot/injected').write_text('bad')
except OSError: pass
else: raise AssertionError('snapshot was writable')
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
sock.connect(('192.0.2.1', 9))
except OSError: pass
else: raise AssertionError('network was reachable')
finally: sock.close()
return outputs
''')
module = ("OUTER_SENTINEL = " + repr(str(sentinel)) + "\n"
+ "HOST_HOME = " + repr(str(Path.home())) + "\n" + module)
(snapshot / "reference.py").write_text(module)
(snapshot / "candidate.py").write_text(module)
(snapshot / "weights.bin").write_bytes(b"weight identity")
recipe = {"root": str(snapshot), "entrypoint": "candidate.py", "weights": "weights.bin",
"input_spec": "1x3x2x2", "reference": {"entrypoint": "reference.py",
"input_spec": "1x3x2x2", "files": {"reference.py": hashlib.sha256(module.encode()).hexdigest()},
"weights_sha256": hashlib.sha256(b"weight identity").hexdigest()}}
python = Path(sys.executable)
image = {"s000": np.ones((2, 2, 3), np.uint8)}
with patch.dict(os.environ, {"RECIPE_TEST_SECRET": "must-not-cross"}):
prepared = runtime.call(python, recipe, "preprocess", image, root / "prepared")
predicted = runtime.call(python, recipe, "predict", prepared, root / "predicted")
decoded = runtime.call(python, recipe, "decode", predicted, root / "decoded")
np.testing.assert_array_equal(decoded["s000/out0"], prepared["s000"])
graph = runtime.export(python, recipe, root / "exported")
graph_recipe = {"root": str(graph.parent), "onnx": graph.name}
metadata = runtime.inspect_onnx(python, graph_recipe, root / "inspected")
self.assertEqual(metadata["inputs"][0]["shape"], [1, 3, 2, 2])
onnx_values = runtime.call(python, graph_recipe, "predict_onnx", prepared, root / "onnx-output")
np.testing.assert_array_equal(onnx_values["s000/out0"], prepared["s000"])
# Candidate siblings cannot shadow imports in the official process.
(snapshot / "torch.py").write_text("raise RuntimeError('candidate poisoned reference')")
reference = runtime.call(python, recipe, "predict", prepared, root / "reference", reference=True)
np.testing.assert_array_equal(reference["s000/out0"], prepared["s000"])
if __name__ == "__main__":
unittest.main()