384 lines
22 KiB
Python
384 lines
22 KiB
Python
"""Boundary regressions; actual namespace execution is a separate Linux check."""
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import sys
|
|
import subprocess
|
|
import textwrap
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import zipfile
|
|
|
|
import numpy as np
|
|
|
|
from harness import recipe_runtime as runtime
|
|
|
|
|
|
class RecipeBoundary(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.temporary.name).resolve()
|
|
self.snapshot = self.root / "snapshot"
|
|
self.snapshot.mkdir()
|
|
(self.snapshot / "candidate.py").write_text("raise RuntimeError('must not import')")
|
|
(self.snapshot / "reference.py").write_text("import common\n")
|
|
(self.snapshot / "common.py").write_text("official = True\n")
|
|
(self.snapshot / "weights.bin").write_bytes(b"actual weights")
|
|
self.recipe = {"root": str(self.snapshot), "entrypoint": "candidate.py", "weights": "weights.bin",
|
|
"input_spec": "1x3x2x2", "reference": {"entrypoint": "reference.py", "files": {
|
|
name: hashlib.sha256((self.snapshot / name).read_bytes()).hexdigest()
|
|
for name in ("reference.py", "common.py")},
|
|
"weights_sha256": hashlib.sha256(b"actual weights").hexdigest()}}
|
|
|
|
def tearDown(self):
|
|
self.temporary.cleanup()
|
|
|
|
def test_batch_numeric_roundtrip(self):
|
|
values = {"sample/out0": np.zeros((1, 2, 3), dtype=np.float32)}
|
|
path = self.root / "result.npz"
|
|
np.savez(path, **values)
|
|
np.testing.assert_array_equal(runtime.load_arrays(path)["sample/out0"], values["sample/out0"])
|
|
|
|
def test_rejects_python_objects_nonfinite_and_unsafe_keys(self):
|
|
for values in ({"s": np.array([object()], dtype=object)}, {"s": np.array([np.nan])},
|
|
{"../../escape": np.zeros(1)}, {"s": np.array(["not numeric"])},
|
|
{"s": np.ones(1, dtype=complex)}):
|
|
with self.subTest(values=list(values)), self.assertRaises(ValueError):
|
|
runtime.arrays_checked(values)
|
|
|
|
def test_rejects_npy_header_allocation_attack_before_numpy_load(self):
|
|
header = io.BytesIO()
|
|
np.lib.format.write_array_header_1_0(header, {"descr": "<f8", "fortran_order": False,
|
|
"shape": (10**12,)})
|
|
path = self.root / "bad.npz"
|
|
with zipfile.ZipFile(path, "w") as archive:
|
|
archive.writestr("s.npy", header.getvalue())
|
|
with patch.object(np, "load", side_effect=AssertionError("unsafe parser reached")):
|
|
with self.assertRaisesRegex(ValueError, "header"):
|
|
runtime.load_arrays(path)
|
|
|
|
def test_rejects_link_and_parent_link_outputs(self):
|
|
path = self.root / "plain"
|
|
path.write_bytes(b"x")
|
|
link = self.root / "link"
|
|
link.symlink_to(path)
|
|
parent = self.root / "alias"
|
|
parent.symlink_to(self.root, target_is_directory=True)
|
|
for candidate in (link, parent / "plain"):
|
|
with self.subTest(candidate=candidate), self.assertRaises(OSError):
|
|
runtime.regular_bytes(candidate, 20)
|
|
hard = self.root / "hard"
|
|
os.link(path, hard)
|
|
with self.assertRaises(ValueError):
|
|
runtime.regular_bytes(hard, 20)
|
|
|
|
def test_request_cannot_escape_snapshot_or_use_unknown_operation(self):
|
|
for operation, changed in (("shell", {}), ("predict", {"entrypoint": "../other.py"}),
|
|
("predict", {"entrypoint": "/tmp/model.py"})):
|
|
with self.assertRaises(ValueError):
|
|
runtime._request({**self.recipe, **changed}, operation, False)
|
|
(self.snapshot / "escape").symlink_to(self.root)
|
|
with self.assertRaises(ValueError):
|
|
runtime._request(self.recipe, "predict", False)
|
|
|
|
def test_reference_only_exposes_pinned_dependencies_and_weights(self):
|
|
(self.snapshot / "torch.py").write_text("raise RuntimeError('poisoned torch')")
|
|
target = runtime._reference_root(self.snapshot, self.recipe, self.root / "reference")
|
|
self.assertEqual(sorted(p.name for p in target.iterdir()), ["common.py", "reference.py", "weights.bin"])
|
|
self.assertFalse((target / "candidate.py").exists())
|
|
self.assertFalse((target / "torch.py").exists())
|
|
|
|
def test_reference_rejects_changed_dependency_or_weights(self):
|
|
(self.snapshot / "common.py").write_text("official = False")
|
|
with self.assertRaisesRegex(ValueError, "digest"):
|
|
runtime._reference_root(self.snapshot, self.recipe, self.root / "reference")
|
|
|
|
def test_reference_rejects_changed_weight_bytes(self):
|
|
(self.snapshot / "weights.bin").write_bytes(b"changed")
|
|
with self.assertRaisesRegex(ValueError, "weight digest"):
|
|
runtime._reference_root(self.snapshot, self.recipe, self.root / "reference")
|
|
|
|
def test_reference_rejects_missing_file_pin_set(self):
|
|
self.recipe["reference"].pop("files")
|
|
with self.assertRaisesRegex(ValueError, "pinned"):
|
|
runtime._reference_root(self.snapshot, self.recipe, self.root / "reference")
|
|
|
|
def test_materials_are_checked_only_when_the_operation_uses_them(self):
|
|
minimal = {"root": str(self.snapshot), "entrypoint": "candidate.py", "input_spec": "1x3x2x2"}
|
|
for operation in ("preprocess", "decode"):
|
|
_, request = runtime._request(minimal, operation, False)
|
|
self.assertNotIn("weights", request)
|
|
with self.assertRaisesRegex(ValueError, "weights"):
|
|
runtime._request(minimal, "predict", False)
|
|
_, request = runtime._request(minimal, "export", False)
|
|
self.assertNotIn("weights", request)
|
|
(self.snapshot / "model.onnx").write_bytes(b"graph")
|
|
for operation in ("inspect_onnx", "predict_onnx", "export"):
|
|
_, request = runtime._request({"root": str(self.snapshot), "onnx": "model.onnx"}, operation, False)
|
|
self.assertNotIn("entrypoint", request)
|
|
self.assertNotIn("weights", request)
|
|
for operation in ("build", "check_onnx"):
|
|
with self.assertRaisesRegex(ValueError, "unknown"):
|
|
runtime._request(self.recipe, operation, False)
|
|
|
|
def test_reference_preprocessing_does_not_require_unused_weights(self):
|
|
self.recipe.pop("weights")
|
|
self.recipe["reference"].pop("weights_sha256")
|
|
target = runtime._reference_root(self.snapshot, self.recipe, self.root / "reference", weights=False)
|
|
self.assertEqual(sorted(p.name for p in target.iterdir()), ["common.py", "reference.py"])
|
|
with self.assertRaises(ValueError):
|
|
runtime._reference_root(self.snapshot, self.recipe, self.root / "model", weights=True)
|
|
|
|
def process(self, code, context=None, operation="preprocess", reference=False, name="out", timeout=5):
|
|
# Exercise real parent capture with an ordinary harmless child. Namespace
|
|
# protection itself is covered by RealRecipeNamespace, not this mock.
|
|
popen = subprocess.Popen
|
|
with patch.object(runtime.sys, "platform", "linux"), \
|
|
patch.object(runtime.shutil, "which", return_value="/usr/bin/unshare"), \
|
|
patch.object(runtime.subprocess, "Popen", side_effect=lambda *a, **kw: popen([sys.executable, "-c", code], **kw)):
|
|
return runtime._execute(Path(sys.executable), self.recipe, operation, {},
|
|
self.root / name, reference, timeout, context)
|
|
|
|
def test_stage_context_reuses_probe_and_copies_weights_only_once(self):
|
|
from harness import recipes
|
|
identity = {"sha256": "test-environment"}
|
|
context = {}
|
|
with patch.object(runtime, "runtime_identity", return_value=identity) as probe, \
|
|
patch.object(recipes, "_stream", wraps=recipes._stream) as copies:
|
|
for number, operation in enumerate(["preprocess", "predict", "decode"] * 3):
|
|
self.process("pass", context, operation, True, f"call-{number}")
|
|
self.assertEqual(probe.call_count, 1)
|
|
self.assertEqual(copies.call_count, 1, "reference weights must not be recopied per callback")
|
|
self.assertEqual(len(context["references"]), 2)
|
|
for path in context["references"].values():
|
|
self.assertFalse((path / "candidate.py").exists())
|
|
|
|
def test_parent_retains_full_stdout_stderr_and_prevents_fd_truncation(self):
|
|
code = "import os; print('before', flush=True); print('x'*20000, flush=True); "
|
|
code += "\ntry: os.ftruncate(1, 0)\nexcept OSError: print('cannot-truncate', flush=True)\n"
|
|
code += "os.write(2,b'failure-evidence'); raise SystemExit(7)"
|
|
with patch.object(runtime, "runtime_identity", return_value={"sha256": "environment"}):
|
|
with self.assertRaisesRegex(RuntimeError, r"failed \(7\)"):
|
|
self.process(code)
|
|
evidence = self.root / "out-process"
|
|
self.assertTrue((evidence / "stdout.log").read_text().startswith("before\n"))
|
|
self.assertGreater((evidence / "stdout.log").stat().st_size, 20000)
|
|
self.assertIn("cannot-truncate", (evidence / "stdout.log").read_text())
|
|
self.assertEqual((evidence / "stderr.log").read_text(), "failure-evidence")
|
|
record = json.loads((evidence / "process.json").read_text())
|
|
self.assertEqual(record["returncode"], 7)
|
|
self.assertFalse(record["timed_out"])
|
|
self.assertEqual(record["request"]["operation"], "preprocess")
|
|
self.assertTrue(record["command"])
|
|
|
|
def test_timeout_keeps_prior_output_and_exit_evidence(self):
|
|
with patch.object(runtime, "runtime_identity", return_value={"sha256": "environment"}):
|
|
with self.assertRaises(TimeoutError):
|
|
self.process("import time; print('before-timeout', flush=True); time.sleep(10)", timeout=1)
|
|
evidence = self.root / "out-process"
|
|
record = json.loads((evidence / "process.json").read_text())
|
|
self.assertTrue(record["timed_out"])
|
|
self.assertLess(record["returncode"], 0)
|
|
self.assertEqual((evidence / "stdout.log").read_text(), "before-timeout\n")
|
|
|
|
def test_child_cannot_escape_timeout_by_closing_its_output_streams(self):
|
|
with patch.object(runtime, "runtime_identity", return_value={"sha256": "environment"}):
|
|
with self.assertRaises(TimeoutError):
|
|
self.process("import os,time; os.close(1); os.close(2); time.sleep(10)", timeout=1)
|
|
record = json.loads((self.root / "out-process/process.json").read_text())
|
|
self.assertTrue(record["timed_out"])
|
|
self.assertLess(record["returncode"], 0)
|
|
|
|
def test_unbounded_candidate_logging_is_stopped_and_reported(self):
|
|
with patch.object(runtime, "runtime_identity", return_value={"sha256": "environment"}), \
|
|
patch.object(runtime, "MAX_GRAPH_BYTES", 1024):
|
|
with self.assertRaisesRegex(RuntimeError, "log limit"):
|
|
self.process("print('x'*20000, flush=True)")
|
|
evidence = self.root / "out-process"
|
|
self.assertEqual((evidence / "stdout.log").stat().st_size, 1024)
|
|
record = json.loads((evidence / "process.json").read_text())
|
|
self.assertTrue(record["log_limit"])
|
|
self.assertIsNotNone(record["returncode"])
|
|
|
|
def test_export_uses_one_operation_without_duplicate_inspector(self):
|
|
with patch.object(runtime, "_execute", return_value=self.root / "out") as execute, \
|
|
patch.object(runtime, "inspect_onnx", side_effect=AssertionError("duplicate inspection")):
|
|
self.assertEqual(runtime.export(Path(sys.executable), self.recipe, self.root / "out"),
|
|
self.root / "out/model.onnx")
|
|
self.assertEqual(execute.call_count, 1)
|
|
|
|
def test_nonlinux_never_runs_plain_subprocess(self):
|
|
with patch.object(runtime.sys, "platform", "darwin"), patch.object(runtime.subprocess, "Popen") as child:
|
|
with self.assertRaisesRegex(RuntimeError, "no unsafe fallback"):
|
|
runtime.call(Path("/usr/bin/python3"), self.recipe, "predict", {}, self.root / "out")
|
|
child.assert_not_called()
|
|
|
|
def test_jail_preserves_intermediate_python_symlinks(self):
|
|
from scripts.recipe_jail import interpreter_links
|
|
host = self.root / "host"
|
|
(host / "venv/bin").mkdir(parents=True)
|
|
(host / "usr/bin").mkdir(parents=True)
|
|
binary = host / "usr/bin/python3.10"
|
|
binary.write_bytes(b"executable")
|
|
alias = host / "usr/bin/python3"
|
|
alias.symlink_to("python3.10")
|
|
launcher = host / "venv/bin/python"
|
|
launcher.symlink_to(alias)
|
|
jail = self.root / "jail"
|
|
jail.mkdir()
|
|
self.assertEqual(interpreter_links(jail, str(launcher)), binary)
|
|
self.assertEqual(os.readlink(jail / str(alias).lstrip("/")), "python3.10")
|
|
self.assertEqual(os.readlink(jail / str(launcher).lstrip("/")), str(alias))
|
|
# Already mounted venv aliases are retained without mutating their inode.
|
|
before = (jail / str(launcher).lstrip("/")).lstat().st_ino
|
|
interpreter_links(jail, str(launcher))
|
|
self.assertEqual((jail / str(launcher).lstrip("/")).lstat().st_ino, before)
|
|
|
|
|
|
class RecipeBatchBudget(unittest.TestCase):
|
|
def runner(self):
|
|
from harness.eval.common import RecipeRunner
|
|
return RecipeRunner({"input_spec": "1x3x2x2", "preprocess": {}}, sys.executable, Path("unused"), "ade20k")
|
|
|
|
def test_large_static_outputs_use_bounded_batches_instead_of_fixed_sixteen(self):
|
|
runner = self.runner()
|
|
# Broadcast views express the real 150 MiB tensor shape without allocating
|
|
# multi-gigabyte test buffers. No serialization is mocked as proof of E2E.
|
|
logits = np.broadcast_to(np.zeros((), np.float32), (1, 150, 512, 512))
|
|
batches = []
|
|
def call(operation, values, **kwargs):
|
|
if operation == "predict":
|
|
batches.append(len(values))
|
|
outputs = {key + "/out0": logits for key in values}
|
|
else:
|
|
outputs = values
|
|
self.assertLess(sum(value.nbytes for value in outputs.values()), runtime.MAX_BYTES)
|
|
return outputs
|
|
with patch.object(runner, "_call", side_effect=call):
|
|
result = runner.batch([np.zeros((1, 3, 2, 2), np.float32)] * 16)
|
|
self.assertEqual(batches, [1, 3, 3, 3, 3, 3])
|
|
self.assertEqual(len(result), 16)
|
|
|
|
def test_516_samples_keep_context_and_report_actual_callback_counts(self):
|
|
runner, counts, contexts = self.runner(), {}, set()
|
|
def call(python, recipe, operation, values, output, **kwargs):
|
|
counts[operation] = counts.get(operation, 0) + 1
|
|
contexts.add(id(kwargs["context"]))
|
|
if operation == "preprocess":
|
|
return {key: value.transpose(2, 0, 1)[None].astype(np.float32) for key, value in values.items()}
|
|
if operation == "predict":
|
|
return {key + "/out0": value for key, value in values.items()}
|
|
return values
|
|
with patch.object(runtime, "call", side_effect=call):
|
|
for count in [16] * 32 + [4]:
|
|
runner.batch(runner.prepare([np.zeros((2, 2, 3), np.uint8)] * count))
|
|
# The single-sample capacity probe adds one predict/decode pair, while
|
|
# all callbacks share one environment probe and one reference copy.
|
|
self.assertEqual(counts, {"preprocess": 33, "predict": 34, "decode": 34})
|
|
self.assertEqual(len(contexts), 1)
|
|
|
|
|
|
@unittest.skipUnless(os.environ.get("BENCH_TEST_RECIPE_NAMESPACE") == "1", "requires explicit Linux namespace test")
|
|
class RealRecipeNamespace(unittest.TestCase):
|
|
"""Use the existing model Python and a disposable directory in the test checkout."""
|
|
|
|
def test_custom_export_accepts_no_weights_and_no_preprocessing(self):
|
|
with tempfile.TemporaryDirectory(prefix="recipe-custom-export-", dir=Path.cwd()) as temporary:
|
|
root = Path(temporary).resolve()
|
|
snapshot = root / "snapshot"
|
|
snapshot.mkdir()
|
|
(snapshot / "export.py").write_text(textwrap.dedent("""
|
|
def export(weights, input_spec, output):
|
|
assert weights is None
|
|
from onnx import TensorProto, helper, save
|
|
value = helper.make_tensor_value_info
|
|
graph = helper.make_graph([helper.make_node('Identity', ['input'], ['output'])], 'custom',
|
|
[value('input', TensorProto.FLOAT, [1,3,2,2])],
|
|
[value('output', TensorProto.FLOAT, [1,3,2,2])])
|
|
save(helper.make_model(graph, opset_imports=[helper.make_opsetid('',11)], ir_version=7), output)
|
|
"""))
|
|
recipe = {"root": str(snapshot), "entrypoint": "export.py", "input_spec": "1x3x2x2"}
|
|
context = {}
|
|
graph = runtime.export(Path(sys.executable), recipe, root / "export", context=context)
|
|
metadata = runtime.inspect_onnx(Path(sys.executable), {"root": str(graph.parent), "onnx": graph.name},
|
|
root / "inspect", context=context)
|
|
self.assertEqual(metadata["inputs"][0]["shape"], [1, 3, 2, 2])
|
|
|
|
def test_real_model_export_ort_reference_and_isolation(self):
|
|
import importlib.util
|
|
for name in ("torch", "onnx", "onnxruntime", "PIL"):
|
|
self.assertIsNotNone(importlib.util.find_spec(name), f"test runtime missing {name}")
|
|
with tempfile.TemporaryDirectory(prefix="recipe-namespace-", dir=Path.cwd()) as temporary:
|
|
root = Path(temporary).resolve()
|
|
snapshot = root / "snapshot"
|
|
snapshot.mkdir()
|
|
sentinel = root / "outside-secret"
|
|
sentinel.write_text("hidden from recipes")
|
|
module = textwrap.dedent('''
|
|
import os, pathlib, socket
|
|
import numpy as np
|
|
import torch
|
|
def build(weights, input_spec):
|
|
return torch.nn.Identity()
|
|
def preprocess(image, input_spec):
|
|
return np.asarray(image, dtype=np.float32).transpose(2,0,1)[None]
|
|
def decode(outputs, input_spec):
|
|
assert os.environ.get('RECIPE_TEST_SECRET') is None
|
|
assert os.environ.get('CUDA_VISIBLE_DEVICES') == ''
|
|
status = pathlib.Path('/proc/self/status').read_text()
|
|
assert 'CapEff:\\t0000000000000000' in status
|
|
assert 'NoNewPrivs:\\t1' in status
|
|
assert not pathlib.Path('/etc/shadow').exists()
|
|
assert not pathlib.Path('/dev/nvidia0').exists()
|
|
assert not pathlib.Path(OUTER_SENTINEL).exists()
|
|
assert not pathlib.Path('/proc/1/root' + OUTER_SENTINEL).exists()
|
|
assert os.environ.get('HOME') == '/tmp/home'
|
|
assert pathlib.Path('/tmp/home').is_dir()
|
|
assert not pathlib.Path(HOST_HOME, '.ssh').exists()
|
|
assert not pathlib.Path(HOST_HOME, '.aws').exists()
|
|
try:
|
|
pathlib.Path('/snapshot/injected').write_text('bad')
|
|
except OSError: pass
|
|
else: raise AssertionError('snapshot was writable')
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
try:
|
|
sock.connect(('192.0.2.1', 9))
|
|
except OSError: pass
|
|
else: raise AssertionError('network was reachable')
|
|
finally: sock.close()
|
|
return outputs
|
|
''')
|
|
module = ("OUTER_SENTINEL = " + repr(str(sentinel)) + "\n"
|
|
+ "HOST_HOME = " + repr(str(Path.home())) + "\n" + module)
|
|
(snapshot / "reference.py").write_text(module)
|
|
(snapshot / "candidate.py").write_text(module)
|
|
(snapshot / "weights.bin").write_bytes(b"weight identity")
|
|
recipe = {"root": str(snapshot), "entrypoint": "candidate.py", "weights": "weights.bin",
|
|
"input_spec": "1x3x2x2", "reference": {"entrypoint": "reference.py",
|
|
"input_spec": "1x3x2x2", "files": {"reference.py": hashlib.sha256(module.encode()).hexdigest()},
|
|
"weights_sha256": hashlib.sha256(b"weight identity").hexdigest()}}
|
|
python = Path(sys.executable)
|
|
image = {"s000": np.ones((2, 2, 3), np.uint8)}
|
|
with patch.dict(os.environ, {"RECIPE_TEST_SECRET": "must-not-cross"}):
|
|
prepared = runtime.call(python, recipe, "preprocess", image, root / "prepared")
|
|
predicted = runtime.call(python, recipe, "predict", prepared, root / "predicted")
|
|
decoded = runtime.call(python, recipe, "decode", predicted, root / "decoded")
|
|
np.testing.assert_array_equal(decoded["s000/out0"], prepared["s000"])
|
|
graph = runtime.export(python, recipe, root / "exported")
|
|
graph_recipe = {"root": str(graph.parent), "onnx": graph.name}
|
|
metadata = runtime.inspect_onnx(python, graph_recipe, root / "inspected")
|
|
self.assertEqual(metadata["inputs"][0]["shape"], [1, 3, 2, 2])
|
|
onnx_values = runtime.call(python, graph_recipe, "predict_onnx", prepared, root / "onnx-output")
|
|
np.testing.assert_array_equal(onnx_values["s000/out0"], prepared["s000"])
|
|
# Candidate siblings cannot shadow imports in the official process.
|
|
(snapshot / "torch.py").write_text("raise RuntimeError('candidate poisoned reference')")
|
|
reference = runtime.call(python, recipe, "predict", prepared, root / "reference", reference=True)
|
|
np.testing.assert_array_equal(reference["s000/out0"], prepared["s000"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|