data-gen-bpu/scripts/test_runner_isolation.py

303 lines
19 KiB
Python

"""Runner registration and trusted jail planning; mount execution is tested on Linux separately."""
import contextlib
import io
import json
import os
import shutil
import subprocess
import sys
import textwrap
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import yaml
from harness import cli, paths, runners
from harness.runners import pi, opencode
from scripts import jail_setup
class RunnerIsolation(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name).resolve()
def test_third_registered_runner_is_accepted_by_cli_and_capabilities(self):
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
patch.object(cli, "cmd_rollout", return_value=17) as rollout:
self.assertEqual(cli.main(["rollout", "job", "--runner", "third", "--dry-run"]), 17)
self.assertEqual(rollout.call_args.args[0].runner, "third")
with patch("harness.registry.boards", return_value={}), \
patch("harness.eval.TASK_MODULES", {}), \
patch.object(paths, "config", return_value={}), \
contextlib.redirect_stdout(io.StringIO()) as output:
self.assertEqual(cli.main(["capabilities"]), 0)
self.assertIn("third", json.loads(output.getvalue())["runners"])
def test_unregistered_or_undeclared_runner_cannot_inherit_pi_policy(self):
with self.assertRaisesRegex(ValueError, "unknown runner"):
runners.isolation_policy("unknown")
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
patch.object(runners, "import_module", return_value=SimpleNamespace()):
with self.assertRaisesRegex(ValueError, "isolation policy"):
runners.isolation_policy("third")
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
patch.object(runners, "import_module", return_value=SimpleNamespace(JAIL_POLICY="legacy-pi")):
with self.assertRaisesRegex(ValueError, "reserved for pi"):
runners.isolation_policy("third")
def test_private_home_policy_is_shared_without_runner_name_dispatch(self):
self.assertEqual(runners.isolation_policy("opencode"), "private-home")
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
patch.object(runners, "import_module", return_value=SimpleNamespace(JAIL_POLICY="private-home")):
self.assertEqual(runners.isolation_policy("third"), "private-home")
def test_rollout_identity_cannot_fall_back_to_pi_when_env_omits_runner(self):
(self.root / "rollout.json").write_text('{"runner": "third"}')
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
patch.object(runners, "import_module", return_value=SimpleNamespace(JAIL_POLICY="private-home")):
self.assertEqual(jail_setup.runner_policy(self.root, {}), ("third", "private-home"))
with self.assertRaisesRegex(ValueError, "runner identity mismatch"):
jail_setup.runner_policy(self.root, {"BENCH_RUNNER": "pi"})
(self.root / "rollout.json").write_text('{}')
self.assertEqual(jail_setup.runner_policy(self.root, {}), ("pi", "legacy-pi"))
def test_only_trusted_snapshot_opts_new_pi_task_into_private_home(self):
(self.root / "rollout.json").write_text('{"runner":"pi","jail_policy":"private-home"}')
self.assertEqual(jail_setup.runner_policy(self.root, {}), ("pi", "private-home"))
(self.root / "rollout.json").write_text('{"runner":"pi"}')
self.assertEqual(jail_setup.runner_policy(self.root, {"BENCH_JAIL_POLICY":"private-home"}),
("pi", "legacy-pi"))
(self.root / "rollout.json").write_text('{"runner":"pi","jail_policy":"unsafe"}')
with self.assertRaisesRegex(ValueError, "recorded jail policy"):
jail_setup.runner_policy(self.root, {})
def test_private_pi_environment_contains_only_current_endpoint_credential(self):
from harness.rollout import agent_env
runs = self.root / "bench/runs"
rd = runs / "run/job/rollouts/01"
rd.mkdir(parents=True)
(rd / "rollout.json").write_text('{"runner":"pi","policy":"teacher","jail_policy":"private-home"}')
cfg = {"paths": {"work_root": str(self.root / "work")},
"rollouts": {"max_turns": 3, "max_tokens": 40},
"endpoints": {"teacher": {"api_key_env":"FIXTURE_API_KEY"}}}
with patch.object(paths, "RUNS", runs), patch.object(paths, "config", return_value=cfg), \
patch.object(paths, "secret", return_value="fixture-key") as secret:
env = agent_env(rd)
secret.assert_called_once_with("FIXTURE_API_KEY")
self.assertEqual(env["HOME"], str(rd / "workspace/.agent-home"))
self.assertEqual(env["BENCH_RUNNER_API_KEY"], "fixture-key")
self.assertNotIn("BENCH_JAIL_POLICY", env) # Only the actual jail may assert success.
def test_explicit_invalid_runner_is_rejected_before_mount_not_legacy_pi(self):
home, bench, rd, stash, env, argv = self.jail_fixture("pi")
for invalid in (None, "", False, 0, [], {}):
with self.subTest(runner=invalid):
(rd / "rollout.json").write_text(json.dumps({"runner": invalid}))
with patch.dict(os.environ, env, clear=True), patch.object(jail_setup.sys, "argv", argv), \
patch.object(jail_setup, "mount") as mount:
with self.assertRaisesRegex(ValueError, "invalid runner identity"):
jail_setup.main()
mount.assert_not_called()
def test_pi_environment_keeps_guard_budget_and_outer_worker(self):
expected = {"BENCH_GUARD_DENY": "deny", "BENCH_GUARD_ALLOW": "allow",
"BENCH_OUTER_WORKER": "1", "BENCH_MAX_TURNS": "4", "HOME": "host-home"}
with patch("harness.rollout.agent_env", return_value=expected.copy()):
actual = pi.environment(self.root, "e2e")
for key, value in expected.items():
self.assertEqual(actual[key], value)
self.assertEqual(runners.isolation_policy("pi"), "legacy-pi")
settings = json.loads((Path(__file__).resolve().parents[1] / "agent/settings.json.tmpl").read_text())
self.assertTrue(any("path-guard.ts" in name for name in settings["extensions"]))
self.assertTrue(any("budget-kill.ts" in name for name in settings["extensions"]))
def test_opencode_home_remains_in_workspace_with_worker_protection(self):
with patch("harness.rollout.agent_env", return_value={"BENCH_OUTER_WORKER": "1"}):
env = opencode.environment(self.root, "e2e")
self.assertEqual(env["HOME"], str(self.root / "workspace/.opencode-home"))
self.assertEqual(env["BENCH_OUTER_WORKER"], "1")
def test_agent_environment_passes_actual_external_state(self):
from harness.rollout import agent_env
runs = self.root / "bench/runs"
actual_state = self.root / "actual-state"
cfg = {"paths": {"work_root": str(self.root / "work")},
"rollouts": {"max_turns": 3, "max_tokens": 40}}
with patch.object(paths, "STATE", actual_state), patch.object(paths, "RUNS", runs), \
patch.object(paths, "config", return_value=cfg):
env = agent_env(runs / "run/job/rollouts/01")
self.assertEqual(env["BENCH_STATE_DIR"], str(actual_state))
self.assertIn(str(actual_state), env["BENCH_GUARD_DENY"])
def test_runtime_inputs_never_restore_home_or_its_ancestor_even_via_symlink(self):
home = self.root / "home"
home.mkdir()
alias = self.root / "alias"
alias.symlink_to(home, target_is_directory=True)
for source in (home, home.parent, alias):
with self.subTest(source=source), self.assertRaisesRegex(ValueError, "hidden root"):
jail_setup.private_home_inputs({"paths": {"venv": {"bench": str(source)}}},
"/usr/bin/true", Path, [home])
def test_runtime_preservation_keeps_only_selected_binary_not_home_parent(self):
binary = self.root / "home/bin/third"
binary.parent.mkdir(parents=True)
binary.write_text("fixture")
with patch.object(jail_setup.shutil, "which", return_value=str(binary)):
inputs = jail_setup.private_home_inputs({"paths": {}}, "third", Path, [binary.parent.parent])
self.assertEqual(inputs, [binary])
def jail_fixture(self, runner):
home = self.root / "home"
bench = home / "bench"
rd = bench / "runs/run/job/rollouts/01"
for directory in (rd / "workspace", rd / "handles", bench / "state"):
directory.mkdir(parents=True)
(rd / "rollout.json").write_text(json.dumps({"runner": runner}))
(bench / "config.yaml").write_text(yaml.safe_dump({"paths": {}}))
stash = self.root / "stash"
stash.mkdir()
env = {"BENCH_HOST_HOME": str(home), "BENCH_OUTER_WORKER": "1"}
argv = ["jail_setup.py", str(bench), str(rd), "e2e", "--", "/usr/bin/true"]
return home, bench, rd, stash, env, argv
def test_each_private_home_name_masks_home_and_keeps_only_workspace_handles_writable(self):
# This observes the mount plan only. It does not claim kernel enforcement.
for runner in ("opencode", "third"):
with self.subTest(runner=runner), tempfile.TemporaryDirectory() as directory:
self.root = Path(directory).resolve()
home, bench, rd, stash, env, argv = self.jail_fixture(runner)
external_state = self.root / "external-state"
external_state.mkdir()
env["BENCH_STATE_DIR"] = str(external_state)
original_read = Path.read_text
def read(path, *args, **kwargs):
if path == Path('/proc/self/mountinfo'):
return '\n'.join(f'1 0 0:1 / {p} rw - tmpfs tmpfs rw'
for p in (home, bench, rd / 'workspace', rd / 'handles'))
return original_read(path, *args, **kwargs)
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
patch.object(runners, "import_module", return_value=SimpleNamespace(JAIL_POLICY="private-home")), \
patch.dict(os.environ, env, clear=True), patch.object(jail_setup.sys, "argv", argv), \
patch.object(jail_setup.tempfile, "mkdtemp", return_value=str(stash)), \
patch.object(Path, "read_text", read), patch.object(Path, "symlink_to"), \
patch.object(jail_setup, "mount") as mount, \
patch.object(jail_setup, "visible_mount", return_value=True), \
patch.object(jail_setup.os, "chdir"), patch.object(jail_setup.os, "execvp") as launch:
jail_setup.main()
calls = [tuple(map(str, call.args)) for call in mount.call_args_list]
self.assertIn(('-t', 'tmpfs', '-o', 'mode=755', 'tmpfs', str(home)), calls)
self.assertIn(('-t', 'tmpfs', '-o', 'mode=755', 'tmpfs', str(external_state)), calls)
self.assertIn(('-t', 'tmpfs', '-o', 'mode=755', 'tmpfs', str(bench / 'state')), calls)
writable = {args[-1] for args in calls if len(args) > 1 and 'remount,bind,rw' in args[1]}
self.assertEqual(writable, {str(rd / 'workspace'), str(rd / 'handles')})
launch.assert_called_once()
self.assertIn('--bounding-set=-all', launch.call_args.args[1])
self.assertIn('--no-new-privs', launch.call_args.args[1])
def test_every_runner_requires_outer_worker_before_mounting(self):
home, bench, rd, stash, env, argv = self.jail_fixture("opencode")
env.pop("BENCH_OUTER_WORKER")
with patch.dict(os.environ, env, clear=True), patch.object(jail_setup.sys, "argv", argv), \
patch.object(jail_setup, "mount") as mount:
with self.assertRaisesRegex(RuntimeError, "outer-worker"):
jail_setup.main()
mount.assert_not_called()
@unittest.skipUnless(sys.platform == "linux" and os.environ.get("BENCH_TEST_AGENT_NAMESPACE") == "1",
"requires explicit Linux mount-namespace probe")
def test_real_private_jail_allows_private_experiments_and_protects_host(self):
source = Path(__file__).resolve().parents[1]
with tempfile.TemporaryDirectory(prefix="agent-namespace-", dir=source.parent) as directory, \
tempfile.TemporaryDirectory(prefix="agent-host-sentinel-", dir=Path.home()) as host_directory:
root = Path(directory)
bench = root / "bench"
rd = bench / "runs/run/job/rollouts/01"
host_home = Path.home()
for path in (rd / "workspace/.agent-home", rd / "handles", bench / "state",
bench / "agent", bench / "harness/adapters", bench / "scripts",
bench / "harness/runners", rd.parent / "00", rd.parent.parent / "oracle"):
path.mkdir(parents=True, exist_ok=True)
for name in ("jail_setup.py",):
shutil.copyfile(source / "scripts" / name, bench / "scripts" / name)
for file in (source / "harness/runners").glob("*.py"):
shutil.copyfile(file, bench / "harness/runners" / file.name)
(bench / "harness/adapters/example.py").write_text("VALUE = 7\n")
(bench / "harness/scorer.py").write_text("protected evaluator")
(bench / "state/jobs.sqlite").write_text("other tasks")
(bench / ".env.local").write_text("platform-secret-fixture")
(rd.parent / "00/claim.json").write_text("other rollout")
(rd.parent.parent / "oracle/result.json").write_text("answer")
(rd.parent.parent / "job.json").write_text("{}")
(rd / "rollout.json").write_text('{"runner":"pi","jail_policy":"private-home"}')
# This host-only fixture is hidden by the real HOME mount, not a regex.
(Path(host_directory) / "host-credential-sentinel").write_text("private-host-fixture")
config = root / "config.yaml"
config.write_text(yaml.safe_dump({"paths": {"venv": {"bench": sys.prefix}}}))
probe = textwrap.dedent("""
import json, os, subprocess, sys, zipfile
from pathlib import Path
rd = Path(os.environ['BENCH_ROLLOUT_DIR'])
bench = Path(os.environ['BENCH_ROOT'])
root = bench.parent
assert os.environ['BENCH_JAIL_POLICY'] == 'private-home'
for hidden in [Path(os.environ['BENCH_TEST_HOST_SENTINEL']), bench/'.env.local',
bench/'state/jobs.sqlite', bench/'harness/scorer.py',
rd.parent/'00/claim.json', rd.parent.parent/'oracle/result.json',
Path(os.environ['BENCH_HOST_HOME'])/'.ssh']:
assert not hidden.exists(), str(hidden)
for protected in [bench/'harness/adapters/example.py', rd/'rollout.json',
Path(sys.prefix)/'shared-write-probe']:
try:
fd = os.open(protected, os.O_WRONLY | os.O_APPEND) if protected.exists() else os.open(protected, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
except OSError:
pass
else:
os.close(fd)
raise AssertionError('shared path writable: '+str(protected))
command = 'mkdir -p src && cd src && DEST=../private.py; cp "$BENCH_ROOT/harness/adapters/example.py" "$DEST"; echo VALUE=9 >> "$DEST"'
subprocess.run(['bash','-c',command], check=True)
assert (rd/'workspace/private.py').read_text().endswith('VALUE=9\\n')
subprocess.run([sys.executable, '-m', 'venv', '--without-pip', '.venv'], check=True)
subprocess.run(['bash','-c','VENV=.venv; "$VENV/bin/python" -c "print(123)"'], check=True)
wheel = Path('private_probe-1.0-py3-none-any.whl')
with zipfile.ZipFile(wheel, 'w') as out:
out.writestr('private_probe.py', 'VALUE=11')
out.writestr('private_probe-1.0.dist-info/METADATA', 'Metadata-Version: 2.1\\nName: private-probe\\nVersion: 1.0\\n')
out.writestr('private_probe-1.0.dist-info/WHEEL', 'Wheel-Version: 1.0\\nRoot-Is-Purelib: true\\nTag: py3-none-any\\n')
out.writestr('private_probe-1.0.dist-info/RECORD', '')
subprocess.run([sys.executable, '-m', 'pip', 'install', '--no-deps', '--no-index',
'--target', 'private-packages', str(wheel)], check=True)
assert Path('private-packages/private_probe.py').exists()
assert (Path.home()).is_relative_to(rd/'workspace')
for device in set(Path('/dev').glob('nvidia*')) | {Path('/dev/dri'), Path('/dev/kfd')}:
if device.is_dir():
assert not list(device.iterdir()), str(device)
elif device.exists():
assert device.stat().st_rdev == Path('/dev/null').stat().st_rdev, str(device)
print('shared GPU device access absent or masked')
assert len(list(Path('/proc').glob('[0-9]*'))) < 15
result = subprocess.run(['mount','-o','remount,bind,rw',str(rd)], capture_output=True)
assert result.returncode != 0
print('private workspace experiments and protected host verified')
""")
env = dict(os.environ, BENCH_CONFIG=str(config), BENCH_HOST_HOME=str(host_home),
BENCH_TEST_HOST_SENTINEL=str(Path(host_directory) / "host-credential-sentinel"),
BENCH_OUTER_WORKER="1", HOME=str(rd / "workspace/.agent-home"),
PYTHONDONTWRITEBYTECODE="1")
result = subprocess.run(['unshare','-Urmpf','--mount-proc',sys.executable,'-B',
str(bench/'scripts/jail_setup.py'),str(bench),str(rd),'e2e','--',
sys.executable,'-B','-c',probe], env=env, capture_output=True,
text=True, timeout=90)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual((bench / "harness/adapters/example.py").read_text(), "VALUE = 7\n")
self.assertEqual((Path(host_directory) / "host-credential-sentinel").read_text(), "private-host-fixture")
if __name__ == "__main__":
unittest.main()