303 lines
19 KiB
Python
303 lines
19 KiB
Python
"""Runner registration and trusted jail planning; mount execution is tested on Linux separately."""
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
from pathlib import Path
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import yaml
|
|
|
|
from harness import cli, paths, runners
|
|
from harness.runners import pi, opencode
|
|
from scripts import jail_setup
|
|
|
|
|
|
class RunnerIsolation(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.root = Path(self.tmp.name).resolve()
|
|
|
|
def test_third_registered_runner_is_accepted_by_cli_and_capabilities(self):
|
|
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
|
|
patch.object(cli, "cmd_rollout", return_value=17) as rollout:
|
|
self.assertEqual(cli.main(["rollout", "job", "--runner", "third", "--dry-run"]), 17)
|
|
self.assertEqual(rollout.call_args.args[0].runner, "third")
|
|
with patch("harness.registry.boards", return_value={}), \
|
|
patch("harness.eval.TASK_MODULES", {}), \
|
|
patch.object(paths, "config", return_value={}), \
|
|
contextlib.redirect_stdout(io.StringIO()) as output:
|
|
self.assertEqual(cli.main(["capabilities"]), 0)
|
|
self.assertIn("third", json.loads(output.getvalue())["runners"])
|
|
|
|
def test_unregistered_or_undeclared_runner_cannot_inherit_pi_policy(self):
|
|
with self.assertRaisesRegex(ValueError, "unknown runner"):
|
|
runners.isolation_policy("unknown")
|
|
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
|
|
patch.object(runners, "import_module", return_value=SimpleNamespace()):
|
|
with self.assertRaisesRegex(ValueError, "isolation policy"):
|
|
runners.isolation_policy("third")
|
|
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
|
|
patch.object(runners, "import_module", return_value=SimpleNamespace(JAIL_POLICY="legacy-pi")):
|
|
with self.assertRaisesRegex(ValueError, "reserved for pi"):
|
|
runners.isolation_policy("third")
|
|
|
|
def test_private_home_policy_is_shared_without_runner_name_dispatch(self):
|
|
self.assertEqual(runners.isolation_policy("opencode"), "private-home")
|
|
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
|
|
patch.object(runners, "import_module", return_value=SimpleNamespace(JAIL_POLICY="private-home")):
|
|
self.assertEqual(runners.isolation_policy("third"), "private-home")
|
|
|
|
def test_rollout_identity_cannot_fall_back_to_pi_when_env_omits_runner(self):
|
|
(self.root / "rollout.json").write_text('{"runner": "third"}')
|
|
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
|
|
patch.object(runners, "import_module", return_value=SimpleNamespace(JAIL_POLICY="private-home")):
|
|
self.assertEqual(jail_setup.runner_policy(self.root, {}), ("third", "private-home"))
|
|
with self.assertRaisesRegex(ValueError, "runner identity mismatch"):
|
|
jail_setup.runner_policy(self.root, {"BENCH_RUNNER": "pi"})
|
|
(self.root / "rollout.json").write_text('{}')
|
|
self.assertEqual(jail_setup.runner_policy(self.root, {}), ("pi", "legacy-pi"))
|
|
|
|
def test_only_trusted_snapshot_opts_new_pi_task_into_private_home(self):
|
|
(self.root / "rollout.json").write_text('{"runner":"pi","jail_policy":"private-home"}')
|
|
self.assertEqual(jail_setup.runner_policy(self.root, {}), ("pi", "private-home"))
|
|
(self.root / "rollout.json").write_text('{"runner":"pi"}')
|
|
self.assertEqual(jail_setup.runner_policy(self.root, {"BENCH_JAIL_POLICY":"private-home"}),
|
|
("pi", "legacy-pi"))
|
|
(self.root / "rollout.json").write_text('{"runner":"pi","jail_policy":"unsafe"}')
|
|
with self.assertRaisesRegex(ValueError, "recorded jail policy"):
|
|
jail_setup.runner_policy(self.root, {})
|
|
|
|
def test_private_pi_environment_contains_only_current_endpoint_credential(self):
|
|
from harness.rollout import agent_env
|
|
runs = self.root / "bench/runs"
|
|
rd = runs / "run/job/rollouts/01"
|
|
rd.mkdir(parents=True)
|
|
(rd / "rollout.json").write_text('{"runner":"pi","policy":"teacher","jail_policy":"private-home"}')
|
|
cfg = {"paths": {"work_root": str(self.root / "work")},
|
|
"rollouts": {"max_turns": 3, "max_tokens": 40},
|
|
"endpoints": {"teacher": {"api_key_env":"FIXTURE_API_KEY"}}}
|
|
with patch.object(paths, "RUNS", runs), patch.object(paths, "config", return_value=cfg), \
|
|
patch.object(paths, "secret", return_value="fixture-key") as secret:
|
|
env = agent_env(rd)
|
|
secret.assert_called_once_with("FIXTURE_API_KEY")
|
|
self.assertEqual(env["HOME"], str(rd / "workspace/.agent-home"))
|
|
self.assertEqual(env["BENCH_RUNNER_API_KEY"], "fixture-key")
|
|
self.assertNotIn("BENCH_JAIL_POLICY", env) # Only the actual jail may assert success.
|
|
|
|
def test_explicit_invalid_runner_is_rejected_before_mount_not_legacy_pi(self):
|
|
home, bench, rd, stash, env, argv = self.jail_fixture("pi")
|
|
for invalid in (None, "", False, 0, [], {}):
|
|
with self.subTest(runner=invalid):
|
|
(rd / "rollout.json").write_text(json.dumps({"runner": invalid}))
|
|
with patch.dict(os.environ, env, clear=True), patch.object(jail_setup.sys, "argv", argv), \
|
|
patch.object(jail_setup, "mount") as mount:
|
|
with self.assertRaisesRegex(ValueError, "invalid runner identity"):
|
|
jail_setup.main()
|
|
mount.assert_not_called()
|
|
|
|
def test_pi_environment_keeps_guard_budget_and_outer_worker(self):
|
|
expected = {"BENCH_GUARD_DENY": "deny", "BENCH_GUARD_ALLOW": "allow",
|
|
"BENCH_OUTER_WORKER": "1", "BENCH_MAX_TURNS": "4", "HOME": "host-home"}
|
|
with patch("harness.rollout.agent_env", return_value=expected.copy()):
|
|
actual = pi.environment(self.root, "e2e")
|
|
for key, value in expected.items():
|
|
self.assertEqual(actual[key], value)
|
|
self.assertEqual(runners.isolation_policy("pi"), "legacy-pi")
|
|
settings = json.loads((Path(__file__).resolve().parents[1] / "agent/settings.json.tmpl").read_text())
|
|
self.assertTrue(any("path-guard.ts" in name for name in settings["extensions"]))
|
|
self.assertTrue(any("budget-kill.ts" in name for name in settings["extensions"]))
|
|
|
|
def test_opencode_home_remains_in_workspace_with_worker_protection(self):
|
|
with patch("harness.rollout.agent_env", return_value={"BENCH_OUTER_WORKER": "1"}):
|
|
env = opencode.environment(self.root, "e2e")
|
|
self.assertEqual(env["HOME"], str(self.root / "workspace/.opencode-home"))
|
|
self.assertEqual(env["BENCH_OUTER_WORKER"], "1")
|
|
|
|
def test_agent_environment_passes_actual_external_state(self):
|
|
from harness.rollout import agent_env
|
|
runs = self.root / "bench/runs"
|
|
actual_state = self.root / "actual-state"
|
|
cfg = {"paths": {"work_root": str(self.root / "work")},
|
|
"rollouts": {"max_turns": 3, "max_tokens": 40}}
|
|
with patch.object(paths, "STATE", actual_state), patch.object(paths, "RUNS", runs), \
|
|
patch.object(paths, "config", return_value=cfg):
|
|
env = agent_env(runs / "run/job/rollouts/01")
|
|
self.assertEqual(env["BENCH_STATE_DIR"], str(actual_state))
|
|
self.assertIn(str(actual_state), env["BENCH_GUARD_DENY"])
|
|
|
|
def test_runtime_inputs_never_restore_home_or_its_ancestor_even_via_symlink(self):
|
|
home = self.root / "home"
|
|
home.mkdir()
|
|
alias = self.root / "alias"
|
|
alias.symlink_to(home, target_is_directory=True)
|
|
for source in (home, home.parent, alias):
|
|
with self.subTest(source=source), self.assertRaisesRegex(ValueError, "hidden root"):
|
|
jail_setup.private_home_inputs({"paths": {"venv": {"bench": str(source)}}},
|
|
"/usr/bin/true", Path, [home])
|
|
|
|
def test_runtime_preservation_keeps_only_selected_binary_not_home_parent(self):
|
|
binary = self.root / "home/bin/third"
|
|
binary.parent.mkdir(parents=True)
|
|
binary.write_text("fixture")
|
|
with patch.object(jail_setup.shutil, "which", return_value=str(binary)):
|
|
inputs = jail_setup.private_home_inputs({"paths": {}}, "third", Path, [binary.parent.parent])
|
|
self.assertEqual(inputs, [binary])
|
|
|
|
def jail_fixture(self, runner):
|
|
home = self.root / "home"
|
|
bench = home / "bench"
|
|
rd = bench / "runs/run/job/rollouts/01"
|
|
for directory in (rd / "workspace", rd / "handles", bench / "state"):
|
|
directory.mkdir(parents=True)
|
|
(rd / "rollout.json").write_text(json.dumps({"runner": runner}))
|
|
(bench / "config.yaml").write_text(yaml.safe_dump({"paths": {}}))
|
|
stash = self.root / "stash"
|
|
stash.mkdir()
|
|
env = {"BENCH_HOST_HOME": str(home), "BENCH_OUTER_WORKER": "1"}
|
|
argv = ["jail_setup.py", str(bench), str(rd), "e2e", "--", "/usr/bin/true"]
|
|
return home, bench, rd, stash, env, argv
|
|
|
|
def test_each_private_home_name_masks_home_and_keeps_only_workspace_handles_writable(self):
|
|
# This observes the mount plan only. It does not claim kernel enforcement.
|
|
for runner in ("opencode", "third"):
|
|
with self.subTest(runner=runner), tempfile.TemporaryDirectory() as directory:
|
|
self.root = Path(directory).resolve()
|
|
home, bench, rd, stash, env, argv = self.jail_fixture(runner)
|
|
external_state = self.root / "external-state"
|
|
external_state.mkdir()
|
|
env["BENCH_STATE_DIR"] = str(external_state)
|
|
original_read = Path.read_text
|
|
def read(path, *args, **kwargs):
|
|
if path == Path('/proc/self/mountinfo'):
|
|
return '\n'.join(f'1 0 0:1 / {p} rw - tmpfs tmpfs rw'
|
|
for p in (home, bench, rd / 'workspace', rd / 'handles'))
|
|
return original_read(path, *args, **kwargs)
|
|
with patch.dict(runners.MODULES, {"third": "fixture.third"}), \
|
|
patch.object(runners, "import_module", return_value=SimpleNamespace(JAIL_POLICY="private-home")), \
|
|
patch.dict(os.environ, env, clear=True), patch.object(jail_setup.sys, "argv", argv), \
|
|
patch.object(jail_setup.tempfile, "mkdtemp", return_value=str(stash)), \
|
|
patch.object(Path, "read_text", read), patch.object(Path, "symlink_to"), \
|
|
patch.object(jail_setup, "mount") as mount, \
|
|
patch.object(jail_setup, "visible_mount", return_value=True), \
|
|
patch.object(jail_setup.os, "chdir"), patch.object(jail_setup.os, "execvp") as launch:
|
|
jail_setup.main()
|
|
calls = [tuple(map(str, call.args)) for call in mount.call_args_list]
|
|
self.assertIn(('-t', 'tmpfs', '-o', 'mode=755', 'tmpfs', str(home)), calls)
|
|
self.assertIn(('-t', 'tmpfs', '-o', 'mode=755', 'tmpfs', str(external_state)), calls)
|
|
self.assertIn(('-t', 'tmpfs', '-o', 'mode=755', 'tmpfs', str(bench / 'state')), calls)
|
|
writable = {args[-1] for args in calls if len(args) > 1 and 'remount,bind,rw' in args[1]}
|
|
self.assertEqual(writable, {str(rd / 'workspace'), str(rd / 'handles')})
|
|
launch.assert_called_once()
|
|
self.assertIn('--bounding-set=-all', launch.call_args.args[1])
|
|
self.assertIn('--no-new-privs', launch.call_args.args[1])
|
|
|
|
def test_every_runner_requires_outer_worker_before_mounting(self):
|
|
home, bench, rd, stash, env, argv = self.jail_fixture("opencode")
|
|
env.pop("BENCH_OUTER_WORKER")
|
|
with patch.dict(os.environ, env, clear=True), patch.object(jail_setup.sys, "argv", argv), \
|
|
patch.object(jail_setup, "mount") as mount:
|
|
with self.assertRaisesRegex(RuntimeError, "outer-worker"):
|
|
jail_setup.main()
|
|
mount.assert_not_called()
|
|
|
|
@unittest.skipUnless(sys.platform == "linux" and os.environ.get("BENCH_TEST_AGENT_NAMESPACE") == "1",
|
|
"requires explicit Linux mount-namespace probe")
|
|
def test_real_private_jail_allows_private_experiments_and_protects_host(self):
|
|
source = Path(__file__).resolve().parents[1]
|
|
with tempfile.TemporaryDirectory(prefix="agent-namespace-", dir=source.parent) as directory, \
|
|
tempfile.TemporaryDirectory(prefix="agent-host-sentinel-", dir=Path.home()) as host_directory:
|
|
root = Path(directory)
|
|
bench = root / "bench"
|
|
rd = bench / "runs/run/job/rollouts/01"
|
|
host_home = Path.home()
|
|
for path in (rd / "workspace/.agent-home", rd / "handles", bench / "state",
|
|
bench / "agent", bench / "harness/adapters", bench / "scripts",
|
|
bench / "harness/runners", rd.parent / "00", rd.parent.parent / "oracle"):
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
for name in ("jail_setup.py",):
|
|
shutil.copyfile(source / "scripts" / name, bench / "scripts" / name)
|
|
for file in (source / "harness/runners").glob("*.py"):
|
|
shutil.copyfile(file, bench / "harness/runners" / file.name)
|
|
(bench / "harness/adapters/example.py").write_text("VALUE = 7\n")
|
|
(bench / "harness/scorer.py").write_text("protected evaluator")
|
|
(bench / "state/jobs.sqlite").write_text("other tasks")
|
|
(bench / ".env.local").write_text("platform-secret-fixture")
|
|
(rd.parent / "00/claim.json").write_text("other rollout")
|
|
(rd.parent.parent / "oracle/result.json").write_text("answer")
|
|
(rd.parent.parent / "job.json").write_text("{}")
|
|
(rd / "rollout.json").write_text('{"runner":"pi","jail_policy":"private-home"}')
|
|
# This host-only fixture is hidden by the real HOME mount, not a regex.
|
|
(Path(host_directory) / "host-credential-sentinel").write_text("private-host-fixture")
|
|
config = root / "config.yaml"
|
|
config.write_text(yaml.safe_dump({"paths": {"venv": {"bench": sys.prefix}}}))
|
|
probe = textwrap.dedent("""
|
|
import json, os, subprocess, sys, zipfile
|
|
from pathlib import Path
|
|
rd = Path(os.environ['BENCH_ROLLOUT_DIR'])
|
|
bench = Path(os.environ['BENCH_ROOT'])
|
|
root = bench.parent
|
|
assert os.environ['BENCH_JAIL_POLICY'] == 'private-home'
|
|
for hidden in [Path(os.environ['BENCH_TEST_HOST_SENTINEL']), bench/'.env.local',
|
|
bench/'state/jobs.sqlite', bench/'harness/scorer.py',
|
|
rd.parent/'00/claim.json', rd.parent.parent/'oracle/result.json',
|
|
Path(os.environ['BENCH_HOST_HOME'])/'.ssh']:
|
|
assert not hidden.exists(), str(hidden)
|
|
for protected in [bench/'harness/adapters/example.py', rd/'rollout.json',
|
|
Path(sys.prefix)/'shared-write-probe']:
|
|
try:
|
|
fd = os.open(protected, os.O_WRONLY | os.O_APPEND) if protected.exists() else os.open(protected, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
except OSError:
|
|
pass
|
|
else:
|
|
os.close(fd)
|
|
raise AssertionError('shared path writable: '+str(protected))
|
|
command = 'mkdir -p src && cd src && DEST=../private.py; cp "$BENCH_ROOT/harness/adapters/example.py" "$DEST"; echo VALUE=9 >> "$DEST"'
|
|
subprocess.run(['bash','-c',command], check=True)
|
|
assert (rd/'workspace/private.py').read_text().endswith('VALUE=9\\n')
|
|
subprocess.run([sys.executable, '-m', 'venv', '--without-pip', '.venv'], check=True)
|
|
subprocess.run(['bash','-c','VENV=.venv; "$VENV/bin/python" -c "print(123)"'], check=True)
|
|
wheel = Path('private_probe-1.0-py3-none-any.whl')
|
|
with zipfile.ZipFile(wheel, 'w') as out:
|
|
out.writestr('private_probe.py', 'VALUE=11')
|
|
out.writestr('private_probe-1.0.dist-info/METADATA', 'Metadata-Version: 2.1\\nName: private-probe\\nVersion: 1.0\\n')
|
|
out.writestr('private_probe-1.0.dist-info/WHEEL', 'Wheel-Version: 1.0\\nRoot-Is-Purelib: true\\nTag: py3-none-any\\n')
|
|
out.writestr('private_probe-1.0.dist-info/RECORD', '')
|
|
subprocess.run([sys.executable, '-m', 'pip', 'install', '--no-deps', '--no-index',
|
|
'--target', 'private-packages', str(wheel)], check=True)
|
|
assert Path('private-packages/private_probe.py').exists()
|
|
assert (Path.home()).is_relative_to(rd/'workspace')
|
|
for device in set(Path('/dev').glob('nvidia*')) | {Path('/dev/dri'), Path('/dev/kfd')}:
|
|
if device.is_dir():
|
|
assert not list(device.iterdir()), str(device)
|
|
elif device.exists():
|
|
assert device.stat().st_rdev == Path('/dev/null').stat().st_rdev, str(device)
|
|
print('shared GPU device access absent or masked')
|
|
assert len(list(Path('/proc').glob('[0-9]*'))) < 15
|
|
result = subprocess.run(['mount','-o','remount,bind,rw',str(rd)], capture_output=True)
|
|
assert result.returncode != 0
|
|
print('private workspace experiments and protected host verified')
|
|
""")
|
|
env = dict(os.environ, BENCH_CONFIG=str(config), BENCH_HOST_HOME=str(host_home),
|
|
BENCH_TEST_HOST_SENTINEL=str(Path(host_directory) / "host-credential-sentinel"),
|
|
BENCH_OUTER_WORKER="1", HOME=str(rd / "workspace/.agent-home"),
|
|
PYTHONDONTWRITEBYTECODE="1")
|
|
result = subprocess.run(['unshare','-Urmpf','--mount-proc',sys.executable,'-B',
|
|
str(bench/'scripts/jail_setup.py'),str(bench),str(rd),'e2e','--',
|
|
sys.executable,'-B','-c',probe], env=env, capture_output=True,
|
|
text=True, timeout=90)
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
self.assertEqual((bench / "harness/adapters/example.py").read_text(), "VALUE = 7\n")
|
|
self.assertEqual((Path(host_directory) / "host-credential-sentinel").read_text(), "private-host-fixture")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|