data-gen-ebd/tests/test_bos_isolation.py
amos de3755c14b fix: keep the Bos grader away from candidate processes
The Bos wrapper ran check.py and every candidate process as UID 65534, so a
candidate could read the grader copy through /proc/<checker>/cwd and pass
without implementing anything. check.py now runs as root on a 0700 grader
copy; its subprocesses drop to 65534 and only the work copy and mkdtemp
scratch directories are handed over. An audit hook ends the check as
UNVERIFIED if candidate code would run inside the checker.

36 tasks run hidden tests inside the candidate process (tests.py style and
S500_a2_426889b9a968); they carry isolation-blocked.txt and always grade
UNVERIFIED. The importer adds the marker for any extra root *.py. All 74 Bos
tasks get the new verify.py and a version bump.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 12:33:34 +08:00

74 lines
2.9 KiB
Python

"""Formal Docker grading must keep the Bos grader away from candidate processes (opt-in: ROLLOUT_TEST_IMAGE)."""
import json
import os
from pathlib import Path
import shutil
import tempfile
import unittest
from unittest.mock import patch
from rollout import bos_verify, core
TASK = Path(__file__).resolve().parents[1] / "tasks" / "S500_n2_070"
# Answers from the check's own fixtures(): reachable through the checker's cwd or command line when
# the candidate shares its identity (reproduced 5/5 PASS on 36, 2026-10-15).
LOOKUP = '''import glob, json, os
def paths():
yield "/proc/%d/cwd/grader/check.py" % os.getppid()
for cmdline in glob.glob("/proc/[0-9]*/cmdline"):
try:
args = open(cmdline, "rb").read().split(b"\\0")
except OSError:
continue
yield from (arg.decode() for arg in args if arg.endswith(b"/grader/check.py"))
def answers():
for path in paths():
try:
source = open(path).read()
except OSError:
continue
scope = {}
exec(source.split("import pathlib,sys,tempfile")[0], scope)
cases, expected = scope["fixtures"]()
return {json.dumps(case, sort_keys=True): row for case, row in zip(cases, expected)}
raise RuntimeError("grader unreadable")
ANSWERS = answers()
def solve(data):
row = ANSWERS[json.dumps(data, sort_keys=True)]
if "error" in row:
raise ValueError(row["error"])
return row["ok"]
'''
@unittest.skipUnless(os.environ.get("ROLLOUT_TEST_IMAGE"), "set ROLLOUT_TEST_IMAGE to a local runtime image")
class BosIsolationTests(unittest.TestCase):
def test_answer_lookup_candidate_fails_under_formal_isolation(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
task = root / "task"
shutil.copytree(TASK, task)
shutil.copyfile(bos_verify.__file__, task / "verify.py")
(task / "cheat_lookup").mkdir()
(task / "cheat_lookup" / "task.py").write_text(LOOKUP)
meta = json.loads((task / "task.json").read_text())
meta["fixtures"] = dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL", cheat_lookup="FAIL")
meta["grader_files"] = ["check.py"]
(task / "task.json").write_text(json.dumps(meta))
with patch.object(core, "MIN_FREE_BYTES", 0):
attempt = core.calibrate(task, root / "runs", image=os.environ["ROLLOUT_TEST_IMAGE"])
receipt = json.loads((attempt / "receipt.json").read_text())
lookups = [trial["verdict"]["status"] for trial in receipt["trials"] if trial["fixture"] == "cheat_lookup"]
self.assertEqual(lookups, ["FAIL"] * 5)
self.assertIn("grader unreadable", (attempt / "cheat_lookup-1.stderr").read_text())
self.assertEqual(receipt["status"], "PASS")
if __name__ == "__main__":
unittest.main()