The Bos wrapper ran check.py and every candidate process as UID 65534, so a candidate could read the grader copy through /proc/<checker>/cwd and pass without implementing anything. check.py now runs as root on a 0700 grader copy; its subprocesses drop to 65534 and only the work copy and mkdtemp scratch directories are handed over. An audit hook ends the check as UNVERIFIED if candidate code would run inside the checker. 36 tasks run hidden tests inside the candidate process (tests.py style and S500_a2_426889b9a968); they carry isolation-blocked.txt and always grade UNVERIFIED. The importer adds the marker for any extra root *.py. All 74 Bos tasks get the new verify.py and a version bump. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
74 lines
2.9 KiB
Python
74 lines
2.9 KiB
Python
"""Formal Docker grading must keep the Bos grader away from candidate processes (opt-in: ROLLOUT_TEST_IMAGE)."""
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from rollout import bos_verify, core
|
|
|
|
TASK = Path(__file__).resolve().parents[1] / "tasks" / "S500_n2_070"
|
|
# Answers from the check's own fixtures(): reachable through the checker's cwd or command line when
|
|
# the candidate shares its identity (reproduced 5/5 PASS on 36, 2026-10-15).
|
|
LOOKUP = '''import glob, json, os
|
|
|
|
def paths():
|
|
yield "/proc/%d/cwd/grader/check.py" % os.getppid()
|
|
for cmdline in glob.glob("/proc/[0-9]*/cmdline"):
|
|
try:
|
|
args = open(cmdline, "rb").read().split(b"\\0")
|
|
except OSError:
|
|
continue
|
|
yield from (arg.decode() for arg in args if arg.endswith(b"/grader/check.py"))
|
|
|
|
def answers():
|
|
for path in paths():
|
|
try:
|
|
source = open(path).read()
|
|
except OSError:
|
|
continue
|
|
scope = {}
|
|
exec(source.split("import pathlib,sys,tempfile")[0], scope)
|
|
cases, expected = scope["fixtures"]()
|
|
return {json.dumps(case, sort_keys=True): row for case, row in zip(cases, expected)}
|
|
raise RuntimeError("grader unreadable")
|
|
|
|
ANSWERS = answers()
|
|
|
|
def solve(data):
|
|
row = ANSWERS[json.dumps(data, sort_keys=True)]
|
|
if "error" in row:
|
|
raise ValueError(row["error"])
|
|
return row["ok"]
|
|
'''
|
|
|
|
|
|
@unittest.skipUnless(os.environ.get("ROLLOUT_TEST_IMAGE"), "set ROLLOUT_TEST_IMAGE to a local runtime image")
|
|
class BosIsolationTests(unittest.TestCase):
|
|
def test_answer_lookup_candidate_fails_under_formal_isolation(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
task = root / "task"
|
|
shutil.copytree(TASK, task)
|
|
shutil.copyfile(bos_verify.__file__, task / "verify.py")
|
|
(task / "cheat_lookup").mkdir()
|
|
(task / "cheat_lookup" / "task.py").write_text(LOOKUP)
|
|
meta = json.loads((task / "task.json").read_text())
|
|
meta["fixtures"] = dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL", cheat_lookup="FAIL")
|
|
meta["grader_files"] = ["check.py"]
|
|
(task / "task.json").write_text(json.dumps(meta))
|
|
with patch.object(core, "MIN_FREE_BYTES", 0):
|
|
attempt = core.calibrate(task, root / "runs", image=os.environ["ROLLOUT_TEST_IMAGE"])
|
|
receipt = json.loads((attempt / "receipt.json").read_text())
|
|
lookups = [trial["verdict"]["status"] for trial in receipt["trials"] if trial["fixture"] == "cheat_lookup"]
|
|
self.assertEqual(lookups, ["FAIL"] * 5)
|
|
self.assertIn("grader unreadable", (attempt / "cheat_lookup-1.stderr").read_text())
|
|
self.assertEqual(receipt["status"], "PASS")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|