The Bos wrapper ran check.py and every candidate process as UID 65534, so a candidate could read the grader copy through /proc/<checker>/cwd and pass without implementing anything. check.py now runs as root on a 0700 grader copy; its subprocesses drop to 65534 and only the work copy and mkdtemp scratch directories are handed over. An audit hook ends the check as UNVERIFIED if candidate code would run inside the checker. 36 tasks run hidden tests inside the candidate process (tests.py style and S500_a2_426889b9a968); they carry isolation-blocked.txt and always grade UNVERIFIED. The importer adds the marker for any extra root *.py. All 74 Bos tasks get the new verify.py and a version bump. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
168 lines
9.9 KiB
Python
168 lines
9.9 KiB
Python
import contextlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from rollout import bos_import, bos_verify, core
|
|
|
|
CHECK = """import pathlib,subprocess,sys
|
|
work=pathlib.Path(sys.argv[1])
|
|
ok=(work/'a.c').read_text().strip() in ('gold','alt') and (work/'a.h').read_text()=='h\\n'
|
|
print('PASS' if ok else 'FAIL: behavior');sys.exit(0 if ok else 1)
|
|
"""
|
|
|
|
|
|
class BosImportTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name).resolve()
|
|
self.env = self.root / "S500_x_001"
|
|
files = {
|
|
"task.json": json.dumps(dict(env_id="S500_x_001", task="按 README.md 完成。", final_targets=["a.c", "a.h"],
|
|
image="python@sha256:" + "0" * 64, budget_s=300, origin="original_authored")),
|
|
"template/README.md": "spec\n", "template/a.c": "todo\n", "template/a.h": "h\n",
|
|
"check.py": CHECK,
|
|
"gold.sh": "set -eu\nmkdir -p .\ncat > 'a.c' <<'EOF_A'\ngold\nEOF_A\n",
|
|
"cheat_exit_zero.sh": "set -eu\npython3 -B - <<'W'\nfrom pathlib import Path\nPath('a.c').write_text('exit')\nW\n",
|
|
"cheat_wrong.sh": "set -eu\npython3 -B - <<'W'\nfrom pathlib import Path\nfiles={'a.c': 'bad\\n'}\n"
|
|
"for name,body in files.items():\n p=Path(name);p.parent.mkdir(parents=True,exist_ok=True)"
|
|
";p.write_text(body)\nW\n",
|
|
"valid_alt.sh": "cat > a.c <<'X'\nalt\nX\n",
|
|
}
|
|
for name, text in files.items():
|
|
self.env.joinpath(name).parent.mkdir(parents=True, exist_ok=True)
|
|
self.env.joinpath(name).write_text(text)
|
|
|
|
def test_fixture_scripts_are_parsed_not_run_and_calibrate_through_the_wrapper(self):
|
|
task = bos_import.write(self.env, self.root / "task", "family")
|
|
self.assertEqual(task["fixtures"], dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL",
|
|
valid_alt="PASS"))
|
|
self.assertEqual(task["bos"]["fixture_scripts"]["noop"], "cheat_exit_zero.sh")
|
|
self.assertEqual(task["_files"]["gold/a.c"], b"gold\n")
|
|
self.assertEqual(task["_files"]["gold/a.h"], b"h\n")
|
|
self.assertEqual(task["_files"]["noop/a.c"], b"exit")
|
|
self.assertEqual(task["_files"]["cheat/a.c"], b"bad\n")
|
|
self.assertEqual(task["grader_files"], ["check.py"])
|
|
self.assertIn("`a.c`、`a.h`", task["_files"]["prompt.md"].decode())
|
|
result = json.loads(core.calibrate(task["_root"], self.root / "runs", trusted_local=True)
|
|
.joinpath("receipt.json").read_text())
|
|
self.assertEqual((result["status"], len(result["trials"])), ("PASS", 25))
|
|
with self.assertRaises(FileExistsError):
|
|
bos_import.write(self.env, self.root / "task", "family")
|
|
self.env.joinpath("tests.py").write_text("import a\n")
|
|
blocked = bos_import.write(self.env, self.root / "blocked", "family")
|
|
self.assertEqual(blocked["grader_files"], ["check.py", "isolation-blocked.txt", "tests.py"])
|
|
result = json.loads(core.calibrate(blocked["_root"], self.root / "runs", trusted_local=True)
|
|
.joinpath("receipt.json").read_text())
|
|
self.assertEqual(result["status"], "UNVERIFIED")
|
|
|
|
def test_isolated_wrapper_keeps_the_grader_private_and_gives_only_the_work_copy_away(self):
|
|
candidate = self.root / "candidate"
|
|
candidate.mkdir()
|
|
candidate.joinpath("a.c").write_text("gold\n")
|
|
modes, given = {}, []
|
|
|
|
def check(argv, **kwargs):
|
|
root = Path(argv[-1]).parent
|
|
self.assertEqual((argv[-3], Path(argv[-2]).name), ("1", "check.py"))
|
|
self.assertNotIn("preexec_fn", kwargs)
|
|
self.assertEqual((modes[root], modes[root / "grader"]), (0o711, 0o700))
|
|
self.assertEqual({Path(path).relative_to(root).parts[0] for path, *_ in given}, {"work"})
|
|
return SimpleNamespace(returncode=0, stdout="PASS\n", stderr="")
|
|
|
|
with patch.object(bos_verify, "isolation", return_value=lambda: None), \
|
|
patch.object(bos_verify.os, "chmod", side_effect=lambda path, mode, **kwargs: modes.update({Path(path): mode})), \
|
|
patch.object(bos_verify.os, "lchown", side_effect=lambda *args: given.append(args)), \
|
|
patch.object(bos_verify.subprocess, "run", side_effect=check):
|
|
self.assertEqual(bos_verify.verify(candidate)["status"], "PASS")
|
|
self.assertEqual(len(given), 2)
|
|
|
|
def test_checks_that_would_run_candidate_code_in_the_grader_process_are_unverified(self):
|
|
grader, candidate = self.root / "grader", self.root / "candidate"
|
|
grader.mkdir()
|
|
candidate.mkdir()
|
|
candidate.joinpath("cand.py").write_text("print('PASS')\n")
|
|
for check in ("import sys\nsys.path.insert(0,sys.argv[1])\nimport cand\n",
|
|
"import runpy,sys\nrunpy.run_path(sys.argv[1]+'/cand.py')\n",
|
|
"import pathlib,shutil,sys,tempfile\nd=tempfile.mkdtemp()\n"
|
|
"shutil.copy(sys.argv[1]+'/cand.py',d)\nexec(compile(open(d+'/cand.py').read(),d+'/cand.py','exec'))\n"):
|
|
with self.subTest(check=check):
|
|
grader.joinpath("check.py").write_text(check)
|
|
with patch.object(bos_verify, "GRADER", grader), contextlib.redirect_stderr(io.StringIO()), \
|
|
self.assertRaisesRegex(bos_verify.UnsupportedEnvironment, "grader process"):
|
|
bos_verify.verify(candidate)
|
|
grader.joinpath("check.py").write_text("print('PASS')\n")
|
|
grader.joinpath("isolation-blocked.txt").write_text("hidden tests share the candidate process\n")
|
|
with patch.object(bos_verify, "GRADER", grader), \
|
|
self.assertRaisesRegex(bos_verify.UnsupportedEnvironment, "share the candidate process"):
|
|
bos_verify.verify(candidate)
|
|
|
|
def test_wrapper_keeps_the_original_verdict_and_copies_failing_child_stderr(self):
|
|
grader, candidate = self.root / "grader", self.root / "candidate"
|
|
grader.mkdir()
|
|
candidate.mkdir()
|
|
candidate.joinpath("a.c").write_text("x")
|
|
grader.joinpath("tests.py").write_text("import sys\nsys.exit('AssertionError: case 7 mismatch')\n")
|
|
grader.joinpath("check.py").write_text(
|
|
"import pathlib,subprocess,sys\nhere=pathlib.Path(__file__).resolve().parent\n"
|
|
"p=subprocess.run([sys.executable,str(here/'tests.py')],capture_output=True,text=True)\n"
|
|
"print('PASS' if p.returncode==0 else 'FAIL: contract verification failed');sys.exit(p.returncode and 1)\n")
|
|
for tests, verdict in (("import sys\nsys.exit('AssertionError: case 7 mismatch')\n", "FAIL"), ("", "PASS")):
|
|
grader.joinpath("tests.py").write_text(tests)
|
|
stderr = io.StringIO()
|
|
with patch.object(bos_verify, "GRADER", grader), contextlib.redirect_stderr(stderr):
|
|
result = bos_verify.verify(candidate)
|
|
self.assertEqual(result["status"], verdict)
|
|
self.assertEqual("case 7 mismatch" in stderr.getvalue(), verdict == "FAIL")
|
|
self.assertEqual(result, {"status": "PASS", "checks": 1})
|
|
|
|
def test_hidden_names_must_be_published_and_every_admitted_task_complies(self):
|
|
self.assertEqual(bos_import.contract_gaps("实现process(x)与feed", "import mod as m\nm.process(1);m.feed(2)",
|
|
["mod.py"]), [])
|
|
self.assertEqual(bos_import.contract_gaps("实现feed", "import mod as m\nm.process(1)\nfrom mod import (feed,\n other)",
|
|
["mod.py"]), ["other", "process"])
|
|
self.env.joinpath("tests.py").write_text("import a as m\nm.secret_api()\n")
|
|
self.env.joinpath("template/a.py").write_text("")
|
|
self.env.joinpath("task.json").write_text(json.dumps(dict(env_id="S500_x_001", task="按 README.md 完成。",
|
|
final_targets=["a.c", "a.h", "a.py"])))
|
|
for name in ("gold.sh", "cheat_exit_zero.sh", "cheat_wrong.sh", "valid_alt.sh"):
|
|
self.env.joinpath(name).write_text("set -eu\n")
|
|
with self.assertRaises(ValueError) as error:
|
|
bos_import.load(self.env)
|
|
self.assertIn("secret_api", str(error.exception))
|
|
tasks = Path(__file__).resolve().parents[1] / "tasks"
|
|
for path in sorted(tasks.iterdir()):
|
|
with self.subTest(task=path.name):
|
|
self.assertEqual(bos_import.task_contract_gaps(core.load_task(path)), [])
|
|
|
|
def test_any_other_command_or_write_is_rejected(self):
|
|
for name, text in (("gold.sh", "set -eu\nrm -rf /tmp/x\n"),
|
|
("gold.sh", "cat > 'other.c' <<'E'\nx\nE\n"),
|
|
("gold.sh", "cat > a.c <<'E'\nx\n"),
|
|
("gold.sh", "cat > a.c <<E\nx\nE\n"),
|
|
("cheat_wrong.sh", "python3 -B - <<'W'\nfrom pathlib import Path\nimport os\nW\n"),
|
|
("cheat_wrong.sh", "python3 -B - <<'W'\nfrom pathlib import Path\n"
|
|
"Path('a.c').write_text(open('/etc/passwd').read())\nW\n"),
|
|
("helper.sh", "true\n")):
|
|
with self.subTest(name=name, text=text):
|
|
original = self.env.joinpath(name).read_text() if self.env.joinpath(name).exists() else None
|
|
self.env.joinpath(name).write_text(text)
|
|
with self.assertRaises(ValueError):
|
|
bos_import.load(self.env)
|
|
if original is None:
|
|
self.env.joinpath(name).unlink()
|
|
else:
|
|
self.env.joinpath(name).write_text(original)
|
|
self.env.joinpath("cheat_wrong.sh").unlink()
|
|
with self.assertRaises(ValueError):
|
|
bos_import.load(self.env)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|