data-gen-ebd/tests/test_bos_import.py
amos de3755c14b fix: keep the Bos grader away from candidate processes
The Bos wrapper ran check.py and every candidate process as UID 65534, so a
candidate could read the grader copy through /proc/<checker>/cwd and pass
without implementing anything. check.py now runs as root on a 0700 grader
copy; its subprocesses drop to 65534 and only the work copy and mkdtemp
scratch directories are handed over. An audit hook ends the check as
UNVERIFIED if candidate code would run inside the checker.

36 tasks run hidden tests inside the candidate process (tests.py style and
S500_a2_426889b9a968); they carry isolation-blocked.txt and always grade
UNVERIFIED. The importer adds the marker for any extra root *.py. All 74 Bos
tasks get the new verify.py and a version bump.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 12:33:34 +08:00

168 lines
9.9 KiB
Python

import contextlib
import io
import json
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from rollout import bos_import, bos_verify, core
CHECK = """import pathlib,subprocess,sys
work=pathlib.Path(sys.argv[1])
ok=(work/'a.c').read_text().strip() in ('gold','alt') and (work/'a.h').read_text()=='h\\n'
print('PASS' if ok else 'FAIL: behavior');sys.exit(0 if ok else 1)
"""
class BosImportTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name).resolve()
self.env = self.root / "S500_x_001"
files = {
"task.json": json.dumps(dict(env_id="S500_x_001", task="按 README.md 完成。", final_targets=["a.c", "a.h"],
image="python@sha256:" + "0" * 64, budget_s=300, origin="original_authored")),
"template/README.md": "spec\n", "template/a.c": "todo\n", "template/a.h": "h\n",
"check.py": CHECK,
"gold.sh": "set -eu\nmkdir -p .\ncat > 'a.c' <<'EOF_A'\ngold\nEOF_A\n",
"cheat_exit_zero.sh": "set -eu\npython3 -B - <<'W'\nfrom pathlib import Path\nPath('a.c').write_text('exit')\nW\n",
"cheat_wrong.sh": "set -eu\npython3 -B - <<'W'\nfrom pathlib import Path\nfiles={'a.c': 'bad\\n'}\n"
"for name,body in files.items():\n p=Path(name);p.parent.mkdir(parents=True,exist_ok=True)"
";p.write_text(body)\nW\n",
"valid_alt.sh": "cat > a.c <<'X'\nalt\nX\n",
}
for name, text in files.items():
self.env.joinpath(name).parent.mkdir(parents=True, exist_ok=True)
self.env.joinpath(name).write_text(text)
def test_fixture_scripts_are_parsed_not_run_and_calibrate_through_the_wrapper(self):
task = bos_import.write(self.env, self.root / "task", "family")
self.assertEqual(task["fixtures"], dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL",
valid_alt="PASS"))
self.assertEqual(task["bos"]["fixture_scripts"]["noop"], "cheat_exit_zero.sh")
self.assertEqual(task["_files"]["gold/a.c"], b"gold\n")
self.assertEqual(task["_files"]["gold/a.h"], b"h\n")
self.assertEqual(task["_files"]["noop/a.c"], b"exit")
self.assertEqual(task["_files"]["cheat/a.c"], b"bad\n")
self.assertEqual(task["grader_files"], ["check.py"])
self.assertIn("`a.c`、`a.h`", task["_files"]["prompt.md"].decode())
result = json.loads(core.calibrate(task["_root"], self.root / "runs", trusted_local=True)
.joinpath("receipt.json").read_text())
self.assertEqual((result["status"], len(result["trials"])), ("PASS", 25))
with self.assertRaises(FileExistsError):
bos_import.write(self.env, self.root / "task", "family")
self.env.joinpath("tests.py").write_text("import a\n")
blocked = bos_import.write(self.env, self.root / "blocked", "family")
self.assertEqual(blocked["grader_files"], ["check.py", "isolation-blocked.txt", "tests.py"])
result = json.loads(core.calibrate(blocked["_root"], self.root / "runs", trusted_local=True)
.joinpath("receipt.json").read_text())
self.assertEqual(result["status"], "UNVERIFIED")
def test_isolated_wrapper_keeps_the_grader_private_and_gives_only_the_work_copy_away(self):
candidate = self.root / "candidate"
candidate.mkdir()
candidate.joinpath("a.c").write_text("gold\n")
modes, given = {}, []
def check(argv, **kwargs):
root = Path(argv[-1]).parent
self.assertEqual((argv[-3], Path(argv[-2]).name), ("1", "check.py"))
self.assertNotIn("preexec_fn", kwargs)
self.assertEqual((modes[root], modes[root / "grader"]), (0o711, 0o700))
self.assertEqual({Path(path).relative_to(root).parts[0] for path, *_ in given}, {"work"})
return SimpleNamespace(returncode=0, stdout="PASS\n", stderr="")
with patch.object(bos_verify, "isolation", return_value=lambda: None), \
patch.object(bos_verify.os, "chmod", side_effect=lambda path, mode, **kwargs: modes.update({Path(path): mode})), \
patch.object(bos_verify.os, "lchown", side_effect=lambda *args: given.append(args)), \
patch.object(bos_verify.subprocess, "run", side_effect=check):
self.assertEqual(bos_verify.verify(candidate)["status"], "PASS")
self.assertEqual(len(given), 2)
def test_checks_that_would_run_candidate_code_in_the_grader_process_are_unverified(self):
grader, candidate = self.root / "grader", self.root / "candidate"
grader.mkdir()
candidate.mkdir()
candidate.joinpath("cand.py").write_text("print('PASS')\n")
for check in ("import sys\nsys.path.insert(0,sys.argv[1])\nimport cand\n",
"import runpy,sys\nrunpy.run_path(sys.argv[1]+'/cand.py')\n",
"import pathlib,shutil,sys,tempfile\nd=tempfile.mkdtemp()\n"
"shutil.copy(sys.argv[1]+'/cand.py',d)\nexec(compile(open(d+'/cand.py').read(),d+'/cand.py','exec'))\n"):
with self.subTest(check=check):
grader.joinpath("check.py").write_text(check)
with patch.object(bos_verify, "GRADER", grader), contextlib.redirect_stderr(io.StringIO()), \
self.assertRaisesRegex(bos_verify.UnsupportedEnvironment, "grader process"):
bos_verify.verify(candidate)
grader.joinpath("check.py").write_text("print('PASS')\n")
grader.joinpath("isolation-blocked.txt").write_text("hidden tests share the candidate process\n")
with patch.object(bos_verify, "GRADER", grader), \
self.assertRaisesRegex(bos_verify.UnsupportedEnvironment, "share the candidate process"):
bos_verify.verify(candidate)
def test_wrapper_keeps_the_original_verdict_and_copies_failing_child_stderr(self):
grader, candidate = self.root / "grader", self.root / "candidate"
grader.mkdir()
candidate.mkdir()
candidate.joinpath("a.c").write_text("x")
grader.joinpath("tests.py").write_text("import sys\nsys.exit('AssertionError: case 7 mismatch')\n")
grader.joinpath("check.py").write_text(
"import pathlib,subprocess,sys\nhere=pathlib.Path(__file__).resolve().parent\n"
"p=subprocess.run([sys.executable,str(here/'tests.py')],capture_output=True,text=True)\n"
"print('PASS' if p.returncode==0 else 'FAIL: contract verification failed');sys.exit(p.returncode and 1)\n")
for tests, verdict in (("import sys\nsys.exit('AssertionError: case 7 mismatch')\n", "FAIL"), ("", "PASS")):
grader.joinpath("tests.py").write_text(tests)
stderr = io.StringIO()
with patch.object(bos_verify, "GRADER", grader), contextlib.redirect_stderr(stderr):
result = bos_verify.verify(candidate)
self.assertEqual(result["status"], verdict)
self.assertEqual("case 7 mismatch" in stderr.getvalue(), verdict == "FAIL")
self.assertEqual(result, {"status": "PASS", "checks": 1})
def test_hidden_names_must_be_published_and_every_admitted_task_complies(self):
self.assertEqual(bos_import.contract_gaps("实现process(x)与feed", "import mod as m\nm.process(1);m.feed(2)",
["mod.py"]), [])
self.assertEqual(bos_import.contract_gaps("实现feed", "import mod as m\nm.process(1)\nfrom mod import (feed,\n other)",
["mod.py"]), ["other", "process"])
self.env.joinpath("tests.py").write_text("import a as m\nm.secret_api()\n")
self.env.joinpath("template/a.py").write_text("")
self.env.joinpath("task.json").write_text(json.dumps(dict(env_id="S500_x_001", task="按 README.md 完成。",
final_targets=["a.c", "a.h", "a.py"])))
for name in ("gold.sh", "cheat_exit_zero.sh", "cheat_wrong.sh", "valid_alt.sh"):
self.env.joinpath(name).write_text("set -eu\n")
with self.assertRaises(ValueError) as error:
bos_import.load(self.env)
self.assertIn("secret_api", str(error.exception))
tasks = Path(__file__).resolve().parents[1] / "tasks"
for path in sorted(tasks.iterdir()):
with self.subTest(task=path.name):
self.assertEqual(bos_import.task_contract_gaps(core.load_task(path)), [])
def test_any_other_command_or_write_is_rejected(self):
for name, text in (("gold.sh", "set -eu\nrm -rf /tmp/x\n"),
("gold.sh", "cat > 'other.c' <<'E'\nx\nE\n"),
("gold.sh", "cat > a.c <<'E'\nx\n"),
("gold.sh", "cat > a.c <<E\nx\nE\n"),
("cheat_wrong.sh", "python3 -B - <<'W'\nfrom pathlib import Path\nimport os\nW\n"),
("cheat_wrong.sh", "python3 -B - <<'W'\nfrom pathlib import Path\n"
"Path('a.c').write_text(open('/etc/passwd').read())\nW\n"),
("helper.sh", "true\n")):
with self.subTest(name=name, text=text):
original = self.env.joinpath(name).read_text() if self.env.joinpath(name).exists() else None
self.env.joinpath(name).write_text(text)
with self.assertRaises(ValueError):
bos_import.load(self.env)
if original is None:
self.env.joinpath(name).unlink()
else:
self.env.joinpath(name).write_text(original)
self.env.joinpath("cheat_wrong.sh").unlink()
with self.assertRaises(ValueError):
bos_import.load(self.env)
if __name__ == "__main__":
unittest.main()