data-gen-ebd/tests/test_bos_import.py
amos dd69720c15 Let Bos fixture scripts rewrite a target; the last write wins
The fixture grammar is straight-line heredocs with no branches, so a script that
writes the same target twice has a well-defined result: its last write. 55
rglob-type environments in batch 16 were refused only because their no-op
fixture writes a module twice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 11:40:08 +08:00

177 lines
11 KiB
Python

import contextlib
import io
import json
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from rollout import bos_import, bos_verify, core
CHECK = """import pathlib,subprocess,sys
work=pathlib.Path(sys.argv[1])
ok=(work/'a.c').read_text().strip() in ('gold','alt') and (work/'a.h').read_text()=='h\\n'
print('PASS' if ok else 'FAIL: behavior');sys.exit(0 if ok else 1)
"""
class BosImportTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name).resolve()
self.env = self.root / "S500_x_001"
files = {
"task.json": json.dumps(dict(env_id="S500_x_001", task="按 README.md 完成。", final_targets=["a.c", "a.h"],
image="python@sha256:" + "0" * 64, budget_s=300, origin="original_authored")),
"template/README.md": "spec\n", "template/a.c": "todo\n", "template/a.h": "h\n",
"check.py": CHECK,
"gold.sh": "set -eu\nmkdir -p .\ncat > 'a.c' <<'EOF_A'\ngold\nEOF_A\n",
"cheat_exit_zero.sh": "set -eu\npython3 -B - <<'W'\nfrom pathlib import Path\nPath('a.c').write_text('exit')\nW\n",
"cheat_wrong.sh": "set -eu\npython3 -B - <<'W'\nfrom pathlib import Path\nfiles={'a.c': 'bad\\n'}\n"
"for name,body in files.items():\n p=Path(name);p.parent.mkdir(parents=True,exist_ok=True)"
";p.write_text(body)\nW\n",
"valid_alt.sh": "cat > a.c <<'X'\nalt\nX\n",
}
for name, text in files.items():
self.env.joinpath(name).parent.mkdir(parents=True, exist_ok=True)
self.env.joinpath(name).write_text(text)
def test_fixture_scripts_are_parsed_not_run_and_calibrate_through_the_wrapper(self):
task = bos_import.write(self.env, self.root / "task", "family")
self.assertEqual(task["fixtures"], dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL",
valid_alt="PASS"))
self.assertEqual(task["bos"]["fixture_scripts"]["noop"], "cheat_exit_zero.sh")
self.assertEqual(task["_files"]["gold/a.c"], b"gold\n")
self.assertEqual(task["_files"]["gold/a.h"], b"h\n")
self.assertEqual(task["_files"]["noop/a.c"], b"exit")
self.assertEqual(task["_files"]["cheat/a.c"], b"bad\n")
self.assertEqual(task["grader_files"], ["check.py"])
self.assertIn("`a.c`、`a.h`", task["_files"]["prompt.md"].decode())
result = json.loads(core.calibrate(task["_root"], self.root / "runs", trusted_local=True)
.joinpath("receipt.json").read_text())
self.assertEqual((result["status"], len(result["trials"])), ("PASS", 25))
with self.assertRaises(FileExistsError):
bos_import.write(self.env, self.root / "task", "family")
self.env.joinpath("tests.py").write_text("import a\n")
blocked = bos_import.write(self.env, self.root / "blocked", "family")
self.assertEqual(blocked["grader_files"], ["check.py", "isolation-blocked.txt", "tests.py"])
result = json.loads(core.calibrate(blocked["_root"], self.root / "runs", trusted_local=True)
.joinpath("receipt.json").read_text())
self.assertEqual(result["status"], "UNVERIFIED")
def test_isolated_wrapper_keeps_the_grader_private_and_gives_only_the_work_copy_away(self):
candidate = self.root / "candidate"
candidate.mkdir()
candidate.joinpath("a.c").write_text("gold\n")
modes, given = {}, []
def check(argv, **kwargs):
root = Path(argv[-1]).parent
self.assertEqual((argv[-3], Path(argv[-2]).name), ("1", "check.py"))
self.assertNotIn("preexec_fn", kwargs)
self.assertEqual((modes[root], modes[root / "grader"]), (0o711, 0o700))
self.assertEqual({Path(path).relative_to(root).parts[0] for path, *_ in given}, {"work"})
return SimpleNamespace(returncode=0, stdout="PASS\n", stderr="")
with patch.object(bos_verify, "isolation", return_value=lambda: None), \
patch.object(bos_verify.os, "chmod", side_effect=lambda path, mode, **kwargs: modes.update({Path(path): mode})), \
patch.object(bos_verify.os, "lchown", side_effect=lambda *args: given.append(args)), \
patch.object(bos_verify.subprocess, "run", side_effect=check):
self.assertEqual(bos_verify.verify(candidate)["status"], "PASS")
self.assertEqual(len(given), 2)
def test_checks_that_would_run_candidate_code_in_the_grader_process_are_unverified(self):
grader, candidate = self.root / "grader", self.root / "candidate"
grader.mkdir()
candidate.mkdir()
candidate.joinpath("cand.py").write_text("print('PASS')\n")
for check in ("import sys\nsys.path.insert(0,sys.argv[1])\nimport cand\n",
"import runpy,sys\nrunpy.run_path(sys.argv[1]+'/cand.py')\n",
"import pathlib,shutil,sys,tempfile\nd=tempfile.mkdtemp()\n"
"shutil.copy(sys.argv[1]+'/cand.py',d)\nexec(compile(open(d+'/cand.py').read(),d+'/cand.py','exec'))\n"):
with self.subTest(check=check):
grader.joinpath("check.py").write_text(check)
with patch.object(bos_verify, "GRADER", grader), contextlib.redirect_stderr(io.StringIO()), \
self.assertRaisesRegex(bos_verify.UnsupportedEnvironment, "grader process"):
bos_verify.verify(candidate)
grader.joinpath("check.py").write_text("print('PASS')\n")
grader.joinpath("isolation-blocked.txt").write_text("hidden tests share the candidate process\n")
with patch.object(bos_verify, "GRADER", grader), \
self.assertRaisesRegex(bos_verify.UnsupportedEnvironment, "share the candidate process"):
bos_verify.verify(candidate)
def test_wrapper_keeps_the_original_verdict_and_copies_failing_child_stderr(self):
grader, candidate = self.root / "grader", self.root / "candidate"
grader.mkdir()
candidate.mkdir()
candidate.joinpath("a.c").write_text("x")
grader.joinpath("tests.py").write_text("import sys\nsys.exit('AssertionError: case 7 mismatch')\n")
grader.joinpath("check.py").write_text(
"import pathlib,subprocess,sys\nhere=pathlib.Path(__file__).resolve().parent\n"
"p=subprocess.run([sys.executable,str(here/'tests.py')],capture_output=True,text=True)\n"
"print('PASS' if p.returncode==0 else 'FAIL: contract verification failed');sys.exit(p.returncode and 1)\n")
for tests, verdict in (("import sys\nsys.exit('AssertionError: case 7 mismatch')\n", "FAIL"), ("", "PASS")):
grader.joinpath("tests.py").write_text(tests)
stderr = io.StringIO()
with patch.object(bos_verify, "GRADER", grader), contextlib.redirect_stderr(stderr):
result = bos_verify.verify(candidate)
self.assertEqual(result["status"], verdict)
self.assertEqual("case 7 mismatch" in stderr.getvalue(), verdict == "FAIL")
self.assertEqual(result, {"status": "PASS", "checks": 1})
def test_a_fixture_script_that_rewrites_a_target_keeps_the_last_write(self):
script = "set -e\ncat > a.c <<'EOF'\nfirst\nEOF\ncat > ./a.c <<'EOF'\nsecond\nEOF\n"
self.assertEqual(bos_import._script(script, ["a.c"]), {"a.c": "second\n"})
with self.assertRaises(ValueError):
bos_import._script("cat > b.c <<'EOF'\nx\nEOF\n", ["a.c"])
def test_hidden_names_must_be_published_and_every_admitted_task_complies(self):
self.assertEqual(bos_import.contract_gaps("实现process(x)与feed", "import mod as m\nm.process(1);m.feed(2)",
["mod.py"]), [])
self.assertEqual(bos_import.contract_gaps("实现feed", "import mod as m\nm.process(1)\nfrom mod import (feed,\n other)",
["mod.py"]), ["other", "process"])
# an unparenthesized import ends at its line; later statements are not imported names
self.assertEqual(bos_import.contract_gaps("encode/decode", "from mod import encode,decode\nfrom stream import Decoder\n"
"vectors = []\ndef oracle(): pass", ["mod.py"]), [])
self.env.joinpath("tests.py").write_text("import a as m\nm.secret_api()\n")
self.env.joinpath("template/a.py").write_text("")
self.env.joinpath("task.json").write_text(json.dumps(dict(env_id="S500_x_001", task="按 README.md 完成。",
final_targets=["a.c", "a.h", "a.py"])))
for name in ("gold.sh", "cheat_exit_zero.sh", "cheat_wrong.sh", "valid_alt.sh"):
self.env.joinpath(name).write_text("set -eu\n")
with self.assertRaises(ValueError) as error:
bos_import.load(self.env)
self.assertIn("secret_api", str(error.exception))
tasks = Path(__file__).resolve().parents[1] / "tasks"
for path in sorted(tasks.iterdir()):
with self.subTest(task=path.name):
self.assertEqual(bos_import.task_contract_gaps(core.load_task(path)), [])
def test_any_other_command_or_write_is_rejected(self):
for name, text in (("gold.sh", "set -eu\nrm -rf /tmp/x\n"),
("gold.sh", "cat > 'other.c' <<'E'\nx\nE\n"),
("gold.sh", "cat > a.c <<'E'\nx\n"),
("gold.sh", "cat > a.c <<E\nx\nE\n"),
("cheat_wrong.sh", "python3 -B - <<'W'\nfrom pathlib import Path\nimport os\nW\n"),
("cheat_wrong.sh", "python3 -B - <<'W'\nfrom pathlib import Path\n"
"Path('a.c').write_text(open('/etc/passwd').read())\nW\n"),
("helper.sh", "true\n")):
with self.subTest(name=name, text=text):
original = self.env.joinpath(name).read_text() if self.env.joinpath(name).exists() else None
self.env.joinpath(name).write_text(text)
with self.assertRaises(ValueError):
bos_import.load(self.env)
if original is None:
self.env.joinpath(name).unlink()
else:
self.env.joinpath(name).write_text(original)
self.env.joinpath("cheat_wrong.sh").unlink()
with self.assertRaises(ValueError):
bos_import.load(self.env)
if __name__ == "__main__":
unittest.main()