The fixture grammar is straight-line heredocs with no branches, so a script that writes the same target twice has a well-defined result: its last write. 55 rglob-type environments in batch 16 were refused only because their no-op fixture writes a module twice. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
177 lines
11 KiB
Python
177 lines
11 KiB
Python
import contextlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from rollout import bos_import, bos_verify, core
|
|
|
|
CHECK = """import pathlib,subprocess,sys
|
|
work=pathlib.Path(sys.argv[1])
|
|
ok=(work/'a.c').read_text().strip() in ('gold','alt') and (work/'a.h').read_text()=='h\\n'
|
|
print('PASS' if ok else 'FAIL: behavior');sys.exit(0 if ok else 1)
|
|
"""
|
|
|
|
|
|
class BosImportTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name).resolve()
|
|
self.env = self.root / "S500_x_001"
|
|
files = {
|
|
"task.json": json.dumps(dict(env_id="S500_x_001", task="按 README.md 完成。", final_targets=["a.c", "a.h"],
|
|
image="python@sha256:" + "0" * 64, budget_s=300, origin="original_authored")),
|
|
"template/README.md": "spec\n", "template/a.c": "todo\n", "template/a.h": "h\n",
|
|
"check.py": CHECK,
|
|
"gold.sh": "set -eu\nmkdir -p .\ncat > 'a.c' <<'EOF_A'\ngold\nEOF_A\n",
|
|
"cheat_exit_zero.sh": "set -eu\npython3 -B - <<'W'\nfrom pathlib import Path\nPath('a.c').write_text('exit')\nW\n",
|
|
"cheat_wrong.sh": "set -eu\npython3 -B - <<'W'\nfrom pathlib import Path\nfiles={'a.c': 'bad\\n'}\n"
|
|
"for name,body in files.items():\n p=Path(name);p.parent.mkdir(parents=True,exist_ok=True)"
|
|
";p.write_text(body)\nW\n",
|
|
"valid_alt.sh": "cat > a.c <<'X'\nalt\nX\n",
|
|
}
|
|
for name, text in files.items():
|
|
self.env.joinpath(name).parent.mkdir(parents=True, exist_ok=True)
|
|
self.env.joinpath(name).write_text(text)
|
|
|
|
def test_fixture_scripts_are_parsed_not_run_and_calibrate_through_the_wrapper(self):
|
|
task = bos_import.write(self.env, self.root / "task", "family")
|
|
self.assertEqual(task["fixtures"], dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL",
|
|
valid_alt="PASS"))
|
|
self.assertEqual(task["bos"]["fixture_scripts"]["noop"], "cheat_exit_zero.sh")
|
|
self.assertEqual(task["_files"]["gold/a.c"], b"gold\n")
|
|
self.assertEqual(task["_files"]["gold/a.h"], b"h\n")
|
|
self.assertEqual(task["_files"]["noop/a.c"], b"exit")
|
|
self.assertEqual(task["_files"]["cheat/a.c"], b"bad\n")
|
|
self.assertEqual(task["grader_files"], ["check.py"])
|
|
self.assertIn("`a.c`、`a.h`", task["_files"]["prompt.md"].decode())
|
|
result = json.loads(core.calibrate(task["_root"], self.root / "runs", trusted_local=True)
|
|
.joinpath("receipt.json").read_text())
|
|
self.assertEqual((result["status"], len(result["trials"])), ("PASS", 25))
|
|
with self.assertRaises(FileExistsError):
|
|
bos_import.write(self.env, self.root / "task", "family")
|
|
self.env.joinpath("tests.py").write_text("import a\n")
|
|
blocked = bos_import.write(self.env, self.root / "blocked", "family")
|
|
self.assertEqual(blocked["grader_files"], ["check.py", "isolation-blocked.txt", "tests.py"])
|
|
result = json.loads(core.calibrate(blocked["_root"], self.root / "runs", trusted_local=True)
|
|
.joinpath("receipt.json").read_text())
|
|
self.assertEqual(result["status"], "UNVERIFIED")
|
|
|
|
def test_isolated_wrapper_keeps_the_grader_private_and_gives_only_the_work_copy_away(self):
|
|
candidate = self.root / "candidate"
|
|
candidate.mkdir()
|
|
candidate.joinpath("a.c").write_text("gold\n")
|
|
modes, given = {}, []
|
|
|
|
def check(argv, **kwargs):
|
|
root = Path(argv[-1]).parent
|
|
self.assertEqual((argv[-3], Path(argv[-2]).name), ("1", "check.py"))
|
|
self.assertNotIn("preexec_fn", kwargs)
|
|
self.assertEqual((modes[root], modes[root / "grader"]), (0o711, 0o700))
|
|
self.assertEqual({Path(path).relative_to(root).parts[0] for path, *_ in given}, {"work"})
|
|
return SimpleNamespace(returncode=0, stdout="PASS\n", stderr="")
|
|
|
|
with patch.object(bos_verify, "isolation", return_value=lambda: None), \
|
|
patch.object(bos_verify.os, "chmod", side_effect=lambda path, mode, **kwargs: modes.update({Path(path): mode})), \
|
|
patch.object(bos_verify.os, "lchown", side_effect=lambda *args: given.append(args)), \
|
|
patch.object(bos_verify.subprocess, "run", side_effect=check):
|
|
self.assertEqual(bos_verify.verify(candidate)["status"], "PASS")
|
|
self.assertEqual(len(given), 2)
|
|
|
|
def test_checks_that_would_run_candidate_code_in_the_grader_process_are_unverified(self):
|
|
grader, candidate = self.root / "grader", self.root / "candidate"
|
|
grader.mkdir()
|
|
candidate.mkdir()
|
|
candidate.joinpath("cand.py").write_text("print('PASS')\n")
|
|
for check in ("import sys\nsys.path.insert(0,sys.argv[1])\nimport cand\n",
|
|
"import runpy,sys\nrunpy.run_path(sys.argv[1]+'/cand.py')\n",
|
|
"import pathlib,shutil,sys,tempfile\nd=tempfile.mkdtemp()\n"
|
|
"shutil.copy(sys.argv[1]+'/cand.py',d)\nexec(compile(open(d+'/cand.py').read(),d+'/cand.py','exec'))\n"):
|
|
with self.subTest(check=check):
|
|
grader.joinpath("check.py").write_text(check)
|
|
with patch.object(bos_verify, "GRADER", grader), contextlib.redirect_stderr(io.StringIO()), \
|
|
self.assertRaisesRegex(bos_verify.UnsupportedEnvironment, "grader process"):
|
|
bos_verify.verify(candidate)
|
|
grader.joinpath("check.py").write_text("print('PASS')\n")
|
|
grader.joinpath("isolation-blocked.txt").write_text("hidden tests share the candidate process\n")
|
|
with patch.object(bos_verify, "GRADER", grader), \
|
|
self.assertRaisesRegex(bos_verify.UnsupportedEnvironment, "share the candidate process"):
|
|
bos_verify.verify(candidate)
|
|
|
|
def test_wrapper_keeps_the_original_verdict_and_copies_failing_child_stderr(self):
|
|
grader, candidate = self.root / "grader", self.root / "candidate"
|
|
grader.mkdir()
|
|
candidate.mkdir()
|
|
candidate.joinpath("a.c").write_text("x")
|
|
grader.joinpath("tests.py").write_text("import sys\nsys.exit('AssertionError: case 7 mismatch')\n")
|
|
grader.joinpath("check.py").write_text(
|
|
"import pathlib,subprocess,sys\nhere=pathlib.Path(__file__).resolve().parent\n"
|
|
"p=subprocess.run([sys.executable,str(here/'tests.py')],capture_output=True,text=True)\n"
|
|
"print('PASS' if p.returncode==0 else 'FAIL: contract verification failed');sys.exit(p.returncode and 1)\n")
|
|
for tests, verdict in (("import sys\nsys.exit('AssertionError: case 7 mismatch')\n", "FAIL"), ("", "PASS")):
|
|
grader.joinpath("tests.py").write_text(tests)
|
|
stderr = io.StringIO()
|
|
with patch.object(bos_verify, "GRADER", grader), contextlib.redirect_stderr(stderr):
|
|
result = bos_verify.verify(candidate)
|
|
self.assertEqual(result["status"], verdict)
|
|
self.assertEqual("case 7 mismatch" in stderr.getvalue(), verdict == "FAIL")
|
|
self.assertEqual(result, {"status": "PASS", "checks": 1})
|
|
|
|
def test_a_fixture_script_that_rewrites_a_target_keeps_the_last_write(self):
|
|
script = "set -e\ncat > a.c <<'EOF'\nfirst\nEOF\ncat > ./a.c <<'EOF'\nsecond\nEOF\n"
|
|
self.assertEqual(bos_import._script(script, ["a.c"]), {"a.c": "second\n"})
|
|
with self.assertRaises(ValueError):
|
|
bos_import._script("cat > b.c <<'EOF'\nx\nEOF\n", ["a.c"])
|
|
|
|
def test_hidden_names_must_be_published_and_every_admitted_task_complies(self):
|
|
self.assertEqual(bos_import.contract_gaps("实现process(x)与feed", "import mod as m\nm.process(1);m.feed(2)",
|
|
["mod.py"]), [])
|
|
self.assertEqual(bos_import.contract_gaps("实现feed", "import mod as m\nm.process(1)\nfrom mod import (feed,\n other)",
|
|
["mod.py"]), ["other", "process"])
|
|
# an unparenthesized import ends at its line; later statements are not imported names
|
|
self.assertEqual(bos_import.contract_gaps("encode/decode", "from mod import encode,decode\nfrom stream import Decoder\n"
|
|
"vectors = []\ndef oracle(): pass", ["mod.py"]), [])
|
|
self.env.joinpath("tests.py").write_text("import a as m\nm.secret_api()\n")
|
|
self.env.joinpath("template/a.py").write_text("")
|
|
self.env.joinpath("task.json").write_text(json.dumps(dict(env_id="S500_x_001", task="按 README.md 完成。",
|
|
final_targets=["a.c", "a.h", "a.py"])))
|
|
for name in ("gold.sh", "cheat_exit_zero.sh", "cheat_wrong.sh", "valid_alt.sh"):
|
|
self.env.joinpath(name).write_text("set -eu\n")
|
|
with self.assertRaises(ValueError) as error:
|
|
bos_import.load(self.env)
|
|
self.assertIn("secret_api", str(error.exception))
|
|
tasks = Path(__file__).resolve().parents[1] / "tasks"
|
|
for path in sorted(tasks.iterdir()):
|
|
with self.subTest(task=path.name):
|
|
self.assertEqual(bos_import.task_contract_gaps(core.load_task(path)), [])
|
|
|
|
def test_any_other_command_or_write_is_rejected(self):
|
|
for name, text in (("gold.sh", "set -eu\nrm -rf /tmp/x\n"),
|
|
("gold.sh", "cat > 'other.c' <<'E'\nx\nE\n"),
|
|
("gold.sh", "cat > a.c <<'E'\nx\n"),
|
|
("gold.sh", "cat > a.c <<E\nx\nE\n"),
|
|
("cheat_wrong.sh", "python3 -B - <<'W'\nfrom pathlib import Path\nimport os\nW\n"),
|
|
("cheat_wrong.sh", "python3 -B - <<'W'\nfrom pathlib import Path\n"
|
|
"Path('a.c').write_text(open('/etc/passwd').read())\nW\n"),
|
|
("helper.sh", "true\n")):
|
|
with self.subTest(name=name, text=text):
|
|
original = self.env.joinpath(name).read_text() if self.env.joinpath(name).exists() else None
|
|
self.env.joinpath(name).write_text(text)
|
|
with self.assertRaises(ValueError):
|
|
bos_import.load(self.env)
|
|
if original is None:
|
|
self.env.joinpath(name).unlink()
|
|
else:
|
|
self.env.joinpath(name).write_text(original)
|
|
self.env.joinpath("cheat_wrong.sh").unlink()
|
|
with self.assertRaises(ValueError):
|
|
bos_import.load(self.env)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|