data-gen-ebd/tests/test_core.py
amos acfa601a56 feat: multi-file task contract, model table and shared I/O helpers
- Tasks may declare up to 16 candidate files; the whole template becomes
  the Agent workspace while only candidate files are frozen and graded.
- Optional fixtures add more cheats or legal alternatives (five trials
  each); optional grader_files ship private tests next to verify.py.
- Tasks without an argv oracle export call_type=verifier ground truth.
- Upstream model routes live in one MODELS table.
- One bounded no-follow reader, one exclusive writer and one SIGTERM
  helper replace per-module copies; drop unused validate_task.

frame-decoder keeps its task digest, calibration receipts and exported
query. Verified on host 36: internal network probe, 20-trial calibration
and a real GLM rollout (PASS, evidence VERIFIED, cleanup confirmed).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 21:25:09 +08:00

401 lines
23 KiB
Python

import json
import hashlib
from pathlib import Path
import subprocess
import tempfile
import unittest
from unittest.mock import patch
from rollout import core
class CoreTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name).resolve()
self.task = self.root / "task"
self.task.mkdir()
self.output = self.root / "out"
self.manifest = dict(id="parser", version="1", family="protocol", source="synthetic",
template="template", verifier="verify.py", candidate_files=["decoder.c"],
timeout_seconds=15)
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
self.task.joinpath("prompt.md").write_text("Repair decoder.c")
self.task.joinpath("verify.py").write_text(
"import json,sys\nfrom pathlib import Path\n"
"ok=Path(sys.argv[1], 'decoder.c').read_text() == 'gold'\n"
"print(json.dumps({'status':'PASS' if ok else 'FAIL','checks':1}))\n"
"sys.exit(0 if ok else 1)\n")
for relative, value in (("template", "buggy"), ("gold", "gold"),
("noop", "noop"), ("cheat", "cheat")):
directory = self.task / relative
directory.mkdir(parents=True)
directory.joinpath("decoder.c").write_text(value)
def receipt(self, attempt):
return json.loads(attempt.joinpath("receipt.json").read_text())
def mock_calibration(self):
def grade(task, attempt, candidate, prefix, image_id):
ok = candidate.joinpath("decoder.c").read_text() == "gold"
return dict(verdict=dict(status="PASS" if ok else "FAIL", checks=1),
process=dict(returncode=0 if ok else 1, termination="exit", cleanup="confirmed"))
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object(
core, "_grade", side_effect=grade):
return core.calibrate(self.task, self.output, image="tag")
def test_task_rejects_unsafe_and_missing_material(self):
self.assertEqual(core.load_task(self.task)["id"], "parser")
for bad in (None, [], ["../decoder.c"], ["/tmp/decoder.c"]):
self.manifest["candidate_files"] = bad
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
with self.assertRaises(ValueError):
core.load_task(self.task)
self.manifest["candidate_files"] = ["decoder.c"]
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
fixture = self.task / "gold/decoder.c"
fixture.unlink()
fixture.symlink_to(self.task / "template/decoder.c")
with self.assertRaises(ValueError):
core.load_task(self.task)
fixture.unlink()
with self.assertRaises(ValueError):
core.load_task(self.task)
def test_size_and_output_boundaries(self):
self.task.joinpath("template/decoder.c").write_bytes(b"x" * (core.MAX_FILE_BYTES + 1))
with self.assertRaises(ValueError):
core.load_task(self.task)
self.task.joinpath("template/decoder.c").write_text("buggy")
with self.assertRaises(ValueError):
core.calibrate(self.task, self.task / "out", trusted_local=True)
with self.assertRaises(ValueError):
core.calibrate(self.task, self.output)
with self.assertRaises(ValueError):
core.calibrate(self.task, self.output, trusted_local=True, image="image")
def test_trusted_calibration_is_five_fresh_trials_per_fixture(self):
attempt = core.calibrate(self.task, self.output, trusted_local=True)
result = self.receipt(attempt)
self.assertEqual(result["status"], "PASS")
self.assertEqual(len(result["trials"]), 20)
self.assertEqual(result["cleanup"], "confirmed")
self.assertEqual(result["training_disposition"], "NOT_TRAIN_READY")
second = core.calibrate(self.task, self.output, trusted_local=True)
self.assertNotEqual(attempt, second)
self.assertTrue(attempt.joinpath("events.jsonl").exists())
def test_grader_cannot_pass_empty_conflicting_or_malformed_output(self):
stdout = self.root / "stdout"
facts = dict(returncode=0, termination="exit", cleanup="confirmed")
for output in ('{"status":"PASS","checks":0}', '{"status":"FAIL","checks":1}',
'noise\n{"status":"PASS","checks":1}', '[]', 'not json'):
stdout.write_text(output)
self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED")
stdout.write_text('{"status":"PASS","checks":1}')
self.assertEqual(core._checker(facts, stdout)["status"], "PASS")
facts["cleanup"] = "unknown"
self.assertEqual(core._checker(facts, stdout)["status"], "INFRA_ERROR")
facts["cleanup"] = "confirmed"
facts["returncode"] = 2
stdout.write_text('{"status":"UNVERIFIED","checks":0}')
self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED")
stdout.write_text("x" * (core.MAX_CHECKER_BYTES + 1))
self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED")
def test_interrupt_stops_calibration_and_preserves_terminal_receipt(self):
with patch.object(core, "_grade", side_effect=KeyboardInterrupt) as grade:
attempt = core.calibrate(self.task, self.output, trusted_local=True)
result = self.receipt(attempt)
self.assertEqual(result["status"], "UNVERIFIED")
self.assertEqual(result["termination"], "interrupted")
self.assertEqual(grade.call_count, 1)
self.assertEqual(result["cleanup"], "unknown")
def test_sigterm_is_recorded_after_active_grader_cleanup(self):
script = ("import os,signal,time\n"
"os.kill(os.getppid(),signal.SIGTERM)\ntime.sleep(10)\n")
self.task.joinpath("verify.py").write_text(script)
attempt = core.calibrate(self.task, self.output, trusted_local=True)
result = self.receipt(attempt)
self.assertEqual(result["status"], "UNVERIFIED")
self.assertEqual(result["termination"], "interrupted")
self.assertEqual(result["cleanup"], "confirmed")
self.assertEqual(len(result["trials"]), 1)
def test_timeout_is_not_a_pass_and_has_process_cleanup(self):
self.task.joinpath("verify.py").write_text("import time\ntime.sleep(10)\n")
self.manifest["timeout_seconds"] = 1
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
task = core.load_task(self.task)
attempt = core._attempt(task, self.output, "test")
candidate = attempt / "candidate"
candidate.mkdir()
candidate.joinpath("decoder.c").write_text("gold")
result = core._grade(task, attempt, candidate, "grade", None)
self.assertEqual(result["verdict"]["status"], "INFRA_ERROR")
self.assertEqual(result["process"]["termination"], "timeout")
self.assertEqual(result["process"]["cleanup"], "confirmed")
def test_docker_run_freezes_and_grades_independent_mounts(self):
commands = []
deadlines = []
image_id = "sha256:" + "a" * 64
calibration = self.mock_calibration()
def cli(argv):
commands.append(argv)
if argv[1:3] == ["image", "inspect"]:
return subprocess.CompletedProcess(argv, 0, image_id + "\n", "")
if argv[1:3] == ["network", "inspect"]:
return subprocess.CompletedProcess(argv, 0, "true\n", "")
if argv[1:3] == ["container", "inspect"]:
return subprocess.CompletedProcess(argv, 1, "", "No such container")
return subprocess.CompletedProcess(argv, 0, "", "")
def process(argv, cwd, timeout, stdout, stderr, stdin=None, controller=False):
commands.append(argv)
deadlines.append(timeout)
stdout.write_text("")
stderr.write_text("")
if stdout.name == "agent.stdout":
cwd.joinpath("workspace/decoder.c").write_text("gold")
cwd.joinpath("trace/native.jsonl").write_text('{"event":"test"}\n')
return dict(returncode=1, termination="exit", cleanup="confirmed")
stdout.write_text('{"status":"PASS","checks":1}\n')
return dict(returncode=0, termination="exit", cleanup="confirmed")
with patch.object(core, "_cli", side_effect=cli), patch.object(core, "_process", side_effect=process):
attempt = core.run(self.task, self.output, "tag", ["agent", "--arg"], "fake-agent@1", calibration,
agent_timeout=120, model_base_url="http://172.22.0.1:8123/v1",
agent_network="rollout-test-network")
result = self.receipt(attempt)
self.assertEqual(result["status"], "PASS")
self.assertEqual(result["agent_exit"], 1)
self.assertEqual(result["image_id"], image_id)
self.assertEqual(result["agent_timeout_seconds"], 120)
self.assertEqual(deadlines, [120, 15])
self.assertEqual(result["native_trace"]["status"], "AVAILABLE")
self.assertEqual(result["native_trace"]["sha256"], hashlib.sha256(b'{"event":"test"}\n').hexdigest())
self.assertEqual(result["training_disposition"], "NOT_TRAIN_READY")
self.assertEqual(attempt.joinpath("candidate/decoder.c").read_text(), "gold")
self.assertEqual(sum(c[1:3] == ["image", "inspect"] for c in commands), 1)
runs = [c for c in commands if c[1] == "run"]
for command in runs:
self.assertIn("exec", command[command.index("--tmpfs") + 1].split(":", 1)[1].split(","))
self.assertNotIn("/grader", " ".join(runs[0]))
self.assertNotIn("/gold", " ".join(runs[0]))
self.assertNotIn("docker.sock", " ".join(runs[0]))
self.assertIn("dst=/trace", " ".join(runs[0]))
self.assertNotIn("dst=/trace", " ".join(runs[1]))
self.assertEqual(runs[0][runs[0].index("--memory") + 1], "2g")
self.assertEqual(runs[1][runs[1].index("--memory") + 1], "512m")
self.assertIn("rollout-test-network", runs[0])
self.assertNotIn("--add-host", runs[0])
self.assertIn("ROLLOUT_MODEL_BASE_URL=http://172.22.0.1:8123/v1", runs[0])
self.assertNotIn("ROLLOUT_MODEL_BASE_URL", " ".join(runs[1]))
self.assertIn("none", runs[1])
self.assertIn("ROLLOUT_GRADE_ISOLATION=uid65534", runs[1])
self.assertIn("SETUID", runs[1])
self.assertIn("SETGID", runs[1])
self.assertIn("CHOWN", runs[1])
self.assertEqual(attempt.stat().st_mode & 0o777, 0o700)
self.assertEqual(attempt.joinpath("workspace").stat().st_mode & 0o777, 0o777)
self.assertEqual(attempt.joinpath("workspace/decoder.c").stat().st_mode & 0o777, 0o666)
self.assertEqual(attempt.joinpath("grader").stat().st_mode & 0o777, 0o700)
self.assertEqual(attempt.joinpath("grader/verify.py").stat().st_mode & 0o777, 0o600)
self.assertEqual(runs[0][-2:], ["agent", "--arg"])
def test_early_absence_cannot_confirm_an_unfinished_docker_run(self):
for termination, code, expected in (("timeout", -9, "unknown"),
("interrupted", -9, "unknown"),
("exit", -9, "unknown"),
("exit", 0, "confirmed"),
("exit", 1, "confirmed")):
with self.subTest(termination=termination, returncode=code):
task = core.load_task(self.task)
attempt = core._attempt(task, self.output, "test")
missing = subprocess.CompletedProcess([], 1, "", "Error: No such container")
with patch.object(core, "_process", return_value=dict(
returncode=code, termination=termination, cleanup="confirmed")), \
patch.object(core, "_cli", return_value=missing) as cli:
facts = core._docker(attempt, "sha256:" + "a" * 64, ["true"], [], "grade", 15)
self.assertEqual(facts["cleanup"], expected)
self.assertEqual(facts["container_cleanup"], expected)
self.assertEqual(cli.call_args_list[0].args[0][1:3], ["rm", "-f"])
self.assertEqual(cli.call_args_list[1].args[0][1:3], ["container", "inspect"])
recorded = json.loads(attempt.joinpath("events.jsonl").read_text().splitlines()[-1])
self.assertEqual(recorded["cleanup"], expected)
if expected == "unknown":
self.assertIn("may still be pending", facts["cleanup_detail"])
def test_agent_configuration_is_validated_before_docker(self):
with patch.object(core, "_image") as image:
for timeout in (0, True, 86401, "900"):
with self.assertRaises(ValueError):
core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", agent_timeout=timeout)
for url in ("file:///tmp/key", "http://user:secret@localhost/v1", "http://localhost?token=secret",
"http://localhost#secret", "http://localhost:bad", "http:///v1", 12):
with self.assertRaises(ValueError):
core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", model_base_url=url)
for network in ("", "--host", "a,b", 12):
with self.assertRaises(ValueError):
core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", agent_network=network)
image.assert_not_called()
def test_agent_gets_no_network_or_only_an_internal_one(self):
calibration = self.mock_calibration()
def cli(internal):
def call(argv):
if argv[1:3] == ["image", "inspect"]:
return subprocess.CompletedProcess(argv, 0, "sha256:" + "a" * 64 + "\n", "")
if argv[1:3] == ["network", "inspect"]:
return subprocess.CompletedProcess(argv, 0, internal, "")
return subprocess.CompletedProcess(argv, 1, "", "No such container")
return call
for internal in ("false\n", ""):
with patch.object(core, "_cli", side_effect=cli(internal)), \
patch.object(core, "_process") as process:
with self.assertRaises(ValueError):
core.run(self.task, self.output, "tag", ["agent"], "fake@1", calibration,
agent_network="open-bridge")
process.assert_not_called()
launch = dict(returncode=None, termination="launch_error", cleanup="confirmed")
with patch.object(core, "_cli", side_effect=cli("")), \
patch.object(core, "_process", return_value=launch) as process:
core.run(self.task, self.output, "tag", ["agent"], "fake@1", calibration)
agent = process.call_args_list[0].args[0]
self.assertEqual(agent[agent.index("--network") + 1], "none")
def test_native_trace_rejects_missing_symlink_and_oversized_file(self):
attempt = self.root / "attempt"
trace = attempt / "trace"
trace.mkdir(parents=True)
path = trace / "native.jsonl"
self.assertEqual(core._native_trace(attempt)["status"], "MISSING")
path.symlink_to(self.task / "gold/decoder.c")
self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED")
path.unlink()
path.mkdir()
self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED")
path.rmdir()
with path.open("wb") as stream:
stream.truncate(core.MAX_TRACE_BYTES + 1)
self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED")
def test_identity_docker_dependency_and_cleanup_failure(self):
with self.assertRaises(ValueError):
core.run(self.task, self.output, "image", ["agent"], "", "unused")
with patch.object(core.shutil, "which", return_value=None):
self.assertEqual(core.doctor()["status"], "UNAVAILABLE")
calibration = self.mock_calibration()
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object(
core, "_docker", return_value=dict(returncode=0, termination="exit", cleanup="unknown")):
attempt = core.run(self.task, self.output, "image", ["agent"], "fake@1", calibration)
result = self.receipt(attempt)
self.assertEqual(result["status"], "INFRA_ERROR")
self.assertEqual(result["cleanup"], "unknown")
self.assertFalse(attempt.joinpath("candidate").exists())
def multi_file_task(self):
root = self.root / "multi"
manifest = dict(id="multi", version="1", family="driver", source="synthetic", template="template",
verifier="verify.py", candidate_files=["src/a.c", "inc/a.h"], timeout_seconds=15,
fixtures=dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL", alternative="PASS"),
grader_files=["tests.py"])
files = {"task.json": json.dumps(manifest), "prompt.md": "Implement src/a.c", "tests.py": "secret",
"verify.py": "import json,sys\nfrom pathlib import Path\nc=Path(sys.argv[1])\n"
"ok=(c/'src/a.c').read_text() in ('ok','alt') and (c/'inc/a.h').read_text()=='h' and "
"Path(__file__).with_name('tests.py').read_text()=='secret' and not (c/'README.md').exists()\n"
"print(json.dumps({'status':'PASS' if ok else 'FAIL','checks':1}))\nsys.exit(0 if ok else 1)\n",
"template/README.md": "spec", "template/src/a.c": "todo", "template/inc/a.h": "h"}
for fixture, value in (("gold", "ok"), ("noop", ""), ("cheat", "puts"), ("alternative", "alt")):
files[fixture + "/src/a.c"] = value
files[fixture + "/inc/a.h"] = "h"
for name, value in files.items():
root.joinpath(name).parent.mkdir(parents=True, exist_ok=True)
root.joinpath(name).write_text(value)
return root, manifest
def test_multi_file_task_calibrates_every_fixture_and_freezes_only_candidates(self):
root, manifest = self.multi_file_task()
task = core.load_task(root)
self.assertEqual(sorted(core._template(task)), ["README.md", "inc/a.h", "src/a.c"])
receipt = self.receipt(core.calibrate(root, self.output, trusted_local=True))
self.assertEqual((receipt["status"], len(receipt["trials"])), ("PASS", 25))
self.assertEqual({t["fixture"]: t["verdict"]["status"] for t in receipt["trials"]}["alternative"], "PASS")
def process(argv, cwd, timeout, stdout, stderr, stdin=None, controller=False):
stdout.write_text("")
stderr.write_text("")
if stdout.name == "agent.stdout":
cwd.joinpath("workspace/src/a.c").write_text("ok")
cwd.joinpath("workspace/extra.c").write_text("ignored")
return dict(returncode=0, termination="exit", cleanup="confirmed")
ok = cwd.joinpath("candidate/src/a.c").read_text() == "ok"
stdout.write_text(json.dumps(dict(status="PASS" if ok else "FAIL", checks=1)) + "\n")
return dict(returncode=0 if ok else 1, termination="exit", cleanup="confirmed")
receipt.update(trusted_local=False, image_id="sha256:" + "a" * 64)
calibration = self.root / "docker-calibration.json"
calibration.write_text(json.dumps(receipt))
missing = subprocess.CompletedProcess([], 1, "", "No such container")
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), \
patch.object(core, "_cli", return_value=missing), patch.object(core, "_process", side_effect=process):
attempt = core.run(root, self.output, "tag", ["agent"], "fake@1", calibration)
result = self.receipt(attempt)
self.assertEqual(result["status"], "PASS")
self.assertEqual(attempt.joinpath("workspace/README.md").read_text(), "spec")
self.assertEqual(attempt.joinpath("workspace/src").stat().st_mode & 0o777, 0o777)
self.assertEqual(sorted(p.relative_to(attempt / "candidate").as_posix()
for p in attempt.joinpath("candidate").rglob("*") if p.is_file()), ["inc/a.h", "src/a.c"])
self.assertEqual(result["candidate_digest"], core._digest({"src/a.c": b"ok", "inc/a.h": b"h"}))
self.assertEqual(attempt.joinpath("grader/tests.py").stat().st_mode & 0o777, 0o600)
altered = dict(receipt, trials=receipt["trials"][:20])
calibration.write_text(json.dumps(altered))
with self.assertRaises(ValueError):
core._calibration(task, "sha256:" + "a" * 64, calibration)
def test_multi_file_contract_rejects_unsafe_or_incomplete_declarations(self):
root, manifest = self.multi_file_task()
for key, value in (("candidate_files", ["src/missing.c"]), ("candidate_files", ["src/a.c", "src/a.c"]),
("fixtures", dict(gold="FAIL", template="FAIL", noop="FAIL", cheat="FAIL")),
("fixtures", dict(gold="PASS", template="FAIL", noop="FAIL")),
("fixtures", dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL", Bad="FAIL")),
("grader_files", ["template/src/a.c"]), ("grader_files", ["../x"]),
("grader_files", ["gold/x"])):
with self.subTest(key=key, value=value):
root.joinpath("task.json").write_text(json.dumps(dict(manifest, **{key: value})))
with self.assertRaises(ValueError):
core.load_task(root)
root.joinpath("task.json").write_text(json.dumps(manifest))
root.joinpath("alternative/inc/a.h").unlink()
with self.assertRaises(ValueError):
core.load_task(root)
def test_run_requires_full_matching_docker_calibration(self):
calibration = self.mock_calibration()
receipt = calibration / "receipt.json"
original = receipt.read_text()
for key, value in (("trusted_local", True), ("status", "FAIL"), ("image_id", "other"),
("task_digest", "other"), ("verifier_digest", "other"), ("trials", [])):
altered = json.loads(original)
altered[key] = value
receipt.write_text(json.dumps(altered))
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object(core, "_docker") as docker:
with self.assertRaises(ValueError):
core.run(self.task, self.output, "image", ["agent"], "fake@1", calibration)
docker.assert_not_called()
altered = json.loads(original)
altered["trials"][0]["verdict"]["checks"] = 0
receipt.write_text(json.dumps(altered))
with self.assertRaises(ValueError):
core._calibration(core.load_task(self.task), "sha256:" + "a" * 64, calibration)
if __name__ == "__main__":
unittest.main()