data-gen-ebd/tests/test_core.py
amos 141f3d06aa fix: cap disk use per attempt and refuse to start on a low output disk
Agent containers wrote to bind-mounted /workspace and /trace with no
size limit, so a runaway Agent could fill the shared /home. Containers
no longer start below 50 GiB free; a running Agent is stopped when the
disk drops below that floor or its attempt directory passes 2 GiB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 15:22:35 +08:00

425 lines
24 KiB
Python

import json
import hashlib
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
from unittest.mock import patch
from rollout import core
class CoreTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name).resolve()
floor = patch.object(core, "MIN_FREE_BYTES", 0) # Development disks may hold less than the 36 floor.
floor.start()
self.addCleanup(floor.stop)
self.task = self.root / "task"
self.task.mkdir()
self.output = self.root / "out"
self.manifest = dict(id="parser", version="1", family="protocol", source="synthetic",
template="template", verifier="verify.py", candidate_files=["decoder.c"],
timeout_seconds=15)
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
self.task.joinpath("prompt.md").write_text("Repair decoder.c")
self.task.joinpath("verify.py").write_text(
"import json,sys\nfrom pathlib import Path\n"
"ok=Path(sys.argv[1], 'decoder.c').read_text() == 'gold'\n"
"print(json.dumps({'status':'PASS' if ok else 'FAIL','checks':1}))\n"
"sys.exit(0 if ok else 1)\n")
for relative, value in (("template", "buggy"), ("gold", "gold"),
("noop", "noop"), ("cheat", "cheat")):
directory = self.task / relative
directory.mkdir(parents=True)
directory.joinpath("decoder.c").write_text(value)
def receipt(self, attempt):
return json.loads(attempt.joinpath("receipt.json").read_text())
def mock_calibration(self):
def grade(task, attempt, candidate, prefix, image_id):
ok = candidate.joinpath("decoder.c").read_text() == "gold"
return dict(verdict=dict(status="PASS" if ok else "FAIL", checks=1),
process=dict(returncode=0 if ok else 1, termination="exit", cleanup="confirmed"))
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object(
core, "_grade", side_effect=grade):
return core.calibrate(self.task, self.output, image="tag")
def test_task_rejects_unsafe_and_missing_material(self):
self.assertEqual(core.load_task(self.task)["id"], "parser")
for bad in (None, [], ["../decoder.c"], ["/tmp/decoder.c"]):
self.manifest["candidate_files"] = bad
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
with self.assertRaises(ValueError):
core.load_task(self.task)
self.manifest["candidate_files"] = ["decoder.c"]
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
fixture = self.task / "gold/decoder.c"
fixture.unlink()
fixture.symlink_to(self.task / "template/decoder.c")
with self.assertRaises(ValueError):
core.load_task(self.task)
fixture.unlink()
with self.assertRaises(ValueError):
core.load_task(self.task)
def test_size_and_output_boundaries(self):
self.task.joinpath("template/decoder.c").write_bytes(b"x" * (core.MAX_FILE_BYTES + 1))
with self.assertRaises(ValueError):
core.load_task(self.task)
self.task.joinpath("template/decoder.c").write_text("buggy")
with self.assertRaises(ValueError):
core.calibrate(self.task, self.task / "out", trusted_local=True)
with self.assertRaises(ValueError):
core.calibrate(self.task, self.output)
with self.assertRaises(ValueError):
core.calibrate(self.task, self.output, trusted_local=True, image="image")
def test_trusted_calibration_is_five_fresh_trials_per_fixture(self):
attempt = core.calibrate(self.task, self.output, trusted_local=True)
result = self.receipt(attempt)
self.assertEqual(result["status"], "PASS")
self.assertEqual(len(result["trials"]), 20)
self.assertEqual(result["cleanup"], "confirmed")
self.assertEqual(result["training_disposition"], "NOT_TRAIN_READY")
second = core.calibrate(self.task, self.output, trusted_local=True)
self.assertNotEqual(attempt, second)
self.assertTrue(attempt.joinpath("events.jsonl").exists())
def test_grader_cannot_pass_empty_conflicting_or_malformed_output(self):
stdout = self.root / "stdout"
facts = dict(returncode=0, termination="exit", cleanup="confirmed")
for output in ('{"status":"PASS","checks":0}', '{"status":"FAIL","checks":1}',
'noise\n{"status":"PASS","checks":1}', '[]', 'not json'):
stdout.write_text(output)
self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED")
stdout.write_text('{"status":"PASS","checks":1}')
self.assertEqual(core._checker(facts, stdout)["status"], "PASS")
facts["cleanup"] = "unknown"
self.assertEqual(core._checker(facts, stdout)["status"], "INFRA_ERROR")
facts["cleanup"] = "confirmed"
facts["returncode"] = 2
stdout.write_text('{"status":"UNVERIFIED","checks":0}')
self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED")
stdout.write_text("x" * (core.MAX_CHECKER_BYTES + 1))
self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED")
def test_interrupt_stops_calibration_and_preserves_terminal_receipt(self):
with patch.object(core, "_grade", side_effect=KeyboardInterrupt) as grade:
attempt = core.calibrate(self.task, self.output, trusted_local=True)
result = self.receipt(attempt)
self.assertEqual(result["status"], "UNVERIFIED")
self.assertEqual(result["termination"], "interrupted")
self.assertEqual(grade.call_count, 1)
self.assertEqual(result["cleanup"], "unknown")
def test_sigterm_is_recorded_after_active_grader_cleanup(self):
script = ("import os,signal,time\n"
"os.kill(os.getppid(),signal.SIGTERM)\ntime.sleep(10)\n")
self.task.joinpath("verify.py").write_text(script)
attempt = core.calibrate(self.task, self.output, trusted_local=True)
result = self.receipt(attempt)
self.assertEqual(result["status"], "UNVERIFIED")
self.assertEqual(result["termination"], "interrupted")
self.assertEqual(result["cleanup"], "confirmed")
self.assertEqual(len(result["trials"]), 1)
def test_timeout_is_not_a_pass_and_has_process_cleanup(self):
self.task.joinpath("verify.py").write_text("import time\ntime.sleep(10)\n")
self.manifest["timeout_seconds"] = 1
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
task = core.load_task(self.task)
attempt = core._attempt(task, self.output, "test")
candidate = attempt / "candidate"
candidate.mkdir()
candidate.joinpath("decoder.c").write_text("gold")
result = core._grade(task, attempt, candidate, "grade", None)
self.assertEqual(result["verdict"]["status"], "INFRA_ERROR")
self.assertEqual(result["process"]["termination"], "timeout")
self.assertEqual(result["process"]["cleanup"], "confirmed")
def test_docker_run_freezes_and_grades_independent_mounts(self):
commands = []
deadlines, watches = [], []
image_id = "sha256:" + "a" * 64
calibration = self.mock_calibration()
def cli(argv):
commands.append(argv)
if argv[1:3] == ["image", "inspect"]:
return subprocess.CompletedProcess(argv, 0, image_id + "\n", "")
if argv[1:3] == ["network", "inspect"]:
return subprocess.CompletedProcess(argv, 0, "true\n", "")
if argv[1:3] == ["container", "inspect"]:
return subprocess.CompletedProcess(argv, 1, "", "No such container")
return subprocess.CompletedProcess(argv, 0, "", "")
def process(argv, cwd, timeout, stdout, stderr, stdin=None, controller=False, watch=None):
commands.append(argv)
deadlines.append(timeout)
watches.append(watch == cwd)
stdout.write_text("")
stderr.write_text("")
if stdout.name == "agent.stdout":
cwd.joinpath("workspace/decoder.c").write_text("gold")
cwd.joinpath("trace/native.jsonl").write_text('{"event":"test"}\n')
return dict(returncode=1, termination="exit", cleanup="confirmed")
stdout.write_text('{"status":"PASS","checks":1}\n')
return dict(returncode=0, termination="exit", cleanup="confirmed")
with patch.object(core, "_cli", side_effect=cli), patch.object(core, "_process", side_effect=process):
attempt = core.run(self.task, self.output, "tag", ["agent", "--arg"], "fake-agent@1", calibration,
agent_timeout=120, model_base_url="http://172.22.0.1:8123/v1",
agent_network="rollout-test-network")
result = self.receipt(attempt)
self.assertEqual(result["status"], "PASS")
self.assertEqual(result["agent_exit"], 1)
self.assertEqual(result["image_id"], image_id)
self.assertEqual(result["agent_timeout_seconds"], 120)
self.assertEqual(deadlines, [120, 15])
self.assertEqual(watches, [True, False])
self.assertEqual(result["native_trace"]["status"], "AVAILABLE")
self.assertEqual(result["native_trace"]["sha256"], hashlib.sha256(b'{"event":"test"}\n').hexdigest())
self.assertEqual(result["training_disposition"], "NOT_TRAIN_READY")
self.assertEqual(attempt.joinpath("candidate/decoder.c").read_text(), "gold")
self.assertEqual(sum(c[1:3] == ["image", "inspect"] for c in commands), 1)
runs = [c for c in commands if c[1] == "run"]
for command in runs:
self.assertIn("exec", command[command.index("--tmpfs") + 1].split(":", 1)[1].split(","))
self.assertNotIn("/grader", " ".join(runs[0]))
self.assertNotIn("/gold", " ".join(runs[0]))
self.assertNotIn("docker.sock", " ".join(runs[0]))
self.assertIn("dst=/trace", " ".join(runs[0]))
self.assertNotIn("dst=/trace", " ".join(runs[1]))
self.assertEqual(runs[0][runs[0].index("--memory") + 1], "2g")
self.assertEqual(runs[1][runs[1].index("--memory") + 1], "512m")
self.assertIn("rollout-test-network", runs[0])
self.assertNotIn("--add-host", runs[0])
self.assertIn("ROLLOUT_MODEL_BASE_URL=http://172.22.0.1:8123/v1", runs[0])
self.assertNotIn("ROLLOUT_MODEL_BASE_URL", " ".join(runs[1]))
self.assertIn("none", runs[1])
self.assertIn("ROLLOUT_GRADE_ISOLATION=uid65534", runs[1])
self.assertIn("SETUID", runs[1])
self.assertIn("SETGID", runs[1])
self.assertIn("CHOWN", runs[1])
self.assertEqual(attempt.stat().st_mode & 0o777, 0o700)
self.assertEqual(attempt.joinpath("workspace").stat().st_mode & 0o777, 0o777)
self.assertEqual(attempt.joinpath("workspace/decoder.c").stat().st_mode & 0o777, 0o666)
self.assertEqual(attempt.joinpath("grader").stat().st_mode & 0o777, 0o700)
self.assertEqual(attempt.joinpath("grader/verify.py").stat().st_mode & 0o777, 0o600)
self.assertEqual(runs[0][-2:], ["agent", "--arg"])
def test_disk_limits_stop_or_skip_containers(self):
writer = "import time\nwhile True:\n open('junk', 'ab').write(b'x' * 65536); time.sleep(0.01)\n"
with patch.object(core, "MAX_ATTEMPT_BYTES", 1 << 20):
facts = core._process([sys.executable, "-c", writer], self.root, 60, self.root / "out.log",
self.root / "err.log", watch=self.root)
self.assertEqual(facts["termination"], "attempt_size")
self.assertLess(self.root.joinpath("junk").stat().st_size, 64 << 20)
with patch.object(core, "MIN_FREE_BYTES", 1 << 62):
facts = core._process([sys.executable, "-c", "import time; time.sleep(60)"], self.root, 60,
self.root / "out.log", self.root / "err.log", watch=self.root)
self.assertEqual(facts["termination"], "disk_low")
attempt = core._attempt(core.load_task(self.task), self.output, "test")
with patch.object(core, "_process") as process:
facts = core._docker(attempt, "sha256:" + "a" * 64, ["true"], [], "grade", 15)
process.assert_not_called()
self.assertEqual((facts["termination"], facts["cleanup"]), ("disk_low", "confirmed"))
self.assertEqual(core._checker(facts, attempt / "grade.stdout")["status"], "INFRA_ERROR")
def test_early_absence_cannot_confirm_an_unfinished_docker_run(self):
for termination, code, expected in (("timeout", -9, "unknown"),
("interrupted", -9, "unknown"),
("exit", -9, "unknown"),
("exit", 0, "confirmed"),
("exit", 1, "confirmed")):
with self.subTest(termination=termination, returncode=code):
task = core.load_task(self.task)
attempt = core._attempt(task, self.output, "test")
missing = subprocess.CompletedProcess([], 1, "", "Error: No such container")
with patch.object(core, "_process", return_value=dict(
returncode=code, termination=termination, cleanup="confirmed")), \
patch.object(core, "_cli", return_value=missing) as cli:
facts = core._docker(attempt, "sha256:" + "a" * 64, ["true"], [], "grade", 15)
self.assertEqual(facts["cleanup"], expected)
self.assertEqual(facts["container_cleanup"], expected)
self.assertEqual(cli.call_args_list[0].args[0][1:3], ["rm", "-f"])
self.assertEqual(cli.call_args_list[1].args[0][1:3], ["container", "inspect"])
recorded = json.loads(attempt.joinpath("events.jsonl").read_text().splitlines()[-1])
self.assertEqual(recorded["cleanup"], expected)
if expected == "unknown":
self.assertIn("may still be pending", facts["cleanup_detail"])
def test_agent_configuration_is_validated_before_docker(self):
with patch.object(core, "_image") as image:
for timeout in (0, True, 86401, "900"):
with self.assertRaises(ValueError):
core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", agent_timeout=timeout)
for url in ("file:///tmp/key", "http://user:secret@localhost/v1", "http://localhost?token=secret",
"http://localhost#secret", "http://localhost:bad", "http:///v1", 12):
with self.assertRaises(ValueError):
core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", model_base_url=url)
for network in ("", "--host", "a,b", 12):
with self.assertRaises(ValueError):
core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", agent_network=network)
image.assert_not_called()
def test_agent_gets_no_network_or_only_an_internal_one(self):
calibration = self.mock_calibration()
def cli(internal):
def call(argv):
if argv[1:3] == ["image", "inspect"]:
return subprocess.CompletedProcess(argv, 0, "sha256:" + "a" * 64 + "\n", "")
if argv[1:3] == ["network", "inspect"]:
return subprocess.CompletedProcess(argv, 0, internal, "")
return subprocess.CompletedProcess(argv, 1, "", "No such container")
return call
for internal in ("false\n", ""):
with patch.object(core, "_cli", side_effect=cli(internal)), \
patch.object(core, "_process") as process:
with self.assertRaises(ValueError):
core.run(self.task, self.output, "tag", ["agent"], "fake@1", calibration,
agent_network="open-bridge")
process.assert_not_called()
launch = dict(returncode=None, termination="launch_error", cleanup="confirmed")
with patch.object(core, "_cli", side_effect=cli("")), \
patch.object(core, "_process", return_value=launch) as process:
core.run(self.task, self.output, "tag", ["agent"], "fake@1", calibration)
agent = process.call_args_list[0].args[0]
self.assertEqual(agent[agent.index("--network") + 1], "none")
def test_native_trace_rejects_missing_symlink_and_oversized_file(self):
attempt = self.root / "attempt"
trace = attempt / "trace"
trace.mkdir(parents=True)
path = trace / "native.jsonl"
self.assertEqual(core._native_trace(attempt)["status"], "MISSING")
path.symlink_to(self.task / "gold/decoder.c")
self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED")
path.unlink()
path.mkdir()
self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED")
path.rmdir()
with path.open("wb") as stream:
stream.truncate(core.MAX_TRACE_BYTES + 1)
self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED")
def test_identity_docker_dependency_and_cleanup_failure(self):
with self.assertRaises(ValueError):
core.run(self.task, self.output, "image", ["agent"], "", "unused")
with patch.object(core.shutil, "which", return_value=None):
self.assertEqual(core.doctor()["status"], "UNAVAILABLE")
calibration = self.mock_calibration()
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object(
core, "_docker", return_value=dict(returncode=0, termination="exit", cleanup="unknown")):
attempt = core.run(self.task, self.output, "image", ["agent"], "fake@1", calibration)
result = self.receipt(attempt)
self.assertEqual(result["status"], "INFRA_ERROR")
self.assertEqual(result["cleanup"], "unknown")
self.assertFalse(attempt.joinpath("candidate").exists())
def multi_file_task(self):
root = self.root / "multi"
manifest = dict(id="multi", version="1", family="driver", source="synthetic", template="template",
verifier="verify.py", candidate_files=["src/a.c", "inc/a.h"], timeout_seconds=15,
fixtures=dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL", alternative="PASS"),
grader_files=["tests.py"])
files = {"task.json": json.dumps(manifest), "prompt.md": "Implement src/a.c", "tests.py": "secret",
"verify.py": "import json,sys\nfrom pathlib import Path\nc=Path(sys.argv[1])\n"
"ok=(c/'src/a.c').read_text() in ('ok','alt') and (c/'inc/a.h').read_text()=='h' and "
"Path(__file__).with_name('tests.py').read_text()=='secret' and not (c/'README.md').exists()\n"
"print(json.dumps({'status':'PASS' if ok else 'FAIL','checks':1}))\nsys.exit(0 if ok else 1)\n",
"template/README.md": "spec", "template/src/a.c": "todo", "template/inc/a.h": "h"}
for fixture, value in (("gold", "ok"), ("noop", ""), ("cheat", "puts"), ("alternative", "alt")):
files[fixture + "/src/a.c"] = value
files[fixture + "/inc/a.h"] = "h"
for name, value in files.items():
root.joinpath(name).parent.mkdir(parents=True, exist_ok=True)
root.joinpath(name).write_text(value)
return root, manifest
def test_multi_file_task_calibrates_every_fixture_and_freezes_only_candidates(self):
root, manifest = self.multi_file_task()
task = core.load_task(root)
self.assertEqual(sorted(core._template(task)), ["README.md", "inc/a.h", "src/a.c"])
receipt = self.receipt(core.calibrate(root, self.output, trusted_local=True))
self.assertEqual((receipt["status"], len(receipt["trials"])), ("PASS", 25))
self.assertEqual({t["fixture"]: t["verdict"]["status"] for t in receipt["trials"]}["alternative"], "PASS")
def process(argv, cwd, timeout, stdout, stderr, stdin=None, controller=False, watch=None):
stdout.write_text("")
stderr.write_text("")
if stdout.name == "agent.stdout":
cwd.joinpath("workspace/src/a.c").write_text("ok")
cwd.joinpath("workspace/extra.c").write_text("ignored")
return dict(returncode=0, termination="exit", cleanup="confirmed")
ok = cwd.joinpath("candidate/src/a.c").read_text() == "ok"
stdout.write_text(json.dumps(dict(status="PASS" if ok else "FAIL", checks=1)) + "\n")
return dict(returncode=0 if ok else 1, termination="exit", cleanup="confirmed")
receipt.update(trusted_local=False, image_id="sha256:" + "a" * 64)
calibration = self.root / "docker-calibration.json"
calibration.write_text(json.dumps(receipt))
missing = subprocess.CompletedProcess([], 1, "", "No such container")
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), \
patch.object(core, "_cli", return_value=missing), patch.object(core, "_process", side_effect=process):
attempt = core.run(root, self.output, "tag", ["agent"], "fake@1", calibration)
result = self.receipt(attempt)
self.assertEqual(result["status"], "PASS")
self.assertEqual(attempt.joinpath("workspace/README.md").read_text(), "spec")
self.assertEqual(attempt.joinpath("workspace/src").stat().st_mode & 0o777, 0o777)
self.assertEqual(sorted(p.relative_to(attempt / "candidate").as_posix()
for p in attempt.joinpath("candidate").rglob("*") if p.is_file()), ["inc/a.h", "src/a.c"])
self.assertEqual(result["candidate_digest"], core._digest({"src/a.c": b"ok", "inc/a.h": b"h"}))
self.assertEqual(attempt.joinpath("grader/tests.py").stat().st_mode & 0o777, 0o600)
altered = dict(receipt, trials=receipt["trials"][:20])
calibration.write_text(json.dumps(altered))
with self.assertRaises(ValueError):
core._calibration(task, "sha256:" + "a" * 64, calibration)
def test_multi_file_contract_rejects_unsafe_or_incomplete_declarations(self):
root, manifest = self.multi_file_task()
for key, value in (("candidate_files", ["src/missing.c"]), ("candidate_files", ["src/a.c", "src/a.c"]),
("fixtures", dict(gold="FAIL", template="FAIL", noop="FAIL", cheat="FAIL")),
("fixtures", dict(gold="PASS", template="FAIL", noop="FAIL")),
("fixtures", dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL", Bad="FAIL")),
("grader_files", ["template/src/a.c"]), ("grader_files", ["../x"]),
("grader_files", ["gold/x"])):
with self.subTest(key=key, value=value):
root.joinpath("task.json").write_text(json.dumps(dict(manifest, **{key: value})))
with self.assertRaises(ValueError):
core.load_task(root)
root.joinpath("task.json").write_text(json.dumps(manifest))
root.joinpath("alternative/inc/a.h").unlink()
with self.assertRaises(ValueError):
core.load_task(root)
def test_run_requires_full_matching_docker_calibration(self):
calibration = self.mock_calibration()
receipt = calibration / "receipt.json"
original = receipt.read_text()
for key, value in (("trusted_local", True), ("status", "FAIL"), ("image_id", "other"),
("task_digest", "other"), ("verifier_digest", "other"), ("trials", [])):
altered = json.loads(original)
altered[key] = value
receipt.write_text(json.dumps(altered))
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object(core, "_docker") as docker:
with self.assertRaises(ValueError):
core.run(self.task, self.output, "image", ["agent"], "fake@1", calibration)
docker.assert_not_called()
altered = json.loads(original)
altered["trials"][0]["verdict"]["checks"] = 0
receipt.write_text(json.dumps(altered))
with self.assertRaises(ValueError):
core._calibration(core.load_task(self.task), "sha256:" + "a" * 64, calibration)
if __name__ == "__main__":
unittest.main()