import json import hashlib from pathlib import Path import subprocess import sys import tempfile import unittest from unittest.mock import patch from rollout import core class CoreTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name).resolve() floor = patch.object(core, "MIN_FREE_BYTES", 0) # Development disks may hold less than the 36 floor. floor.start() self.addCleanup(floor.stop) self.task = self.root / "task" self.task.mkdir() self.output = self.root / "out" self.manifest = dict(id="parser", version="1", family="protocol", source="synthetic", template="template", verifier="verify.py", candidate_files=["decoder.c"], timeout_seconds=15) self.task.joinpath("task.json").write_text(json.dumps(self.manifest)) self.task.joinpath("prompt.md").write_text("Repair decoder.c") self.task.joinpath("verify.py").write_text( "import json,sys\nfrom pathlib import Path\n" "ok=Path(sys.argv[1], 'decoder.c').read_text() == 'gold'\n" "print(json.dumps({'status':'PASS' if ok else 'FAIL','checks':1}))\n" "sys.exit(0 if ok else 1)\n") for relative, value in (("template", "buggy"), ("gold", "gold"), ("noop", "noop"), ("cheat", "cheat")): directory = self.task / relative directory.mkdir(parents=True) directory.joinpath("decoder.c").write_text(value) def receipt(self, attempt): return json.loads(attempt.joinpath("receipt.json").read_text()) def mock_calibration(self): def grade(task, attempt, candidate, prefix, image_id): ok = candidate.joinpath("decoder.c").read_text() == "gold" return dict(verdict=dict(status="PASS" if ok else "FAIL", checks=1), process=dict(returncode=0 if ok else 1, termination="exit", cleanup="confirmed")) with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object( core, "_grade", side_effect=grade): return core.calibrate(self.task, self.output, image="tag") def test_task_rejects_unsafe_and_missing_material(self): self.assertEqual(core.load_task(self.task)["id"], "parser") for bad in (None, [], ["../decoder.c"], ["/tmp/decoder.c"]): self.manifest["candidate_files"] = bad self.task.joinpath("task.json").write_text(json.dumps(self.manifest)) with self.assertRaises(ValueError): core.load_task(self.task) self.manifest["candidate_files"] = ["decoder.c"] self.task.joinpath("task.json").write_text(json.dumps(self.manifest)) fixture = self.task / "gold/decoder.c" fixture.unlink() fixture.symlink_to(self.task / "template/decoder.c") with self.assertRaises(ValueError): core.load_task(self.task) fixture.unlink() with self.assertRaises(ValueError): core.load_task(self.task) def test_size_and_output_boundaries(self): self.task.joinpath("template/decoder.c").write_bytes(b"x" * (core.MAX_FILE_BYTES + 1)) with self.assertRaises(ValueError): core.load_task(self.task) self.task.joinpath("template/decoder.c").write_text("buggy") with self.assertRaises(ValueError): core.calibrate(self.task, self.task / "out", trusted_local=True) with self.assertRaises(ValueError): core.calibrate(self.task, self.output) with self.assertRaises(ValueError): core.calibrate(self.task, self.output, trusted_local=True, image="image") def test_trusted_calibration_is_five_fresh_trials_per_fixture(self): attempt = core.calibrate(self.task, self.output, trusted_local=True) result = self.receipt(attempt) self.assertEqual(result["status"], "PASS") self.assertEqual(len(result["trials"]), 20) self.assertEqual(result["cleanup"], "confirmed") self.assertEqual(result["training_disposition"], "NOT_TRAIN_READY") second = core.calibrate(self.task, self.output, trusted_local=True) self.assertNotEqual(attempt, second) self.assertTrue(attempt.joinpath("events.jsonl").exists()) def test_grader_cannot_pass_empty_conflicting_or_malformed_output(self): stdout = self.root / "stdout" facts = dict(returncode=0, termination="exit", cleanup="confirmed") for output in ('{"status":"PASS","checks":0}', '{"status":"FAIL","checks":1}', 'noise\n{"status":"PASS","checks":1}', '[]', 'not json'): stdout.write_text(output) self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED") stdout.write_text('{"status":"PASS","checks":1}') self.assertEqual(core._checker(facts, stdout)["status"], "PASS") facts["cleanup"] = "unknown" self.assertEqual(core._checker(facts, stdout)["status"], "INFRA_ERROR") facts["cleanup"] = "confirmed" facts["returncode"] = 2 stdout.write_text('{"status":"UNVERIFIED","checks":0}') self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED") stdout.write_text("x" * (core.MAX_CHECKER_BYTES + 1)) self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED") def test_interrupt_stops_calibration_and_preserves_terminal_receipt(self): with patch.object(core, "_grade", side_effect=KeyboardInterrupt) as grade: attempt = core.calibrate(self.task, self.output, trusted_local=True) result = self.receipt(attempt) self.assertEqual(result["status"], "UNVERIFIED") self.assertEqual(result["termination"], "interrupted") self.assertEqual(grade.call_count, 1) self.assertEqual(result["cleanup"], "unknown") def test_sigterm_is_recorded_after_active_grader_cleanup(self): script = ("import os,signal,time\n" "os.kill(os.getppid(),signal.SIGTERM)\ntime.sleep(10)\n") self.task.joinpath("verify.py").write_text(script) attempt = core.calibrate(self.task, self.output, trusted_local=True) result = self.receipt(attempt) self.assertEqual(result["status"], "UNVERIFIED") self.assertEqual(result["termination"], "interrupted") self.assertEqual(result["cleanup"], "confirmed") self.assertEqual(len(result["trials"]), 1) def test_timeout_is_not_a_pass_and_has_process_cleanup(self): self.task.joinpath("verify.py").write_text("import time\ntime.sleep(10)\n") self.manifest["timeout_seconds"] = 1 self.task.joinpath("task.json").write_text(json.dumps(self.manifest)) task = core.load_task(self.task) attempt = core._attempt(task, self.output, "test") candidate = attempt / "candidate" candidate.mkdir() candidate.joinpath("decoder.c").write_text("gold") result = core._grade(task, attempt, candidate, "grade", None) self.assertEqual(result["verdict"]["status"], "INFRA_ERROR") self.assertEqual(result["process"]["termination"], "timeout") self.assertEqual(result["process"]["cleanup"], "confirmed") def test_docker_run_freezes_and_grades_independent_mounts(self): commands = [] deadlines, watches = [], [] image_id = "sha256:" + "a" * 64 calibration = self.mock_calibration() def cli(argv): commands.append(argv) if argv[1:3] == ["image", "inspect"]: return subprocess.CompletedProcess(argv, 0, image_id + "\n", "") if argv[1:3] == ["network", "inspect"]: return subprocess.CompletedProcess(argv, 0, "true\n", "") if argv[1:3] == ["container", "inspect"]: return subprocess.CompletedProcess(argv, 1, "", "No such container") return subprocess.CompletedProcess(argv, 0, "", "") def process(argv, cwd, timeout, stdout, stderr, stdin=None, controller=False, watch=None): commands.append(argv) deadlines.append(timeout) watches.append(watch == cwd) stdout.write_text("") stderr.write_text("") if stdout.name == "agent.stdout": cwd.joinpath("workspace/decoder.c").write_text("gold") cwd.joinpath("trace/native.jsonl").write_text('{"event":"test"}\n') return dict(returncode=1, termination="exit", cleanup="confirmed") stdout.write_text('{"status":"PASS","checks":1}\n') return dict(returncode=0, termination="exit", cleanup="confirmed") with patch.object(core, "_cli", side_effect=cli), patch.object(core, "_process", side_effect=process): attempt = core.run(self.task, self.output, "tag", ["agent", "--arg"], "fake-agent@1", calibration, agent_timeout=120, model_base_url="http://172.22.0.1:8123/v1", agent_network="rollout-test-network") result = self.receipt(attempt) self.assertEqual(result["status"], "PASS") self.assertEqual(result["agent_exit"], 1) self.assertEqual(result["image_id"], image_id) self.assertEqual(result["agent_timeout_seconds"], 120) self.assertEqual(deadlines, [120, 15]) self.assertEqual(watches, [True, False]) self.assertEqual(result["native_trace"]["status"], "AVAILABLE") self.assertEqual(result["native_trace"]["sha256"], hashlib.sha256(b'{"event":"test"}\n').hexdigest()) self.assertEqual(result["training_disposition"], "NOT_TRAIN_READY") self.assertEqual(attempt.joinpath("candidate/decoder.c").read_text(), "gold") self.assertEqual(sum(c[1:3] == ["image", "inspect"] for c in commands), 1) runs = [c for c in commands if c[1] == "run"] for command in runs: self.assertIn("exec", command[command.index("--tmpfs") + 1].split(":", 1)[1].split(",")) self.assertNotIn("/grader", " ".join(runs[0])) self.assertNotIn("/gold", " ".join(runs[0])) self.assertNotIn("docker.sock", " ".join(runs[0])) self.assertIn("dst=/trace", " ".join(runs[0])) self.assertNotIn("dst=/trace", " ".join(runs[1])) self.assertEqual(runs[0][runs[0].index("--memory") + 1], "2g") self.assertEqual(runs[1][runs[1].index("--memory") + 1], "512m") self.assertIn("rollout-test-network", runs[0]) self.assertNotIn("--add-host", runs[0]) self.assertIn("ROLLOUT_MODEL_BASE_URL=http://172.22.0.1:8123/v1", runs[0]) self.assertNotIn("ROLLOUT_MODEL_BASE_URL", " ".join(runs[1])) self.assertIn("none", runs[1]) self.assertIn("ROLLOUT_GRADE_ISOLATION=uid65534", runs[1]) self.assertIn("SETUID", runs[1]) self.assertIn("SETGID", runs[1]) self.assertIn("CHOWN", runs[1]) self.assertEqual(attempt.stat().st_mode & 0o777, 0o700) self.assertEqual(attempt.joinpath("workspace").stat().st_mode & 0o777, 0o777) self.assertEqual(attempt.joinpath("workspace/decoder.c").stat().st_mode & 0o777, 0o666) self.assertEqual(attempt.joinpath("grader").stat().st_mode & 0o777, 0o700) self.assertEqual(attempt.joinpath("grader/verify.py").stat().st_mode & 0o777, 0o600) self.assertEqual(runs[0][-2:], ["agent", "--arg"]) def test_disk_limits_stop_or_skip_containers(self): writer = "import time\nwhile True:\n open('junk', 'ab').write(b'x' * 65536); time.sleep(0.01)\n" with patch.object(core, "MAX_ATTEMPT_BYTES", 1 << 20): facts = core._process([sys.executable, "-c", writer], self.root, 60, self.root / "out.log", self.root / "err.log", watch=self.root) self.assertEqual(facts["termination"], "attempt_size") self.assertLess(self.root.joinpath("junk").stat().st_size, 64 << 20) with patch.object(core, "MIN_FREE_BYTES", 1 << 62): facts = core._process([sys.executable, "-c", "import time; time.sleep(60)"], self.root, 60, self.root / "out.log", self.root / "err.log", watch=self.root) self.assertEqual(facts["termination"], "disk_low") attempt = core._attempt(core.load_task(self.task), self.output, "test") with patch.object(core, "_process") as process: facts = core._docker(attempt, "sha256:" + "a" * 64, ["true"], [], "grade", 15) process.assert_not_called() self.assertEqual((facts["termination"], facts["cleanup"]), ("disk_low", "confirmed")) self.assertEqual(core._checker(facts, attempt / "grade.stdout")["status"], "INFRA_ERROR") def test_early_absence_cannot_confirm_an_unfinished_docker_run(self): for termination, code, expected in (("timeout", -9, "unknown"), ("interrupted", -9, "unknown"), ("exit", -9, "unknown"), ("exit", 0, "confirmed"), ("exit", 1, "confirmed")): with self.subTest(termination=termination, returncode=code): task = core.load_task(self.task) attempt = core._attempt(task, self.output, "test") missing = subprocess.CompletedProcess([], 1, "", "Error: No such container") with patch.object(core, "_process", return_value=dict( returncode=code, termination=termination, cleanup="confirmed")), \ patch.object(core, "_cli", return_value=missing) as cli: facts = core._docker(attempt, "sha256:" + "a" * 64, ["true"], [], "grade", 15) self.assertEqual(facts["cleanup"], expected) self.assertEqual(facts["container_cleanup"], expected) self.assertEqual(cli.call_args_list[0].args[0][1:3], ["rm", "-f"]) self.assertEqual(cli.call_args_list[1].args[0][1:3], ["container", "inspect"]) recorded = json.loads(attempt.joinpath("events.jsonl").read_text().splitlines()[-1]) self.assertEqual(recorded["cleanup"], expected) if expected == "unknown": self.assertIn("may still be pending", facts["cleanup_detail"]) def test_agent_configuration_is_validated_before_docker(self): with patch.object(core, "_image") as image: for timeout in (0, True, 86401, "900"): with self.assertRaises(ValueError): core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", agent_timeout=timeout) for url in ("file:///tmp/key", "http://user:secret@localhost/v1", "http://localhost?token=secret", "http://localhost#secret", "http://localhost:bad", "http:///v1", 12): with self.assertRaises(ValueError): core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", model_base_url=url) for network in ("", "--host", "a,b", 12): with self.assertRaises(ValueError): core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", agent_network=network) image.assert_not_called() def test_agent_gets_no_network_or_only_an_internal_one(self): calibration = self.mock_calibration() def cli(internal): def call(argv): if argv[1:3] == ["image", "inspect"]: return subprocess.CompletedProcess(argv, 0, "sha256:" + "a" * 64 + "\n", "") if argv[1:3] == ["network", "inspect"]: return subprocess.CompletedProcess(argv, 0, internal, "") return subprocess.CompletedProcess(argv, 1, "", "No such container") return call for internal in ("false\n", ""): with patch.object(core, "_cli", side_effect=cli(internal)), \ patch.object(core, "_process") as process: with self.assertRaises(ValueError): core.run(self.task, self.output, "tag", ["agent"], "fake@1", calibration, agent_network="open-bridge") process.assert_not_called() launch = dict(returncode=None, termination="launch_error", cleanup="confirmed") with patch.object(core, "_cli", side_effect=cli("")), \ patch.object(core, "_process", return_value=launch) as process: core.run(self.task, self.output, "tag", ["agent"], "fake@1", calibration) agent = process.call_args_list[0].args[0] self.assertEqual(agent[agent.index("--network") + 1], "none") def test_native_trace_rejects_missing_symlink_and_oversized_file(self): attempt = self.root / "attempt" trace = attempt / "trace" trace.mkdir(parents=True) path = trace / "native.jsonl" self.assertEqual(core._native_trace(attempt)["status"], "MISSING") path.symlink_to(self.task / "gold/decoder.c") self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED") path.unlink() path.mkdir() self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED") path.rmdir() with path.open("wb") as stream: stream.truncate(core.MAX_TRACE_BYTES + 1) self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED") def test_identity_docker_dependency_and_cleanup_failure(self): with self.assertRaises(ValueError): core.run(self.task, self.output, "image", ["agent"], "", "unused") with patch.object(core.shutil, "which", return_value=None): self.assertEqual(core.doctor()["status"], "UNAVAILABLE") calibration = self.mock_calibration() with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object( core, "_docker", return_value=dict(returncode=0, termination="exit", cleanup="unknown")): attempt = core.run(self.task, self.output, "image", ["agent"], "fake@1", calibration) result = self.receipt(attempt) self.assertEqual(result["status"], "INFRA_ERROR") self.assertEqual(result["cleanup"], "unknown") self.assertFalse(attempt.joinpath("candidate").exists()) def multi_file_task(self): root = self.root / "multi" manifest = dict(id="multi", version="1", family="driver", source="synthetic", template="template", verifier="verify.py", candidate_files=["src/a.c", "inc/a.h"], timeout_seconds=15, fixtures=dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL", alternative="PASS"), grader_files=["tests.py"]) files = {"task.json": json.dumps(manifest), "prompt.md": "Implement src/a.c", "tests.py": "secret", "verify.py": "import json,sys\nfrom pathlib import Path\nc=Path(sys.argv[1])\n" "ok=(c/'src/a.c').read_text() in ('ok','alt') and (c/'inc/a.h').read_text()=='h' and " "Path(__file__).with_name('tests.py').read_text()=='secret' and not (c/'README.md').exists()\n" "print(json.dumps({'status':'PASS' if ok else 'FAIL','checks':1}))\nsys.exit(0 if ok else 1)\n", "template/README.md": "spec", "template/src/a.c": "todo", "template/inc/a.h": "h"} for fixture, value in (("gold", "ok"), ("noop", ""), ("cheat", "puts"), ("alternative", "alt")): files[fixture + "/src/a.c"] = value files[fixture + "/inc/a.h"] = "h" for name, value in files.items(): root.joinpath(name).parent.mkdir(parents=True, exist_ok=True) root.joinpath(name).write_text(value) return root, manifest def test_multi_file_task_calibrates_every_fixture_and_freezes_only_candidates(self): root, manifest = self.multi_file_task() task = core.load_task(root) self.assertEqual(sorted(core._template(task)), ["README.md", "inc/a.h", "src/a.c"]) receipt = self.receipt(core.calibrate(root, self.output, trusted_local=True)) self.assertEqual((receipt["status"], len(receipt["trials"])), ("PASS", 25)) self.assertEqual({t["fixture"]: t["verdict"]["status"] for t in receipt["trials"]}["alternative"], "PASS") def process(argv, cwd, timeout, stdout, stderr, stdin=None, controller=False, watch=None): stdout.write_text("") stderr.write_text("") if stdout.name == "agent.stdout": cwd.joinpath("workspace/src/a.c").write_text("ok") cwd.joinpath("workspace/extra.c").write_text("ignored") return dict(returncode=0, termination="exit", cleanup="confirmed") ok = cwd.joinpath("candidate/src/a.c").read_text() == "ok" stdout.write_text(json.dumps(dict(status="PASS" if ok else "FAIL", checks=1)) + "\n") return dict(returncode=0 if ok else 1, termination="exit", cleanup="confirmed") receipt.update(trusted_local=False, image_id="sha256:" + "a" * 64) calibration = self.root / "docker-calibration.json" calibration.write_text(json.dumps(receipt)) missing = subprocess.CompletedProcess([], 1, "", "No such container") with patch.object(core, "_image", return_value="sha256:" + "a" * 64), \ patch.object(core, "_cli", return_value=missing), patch.object(core, "_process", side_effect=process): attempt = core.run(root, self.output, "tag", ["agent"], "fake@1", calibration) result = self.receipt(attempt) self.assertEqual(result["status"], "PASS") self.assertEqual(attempt.joinpath("workspace/README.md").read_text(), "spec") self.assertEqual(attempt.joinpath("workspace/src").stat().st_mode & 0o777, 0o777) self.assertEqual(sorted(p.relative_to(attempt / "candidate").as_posix() for p in attempt.joinpath("candidate").rglob("*") if p.is_file()), ["inc/a.h", "src/a.c"]) self.assertEqual(result["candidate_digest"], core._digest({"src/a.c": b"ok", "inc/a.h": b"h"})) self.assertEqual(attempt.joinpath("grader/tests.py").stat().st_mode & 0o777, 0o600) altered = dict(receipt, trials=receipt["trials"][:20]) calibration.write_text(json.dumps(altered)) with self.assertRaises(ValueError): core._calibration(task, "sha256:" + "a" * 64, calibration) def test_multi_file_contract_rejects_unsafe_or_incomplete_declarations(self): root, manifest = self.multi_file_task() for key, value in (("candidate_files", ["src/missing.c"]), ("candidate_files", ["src/a.c", "src/a.c"]), ("fixtures", dict(gold="FAIL", template="FAIL", noop="FAIL", cheat="FAIL")), ("fixtures", dict(gold="PASS", template="FAIL", noop="FAIL")), ("fixtures", dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL", Bad="FAIL")), ("grader_files", ["template/src/a.c"]), ("grader_files", ["../x"]), ("grader_files", ["gold/x"])): with self.subTest(key=key, value=value): root.joinpath("task.json").write_text(json.dumps(dict(manifest, **{key: value}))) with self.assertRaises(ValueError): core.load_task(root) root.joinpath("task.json").write_text(json.dumps(manifest)) root.joinpath("alternative/inc/a.h").unlink() with self.assertRaises(ValueError): core.load_task(root) def test_run_requires_full_matching_docker_calibration(self): calibration = self.mock_calibration() receipt = calibration / "receipt.json" original = receipt.read_text() for key, value in (("trusted_local", True), ("status", "FAIL"), ("image_id", "other"), ("task_digest", "other"), ("verifier_digest", "other"), ("trials", [])): altered = json.loads(original) altered[key] = value receipt.write_text(json.dumps(altered)) with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object(core, "_docker") as docker: with self.assertRaises(ValueError): core.run(self.task, self.output, "image", ["agent"], "fake@1", calibration) docker.assert_not_called() altered = json.loads(original) altered["trials"][0]["verdict"]["checks"] = 0 receipt.write_text(json.dumps(altered)) with self.assertRaises(ValueError): core._calibration(core.load_task(self.task), "sha256:" + "a" * 64, calibration) if __name__ == "__main__": unittest.main()