import hashlib import json from pathlib import Path import tarfile import unittest from unittest.mock import patch import test_core as core_fixture import test_pi_run as pi_fixture from rollout import archive, core, pi_run class ArchiveTests(unittest.TestCase): def setUp(self): helper = core_fixture.CoreTests("runTest") helper.setUp() self.addCleanup(helper.doCleanups) self.root, self.task = helper.root, helper.task self.calibration = helper.mock_calibration() self.control = self.root / "controller" self.control.mkdir() self.attempt = self.control / "run-fixture" pi_fixture.fixture(self.attempt, self.control / "broker.jsonl") self.output = self.root / "archive.tar.gz" self.finish("PASS") def finish(self, verdict): task = core.load_task(self.task) (self.task / "dataset.json").write_text(json.dumps(dict(task_digest=task["_digest"], verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest(), ground_truth=dict(call_type="argv", fn_name=None, inputs=[[]], outputs=[""], exit_codes=[0])))) calibration = core._calibration(task, pi_fixture.IMAGE, self.calibration) candidate = self.attempt / "candidate" candidate.mkdir(exist_ok=True) candidate.joinpath("decoder.c").write_text("gold" if verdict == "PASS" else "buggy") self.attempt.joinpath("prompt.md").write_bytes(task["_files"]["prompt.md"]) for name in ("agent.stderr", "grade.stderr"): self.attempt.joinpath(name).write_bytes(b"") grade = dict(verdict=dict(status=verdict, checks=1), process=dict( returncode=0 if verdict == "PASS" else 1, termination="exit", cleanup="confirmed")) self.attempt.joinpath("grade.stdout").write_text(json.dumps(grade["verdict"]) + "\n") receipt = core._receipt(task, "run", pi_fixture.IMAGE) receipt.update(status=verdict, task_verdict=verdict, calibration=calibration, candidate_digest=core._digest({"decoder.c": core._read(candidate, "decoder.c")}), agent_exit=0, termination="exit", agent_process=dict(returncode=0, termination="exit", cleanup="confirmed"), grade=grade) core._finish(self.attempt, receipt) controller = dict(schema="embedded-rollout.pi-control/1", status=verdict, task_verdict=verdict, task_id=task["id"], task_digest=task["_digest"], image_id=pi_fixture.IMAGE, attempt=str(self.attempt), calibration=calibration, cleanup="confirmed", agent_cleanup="confirmed", network_cleanup="confirmed", proxy_cleanup="confirmed", agent_exit=0, agent_termination="exit", pi_evidence=pi_run.evidence(self.attempt, self.control / "broker.jsonl")) core._finish(self.control, controller) def test_pass_and_fail_archive_only_allowlisted_files_and_manifest_hashes(self): for verdict in ("PASS", "FAIL"): with self.subTest(verdict=verdict): self.finish(verdict) for name in ("gold/decoder.c", "grader/verify.py", ".private/secret.json", "workspace/private"): hidden = self.control / name hidden.parent.mkdir(parents=True, exist_ok=True) hidden.write_text("DO_NOT_EXPORT_SECRET") output = archive.archive(self.control, self.task, self.output) self.assertEqual(output.stat().st_mode & 0o777, 0o600) with tarfile.open(output, "r:gz") as bundle: members = {member.name: bundle.extractfile(member).read() for member in bundle.getmembers()} manifest = json.loads(members.pop("manifest.json")) self.assertEqual(manifest["task_verdict"], verdict) self.assertEqual(manifest["pi_evidence"], "VERIFIED") self.assertEqual(set(members), set(manifest["files"])) self.assertEqual(len(members), 17) row = json.loads(members["dataset.jsonl"]) self.assertEqual(set(row), {"uuid", "query", "ground_truth", "source", "domain"}) self.assertEqual(row["ground_truth"]["call_type"], "argv") self.assertFalse(any(part in name.split("/") for name in members for part in ("gold", "grader", ".private", "workspace"))) self.assertNotIn(b"DO_NOT_EXPORT_SECRET", b"".join(members.values())) for name, data in members.items(): self.assertEqual(manifest["files"][name], dict(bytes=len(data), sha256=hashlib.sha256(data).hexdigest())) output.unlink() def test_tampered_candidate_feedback_calibration_or_grade_is_rejected(self): paths = [self.attempt / "candidate/decoder.c", self.attempt / "agent.stdout", self.attempt / "trace/native.jsonl", self.calibration / "receipt.json", self.attempt / "grade.stdout"] originals = {path: path.read_bytes() for path in paths} for kind in ("candidate", "feedback", "calibration", "grade"): with self.subTest(kind=kind): if kind == "candidate": paths[0].write_text("changed after grading") elif kind == "feedback": events = [json.loads(line) for line in originals[paths[1]].decode().splitlines()] events[-3]["context"]["messages"] = [] raw = "".join(json.dumps(event) + "\n" for event in events) paths[1].write_text(raw) paths[2].write_text(raw) elif kind == "calibration": calibration = json.loads(originals[paths[3]]) calibration["additional_note"] = "receipt drift" paths[3].write_text(json.dumps(calibration)) else: paths[4].write_text('{"status":"FAIL","checks":1}\n') with self.assertRaises(ValueError): archive.archive(self.control, self.task, self.output) self.assertFalse(self.output.exists()) for path, raw in originals.items(): path.write_bytes(raw) def test_symlinks_multiple_attempts_and_outputs_inside_inputs_are_rejected(self): candidate = self.attempt / "candidate/decoder.c" raw = candidate.read_bytes() candidate.unlink() candidate.symlink_to(self.task / "gold/decoder.c") with self.assertRaises(ValueError): archive.archive(self.control, self.task, self.output) candidate.unlink() candidate.write_bytes(raw) second = self.control / "run-extra" second.mkdir() with self.assertRaises(ValueError): archive.archive(self.control, self.task, self.output) second.rmdir() for directory in (self.control, self.task, self.calibration): with self.subTest(directory=directory), self.assertRaises(ValueError): archive.archive(self.control, self.task, directory / "archive.tar.gz") link = self.root / "controller-link" link.symlink_to(self.control, target_is_directory=True) with self.assertRaises(ValueError): archive.archive(link, self.task, self.output) def test_incomplete_cleanup_or_identity_cannot_be_archived(self): path = self.control / "receipt.json" raw = path.read_bytes() for key, value in (("cleanup", "unknown"), ("proxy_cleanup", "unknown"), ("agent_exit", 1), ("status", "UNVERIFIED"), ("image_id", "sha256:" + "b" * 64), ("attempt", str(self.root / "elsewhere"))): with self.subTest(key=key): receipt = json.loads(raw) receipt[key] = value path.write_text(json.dumps(receipt)) with self.assertRaises(ValueError): archive.archive(self.control, self.task, self.output) path.write_bytes(raw) def test_existing_output_is_preserved_and_failed_new_output_is_removed(self): self.output.write_bytes(b"existing result") with self.assertRaises(FileExistsError): archive.archive(self.control, self.task, self.output) self.assertEqual(self.output.read_bytes(), b"existing result") self.output.unlink() with patch.object(archive.tarfile, "open", side_effect=OSError("write failed")): with self.assertRaises(OSError): archive.archive(self.control, self.task, self.output) self.assertFalse(self.output.exists()) def test_trace_size_and_symlink_output_boundaries(self): trace = self.attempt / "trace/native.jsonl" with trace.open("wb") as stream: stream.truncate(core.MAX_TRACE_BYTES + 1) with self.assertRaises(ValueError): archive.archive(self.control, self.task, self.output) self.assertFalse(self.output.exists()) trace.write_bytes(self.attempt.joinpath("agent.stdout").read_bytes()) existing = self.root / "existing.tar.gz" existing.write_bytes(b"existing result") self.output.symlink_to(existing) with self.assertRaises(ValueError): archive.archive(self.control, self.task, self.output) self.assertTrue(self.output.is_symlink()) self.assertEqual(existing.read_bytes(), b"existing result") def test_qwen_archive_cannot_skip_model_checks_by_deleting_identity(self): source = self.root / "qwen-fixture" broker = self.root / "qwen-broker.jsonl" pi_fixture.fixture(source, broker, model=pi_run.QWEN_MODEL) for name in ("agent.stdout", "trace/native.jsonl"): (self.attempt / name).write_bytes((source / name).read_bytes()) (self.control / "broker.jsonl").write_bytes(broker.read_bytes()) self.finish("PASS") path = self.control / "receipt.json" control = json.loads(path.read_text()) control["model_id"] = pi_run.QWEN_MODEL control["model_api"] = "openai-responses" path.write_text(json.dumps(control)) archive.archive(self.control, self.task, self.output) self.output.unlink() for value in (None, "unknown", "missing"): if value == "missing": control.pop("model_id") else: control["model_id"] = value path.write_text(json.dumps(control)) with self.assertRaises(ValueError): archive.archive(self.control, self.task, self.output) self.assertFalse(self.output.exists()) if __name__ == "__main__": unittest.main()