import contextlib import copy import hashlib import io import json import os from pathlib import Path import shutil import tempfile import unittest from unittest.mock import patch from rollout import core, export TASK = Path(__file__).resolve().parents[1] / "tasks/frame-decoder" class ExportTests(unittest.TestCase): def setUp(self): temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup) self.root = Path(temporary.name).resolve() self.task = self.root / "task" shutil.copytree(TASK, self.task) self.output = self.root / "dataset.jsonl" self.snapshot = json.loads((self.task / "dataset.json").read_bytes()) def test_task_without_oracle_snapshot_exports_the_protected_verifier_identity(self): import test_core helper = test_core.CoreTests("runTest") helper.setUp() self.addCleanup(helper.doCleanups) root, _ = helper.multi_file_task() task = core.load_task(root) row = export.record(task) self.assertEqual(row["ground_truth"], dict(call_type="verifier", fn_name=None, task_digest=task["_digest"], verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest())) for name in ("README.md", "src/a.c", "inc/a.h"): self.assertIn("## 初始 " + name, row["query"]) self.assertNotIn("secret", row["query"]) self.assertNotIn("alt", row["query"]) root.joinpath("dataset.json").symlink_to(root / "tests.py") with self.assertRaises(ValueError): export.record(task) def write_snapshot(self, value): (self.task / "dataset.json").write_text(json.dumps(value), encoding="utf-8") def test_five_fields_frozen_identity_initial_state_and_complete_oracle(self): task = core.load_task(self.task) row = export.record(task) self.assertEqual(set(row), {"uuid", "query", "ground_truth", "source", "domain"}) self.assertEqual(row["uuid"], "Code_" + task["_digest"]) self.assertEqual(row, export.record(core.load_task(TASK))) self.assertEqual(row["source"], "authored-bootstrap-fixture") self.assertEqual(row["domain"], "Code") self.assertIn(task["_files"]["prompt.md"].decode(), row["query"]) self.assertIn(task["_files"]["template/decoder.c"].decode(), row["query"]) self.assertNotIn(task["_files"]["gold/decoder.c"].decode(), row["query"]) self.assertNotIn(task["_files"]["verify.py"].decode(), row["query"]) # Rows exported before multi-file support must keep their exact query text. self.assertEqual(row["query"], task["_files"]["prompt.md"].decode() + "\n## 初始 decoder.c\n\n```c\n" + task["_files"]["template/decoder.c"].decode() + "\n```\n") truth = row["ground_truth"] self.assertEqual(truth["call_type"], "argv") self.assertIsNone(truth["fn_name"]) self.assertEqual(len(truth["inputs"]), 87) cases = list(zip(truth["inputs"], truth["outputs"], truth["exit_codes"])) for case in (([], "", 0), (["02", "aa", "bb", "13"], "aabb\n", 0), (["02aa"], "INCOMPLETE\n", 1), (["f"], "", 2), (["zz"], "", 2)): self.assertIn(case, cases) self.assertEqual(self.snapshot["verifier_digest"], hashlib.sha256(task["_files"]["verify.py"]).hexdigest()) def test_jsonl_export_preserves_one_row_and_excludes_private_material(self): private = self.task / ".private" private.mkdir() (private / "access.json").write_text("DO_NOT_EXPORT_SECRET") output = export.export(self.task, self.output) self.assertEqual(output, self.output) self.assertEqual(output.stat().st_mode & 0o777, 0o600) raw = output.read_bytes() self.assertEqual(len(raw.splitlines()), 1) self.assertNotIn(b"DO_NOT_EXPORT_SECRET", raw) self.assertEqual(json.loads(raw), export.record(core.load_task(self.task))) def test_task_or_verifier_identity_drift_is_rejected(self): for name in ("prompt.md", "verify.py", "template/decoder.c"): with self.subTest(name=name): path = self.task / name original = path.read_bytes() path.write_bytes(original + b"\n") with self.assertRaises(ValueError): export.export(self.task, self.output) self.assertFalse(self.output.exists()) path.write_bytes(original) for key in ("task_digest", "verifier_digest"): with self.subTest(key=key): value = copy.deepcopy(self.snapshot) value[key] = "0" * 64 self.write_snapshot(value) with self.assertRaises(ValueError): export.record(core.load_task(self.task)) def test_invalid_fields_types_lengths_or_limits_are_rejected(self): values = [] for key, value in (("call_type", "std"), ("fn_name", "main"), ("inputs", []), ("outputs", []), ("exit_codes", []), ("inputs", "argv"), ("inputs", [["nul\0argument"]] * 87), ("inputs", [["x"] * 129] * 87), ("inputs", [["x" * (core.MAX_CHECKER_BYTES + 1)]] * 87), ("outputs", ["x" * (core.MAX_CHECKER_BYTES + 1)] * 87), ("outputs", [None] * 87), ("exit_codes", [True] * 87), ("exit_codes", [3] * 87)): value_snapshot = copy.deepcopy(self.snapshot) value_snapshot["ground_truth"][key] = value values.append(value_snapshot) extra = copy.deepcopy(self.snapshot) extra["ground_truth"]["extra"] = "unused" values += [extra, {**self.snapshot, "schema": "unused"}, [], {**self.snapshot, "ground_truth": json.dumps(self.snapshot["ground_truth"])}] for value in values: with self.subTest(value_type=type(value).__name__): self.write_snapshot(value) with self.assertRaises(ValueError): export.export(self.task, self.output) self.assertFalse(self.output.exists()) def test_snapshot_must_be_bounded_regular_file_without_symlinks(self): path = self.task / "dataset.json" path.unlink() path.symlink_to(TASK / "dataset.json") with self.assertRaises(ValueError): export.record(core.load_task(self.task)) path.unlink() os.mkfifo(path) with self.assertRaises(ValueError): export.record(core.load_task(self.task)) path.unlink() with path.open("wb") as stream: stream.truncate(core.MAX_FILE_BYTES + 1) with self.assertRaises(ValueError): export.record(core.load_task(self.task)) def test_output_cannot_overwrite_follow_symlink_or_write_inside_task(self): self.output.write_bytes(b"existing result") with self.assertRaises(FileExistsError): export.export(self.task, self.output) self.assertEqual(self.output.read_bytes(), b"existing result") self.output.unlink() existing = self.root / "existing.jsonl" existing.write_bytes(b"existing result") self.output.symlink_to(existing) with self.assertRaises(ValueError): export.export(self.task, self.output) self.assertTrue(self.output.is_symlink()) self.assertEqual(existing.read_bytes(), b"existing result") with self.assertRaises(ValueError): export.export(self.task, self.task / "export.jsonl") def test_failed_write_removes_only_its_own_output_inode(self): with patch.object(export.os, "fsync", side_effect=OSError("disk error")): with self.assertRaises(OSError): export.export(self.task, self.output) self.assertFalse(self.output.exists()) def replace_output(descriptor): self.output.rename(self.root / "original.jsonl") self.output.write_bytes(b"concurrent replacement") raise OSError("disk error") with patch.object(export.os, "fsync", side_effect=replace_output): with self.assertRaises(OSError): export.export(self.task, self.output) self.assertEqual(self.output.read_bytes(), b"concurrent replacement") def test_cli_returns_success_and_rejection_without_overwriting(self): argv = [str(self.task), "--output", str(self.output)] with contextlib.redirect_stdout(io.StringIO()) as result: self.assertEqual(export.main(argv), 0) self.assertEqual(json.loads(result.getvalue()), dict(dataset=str(self.output))) raw = self.output.read_bytes() with contextlib.redirect_stderr(io.StringIO()): self.assertEqual(export.main(argv), 2) self.assertEqual(self.output.read_bytes(), raw) if __name__ == "__main__": unittest.main()