data-gen-ebd/tests/test_export.py
amos acfa601a56 feat: multi-file task contract, model table and shared I/O helpers
- Tasks may declare up to 16 candidate files; the whole template becomes
  the Agent workspace while only candidate files are frozen and graded.
- Optional fixtures add more cheats or legal alternatives (five trials
  each); optional grader_files ship private tests next to verify.py.
- Tasks without an argv oracle export call_type=verifier ground truth.
- Upstream model routes live in one MODELS table.
- One bounded no-follow reader, one exclusive writer and one SIGTERM
  helper replace per-module copies; drop unused validate_task.

frame-decoder keeps its task digest, calibration receipts and exported
query. Verified on host 36: internal network probe, 20-trial calibration
and a real GLM rollout (PASS, evidence VERIFIED, cleanup confirmed).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 21:25:09 +08:00

191 lines
8.9 KiB
Python

import contextlib
import copy
import hashlib
import io
import json
import os
from pathlib import Path
import shutil
import tempfile
import unittest
from unittest.mock import patch
from rollout import core, export
TASK = Path(__file__).resolve().parents[1] / "tasks/frame-decoder"
class ExportTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name).resolve()
self.task = self.root / "task"
shutil.copytree(TASK, self.task)
self.output = self.root / "dataset.jsonl"
self.snapshot = json.loads((self.task / "dataset.json").read_bytes())
def test_task_without_oracle_snapshot_exports_the_protected_verifier_identity(self):
import test_core
helper = test_core.CoreTests("runTest")
helper.setUp()
self.addCleanup(helper.doCleanups)
root, _ = helper.multi_file_task()
task = core.load_task(root)
row = export.record(task)
self.assertEqual(row["ground_truth"], dict(call_type="verifier", fn_name=None, task_digest=task["_digest"],
verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest()))
for name in ("README.md", "src/a.c", "inc/a.h"):
self.assertIn("## 初始 " + name, row["query"])
self.assertNotIn("secret", row["query"])
self.assertNotIn("alt", row["query"])
root.joinpath("dataset.json").symlink_to(root / "tests.py")
with self.assertRaises(ValueError):
export.record(task)
def write_snapshot(self, value):
(self.task / "dataset.json").write_text(json.dumps(value), encoding="utf-8")
def test_five_fields_frozen_identity_initial_state_and_complete_oracle(self):
task = core.load_task(self.task)
row = export.record(task)
self.assertEqual(set(row), {"uuid", "query", "ground_truth", "source", "domain"})
self.assertEqual(row["uuid"], "Code_" + task["_digest"])
self.assertEqual(row, export.record(core.load_task(TASK)))
self.assertEqual(row["source"], "authored-bootstrap-fixture")
self.assertEqual(row["domain"], "Code")
self.assertIn(task["_files"]["prompt.md"].decode(), row["query"])
self.assertIn(task["_files"]["template/decoder.c"].decode(), row["query"])
self.assertNotIn(task["_files"]["gold/decoder.c"].decode(), row["query"])
self.assertNotIn(task["_files"]["verify.py"].decode(), row["query"])
# Rows exported before multi-file support must keep their exact query text.
self.assertEqual(row["query"], task["_files"]["prompt.md"].decode() + "\n## 初始 decoder.c\n\n```c\n" +
task["_files"]["template/decoder.c"].decode() + "\n```\n")
truth = row["ground_truth"]
self.assertEqual(truth["call_type"], "argv")
self.assertIsNone(truth["fn_name"])
self.assertEqual(len(truth["inputs"]), 87)
cases = list(zip(truth["inputs"], truth["outputs"], truth["exit_codes"]))
for case in (([], "", 0), (["02", "aa", "bb", "13"], "aabb\n", 0),
(["02aa"], "INCOMPLETE\n", 1), (["f"], "", 2), (["zz"], "", 2)):
self.assertIn(case, cases)
self.assertEqual(self.snapshot["verifier_digest"],
hashlib.sha256(task["_files"]["verify.py"]).hexdigest())
def test_jsonl_export_preserves_one_row_and_excludes_private_material(self):
private = self.task / ".private"
private.mkdir()
(private / "access.json").write_text("DO_NOT_EXPORT_SECRET")
output = export.export(self.task, self.output)
self.assertEqual(output, self.output)
self.assertEqual(output.stat().st_mode & 0o777, 0o600)
raw = output.read_bytes()
self.assertEqual(len(raw.splitlines()), 1)
self.assertNotIn(b"DO_NOT_EXPORT_SECRET", raw)
self.assertEqual(json.loads(raw), export.record(core.load_task(self.task)))
def test_task_or_verifier_identity_drift_is_rejected(self):
for name in ("prompt.md", "verify.py", "template/decoder.c"):
with self.subTest(name=name):
path = self.task / name
original = path.read_bytes()
path.write_bytes(original + b"\n")
with self.assertRaises(ValueError):
export.export(self.task, self.output)
self.assertFalse(self.output.exists())
path.write_bytes(original)
for key in ("task_digest", "verifier_digest"):
with self.subTest(key=key):
value = copy.deepcopy(self.snapshot)
value[key] = "0" * 64
self.write_snapshot(value)
with self.assertRaises(ValueError):
export.record(core.load_task(self.task))
def test_invalid_fields_types_lengths_or_limits_are_rejected(self):
values = []
for key, value in (("call_type", "std"), ("fn_name", "main"), ("inputs", []),
("outputs", []), ("exit_codes", []), ("inputs", "argv"),
("inputs", [["nul\0argument"]] * 87),
("inputs", [["x"] * 129] * 87),
("inputs", [["x" * (core.MAX_CHECKER_BYTES + 1)]] * 87),
("outputs", ["x" * (core.MAX_CHECKER_BYTES + 1)] * 87),
("outputs", [None] * 87), ("exit_codes", [True] * 87),
("exit_codes", [3] * 87)):
value_snapshot = copy.deepcopy(self.snapshot)
value_snapshot["ground_truth"][key] = value
values.append(value_snapshot)
extra = copy.deepcopy(self.snapshot)
extra["ground_truth"]["extra"] = "unused"
values += [extra, {**self.snapshot, "schema": "unused"}, [],
{**self.snapshot, "ground_truth": json.dumps(self.snapshot["ground_truth"])}]
for value in values:
with self.subTest(value_type=type(value).__name__):
self.write_snapshot(value)
with self.assertRaises(ValueError):
export.export(self.task, self.output)
self.assertFalse(self.output.exists())
def test_snapshot_must_be_bounded_regular_file_without_symlinks(self):
path = self.task / "dataset.json"
path.unlink()
path.symlink_to(TASK / "dataset.json")
with self.assertRaises(ValueError):
export.record(core.load_task(self.task))
path.unlink()
os.mkfifo(path)
with self.assertRaises(ValueError):
export.record(core.load_task(self.task))
path.unlink()
with path.open("wb") as stream:
stream.truncate(core.MAX_FILE_BYTES + 1)
with self.assertRaises(ValueError):
export.record(core.load_task(self.task))
def test_output_cannot_overwrite_follow_symlink_or_write_inside_task(self):
self.output.write_bytes(b"existing result")
with self.assertRaises(FileExistsError):
export.export(self.task, self.output)
self.assertEqual(self.output.read_bytes(), b"existing result")
self.output.unlink()
existing = self.root / "existing.jsonl"
existing.write_bytes(b"existing result")
self.output.symlink_to(existing)
with self.assertRaises(ValueError):
export.export(self.task, self.output)
self.assertTrue(self.output.is_symlink())
self.assertEqual(existing.read_bytes(), b"existing result")
with self.assertRaises(ValueError):
export.export(self.task, self.task / "export.jsonl")
def test_failed_write_removes_only_its_own_output_inode(self):
with patch.object(export.os, "fsync", side_effect=OSError("disk error")):
with self.assertRaises(OSError):
export.export(self.task, self.output)
self.assertFalse(self.output.exists())
def replace_output(descriptor):
self.output.rename(self.root / "original.jsonl")
self.output.write_bytes(b"concurrent replacement")
raise OSError("disk error")
with patch.object(export.os, "fsync", side_effect=replace_output):
with self.assertRaises(OSError):
export.export(self.task, self.output)
self.assertEqual(self.output.read_bytes(), b"concurrent replacement")
def test_cli_returns_success_and_rejection_without_overwriting(self):
argv = [str(self.task), "--output", str(self.output)]
with contextlib.redirect_stdout(io.StringIO()) as result:
self.assertEqual(export.main(argv), 0)
self.assertEqual(json.loads(result.getvalue()), dict(dataset=str(self.output)))
raw = self.output.read_bytes()
with contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(export.main(argv), 2)
self.assertEqual(self.output.read_bytes(), raw)
if __name__ == "__main__":
unittest.main()