- Tasks may declare up to 16 candidate files; the whole template becomes the Agent workspace while only candidate files are frozen and graded. - Optional fixtures add more cheats or legal alternatives (five trials each); optional grader_files ship private tests next to verify.py. - Tasks without an argv oracle export call_type=verifier ground truth. - Upstream model routes live in one MODELS table. - One bounded no-follow reader, one exclusive writer and one SIGTERM helper replace per-module copies; drop unused validate_task. frame-decoder keeps its task digest, calibration receipts and exported query. Verified on host 36: internal network probe, 20-trial calibration and a real GLM rollout (PASS, evidence VERIFIED, cleanup confirmed). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
191 lines
8.9 KiB
Python
191 lines
8.9 KiB
Python
import contextlib
|
|
import copy
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from rollout import core, export
|
|
|
|
|
|
TASK = Path(__file__).resolve().parents[1] / "tasks/frame-decoder"
|
|
|
|
|
|
class ExportTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name).resolve()
|
|
self.task = self.root / "task"
|
|
shutil.copytree(TASK, self.task)
|
|
self.output = self.root / "dataset.jsonl"
|
|
self.snapshot = json.loads((self.task / "dataset.json").read_bytes())
|
|
|
|
def test_task_without_oracle_snapshot_exports_the_protected_verifier_identity(self):
|
|
import test_core
|
|
helper = test_core.CoreTests("runTest")
|
|
helper.setUp()
|
|
self.addCleanup(helper.doCleanups)
|
|
root, _ = helper.multi_file_task()
|
|
task = core.load_task(root)
|
|
row = export.record(task)
|
|
self.assertEqual(row["ground_truth"], dict(call_type="verifier", fn_name=None, task_digest=task["_digest"],
|
|
verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest()))
|
|
for name in ("README.md", "src/a.c", "inc/a.h"):
|
|
self.assertIn("## 初始 " + name, row["query"])
|
|
self.assertNotIn("secret", row["query"])
|
|
self.assertNotIn("alt", row["query"])
|
|
root.joinpath("dataset.json").symlink_to(root / "tests.py")
|
|
with self.assertRaises(ValueError):
|
|
export.record(task)
|
|
|
|
def write_snapshot(self, value):
|
|
(self.task / "dataset.json").write_text(json.dumps(value), encoding="utf-8")
|
|
|
|
def test_five_fields_frozen_identity_initial_state_and_complete_oracle(self):
|
|
task = core.load_task(self.task)
|
|
row = export.record(task)
|
|
self.assertEqual(set(row), {"uuid", "query", "ground_truth", "source", "domain"})
|
|
self.assertEqual(row["uuid"], "Code_" + task["_digest"])
|
|
self.assertEqual(row, export.record(core.load_task(TASK)))
|
|
self.assertEqual(row["source"], "authored-bootstrap-fixture")
|
|
self.assertEqual(row["domain"], "Code")
|
|
self.assertIn(task["_files"]["prompt.md"].decode(), row["query"])
|
|
self.assertIn(task["_files"]["template/decoder.c"].decode(), row["query"])
|
|
self.assertNotIn(task["_files"]["gold/decoder.c"].decode(), row["query"])
|
|
self.assertNotIn(task["_files"]["verify.py"].decode(), row["query"])
|
|
# Rows exported before multi-file support must keep their exact query text.
|
|
self.assertEqual(row["query"], task["_files"]["prompt.md"].decode() + "\n## 初始 decoder.c\n\n```c\n" +
|
|
task["_files"]["template/decoder.c"].decode() + "\n```\n")
|
|
truth = row["ground_truth"]
|
|
self.assertEqual(truth["call_type"], "argv")
|
|
self.assertIsNone(truth["fn_name"])
|
|
self.assertEqual(len(truth["inputs"]), 87)
|
|
cases = list(zip(truth["inputs"], truth["outputs"], truth["exit_codes"]))
|
|
for case in (([], "", 0), (["02", "aa", "bb", "13"], "aabb\n", 0),
|
|
(["02aa"], "INCOMPLETE\n", 1), (["f"], "", 2), (["zz"], "", 2)):
|
|
self.assertIn(case, cases)
|
|
self.assertEqual(self.snapshot["verifier_digest"],
|
|
hashlib.sha256(task["_files"]["verify.py"]).hexdigest())
|
|
|
|
def test_jsonl_export_preserves_one_row_and_excludes_private_material(self):
|
|
private = self.task / ".private"
|
|
private.mkdir()
|
|
(private / "access.json").write_text("DO_NOT_EXPORT_SECRET")
|
|
output = export.export(self.task, self.output)
|
|
self.assertEqual(output, self.output)
|
|
self.assertEqual(output.stat().st_mode & 0o777, 0o600)
|
|
raw = output.read_bytes()
|
|
self.assertEqual(len(raw.splitlines()), 1)
|
|
self.assertNotIn(b"DO_NOT_EXPORT_SECRET", raw)
|
|
self.assertEqual(json.loads(raw), export.record(core.load_task(self.task)))
|
|
|
|
def test_task_or_verifier_identity_drift_is_rejected(self):
|
|
for name in ("prompt.md", "verify.py", "template/decoder.c"):
|
|
with self.subTest(name=name):
|
|
path = self.task / name
|
|
original = path.read_bytes()
|
|
path.write_bytes(original + b"\n")
|
|
with self.assertRaises(ValueError):
|
|
export.export(self.task, self.output)
|
|
self.assertFalse(self.output.exists())
|
|
path.write_bytes(original)
|
|
for key in ("task_digest", "verifier_digest"):
|
|
with self.subTest(key=key):
|
|
value = copy.deepcopy(self.snapshot)
|
|
value[key] = "0" * 64
|
|
self.write_snapshot(value)
|
|
with self.assertRaises(ValueError):
|
|
export.record(core.load_task(self.task))
|
|
|
|
def test_invalid_fields_types_lengths_or_limits_are_rejected(self):
|
|
values = []
|
|
for key, value in (("call_type", "std"), ("fn_name", "main"), ("inputs", []),
|
|
("outputs", []), ("exit_codes", []), ("inputs", "argv"),
|
|
("inputs", [["nul\0argument"]] * 87),
|
|
("inputs", [["x"] * 129] * 87),
|
|
("inputs", [["x" * (core.MAX_CHECKER_BYTES + 1)]] * 87),
|
|
("outputs", ["x" * (core.MAX_CHECKER_BYTES + 1)] * 87),
|
|
("outputs", [None] * 87), ("exit_codes", [True] * 87),
|
|
("exit_codes", [3] * 87)):
|
|
value_snapshot = copy.deepcopy(self.snapshot)
|
|
value_snapshot["ground_truth"][key] = value
|
|
values.append(value_snapshot)
|
|
extra = copy.deepcopy(self.snapshot)
|
|
extra["ground_truth"]["extra"] = "unused"
|
|
values += [extra, {**self.snapshot, "schema": "unused"}, [],
|
|
{**self.snapshot, "ground_truth": json.dumps(self.snapshot["ground_truth"])}]
|
|
for value in values:
|
|
with self.subTest(value_type=type(value).__name__):
|
|
self.write_snapshot(value)
|
|
with self.assertRaises(ValueError):
|
|
export.export(self.task, self.output)
|
|
self.assertFalse(self.output.exists())
|
|
|
|
def test_snapshot_must_be_bounded_regular_file_without_symlinks(self):
|
|
path = self.task / "dataset.json"
|
|
path.unlink()
|
|
path.symlink_to(TASK / "dataset.json")
|
|
with self.assertRaises(ValueError):
|
|
export.record(core.load_task(self.task))
|
|
path.unlink()
|
|
os.mkfifo(path)
|
|
with self.assertRaises(ValueError):
|
|
export.record(core.load_task(self.task))
|
|
path.unlink()
|
|
with path.open("wb") as stream:
|
|
stream.truncate(core.MAX_FILE_BYTES + 1)
|
|
with self.assertRaises(ValueError):
|
|
export.record(core.load_task(self.task))
|
|
|
|
def test_output_cannot_overwrite_follow_symlink_or_write_inside_task(self):
|
|
self.output.write_bytes(b"existing result")
|
|
with self.assertRaises(FileExistsError):
|
|
export.export(self.task, self.output)
|
|
self.assertEqual(self.output.read_bytes(), b"existing result")
|
|
self.output.unlink()
|
|
existing = self.root / "existing.jsonl"
|
|
existing.write_bytes(b"existing result")
|
|
self.output.symlink_to(existing)
|
|
with self.assertRaises(ValueError):
|
|
export.export(self.task, self.output)
|
|
self.assertTrue(self.output.is_symlink())
|
|
self.assertEqual(existing.read_bytes(), b"existing result")
|
|
with self.assertRaises(ValueError):
|
|
export.export(self.task, self.task / "export.jsonl")
|
|
|
|
def test_failed_write_removes_only_its_own_output_inode(self):
|
|
with patch.object(export.os, "fsync", side_effect=OSError("disk error")):
|
|
with self.assertRaises(OSError):
|
|
export.export(self.task, self.output)
|
|
self.assertFalse(self.output.exists())
|
|
|
|
def replace_output(descriptor):
|
|
self.output.rename(self.root / "original.jsonl")
|
|
self.output.write_bytes(b"concurrent replacement")
|
|
raise OSError("disk error")
|
|
|
|
with patch.object(export.os, "fsync", side_effect=replace_output):
|
|
with self.assertRaises(OSError):
|
|
export.export(self.task, self.output)
|
|
self.assertEqual(self.output.read_bytes(), b"concurrent replacement")
|
|
|
|
def test_cli_returns_success_and_rejection_without_overwriting(self):
|
|
argv = [str(self.task), "--output", str(self.output)]
|
|
with contextlib.redirect_stdout(io.StringIO()) as result:
|
|
self.assertEqual(export.main(argv), 0)
|
|
self.assertEqual(json.loads(result.getvalue()), dict(dataset=str(self.output)))
|
|
raw = self.output.read_bytes()
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(export.main(argv), 2)
|
|
self.assertEqual(self.output.read_bytes(), raw)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|