data-gen-ebd/tests/test_export.py
amos 714f345b63 feat: add Qwen Responses route, five-field task export and pending-create cleanup
- Pi controller accepts a fixed Qwen qwen3.8-27b-nvfp4 Responses route next
  to GLM-5.3; the broker requires exactly one response.completed per call
  and evidence checks the native model identity.
- rollout.export writes the UltraData five-field JSONL from a frozen
  dataset.json snapshot; every archive now carries dataset.jsonl.
- A Docker CLI that did not finish cannot confirm container or network
  cleanup, and auto-calibration cleanup uncertainty reaches the controller.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 21:05:56 +08:00

170 lines
7.7 KiB
Python

import contextlib
import copy
import hashlib
import io
import json
import os
from pathlib import Path
import shutil
import tempfile
import unittest
from unittest.mock import patch
from rollout import core, export
TASK = Path(__file__).resolve().parents[1] / "tasks/frame-decoder"
class ExportTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name).resolve()
self.task = self.root / "task"
shutil.copytree(TASK, self.task)
self.output = self.root / "dataset.jsonl"
self.snapshot = json.loads((self.task / "dataset.json").read_bytes())
def write_snapshot(self, value):
(self.task / "dataset.json").write_text(json.dumps(value), encoding="utf-8")
def test_five_fields_frozen_identity_initial_state_and_complete_oracle(self):
task = core.load_task(self.task)
row = export.record(task)
self.assertEqual(set(row), {"uuid", "query", "ground_truth", "source", "domain"})
self.assertEqual(row["uuid"], "Code_" + task["_digest"])
self.assertEqual(row, export.record(core.load_task(TASK)))
self.assertEqual(row["source"], "authored-bootstrap-fixture")
self.assertEqual(row["domain"], "Code")
self.assertIn(task["_files"]["prompt.md"].decode(), row["query"])
self.assertIn(task["_files"]["template/decoder.c"].decode(), row["query"])
self.assertNotIn(task["_files"]["gold/decoder.c"].decode(), row["query"])
self.assertNotIn(task["_files"]["verify.py"].decode(), row["query"])
truth = row["ground_truth"]
self.assertEqual(truth["call_type"], "argv")
self.assertIsNone(truth["fn_name"])
self.assertEqual(len(truth["inputs"]), 87)
cases = list(zip(truth["inputs"], truth["outputs"], truth["exit_codes"]))
for case in (([], "", 0), (["02", "aa", "bb", "13"], "aabb\n", 0),
(["02aa"], "INCOMPLETE\n", 1), (["f"], "", 2), (["zz"], "", 2)):
self.assertIn(case, cases)
self.assertEqual(self.snapshot["verifier_digest"],
hashlib.sha256(task["_files"]["verify.py"]).hexdigest())
def test_jsonl_export_preserves_one_row_and_excludes_private_material(self):
private = self.task / ".private"
private.mkdir()
(private / "access.json").write_text("DO_NOT_EXPORT_SECRET")
output = export.export(self.task, self.output)
self.assertEqual(output, self.output)
self.assertEqual(output.stat().st_mode & 0o777, 0o600)
raw = output.read_bytes()
self.assertEqual(len(raw.splitlines()), 1)
self.assertNotIn(b"DO_NOT_EXPORT_SECRET", raw)
self.assertEqual(json.loads(raw), export.record(core.load_task(self.task)))
def test_task_or_verifier_identity_drift_is_rejected(self):
for name in ("prompt.md", "verify.py", "template/decoder.c"):
with self.subTest(name=name):
path = self.task / name
original = path.read_bytes()
path.write_bytes(original + b"\n")
with self.assertRaises(ValueError):
export.export(self.task, self.output)
self.assertFalse(self.output.exists())
path.write_bytes(original)
for key in ("task_digest", "verifier_digest"):
with self.subTest(key=key):
value = copy.deepcopy(self.snapshot)
value[key] = "0" * 64
self.write_snapshot(value)
with self.assertRaises(ValueError):
export.record(core.load_task(self.task))
def test_invalid_fields_types_lengths_or_limits_are_rejected(self):
values = []
for key, value in (("call_type", "std"), ("fn_name", "main"), ("inputs", []),
("outputs", []), ("exit_codes", []), ("inputs", "argv"),
("inputs", [["nul\0argument"]] * 87),
("inputs", [["x"] * 129] * 87),
("inputs", [["x" * (core.MAX_CHECKER_BYTES + 1)]] * 87),
("outputs", ["x" * (core.MAX_CHECKER_BYTES + 1)] * 87),
("outputs", [None] * 87), ("exit_codes", [True] * 87),
("exit_codes", [3] * 87)):
value_snapshot = copy.deepcopy(self.snapshot)
value_snapshot["ground_truth"][key] = value
values.append(value_snapshot)
extra = copy.deepcopy(self.snapshot)
extra["ground_truth"]["extra"] = "unused"
values += [extra, {**self.snapshot, "schema": "unused"}, [],
{**self.snapshot, "ground_truth": json.dumps(self.snapshot["ground_truth"])}]
for value in values:
with self.subTest(value_type=type(value).__name__):
self.write_snapshot(value)
with self.assertRaises(ValueError):
export.export(self.task, self.output)
self.assertFalse(self.output.exists())
def test_snapshot_must_be_bounded_regular_file_without_symlinks(self):
path = self.task / "dataset.json"
path.unlink()
path.symlink_to(TASK / "dataset.json")
with self.assertRaises(ValueError):
export.record(core.load_task(self.task))
path.unlink()
os.mkfifo(path)
with self.assertRaises(ValueError):
export.record(core.load_task(self.task))
path.unlink()
with path.open("wb") as stream:
stream.truncate(core.MAX_FILE_BYTES + 1)
with self.assertRaises(ValueError):
export.record(core.load_task(self.task))
def test_output_cannot_overwrite_follow_symlink_or_write_inside_task(self):
self.output.write_bytes(b"existing result")
with self.assertRaises(FileExistsError):
export.export(self.task, self.output)
self.assertEqual(self.output.read_bytes(), b"existing result")
self.output.unlink()
existing = self.root / "existing.jsonl"
existing.write_bytes(b"existing result")
self.output.symlink_to(existing)
with self.assertRaises(ValueError):
export.export(self.task, self.output)
self.assertTrue(self.output.is_symlink())
self.assertEqual(existing.read_bytes(), b"existing result")
with self.assertRaises(ValueError):
export.export(self.task, self.task / "export.jsonl")
def test_failed_write_removes_only_its_own_output_inode(self):
with patch.object(export.os, "fsync", side_effect=OSError("disk error")):
with self.assertRaises(OSError):
export.export(self.task, self.output)
self.assertFalse(self.output.exists())
def replace_output(descriptor):
self.output.rename(self.root / "original.jsonl")
self.output.write_bytes(b"concurrent replacement")
raise OSError("disk error")
with patch.object(export.os, "fsync", side_effect=replace_output):
with self.assertRaises(OSError):
export.export(self.task, self.output)
self.assertEqual(self.output.read_bytes(), b"concurrent replacement")
def test_cli_returns_success_and_rejection_without_overwriting(self):
argv = [str(self.task), "--output", str(self.output)]
with contextlib.redirect_stdout(io.StringIO()) as result:
self.assertEqual(export.main(argv), 0)
self.assertEqual(json.loads(result.getvalue()), dict(dataset=str(self.output)))
raw = self.output.read_bytes()
with contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(export.main(argv), 2)
self.assertEqual(self.output.read_bytes(), raw)
if __name__ == "__main__":
unittest.main()