data-gen-ebd/tests/test_archive.py
amos 714f345b63 feat: add Qwen Responses route, five-field task export and pending-create cleanup
- Pi controller accepts a fixed Qwen qwen3.8-27b-nvfp4 Responses route next
  to GLM-5.3; the broker requires exactly one response.completed per call
  and evidence checks the native model identity.
- rollout.export writes the UltraData five-field JSONL from a frozen
  dataset.json snapshot; every archive now carries dataset.jsonl.
- A Docker CLI that did not finish cannot confirm container or network
  cleanup, and auto-calibration cleanup uncertainty reaches the controller.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 21:05:56 +08:00

202 lines
11 KiB
Python

import hashlib
import json
from pathlib import Path
import tarfile
import unittest
from unittest.mock import patch
import test_core as core_fixture
import test_pi_run as pi_fixture
from rollout import archive, core, pi_run
class ArchiveTests(unittest.TestCase):
def setUp(self):
helper = core_fixture.CoreTests("runTest")
helper.setUp()
self.addCleanup(helper.doCleanups)
self.root, self.task = helper.root, helper.task
self.calibration = helper.mock_calibration()
self.control = self.root / "controller"
self.control.mkdir()
self.attempt = self.control / "run-fixture"
pi_fixture.fixture(self.attempt, self.control / "broker.jsonl")
self.output = self.root / "archive.tar.gz"
self.finish("PASS")
def finish(self, verdict):
task = core.load_task(self.task)
(self.task / "dataset.json").write_text(json.dumps(dict(task_digest=task["_digest"],
verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest(),
ground_truth=dict(call_type="argv", fn_name=None, inputs=[[]], outputs=[""], exit_codes=[0]))))
calibration = core._calibration(task, pi_fixture.IMAGE, self.calibration)
candidate = self.attempt / "candidate"
candidate.mkdir(exist_ok=True)
candidate.joinpath("decoder.c").write_text("gold" if verdict == "PASS" else "buggy")
self.attempt.joinpath("prompt.md").write_bytes(task["_files"]["prompt.md"])
for name in ("agent.stderr", "grade.stderr"):
self.attempt.joinpath(name).write_bytes(b"")
grade = dict(verdict=dict(status=verdict, checks=1), process=dict(
returncode=0 if verdict == "PASS" else 1, termination="exit", cleanup="confirmed"))
self.attempt.joinpath("grade.stdout").write_text(json.dumps(grade["verdict"]) + "\n")
receipt = core._receipt(task, "run", pi_fixture.IMAGE)
receipt.update(status=verdict, task_verdict=verdict, calibration=calibration,
candidate_digest=core._digest({"decoder.c": core._read(candidate, "decoder.c")}),
agent_exit=0, termination="exit", agent_process=dict(returncode=0,
termination="exit", cleanup="confirmed"), grade=grade)
core._finish(self.attempt, receipt)
controller = dict(schema="embedded-rollout.pi-control/1", status=verdict, task_verdict=verdict,
task_id=task["id"], task_digest=task["_digest"], image_id=pi_fixture.IMAGE,
attempt=str(self.attempt), calibration=calibration, cleanup="confirmed",
agent_cleanup="confirmed", network_cleanup="confirmed", proxy_cleanup="confirmed",
agent_exit=0, agent_termination="exit",
pi_evidence=pi_run.evidence(self.attempt, self.control / "broker.jsonl"))
core._finish(self.control, controller)
def test_pass_and_fail_archive_only_allowlisted_files_and_manifest_hashes(self):
for verdict in ("PASS", "FAIL"):
with self.subTest(verdict=verdict):
self.finish(verdict)
for name in ("gold/decoder.c", "grader/verify.py", ".private/secret.json", "workspace/private"):
hidden = self.control / name
hidden.parent.mkdir(parents=True, exist_ok=True)
hidden.write_text("DO_NOT_EXPORT_SECRET")
output = archive.archive(self.control, self.task, self.output)
self.assertEqual(output.stat().st_mode & 0o777, 0o600)
with tarfile.open(output, "r:gz") as bundle:
members = {member.name: bundle.extractfile(member).read() for member in bundle.getmembers()}
manifest = json.loads(members.pop("manifest.json"))
self.assertEqual(manifest["task_verdict"], verdict)
self.assertEqual(manifest["pi_evidence"], "VERIFIED")
self.assertEqual(set(members), set(manifest["files"]))
self.assertEqual(len(members), 17)
row = json.loads(members["dataset.jsonl"])
self.assertEqual(set(row), {"uuid", "query", "ground_truth", "source", "domain"})
self.assertEqual(row["ground_truth"]["call_type"], "argv")
self.assertFalse(any(part in name.split("/") for name in members
for part in ("gold", "grader", ".private", "workspace")))
self.assertNotIn(b"DO_NOT_EXPORT_SECRET", b"".join(members.values()))
for name, data in members.items():
self.assertEqual(manifest["files"][name], dict(bytes=len(data),
sha256=hashlib.sha256(data).hexdigest()))
output.unlink()
def test_tampered_candidate_feedback_calibration_or_grade_is_rejected(self):
paths = [self.attempt / "candidate/decoder.c", self.attempt / "agent.stdout",
self.attempt / "trace/native.jsonl", self.calibration / "receipt.json",
self.attempt / "grade.stdout"]
originals = {path: path.read_bytes() for path in paths}
for kind in ("candidate", "feedback", "calibration", "grade"):
with self.subTest(kind=kind):
if kind == "candidate":
paths[0].write_text("changed after grading")
elif kind == "feedback":
events = [json.loads(line) for line in originals[paths[1]].decode().splitlines()]
events[-3]["context"]["messages"] = []
raw = "".join(json.dumps(event) + "\n" for event in events)
paths[1].write_text(raw)
paths[2].write_text(raw)
elif kind == "calibration":
calibration = json.loads(originals[paths[3]])
calibration["additional_note"] = "receipt drift"
paths[3].write_text(json.dumps(calibration))
else:
paths[4].write_text('{"status":"FAIL","checks":1}\n')
with self.assertRaises(ValueError):
archive.archive(self.control, self.task, self.output)
self.assertFalse(self.output.exists())
for path, raw in originals.items():
path.write_bytes(raw)
def test_symlinks_multiple_attempts_and_outputs_inside_inputs_are_rejected(self):
candidate = self.attempt / "candidate/decoder.c"
raw = candidate.read_bytes()
candidate.unlink()
candidate.symlink_to(self.task / "gold/decoder.c")
with self.assertRaises(ValueError):
archive.archive(self.control, self.task, self.output)
candidate.unlink()
candidate.write_bytes(raw)
second = self.control / "run-extra"
second.mkdir()
with self.assertRaises(ValueError):
archive.archive(self.control, self.task, self.output)
second.rmdir()
for directory in (self.control, self.task, self.calibration):
with self.subTest(directory=directory), self.assertRaises(ValueError):
archive.archive(self.control, self.task, directory / "archive.tar.gz")
link = self.root / "controller-link"
link.symlink_to(self.control, target_is_directory=True)
with self.assertRaises(ValueError):
archive.archive(link, self.task, self.output)
def test_incomplete_cleanup_or_identity_cannot_be_archived(self):
path = self.control / "receipt.json"
raw = path.read_bytes()
for key, value in (("cleanup", "unknown"), ("proxy_cleanup", "unknown"),
("agent_exit", 1), ("status", "UNVERIFIED"),
("image_id", "sha256:" + "b" * 64), ("attempt", str(self.root / "elsewhere"))):
with self.subTest(key=key):
receipt = json.loads(raw)
receipt[key] = value
path.write_text(json.dumps(receipt))
with self.assertRaises(ValueError):
archive.archive(self.control, self.task, self.output)
path.write_bytes(raw)
def test_existing_output_is_preserved_and_failed_new_output_is_removed(self):
self.output.write_bytes(b"existing result")
with self.assertRaises(FileExistsError):
archive.archive(self.control, self.task, self.output)
self.assertEqual(self.output.read_bytes(), b"existing result")
self.output.unlink()
with patch.object(archive.tarfile, "open", side_effect=OSError("write failed")):
with self.assertRaises(OSError):
archive.archive(self.control, self.task, self.output)
self.assertFalse(self.output.exists())
def test_trace_size_and_symlink_output_boundaries(self):
trace = self.attempt / "trace/native.jsonl"
with trace.open("wb") as stream:
stream.truncate(core.MAX_TRACE_BYTES + 1)
with self.assertRaises(ValueError):
archive.archive(self.control, self.task, self.output)
self.assertFalse(self.output.exists())
trace.write_bytes(self.attempt.joinpath("agent.stdout").read_bytes())
existing = self.root / "existing.tar.gz"
existing.write_bytes(b"existing result")
self.output.symlink_to(existing)
with self.assertRaises(ValueError):
archive.archive(self.control, self.task, self.output)
self.assertTrue(self.output.is_symlink())
self.assertEqual(existing.read_bytes(), b"existing result")
def test_qwen_archive_cannot_skip_model_checks_by_deleting_identity(self):
source = self.root / "qwen-fixture"
broker = self.root / "qwen-broker.jsonl"
pi_fixture.fixture(source, broker, model=pi_run.QWEN_MODEL)
for name in ("agent.stdout", "trace/native.jsonl"):
(self.attempt / name).write_bytes((source / name).read_bytes())
(self.control / "broker.jsonl").write_bytes(broker.read_bytes())
self.finish("PASS")
path = self.control / "receipt.json"
control = json.loads(path.read_text())
control["model_id"] = pi_run.QWEN_MODEL
control["model_api"] = "openai-responses"
path.write_text(json.dumps(control))
archive.archive(self.control, self.task, self.output)
self.output.unlink()
for value in (None, "unknown", "missing"):
if value == "missing":
control.pop("model_id")
else:
control["model_id"] = value
path.write_text(json.dumps(control))
with self.assertRaises(ValueError):
archive.archive(self.control, self.task, self.output)
self.assertFalse(self.output.exists())
if __name__ == "__main__":
unittest.main()