- Pi controller accepts a fixed Qwen qwen3.8-27b-nvfp4 Responses route next to GLM-5.3; the broker requires exactly one response.completed per call and evidence checks the native model identity. - rollout.export writes the UltraData five-field JSONL from a frozen dataset.json snapshot; every archive now carries dataset.jsonl. - A Docker CLI that did not finish cannot confirm container or network cleanup, and auto-calibration cleanup uncertainty reaches the controller. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
202 lines
11 KiB
Python
202 lines
11 KiB
Python
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import tarfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import test_core as core_fixture
|
|
import test_pi_run as pi_fixture
|
|
from rollout import archive, core, pi_run
|
|
|
|
|
|
class ArchiveTests(unittest.TestCase):
|
|
def setUp(self):
|
|
helper = core_fixture.CoreTests("runTest")
|
|
helper.setUp()
|
|
self.addCleanup(helper.doCleanups)
|
|
self.root, self.task = helper.root, helper.task
|
|
self.calibration = helper.mock_calibration()
|
|
self.control = self.root / "controller"
|
|
self.control.mkdir()
|
|
self.attempt = self.control / "run-fixture"
|
|
pi_fixture.fixture(self.attempt, self.control / "broker.jsonl")
|
|
self.output = self.root / "archive.tar.gz"
|
|
self.finish("PASS")
|
|
|
|
def finish(self, verdict):
|
|
task = core.load_task(self.task)
|
|
(self.task / "dataset.json").write_text(json.dumps(dict(task_digest=task["_digest"],
|
|
verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest(),
|
|
ground_truth=dict(call_type="argv", fn_name=None, inputs=[[]], outputs=[""], exit_codes=[0]))))
|
|
calibration = core._calibration(task, pi_fixture.IMAGE, self.calibration)
|
|
candidate = self.attempt / "candidate"
|
|
candidate.mkdir(exist_ok=True)
|
|
candidate.joinpath("decoder.c").write_text("gold" if verdict == "PASS" else "buggy")
|
|
self.attempt.joinpath("prompt.md").write_bytes(task["_files"]["prompt.md"])
|
|
for name in ("agent.stderr", "grade.stderr"):
|
|
self.attempt.joinpath(name).write_bytes(b"")
|
|
grade = dict(verdict=dict(status=verdict, checks=1), process=dict(
|
|
returncode=0 if verdict == "PASS" else 1, termination="exit", cleanup="confirmed"))
|
|
self.attempt.joinpath("grade.stdout").write_text(json.dumps(grade["verdict"]) + "\n")
|
|
receipt = core._receipt(task, "run", pi_fixture.IMAGE)
|
|
receipt.update(status=verdict, task_verdict=verdict, calibration=calibration,
|
|
candidate_digest=core._digest({"decoder.c": core._read(candidate, "decoder.c")}),
|
|
agent_exit=0, termination="exit", agent_process=dict(returncode=0,
|
|
termination="exit", cleanup="confirmed"), grade=grade)
|
|
core._finish(self.attempt, receipt)
|
|
controller = dict(schema="embedded-rollout.pi-control/1", status=verdict, task_verdict=verdict,
|
|
task_id=task["id"], task_digest=task["_digest"], image_id=pi_fixture.IMAGE,
|
|
attempt=str(self.attempt), calibration=calibration, cleanup="confirmed",
|
|
agent_cleanup="confirmed", network_cleanup="confirmed", proxy_cleanup="confirmed",
|
|
agent_exit=0, agent_termination="exit",
|
|
pi_evidence=pi_run.evidence(self.attempt, self.control / "broker.jsonl"))
|
|
core._finish(self.control, controller)
|
|
|
|
def test_pass_and_fail_archive_only_allowlisted_files_and_manifest_hashes(self):
|
|
for verdict in ("PASS", "FAIL"):
|
|
with self.subTest(verdict=verdict):
|
|
self.finish(verdict)
|
|
for name in ("gold/decoder.c", "grader/verify.py", ".private/secret.json", "workspace/private"):
|
|
hidden = self.control / name
|
|
hidden.parent.mkdir(parents=True, exist_ok=True)
|
|
hidden.write_text("DO_NOT_EXPORT_SECRET")
|
|
output = archive.archive(self.control, self.task, self.output)
|
|
self.assertEqual(output.stat().st_mode & 0o777, 0o600)
|
|
with tarfile.open(output, "r:gz") as bundle:
|
|
members = {member.name: bundle.extractfile(member).read() for member in bundle.getmembers()}
|
|
manifest = json.loads(members.pop("manifest.json"))
|
|
self.assertEqual(manifest["task_verdict"], verdict)
|
|
self.assertEqual(manifest["pi_evidence"], "VERIFIED")
|
|
self.assertEqual(set(members), set(manifest["files"]))
|
|
self.assertEqual(len(members), 17)
|
|
row = json.loads(members["dataset.jsonl"])
|
|
self.assertEqual(set(row), {"uuid", "query", "ground_truth", "source", "domain"})
|
|
self.assertEqual(row["ground_truth"]["call_type"], "argv")
|
|
self.assertFalse(any(part in name.split("/") for name in members
|
|
for part in ("gold", "grader", ".private", "workspace")))
|
|
self.assertNotIn(b"DO_NOT_EXPORT_SECRET", b"".join(members.values()))
|
|
for name, data in members.items():
|
|
self.assertEqual(manifest["files"][name], dict(bytes=len(data),
|
|
sha256=hashlib.sha256(data).hexdigest()))
|
|
output.unlink()
|
|
|
|
def test_tampered_candidate_feedback_calibration_or_grade_is_rejected(self):
|
|
paths = [self.attempt / "candidate/decoder.c", self.attempt / "agent.stdout",
|
|
self.attempt / "trace/native.jsonl", self.calibration / "receipt.json",
|
|
self.attempt / "grade.stdout"]
|
|
originals = {path: path.read_bytes() for path in paths}
|
|
for kind in ("candidate", "feedback", "calibration", "grade"):
|
|
with self.subTest(kind=kind):
|
|
if kind == "candidate":
|
|
paths[0].write_text("changed after grading")
|
|
elif kind == "feedback":
|
|
events = [json.loads(line) for line in originals[paths[1]].decode().splitlines()]
|
|
events[-3]["context"]["messages"] = []
|
|
raw = "".join(json.dumps(event) + "\n" for event in events)
|
|
paths[1].write_text(raw)
|
|
paths[2].write_text(raw)
|
|
elif kind == "calibration":
|
|
calibration = json.loads(originals[paths[3]])
|
|
calibration["additional_note"] = "receipt drift"
|
|
paths[3].write_text(json.dumps(calibration))
|
|
else:
|
|
paths[4].write_text('{"status":"FAIL","checks":1}\n')
|
|
with self.assertRaises(ValueError):
|
|
archive.archive(self.control, self.task, self.output)
|
|
self.assertFalse(self.output.exists())
|
|
for path, raw in originals.items():
|
|
path.write_bytes(raw)
|
|
|
|
def test_symlinks_multiple_attempts_and_outputs_inside_inputs_are_rejected(self):
|
|
candidate = self.attempt / "candidate/decoder.c"
|
|
raw = candidate.read_bytes()
|
|
candidate.unlink()
|
|
candidate.symlink_to(self.task / "gold/decoder.c")
|
|
with self.assertRaises(ValueError):
|
|
archive.archive(self.control, self.task, self.output)
|
|
candidate.unlink()
|
|
candidate.write_bytes(raw)
|
|
second = self.control / "run-extra"
|
|
second.mkdir()
|
|
with self.assertRaises(ValueError):
|
|
archive.archive(self.control, self.task, self.output)
|
|
second.rmdir()
|
|
for directory in (self.control, self.task, self.calibration):
|
|
with self.subTest(directory=directory), self.assertRaises(ValueError):
|
|
archive.archive(self.control, self.task, directory / "archive.tar.gz")
|
|
link = self.root / "controller-link"
|
|
link.symlink_to(self.control, target_is_directory=True)
|
|
with self.assertRaises(ValueError):
|
|
archive.archive(link, self.task, self.output)
|
|
|
|
def test_incomplete_cleanup_or_identity_cannot_be_archived(self):
|
|
path = self.control / "receipt.json"
|
|
raw = path.read_bytes()
|
|
for key, value in (("cleanup", "unknown"), ("proxy_cleanup", "unknown"),
|
|
("agent_exit", 1), ("status", "UNVERIFIED"),
|
|
("image_id", "sha256:" + "b" * 64), ("attempt", str(self.root / "elsewhere"))):
|
|
with self.subTest(key=key):
|
|
receipt = json.loads(raw)
|
|
receipt[key] = value
|
|
path.write_text(json.dumps(receipt))
|
|
with self.assertRaises(ValueError):
|
|
archive.archive(self.control, self.task, self.output)
|
|
path.write_bytes(raw)
|
|
|
|
def test_existing_output_is_preserved_and_failed_new_output_is_removed(self):
|
|
self.output.write_bytes(b"existing result")
|
|
with self.assertRaises(FileExistsError):
|
|
archive.archive(self.control, self.task, self.output)
|
|
self.assertEqual(self.output.read_bytes(), b"existing result")
|
|
self.output.unlink()
|
|
with patch.object(archive.tarfile, "open", side_effect=OSError("write failed")):
|
|
with self.assertRaises(OSError):
|
|
archive.archive(self.control, self.task, self.output)
|
|
self.assertFalse(self.output.exists())
|
|
|
|
def test_trace_size_and_symlink_output_boundaries(self):
|
|
trace = self.attempt / "trace/native.jsonl"
|
|
with trace.open("wb") as stream:
|
|
stream.truncate(core.MAX_TRACE_BYTES + 1)
|
|
with self.assertRaises(ValueError):
|
|
archive.archive(self.control, self.task, self.output)
|
|
self.assertFalse(self.output.exists())
|
|
trace.write_bytes(self.attempt.joinpath("agent.stdout").read_bytes())
|
|
existing = self.root / "existing.tar.gz"
|
|
existing.write_bytes(b"existing result")
|
|
self.output.symlink_to(existing)
|
|
with self.assertRaises(ValueError):
|
|
archive.archive(self.control, self.task, self.output)
|
|
self.assertTrue(self.output.is_symlink())
|
|
self.assertEqual(existing.read_bytes(), b"existing result")
|
|
|
|
def test_qwen_archive_cannot_skip_model_checks_by_deleting_identity(self):
|
|
source = self.root / "qwen-fixture"
|
|
broker = self.root / "qwen-broker.jsonl"
|
|
pi_fixture.fixture(source, broker, model=pi_run.QWEN_MODEL)
|
|
for name in ("agent.stdout", "trace/native.jsonl"):
|
|
(self.attempt / name).write_bytes((source / name).read_bytes())
|
|
(self.control / "broker.jsonl").write_bytes(broker.read_bytes())
|
|
self.finish("PASS")
|
|
path = self.control / "receipt.json"
|
|
control = json.loads(path.read_text())
|
|
control["model_id"] = pi_run.QWEN_MODEL
|
|
control["model_api"] = "openai-responses"
|
|
path.write_text(json.dumps(control))
|
|
archive.archive(self.control, self.task, self.output)
|
|
self.output.unlink()
|
|
for value in (None, "unknown", "missing"):
|
|
if value == "missing":
|
|
control.pop("model_id")
|
|
else:
|
|
control["model_id"] = value
|
|
path.write_text(json.dumps(control))
|
|
with self.assertRaises(ValueError):
|
|
archive.archive(self.control, self.task, self.output)
|
|
self.assertFalse(self.output.exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|