241 lines
14 KiB
Python
241 lines
14 KiB
Python
"""Preservation regressions use frozen Pi events, never a model or a board."""
|
|
import copy
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from export import trajectory as tr
|
|
|
|
|
|
def message(role, content, **fields):
|
|
return {"role": role, "content": content, **fields}
|
|
|
|
|
|
def call(ident, command="true"):
|
|
return message("assistant", [{"type": "toolCall", "id": ident, "name": "bash",
|
|
"arguments": {"command": command}}])
|
|
|
|
|
|
def result(ident, text="OK", **fields):
|
|
return message("toolResult", [{"type": "text", "text": text}], toolCallId=ident, **fields)
|
|
|
|
|
|
def segment(messages):
|
|
return [{"type": "agent_start"}] + [
|
|
event for msg in messages for event in
|
|
({"type": "message_start", "message": msg}, {"type": "message_end", "message": msg})
|
|
] + [{"type": "agent_end", "messages": messages}]
|
|
|
|
|
|
class PreservationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.rd = Path(self.temp.name)
|
|
(self.rd / "agent").mkdir()
|
|
(self.rd / "rollout.json").write_text(json.dumps({"runner": "pi", "runner_version": "pi-sdk-v1", "execution_id": "exec-fixture", "state": "DONE"}))
|
|
(self.rd / "agent/system.md").write_text("Recorded system.")
|
|
(self.rd / "agent/tools.json").write_text('[{"name":"bash"}]')
|
|
|
|
def build(self, events, *, complete_context=True):
|
|
if not any(e.get("type") == "session" for e in events):
|
|
header = {"type": "session", "version": 3, "id": "session-fixture"}
|
|
if complete_context:
|
|
header["bench"] = {"runner_version": "pi-sdk-v1", "execution_id": "exec-fixture",
|
|
"system_prompt": "Recorded system.", "tools": [{"name": "bash"}]}
|
|
events = [header] + events
|
|
(self.rd / "agent/events.jsonl").write_text("\n".join(map(json.dumps, events)) + "\n")
|
|
self.frozen = tr.freeze(self.rd)
|
|
return tr.reconstruct(self.frozen)
|
|
|
|
def test_append_only_context_is_preserved_but_never_qualified(self):
|
|
for version in ("legacy-pi", "pi-sdk-v1", None):
|
|
with self.subTest(version=version):
|
|
(self.rd / "rollout.json").write_text(json.dumps({"runner": "pi", "runner_version": version, "state": "DONE"}))
|
|
output = self.build(segment([message("user", "task"), message("assistant", "done")]), complete_context=False)
|
|
check = tr.validate(self.frozen, output)
|
|
self.assertEqual(len(output["messages"]), 3)
|
|
self.assertEqual(output["excluded"], [])
|
|
self.assertTrue(check["checks"]["VALID_MESSAGE_COVERAGE"]["pass"])
|
|
self.assertFalse(check["checks"]["REQUIRED_CONTEXT"]["pass"])
|
|
self.assertFalse(check["pass"])
|
|
|
|
def test_data001_retry_segments_are_all_retained(self):
|
|
err = message("assistant", [], stopReason="error", errorMessage="429")
|
|
first = [message("user", "task"), call("a"), result("a"), err]
|
|
output = self.build(segment(first) + [{"type": "auto_retry_start", "attempt": 1}] +
|
|
segment([err]) + segment([call("b"), result("b"), message("assistant", "done")]))
|
|
self.assertEqual(len(output["messages"]), 7) # system + all six effective event messages
|
|
self.assertEqual(len(output["excluded"]), 2)
|
|
self.assertTrue(tr.validate(self.frozen, output)["pass"])
|
|
broken = copy.deepcopy(output)
|
|
broken["messages"] = broken["messages"][:2] + broken["messages"][4:]
|
|
broken["source_ids"] = broken["source_ids"][:2] + broken["source_ids"][4:]
|
|
check = tr.validate(self.frozen, broken)
|
|
self.assertTrue(check["checks"]["CALL_TOOL_CLOSURE"]["pass"])
|
|
self.assertFalse(check["checks"]["VALID_MESSAGE_COVERAGE"]["pass"])
|
|
|
|
def test_data002_equal_content_different_occurrences_survive(self):
|
|
output = self.build(segment([message("user", "task"), message("assistant", "same"), message("assistant", "same")]))
|
|
self.assertEqual(len(output["messages"]), 4)
|
|
self.assertNotEqual(output["source_ids"][-1], output["source_ids"][-2])
|
|
self.assertTrue(tr.validate(self.frozen, output)["pass"])
|
|
|
|
def test_data003_mirrors_are_identity_aliases(self):
|
|
msgs = [message("user", "task"), call("a"), result("a")]
|
|
events = segment(msgs)
|
|
events.insert(-1, {"type": "turn_end", "message": msgs[1], "toolResults": [msgs[2]]})
|
|
events.append(events[-1])
|
|
output = self.build(events)
|
|
self.assertEqual(len(output["messages"]), 4)
|
|
self.assertTrue(tr.validate(self.frozen, output)["pass"])
|
|
self.assertGreater(len(output["lineage"][-1]["observations"]), 2)
|
|
|
|
def test_data004_mirror_only_context_has_anchored_position(self):
|
|
before, after = message("assistant", "before"), message("assistant", "after")
|
|
custom = message("custom", [{"type": "text", "text": "budget reminder"}], customType="budget")
|
|
events = segment([before, after])
|
|
events[-1]["messages"] = [message("user", "task"), before, custom, after]
|
|
output = self.build(events)
|
|
self.assertEqual([m["role"] for m in output["messages"]], ["system", "user", "assistant", "custom", "assistant"])
|
|
self.assertTrue(tr.validate(self.frozen, output)["pass"])
|
|
|
|
def test_data005_only_proven_empty_errors_are_excluded(self):
|
|
msgs = [message("user", "task"), message("assistant", [], stopReason="error"),
|
|
call("a"), result("a", "error 429 timeout", isError=True), message("assistant", "normal 429")]
|
|
output = self.build(segment(msgs))
|
|
self.assertEqual(len(output["excluded"]), 1)
|
|
self.assertTrue(tr.validate(self.frozen, output)["pass"])
|
|
for content in (" ", [{"type": "thinking", "thinking": "x"}], [{"type": "unknown"}]):
|
|
with self.subTest(content=content):
|
|
output = self.build(segment([message("user", "task"), message("assistant", content, stopReason="error")]))
|
|
self.assertEqual(output["excluded"], [])
|
|
self.assertFalse(tr.validate(self.frozen, output)["pass"])
|
|
|
|
def test_streamed_content_forbids_empty_error_exclusion(self):
|
|
err = message("assistant", [], stopReason="error")
|
|
events = segment([message("user", "task"), err])
|
|
events.insert(-2, {"type": "message_update", "message": err,
|
|
"assistantMessageEvent": {"type": "thinking_delta", "delta": "partial"}})
|
|
output = self.build(events)
|
|
self.assertEqual(output["excluded"], [])
|
|
self.assertFalse(tr.validate(self.frozen, output)["pass"])
|
|
|
|
def test_nonempty_start_forbids_empty_error_exclusion(self):
|
|
err = message("assistant", [], stopReason="error")
|
|
events = segment([message("user", "task"), err, message("assistant", "done")])
|
|
for event in events:
|
|
if event["type"] == "message_start" and event["message"] is err:
|
|
event["message"] = message("assistant", "nonempty initial generation")
|
|
output = self.build(events)
|
|
self.assertEqual(len(output["excluded"]), 0)
|
|
self.assertFalse(tr.validate(self.frozen, output)["pass"])
|
|
|
|
def test_sdk_header_requires_matching_saved_context(self):
|
|
metadata = json.loads((self.rd / "rollout.json").read_text())
|
|
metadata.update(runner_version="pi-sdk-v1", execution_id="exec-fixture")
|
|
(self.rd / "rollout.json").write_text(json.dumps(metadata))
|
|
header = {"type": "session", "version": 3, "id": "session-fixture", "bench": {"runner_version": "pi-sdk-v1", "execution_id": "exec-fixture",
|
|
"system_prompt": "Recorded system.", "tools": [{"name": "bash"}]}}
|
|
output = self.build([header] + segment([message("user", "task"), message("assistant", "done")]))
|
|
self.assertTrue(tr.validate(self.frozen, output)["pass"])
|
|
resumed = [header] + segment([message("user", "task"), message("assistant", "first")]) + [copy.deepcopy(header)] + segment([message("assistant", "resumed")])
|
|
output = self.build(resumed)
|
|
self.assertTrue(tr.validate(self.frozen, output)["pass"])
|
|
self.assertEqual(sum(m["role"] == "user" for m in output["messages"]), 1)
|
|
interrupted = [header] + segment([message("user", "task"), message("assistant", "first")])[:-1] + [copy.deepcopy(header)] + segment([message("assistant", "resumed")])
|
|
output = self.build(interrupted)
|
|
self.assertIn("SDK_RESUME_INCOMPLETE_SEGMENT", output["errors"])
|
|
self.assertFalse(tr.validate(self.frozen, output)["pass"])
|
|
header["bench"]["system_prompt"] = "different current prompt"
|
|
output = self.build([header] + segment([message("user", "task"), message("assistant", "done")]))
|
|
self.assertIn("SDK_CONTEXT_SNAPSHOT_MISMATCH", output["errors"])
|
|
self.assertFalse(tr.validate(self.frozen, output)["pass"])
|
|
|
|
def test_header_cannot_upgrade_legacy_or_be_backfilled_after_generation(self):
|
|
events = segment([message("user", "task"), message("assistant", "done")])
|
|
self.build(events)
|
|
header = json.loads((self.rd / "agent/events.jsonl").read_text().splitlines()[0])
|
|
output = self.build(events + [header])
|
|
self.assertIn("SDK_CONTEXT_HEADER_LATE", output["errors"])
|
|
self.assertFalse(tr.validate(self.frozen, output)["checks"]["REQUIRED_CONTEXT"]["pass"])
|
|
for version, execution_id in (("legacy-pi", "exec-fixture"), (None, "exec-fixture"), ("pi-sdk-v1", None)):
|
|
with self.subTest(version=version, execution_id=execution_id):
|
|
(self.rd / "rollout.json").write_text(json.dumps({"runner": "pi", "runner_version": version,
|
|
"execution_id": execution_id, "state": "DONE"}))
|
|
output = self.build([header] + events)
|
|
self.assertFalse(tr.validate(self.frozen, output)["checks"]["REQUIRED_CONTEXT"]["pass"])
|
|
|
|
def test_empty_matching_system_is_not_complete_context(self):
|
|
events = segment([message("user", "task"), message("assistant", "done")])
|
|
self.build(events)
|
|
header = json.loads((self.rd / "agent/events.jsonl").read_text().splitlines()[0])
|
|
header["bench"]["system_prompt"] = ""
|
|
(self.rd / "agent/system.md").write_text("")
|
|
output = self.build([header] + events)
|
|
self.assertFalse(tr.validate(self.frozen, output)["checks"]["REQUIRED_CONTEXT"]["pass"])
|
|
def test_data007_timeout_never_becomes_later_success(self):
|
|
output = self.build(segment([message("user", "task"), call("a"), result("a", "TIMEOUT"),
|
|
call("b"), result("b", "OK")]))
|
|
broken = copy.deepcopy(output)
|
|
broken["messages"][3]["content"] = broken["messages"][5]["content"]
|
|
self.assertFalse(tr.validate(self.frozen, broken)["checks"]["PAYLOAD_COVERAGE"]["pass"])
|
|
|
|
def test_data006_wait_annotations_never_change_messages(self):
|
|
msgs = [message("user", "task")]
|
|
texts = ["[12:00:00] handle: RUNNING 已 1s", "[12:00:01] handle: RUNNING 已 2s",
|
|
"[12:00:02] handle: RUNNING 已 3s\nmeasurements: {\"x\":1}",
|
|
"result: OK exit_code: 0 error_class: None 4.0s"]
|
|
for i, text in enumerate(texts):
|
|
msgs += [call(str(i), "bench stage wait handle"), result(str(i), text)]
|
|
output = self.build(segment(msgs))
|
|
self.assertEqual([a["tag"] for a in output["annotations"]],
|
|
["FIRST_OBSERVATION", "HEARTBEAT_REPEAT", "SAME_STATE_NEW_EVIDENCE", "STATE_CHANGED"])
|
|
self.assertEqual(output["messages"][1:], msgs)
|
|
self.assertTrue(tr.validate(self.frozen, output)["pass"])
|
|
|
|
def test_data010_removed_explicit_reference_and_paraphrase(self):
|
|
removed = [{"source_message_id": "tool-1", "pointer": "/metric", "sha256": "abc"}]
|
|
explicit = message("assistant", "using the measured metric", evidence_refs=[{"source_message_id": "tool-1", "pointer": "/metric"}])
|
|
self.assertEqual(tr.truncation_annotations(removed, [explicit])[0]["tag"], "REFERENCED_CONTENT_REMOVED")
|
|
self.assertEqual(tr.truncation_annotations(removed, [message("assistant", "accuracy is acceptable")])[0]["tag"], "REFERENCE_UNKNOWN")
|
|
|
|
def test_redaction_is_source_frozen_and_identity_preserving(self):
|
|
from unittest.mock import patch
|
|
msgs = [message("user", "path /home/test.person/data and test.person"), call("test.person", "echo /home/test.person"), result("test.person")]
|
|
output = self.build(segment(msgs))
|
|
with patch.dict("os.environ", {"USER": "different-account", "LOGNAME": "different-account"}):
|
|
self.assertEqual(output, tr.reconstruct(self.frozen))
|
|
self.assertEqual(output["messages"][2]["content"][0]["id"], "test.person")
|
|
self.assertFalse(tr.validate(self.frozen, output)["checks"]["PRIVACY"]["pass"])
|
|
self.assertIn("<USER_1>", output["messages"][1]["content"])
|
|
|
|
def test_data009_closure_and_parallel_completion(self):
|
|
parallel = message("assistant", call("a")["content"] + call("b")["content"])
|
|
output = self.build(segment([message("user", "task"), parallel, result("b"), result("a")]))
|
|
self.assertTrue(tr.validate(self.frozen, output)["pass"])
|
|
for msgs in ([result("missing")], [call("missing")], [call("a"), result("a"), call("a"), result("a")]):
|
|
output = self.build(segment([message("user", "task"), *msgs]))
|
|
self.assertFalse(tr.validate(self.frozen, output)["checks"]["CALL_TOOL_CLOSURE"]["pass"])
|
|
|
|
def test_full_blocks_long_output_and_unknown_event(self):
|
|
msgs = [message("user", "task"), call("a"), result("a", "x" * 15000),
|
|
message("assistant", [{"type": "thinking", "thinking": "keep"}, {"type": "text", "text": "done"}], vendor={"future": True})]
|
|
output = self.build(segment(msgs))
|
|
self.assertEqual(len(output["messages"][3]["content"][0]["text"]), 15000)
|
|
self.assertEqual(output["messages"][-1]["vendor"], {"future": True})
|
|
self.assertTrue(tr.validate(self.frozen, output)["pass"])
|
|
output = self.build(segment(msgs) + [{"type": "future_context", "content": "unknown"}])
|
|
self.assertFalse(tr.validate(self.frozen, output)["pass"])
|
|
|
|
def test_missing_snapshots_never_read_live_files(self):
|
|
(self.rd / "agent/system.md").unlink()
|
|
output = self.build(segment([message("user", "task"), message("assistant", "done")]))
|
|
self.assertIn("CONTEXT_SNAPSHOT_MISSING", output["errors"])
|
|
self.assertFalse(tr.validate(self.frozen, output)["pass"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|