data-gen-bpu/export/test_opencode_source.py

153 lines
7.5 KiB
Python

"""Native OpenCode fixtures use no credentials, models, boards or shared state."""
import copy
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from export import trajectory as tr
from harness.runners.opencode import ALLOWED_TOOLS, VERSION
def fixture(rd):
agent = rd / "agent"
agent.mkdir(exist_ok=True)
meta_path = rd / "rollout.json"
meta = json.loads(meta_path.read_text()) if meta_path.exists() else {"state": "DONE"}
meta.update(runner="opencode", runner_version=VERSION, agent_config_version="1")
meta_path.write_text(json.dumps(meta))
sid = "ses_fixture"
entries = [
{"info": {"id": "msg_user", "role": "user"}, "parts": [{"type": "text", "text": "Compile and evaluate."}]},
{"info": {"id": "msg_call", "role": "assistant", "finish": "tool-calls"}, "parts": [
{"type": "step-start"}, {"type": "reasoning", "text": "Check measured results first."},
{"type": "tool", "callID": "t1", "tool": "bash", "state": {
"status": "completed", "input": {"command": "bench stage status h"}, "output": "DONE"}},
{"type": "step-finish", "reason": "tool-calls"}]},
{"info": {"id": "msg_final", "role": "assistant", "finish": "stop"}, "parts": [
{"type": "step-start"}, {"type": "text", "text": "The measured subset passed within the task tolerance."},
{"type": "step-finish", "reason": "stop"}]}]
events = []
for i, entry in enumerate(entries):
info = entry["info"]
info.update(sessionID=sid, time={"created": 10 + i, "completed": 20 + i})
if info["role"] == "assistant":
info["parentID"] = "msg_user"
for j, part in enumerate(entry["parts"]):
part.update(id=f"prt_{i}{j}", sessionID=sid, messageID=info["id"])
if info["role"] == "assistant" and part["type"] != "reasoning":
events.append({"type": {"tool": "tool_use"}.get(part["type"], part["type"].replace("-", "_")),
"sessionID": sid, "part": part})
tools = [{"name": name, "parameters": {"type": "object"}} for name in ALLOWED_TOOLS]
for name, data in {"session-id.json": {"runner": "opencode", "version": VERSION, "session_id": sid},
"raw-session.json": {"info": {"id": sid}, "messages": entries},
"raw-system.json": ["Actual runner system", "Actual harness instruction"],
"raw-tools.json": tools, "tools.json": tools}.items():
(agent / name).write_text(json.dumps(data))
(agent / "events.jsonl").write_text("\n".join(map(json.dumps, events)))
class OpenCodeSourceTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.rd = Path(self.tmp.name)
fixture(self.rd)
def check(self):
frozen = tr.freeze(self.rd)
output = tr.reconstruct(frozen)
return frozen, output, tr.validate(frozen, output)
def mutate(self, file, change):
path = self.rd / "agent" / file
data = json.loads(path.read_text())
change(data)
path.write_text(json.dumps(data))
def test_complete_native_source_preserves_reasoning_and_identity(self):
_, output, validation = self.check()
self.assertTrue(validation["pass"], validation)
self.assertEqual(output["format"], "opencode-native-messages/1")
self.assertEqual(output["messages"][2]["content"][0]["type"], "thinking")
self.assertEqual(output["source_ids"][2], "ses_fixture/msg_call")
self.assertEqual(output["messages"][2]["opencode"]["info"]["finish"], "tool-calls")
self.assertEqual(validation["checks"]["OBSERVATION_ACCOUNTING"]["expected"], 11)
def test_missing_native_files_cannot_qualify(self):
for name in ("raw-session.json", "raw-system.json", "raw-tools.json", "session-id.json", "tools.json", "events.jsonl"):
with self.subTest(name=name):
fixture(self.rd)
(self.rd / "agent" / name).unlink()
self.assertFalse(self.check()[2]["pass"])
def test_incomplete_or_conflicting_source_cannot_qualify(self):
changes = [
lambda d: d["messages"].pop(1),
lambda d: d["messages"].pop(),
lambda d: d["messages"][1]["info"].pop("id"),
lambda d: d["messages"][1]["info"].update(sessionID="ses_other"),
lambda d: d["messages"][1]["info"]["time"].pop("completed"),
lambda d: d["messages"][1]["info"].update(parentID="msg_unknown"),
lambda d: d["messages"][1]["info"].update(error={"name": "APIError"}),
lambda d: d["messages"][1]["parts"][2]["state"].update(status="running"),
lambda d: d["messages"][1]["parts"][2]["state"].pop("output"),
lambda d: d["messages"][1]["parts"][2]["state"].update(output="tampered"),
lambda d: d["messages"][1]["parts"][2].update(callID=""),
lambda d: d["messages"][1]["parts"][1].update(type="compaction"),
lambda d: d["messages"][1]["parts"].reverse(),
lambda d: d["messages"].reverse(),
]
for index, change in enumerate(changes):
with self.subTest(index=index):
fixture(self.rd)
self.mutate("raw-session.json", change)
self.assertFalse(self.check()[2]["pass"])
def test_output_omissions_and_tampering_are_detected(self):
frozen, output, _ = self.check()
for field in ("messages", "source_ids", "lineage"):
broken = copy.deepcopy(output)
broken[field].pop(2)
self.assertFalse(tr.validate(frozen, broken)["pass"])
broken = copy.deepcopy(output)
broken["messages"][3]["content"][0]["text"] = "invented result"
self.assertFalse(tr.validate(frozen, broken)["pass"])
def test_corrupt_truncated_reordered_or_duplicate_events_fail(self):
path = self.rd / "agent/events.jsonl"
lines = path.read_text().splitlines()
for bad in (lines[:-1], lines[::-1], lines + [lines[0]], lines + ["{"]):
path.write_text("\n".join(bad))
self.assertFalse(self.check()[2]["pass"])
def test_independent_inventory_catches_parser_omission(self):
from export import opencode_source as oc
original = oc.inventory
def omit(source):
result = original(source)
result["messages"].pop(1)
return result
with patch.object(oc, "inventory", side_effect=omit):
self.assertFalse(self.check()[2]["checks"]["OBSERVATION_ACCOUNTING"]["pass"])
def test_frozen_native_bytes_and_context_are_bound(self):
for name in ("raw-session.json", "raw-system.json", "raw-tools.json", "session-id.json"):
fixture(self.rd)
frozen = tr.freeze(self.rd)
path = self.rd / "agent" / name
path.write_bytes(path.read_bytes() + b" ")
self.assertFalse(tr.source_unchanged(self.rd, frozen))
def test_native_payload_and_system_segments_are_redacted(self):
self.mutate("raw-system.json", lambda d: d.append("/home/privateuser/work"))
self.mutate("raw-session.json", lambda d: d["messages"][0]["parts"][0].update(text="/home/privateuser/work"))
_, output, checks = self.check()
self.assertTrue(checks["pass"], checks)
self.assertNotIn("privateuser", json.dumps(output))
if __name__ == "__main__":
unittest.main()