153 lines
7.5 KiB
Python
153 lines
7.5 KiB
Python
"""Native OpenCode fixtures use no credentials, models, boards or shared state."""
|
|
import copy
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from export import trajectory as tr
|
|
from harness.runners.opencode import ALLOWED_TOOLS, VERSION
|
|
|
|
|
|
def fixture(rd):
|
|
agent = rd / "agent"
|
|
agent.mkdir(exist_ok=True)
|
|
meta_path = rd / "rollout.json"
|
|
meta = json.loads(meta_path.read_text()) if meta_path.exists() else {"state": "DONE"}
|
|
meta.update(runner="opencode", runner_version=VERSION, agent_config_version="1")
|
|
meta_path.write_text(json.dumps(meta))
|
|
sid = "ses_fixture"
|
|
entries = [
|
|
{"info": {"id": "msg_user", "role": "user"}, "parts": [{"type": "text", "text": "Compile and evaluate."}]},
|
|
{"info": {"id": "msg_call", "role": "assistant", "finish": "tool-calls"}, "parts": [
|
|
{"type": "step-start"}, {"type": "reasoning", "text": "Check measured results first."},
|
|
{"type": "tool", "callID": "t1", "tool": "bash", "state": {
|
|
"status": "completed", "input": {"command": "bench stage status h"}, "output": "DONE"}},
|
|
{"type": "step-finish", "reason": "tool-calls"}]},
|
|
{"info": {"id": "msg_final", "role": "assistant", "finish": "stop"}, "parts": [
|
|
{"type": "step-start"}, {"type": "text", "text": "The measured subset passed within the task tolerance."},
|
|
{"type": "step-finish", "reason": "stop"}]}]
|
|
events = []
|
|
for i, entry in enumerate(entries):
|
|
info = entry["info"]
|
|
info.update(sessionID=sid, time={"created": 10 + i, "completed": 20 + i})
|
|
if info["role"] == "assistant":
|
|
info["parentID"] = "msg_user"
|
|
for j, part in enumerate(entry["parts"]):
|
|
part.update(id=f"prt_{i}{j}", sessionID=sid, messageID=info["id"])
|
|
if info["role"] == "assistant" and part["type"] != "reasoning":
|
|
events.append({"type": {"tool": "tool_use"}.get(part["type"], part["type"].replace("-", "_")),
|
|
"sessionID": sid, "part": part})
|
|
tools = [{"name": name, "parameters": {"type": "object"}} for name in ALLOWED_TOOLS]
|
|
for name, data in {"session-id.json": {"runner": "opencode", "version": VERSION, "session_id": sid},
|
|
"raw-session.json": {"info": {"id": sid}, "messages": entries},
|
|
"raw-system.json": ["Actual runner system", "Actual harness instruction"],
|
|
"raw-tools.json": tools, "tools.json": tools}.items():
|
|
(agent / name).write_text(json.dumps(data))
|
|
(agent / "events.jsonl").write_text("\n".join(map(json.dumps, events)))
|
|
|
|
|
|
class OpenCodeSourceTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.rd = Path(self.tmp.name)
|
|
fixture(self.rd)
|
|
|
|
def check(self):
|
|
frozen = tr.freeze(self.rd)
|
|
output = tr.reconstruct(frozen)
|
|
return frozen, output, tr.validate(frozen, output)
|
|
|
|
def mutate(self, file, change):
|
|
path = self.rd / "agent" / file
|
|
data = json.loads(path.read_text())
|
|
change(data)
|
|
path.write_text(json.dumps(data))
|
|
|
|
def test_complete_native_source_preserves_reasoning_and_identity(self):
|
|
_, output, validation = self.check()
|
|
self.assertTrue(validation["pass"], validation)
|
|
self.assertEqual(output["format"], "opencode-native-messages/1")
|
|
self.assertEqual(output["messages"][2]["content"][0]["type"], "thinking")
|
|
self.assertEqual(output["source_ids"][2], "ses_fixture/msg_call")
|
|
self.assertEqual(output["messages"][2]["opencode"]["info"]["finish"], "tool-calls")
|
|
self.assertEqual(validation["checks"]["OBSERVATION_ACCOUNTING"]["expected"], 11)
|
|
|
|
def test_missing_native_files_cannot_qualify(self):
|
|
for name in ("raw-session.json", "raw-system.json", "raw-tools.json", "session-id.json", "tools.json", "events.jsonl"):
|
|
with self.subTest(name=name):
|
|
fixture(self.rd)
|
|
(self.rd / "agent" / name).unlink()
|
|
self.assertFalse(self.check()[2]["pass"])
|
|
|
|
def test_incomplete_or_conflicting_source_cannot_qualify(self):
|
|
changes = [
|
|
lambda d: d["messages"].pop(1),
|
|
lambda d: d["messages"].pop(),
|
|
lambda d: d["messages"][1]["info"].pop("id"),
|
|
lambda d: d["messages"][1]["info"].update(sessionID="ses_other"),
|
|
lambda d: d["messages"][1]["info"]["time"].pop("completed"),
|
|
lambda d: d["messages"][1]["info"].update(parentID="msg_unknown"),
|
|
lambda d: d["messages"][1]["info"].update(error={"name": "APIError"}),
|
|
lambda d: d["messages"][1]["parts"][2]["state"].update(status="running"),
|
|
lambda d: d["messages"][1]["parts"][2]["state"].pop("output"),
|
|
lambda d: d["messages"][1]["parts"][2]["state"].update(output="tampered"),
|
|
lambda d: d["messages"][1]["parts"][2].update(callID=""),
|
|
lambda d: d["messages"][1]["parts"][1].update(type="compaction"),
|
|
lambda d: d["messages"][1]["parts"].reverse(),
|
|
lambda d: d["messages"].reverse(),
|
|
]
|
|
for index, change in enumerate(changes):
|
|
with self.subTest(index=index):
|
|
fixture(self.rd)
|
|
self.mutate("raw-session.json", change)
|
|
self.assertFalse(self.check()[2]["pass"])
|
|
|
|
def test_output_omissions_and_tampering_are_detected(self):
|
|
frozen, output, _ = self.check()
|
|
for field in ("messages", "source_ids", "lineage"):
|
|
broken = copy.deepcopy(output)
|
|
broken[field].pop(2)
|
|
self.assertFalse(tr.validate(frozen, broken)["pass"])
|
|
broken = copy.deepcopy(output)
|
|
broken["messages"][3]["content"][0]["text"] = "invented result"
|
|
self.assertFalse(tr.validate(frozen, broken)["pass"])
|
|
|
|
def test_corrupt_truncated_reordered_or_duplicate_events_fail(self):
|
|
path = self.rd / "agent/events.jsonl"
|
|
lines = path.read_text().splitlines()
|
|
for bad in (lines[:-1], lines[::-1], lines + [lines[0]], lines + ["{"]):
|
|
path.write_text("\n".join(bad))
|
|
self.assertFalse(self.check()[2]["pass"])
|
|
|
|
def test_independent_inventory_catches_parser_omission(self):
|
|
from export import opencode_source as oc
|
|
original = oc.inventory
|
|
def omit(source):
|
|
result = original(source)
|
|
result["messages"].pop(1)
|
|
return result
|
|
with patch.object(oc, "inventory", side_effect=omit):
|
|
self.assertFalse(self.check()[2]["checks"]["OBSERVATION_ACCOUNTING"]["pass"])
|
|
|
|
def test_frozen_native_bytes_and_context_are_bound(self):
|
|
for name in ("raw-session.json", "raw-system.json", "raw-tools.json", "session-id.json"):
|
|
fixture(self.rd)
|
|
frozen = tr.freeze(self.rd)
|
|
path = self.rd / "agent" / name
|
|
path.write_bytes(path.read_bytes() + b" ")
|
|
self.assertFalse(tr.source_unchanged(self.rd, frozen))
|
|
|
|
def test_native_payload_and_system_segments_are_redacted(self):
|
|
self.mutate("raw-system.json", lambda d: d.append("/home/privateuser/work"))
|
|
self.mutate("raw-session.json", lambda d: d["messages"][0]["parts"][0].update(text="/home/privateuser/work"))
|
|
_, output, checks = self.check()
|
|
self.assertTrue(checks["pass"], checks)
|
|
self.assertNotIn("privateuser", json.dumps(output))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|