terminal_exec's body moves to agent/extensions/terminal-core.js, shared by Pi's terminal.ts (unchanged schema/behaviour) and OpenCode's bench.ts. OpenCode rollouts now get a managed Execution and a broker-scoped token (the upstream key no longer enters its environment); terminal requires both. No checkpoint/resume for OpenCode yet: the resume scan skips it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
206 lines
11 KiB
Python
206 lines
11 KiB
Python
"""Offline contract tests; no compiler, model download, LLM or physical board."""
|
|
import copy
|
|
from contextlib import ExitStack
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from harness import paths, registry, toolchains
|
|
from harness.board_identity import validate
|
|
from harness.runners import get
|
|
from harness.runners import opencode
|
|
|
|
|
|
class Extensions(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
self.root = Path(self.temp.name)
|
|
|
|
def test_board_march_can_be_derived_but_disagreement_fails(self):
|
|
cfg = {"march_by_board": {"s600": "nash-p"}, "boards": {"test": {"slot": 9, "board_type": "s600"}}}
|
|
with patch.object(registry, "boards", return_value=cfg), patch.object(paths, "secret", return_value=None):
|
|
self.assertEqual(registry.board_slots()[9]["march"], "nash-p")
|
|
cfg["boards"]["test"]["march"] = "nash-m"
|
|
with self.assertRaisesRegex(ValueError, "mismatch"):
|
|
registry.board_slots()
|
|
|
|
def test_new_target_without_backend_is_rejected(self):
|
|
with patch.object(registry, "boards", return_value={"march_by_board": {"x5": "bayes"}}):
|
|
with self.assertRaises(ValueError):
|
|
registry.target_spec("x5")
|
|
|
|
def test_board_identity_does_not_confuse_s100_and_s100p(self):
|
|
validate({"board_type": "D-Robotics RDK S100P V1P0", "ucp": "3.13.6"}, "s100p", "3.13.6")
|
|
with self.assertRaises(ValueError):
|
|
validate({"board_type": "D-Robotics RDK S100P V1P0"}, "s100")
|
|
with self.assertRaises(ValueError):
|
|
validate({}, "s600")
|
|
|
|
def profiles(self):
|
|
return {"paths": {"venv": {"ptq": str(self.root / "old")}},
|
|
"toolchains": {name: {"backend": "ptq", "marches": ["nash-m"], "qcfgs": ["int8"],
|
|
"version": name, "venv": str(self.root / name)} for name in ("v1", "v2")}}
|
|
|
|
def test_same_target_selects_different_versions_explicitly(self):
|
|
with patch.object(paths, "config", return_value=self.profiles()):
|
|
one = toolchains.resolve("ptq", "nash-m", "int8", "v1")
|
|
two = toolchains.resolve("ptq", "nash-m", "int8", "v2")
|
|
self.assertNotEqual(one, two)
|
|
self.assertEqual(one["march"], two["march"])
|
|
with self.assertRaises(ValueError):
|
|
toolchains.resolve("ptq", "nash-p", "int8", "v1")
|
|
with self.assertRaises(ValueError):
|
|
toolchains.resolve("qat", "nash-m", "int8", "v1")
|
|
|
|
def test_frozen_compiler_commands_reject_profile_drift(self):
|
|
from harness.backends import ptq
|
|
cfg = self.profiles()
|
|
with patch.object(paths, "config", return_value=cfg):
|
|
job = {"backend": "ptq", "march": "nash-m", "qcfg": "int8"}
|
|
job["toolchain"] = toolchains.resolve(**job, profile="v1")
|
|
self.assertEqual(ptq.compile_cmd(Path("compile.yaml"), job)[0], str(self.root / "v1/bin/hb_compile"))
|
|
cfg["toolchains"]["v1"]["version"] = "changed"
|
|
with self.assertRaisesRegex(ValueError, "frozen toolchain"):
|
|
ptq.model_info_cmd(Path("model.hbm"), job)
|
|
|
|
def test_installed_package_metadata_drift_is_detected(self):
|
|
cfg = self.profiles()
|
|
meta = self.root / "v1/lib/python3.12/site-packages/hmct-1.dist-info/METADATA"
|
|
meta.parent.mkdir(parents=True); meta.write_text("Version: 1")
|
|
with patch.object(paths, "config", return_value=cfg):
|
|
job = {"backend": "ptq", "march": "nash-m", "qcfg": "int8"}
|
|
job["toolchain"] = toolchains.resolve(**job, profile="v1")
|
|
meta.write_text("Version: 2")
|
|
with self.assertRaises(ValueError):
|
|
toolchains.validate(job)
|
|
|
|
def test_old_jobs_keep_runtime_and_new_jobs_reject_drift(self):
|
|
job = {"model": "toy", "model_spec": {"runtime_python": "/old/bin/python"}}
|
|
cfg = {"model_runtimes": {"toy": "/new/bin/python"}}
|
|
with patch.object(paths, "config", return_value=cfg):
|
|
self.assertEqual(str(paths.model_python(job)), "/old/bin/python")
|
|
job["model_runtime"] = {"configured": "/new/bin/python", "python": "/new/bin/python"}
|
|
self.assertEqual(str(paths.model_python(job)), "/new/bin/python")
|
|
cfg["model_runtimes"]["toy"] = "/third/bin/python"
|
|
with self.assertRaises(ValueError):
|
|
paths.model_python(job)
|
|
|
|
def test_new_task_uses_inputs_protocol_without_task_branch(self):
|
|
from harness.eval import TASK_MODULES, run
|
|
module = SimpleNamespace(inputs=lambda *args: "tensor", run_fp32=lambda *a: None, run_quant=lambda *a: None)
|
|
t1 = SimpleNamespace(g2_subset=lambda n: ["item"])
|
|
with patch.dict(TASK_MODULES, {"toy": "harness.eval.toy"}), patch("importlib.import_module", return_value=module):
|
|
self.assertEqual(run.g2_inputs("toy", t1, "1x3x4x4", 1), {"s000": "tensor"})
|
|
|
|
def test_unknown_runner_is_not_interpreted_as_pi(self):
|
|
with self.assertRaises(ValueError):
|
|
get("unknown")
|
|
|
|
def test_opencode_requires_opt_in_before_any_binary_launch(self):
|
|
cfg = {"endpoints": {"teacher": {"model": "toy", "base_url": "http://localhost"}}}
|
|
with patch.object(paths, "config", return_value=cfg), patch.object(opencode.subprocess, "run") as proc:
|
|
with self.assertRaisesRegex(ValueError, "opt-in"):
|
|
opencode.check("teacher", execute=True)
|
|
proc.assert_not_called()
|
|
|
|
def session(self):
|
|
agent = self.root / "agent"; agent.mkdir()
|
|
(agent / "session-id.json").write_text(json.dumps({"session_id": "ses_test"}))
|
|
raw = {"info": {"id": "ses_test"}, "messages": [
|
|
{"info": {"role": "user"}, "parts": [{"type": "text", "text": "do work"}]},
|
|
{"info": {"role": "assistant", "time": {"completed": 3}}, "parts": [
|
|
{"type": "reasoning", "text": "reason"}, {"type": "text", "text": "claim now"},
|
|
{"type": "tool", "callID": "c1", "tool": "claim", "state": {
|
|
"status": "completed", "input": {"terminal": "PASS"}, "output": "recorded"}}]}]}
|
|
(agent / "raw-session.json").write_text(json.dumps(raw))
|
|
return raw
|
|
|
|
def tool_definitions(self):
|
|
return [{"name": name, "description": f"fixture {name}",
|
|
"parameters": {"type": "object", "properties": {"value": {"type": "string"}},
|
|
"required": ["value"], "additionalProperties": False}}
|
|
for name in (*opencode.ALLOWED_TOOLS, "webfetch", "websearch", "task", "todowrite", "question", "unknown-future-tool")]
|
|
|
|
def finish_fixture(self, definitions):
|
|
raw = self.session()
|
|
workspace = self.root / "workspace"
|
|
workspace.mkdir()
|
|
raw_tools = json.dumps(definitions, indent=1).encode() + b"\n"
|
|
(workspace / "opencode-tools.json").write_bytes(raw_tools)
|
|
(workspace / "opencode-system.json").write_text('["actual system"]')
|
|
(self.root / "agent/events.jsonl").write_text('{"sessionID":"ses_test","type":"step_finish"}\n')
|
|
self.stack.enter_context(patch.object(opencode, "environment", return_value={}))
|
|
self.stack.enter_context(patch.object(opencode, "binary", return_value="never-executed"))
|
|
self.stack.enter_context(patch("harness.rollout.jail_prefix", return_value=[]))
|
|
self.stack.enter_context(patch.object(opencode.subprocess, "run", return_value=SimpleNamespace(stdout=json.dumps(raw))))
|
|
return raw_tools
|
|
|
|
def test_finish_preserves_raw_registry_and_exports_only_actual_allowed_tools(self):
|
|
definitions = self.tool_definitions()
|
|
raw_tools = self.finish_fixture(definitions)
|
|
cfg = {"endpoints": {"teacher": {"model": "fixture", "base_url": "http://unused"}}}
|
|
with patch.object(paths, "config", return_value=cfg):
|
|
opencode.prepare(self.root, "teacher")
|
|
permission = json.loads((self.root / "agent/opencode.json").read_text())["permission"]
|
|
self.assertEqual({k for k, value in permission.items() if value == "allow"}, set(opencode.ALLOWED_TOOLS))
|
|
self.assertEqual(permission["*"], "deny")
|
|
opencode.finish(self.root)
|
|
self.assertEqual((self.root / "agent/raw-tools.json").read_bytes(), raw_tools)
|
|
exported = json.loads((self.root / "agent/tools.json").read_text())
|
|
self.assertEqual(len(exported), len(opencode.ALLOWED_TOOLS))
|
|
self.assertEqual(exported, [d for d in definitions if d["name"] in opencode.ALLOWED_TOOLS])
|
|
self.assertNotIn("unknown-future-tool", {d["name"] for d in exported})
|
|
|
|
def test_duplicate_registered_tool_fails_closed_but_preserves_raw_evidence(self):
|
|
definitions = self.tool_definitions()
|
|
definitions.append(copy.deepcopy(definitions[0]))
|
|
raw_tools = self.finish_fixture(definitions)
|
|
with self.assertRaisesRegex(ValueError, "duplicate tool definition"):
|
|
opencode.finish(self.root)
|
|
self.assertEqual((self.root / "agent/raw-tools.json").read_bytes(), raw_tools)
|
|
self.assertFalse((self.root / "agent/tools.json").exists())
|
|
|
|
def test_missing_or_malformed_allowed_definition_fails_closed(self):
|
|
definitions = self.tool_definitions()
|
|
with self.assertRaisesRegex(ValueError, "missing allowed tool"):
|
|
opencode._allowed_tool_definitions(definitions[1:])
|
|
definitions[0]["parameters"] = None
|
|
with self.assertRaisesRegex(ValueError, "no JSON schema"):
|
|
opencode._allowed_tool_definitions(definitions)
|
|
|
|
def test_opencode_raw_parts_and_tool_pairing_are_preserved(self):
|
|
self.session()
|
|
messages, metadata = opencode.trajectory(self.root)
|
|
self.assertEqual([part["type"] for part in messages[1]["content"]], ["thinking", "text", "toolCall"])
|
|
self.assertEqual(messages[2]["toolCallId"], "c1")
|
|
self.assertFalse(metadata["partial"])
|
|
|
|
def test_opencode_incomplete_tool_is_partial(self):
|
|
raw = self.session()
|
|
raw["messages"][1]["parts"][-1]["state"]["status"] = "running"
|
|
(self.root / "agent/raw-session.json").write_text(json.dumps(raw))
|
|
self.assertTrue(opencode.trajectory(self.root)[1]["partial"])
|
|
|
|
def test_wrong_session_is_never_exported(self):
|
|
raw = self.session(); raw["info"]["id"] = "ses_other"
|
|
(self.root / "agent/raw-session.json").write_text(json.dumps(raw))
|
|
with self.assertRaisesRegex(ValueError, "identity"):
|
|
opencode.trajectory(self.root)
|
|
|
|
def test_cli_progress_is_not_training_transcript(self):
|
|
(self.root / "agent").mkdir()
|
|
(self.root / "agent/events.jsonl").write_text(json.dumps({"type": "text", "part": {"text": "partial"}}))
|
|
messages, meta = opencode.trajectory(self.root)
|
|
self.assertEqual(messages, [])
|
|
self.assertTrue(meta["partial"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|