data-gen-bpu/scripts/test_extensions.py
bench 5fb89d66b1 opencode: terminal_exec and the quota broker, like Pi
terminal_exec's body moves to agent/extensions/terminal-core.js, shared by
Pi's terminal.ts (unchanged schema/behaviour) and OpenCode's bench.ts.
OpenCode rollouts now get a managed Execution and a broker-scoped token
(the upstream key no longer enters its environment); terminal requires both.
No checkpoint/resume for OpenCode yet: the resume scan skips it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 12:46:53 +08:00

206 lines
11 KiB
Python

"""Offline contract tests; no compiler, model download, LLM or physical board."""
import copy
from contextlib import ExitStack
import json
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from harness import paths, registry, toolchains
from harness.board_identity import validate
from harness.runners import get
from harness.runners import opencode
class Extensions(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
self.root = Path(self.temp.name)
def test_board_march_can_be_derived_but_disagreement_fails(self):
cfg = {"march_by_board": {"s600": "nash-p"}, "boards": {"test": {"slot": 9, "board_type": "s600"}}}
with patch.object(registry, "boards", return_value=cfg), patch.object(paths, "secret", return_value=None):
self.assertEqual(registry.board_slots()[9]["march"], "nash-p")
cfg["boards"]["test"]["march"] = "nash-m"
with self.assertRaisesRegex(ValueError, "mismatch"):
registry.board_slots()
def test_new_target_without_backend_is_rejected(self):
with patch.object(registry, "boards", return_value={"march_by_board": {"x5": "bayes"}}):
with self.assertRaises(ValueError):
registry.target_spec("x5")
def test_board_identity_does_not_confuse_s100_and_s100p(self):
validate({"board_type": "D-Robotics RDK S100P V1P0", "ucp": "3.13.6"}, "s100p", "3.13.6")
with self.assertRaises(ValueError):
validate({"board_type": "D-Robotics RDK S100P V1P0"}, "s100")
with self.assertRaises(ValueError):
validate({}, "s600")
def profiles(self):
return {"paths": {"venv": {"ptq": str(self.root / "old")}},
"toolchains": {name: {"backend": "ptq", "marches": ["nash-m"], "qcfgs": ["int8"],
"version": name, "venv": str(self.root / name)} for name in ("v1", "v2")}}
def test_same_target_selects_different_versions_explicitly(self):
with patch.object(paths, "config", return_value=self.profiles()):
one = toolchains.resolve("ptq", "nash-m", "int8", "v1")
two = toolchains.resolve("ptq", "nash-m", "int8", "v2")
self.assertNotEqual(one, two)
self.assertEqual(one["march"], two["march"])
with self.assertRaises(ValueError):
toolchains.resolve("ptq", "nash-p", "int8", "v1")
with self.assertRaises(ValueError):
toolchains.resolve("qat", "nash-m", "int8", "v1")
def test_frozen_compiler_commands_reject_profile_drift(self):
from harness.backends import ptq
cfg = self.profiles()
with patch.object(paths, "config", return_value=cfg):
job = {"backend": "ptq", "march": "nash-m", "qcfg": "int8"}
job["toolchain"] = toolchains.resolve(**job, profile="v1")
self.assertEqual(ptq.compile_cmd(Path("compile.yaml"), job)[0], str(self.root / "v1/bin/hb_compile"))
cfg["toolchains"]["v1"]["version"] = "changed"
with self.assertRaisesRegex(ValueError, "frozen toolchain"):
ptq.model_info_cmd(Path("model.hbm"), job)
def test_installed_package_metadata_drift_is_detected(self):
cfg = self.profiles()
meta = self.root / "v1/lib/python3.12/site-packages/hmct-1.dist-info/METADATA"
meta.parent.mkdir(parents=True); meta.write_text("Version: 1")
with patch.object(paths, "config", return_value=cfg):
job = {"backend": "ptq", "march": "nash-m", "qcfg": "int8"}
job["toolchain"] = toolchains.resolve(**job, profile="v1")
meta.write_text("Version: 2")
with self.assertRaises(ValueError):
toolchains.validate(job)
def test_old_jobs_keep_runtime_and_new_jobs_reject_drift(self):
job = {"model": "toy", "model_spec": {"runtime_python": "/old/bin/python"}}
cfg = {"model_runtimes": {"toy": "/new/bin/python"}}
with patch.object(paths, "config", return_value=cfg):
self.assertEqual(str(paths.model_python(job)), "/old/bin/python")
job["model_runtime"] = {"configured": "/new/bin/python", "python": "/new/bin/python"}
self.assertEqual(str(paths.model_python(job)), "/new/bin/python")
cfg["model_runtimes"]["toy"] = "/third/bin/python"
with self.assertRaises(ValueError):
paths.model_python(job)
def test_new_task_uses_inputs_protocol_without_task_branch(self):
from harness.eval import TASK_MODULES, run
module = SimpleNamespace(inputs=lambda *args: "tensor", run_fp32=lambda *a: None, run_quant=lambda *a: None)
t1 = SimpleNamespace(g2_subset=lambda n: ["item"])
with patch.dict(TASK_MODULES, {"toy": "harness.eval.toy"}), patch("importlib.import_module", return_value=module):
self.assertEqual(run.g2_inputs("toy", t1, "1x3x4x4", 1), {"s000": "tensor"})
def test_unknown_runner_is_not_interpreted_as_pi(self):
with self.assertRaises(ValueError):
get("unknown")
def test_opencode_requires_opt_in_before_any_binary_launch(self):
cfg = {"endpoints": {"teacher": {"model": "toy", "base_url": "http://localhost"}}}
with patch.object(paths, "config", return_value=cfg), patch.object(opencode.subprocess, "run") as proc:
with self.assertRaisesRegex(ValueError, "opt-in"):
opencode.check("teacher", execute=True)
proc.assert_not_called()
def session(self):
agent = self.root / "agent"; agent.mkdir()
(agent / "session-id.json").write_text(json.dumps({"session_id": "ses_test"}))
raw = {"info": {"id": "ses_test"}, "messages": [
{"info": {"role": "user"}, "parts": [{"type": "text", "text": "do work"}]},
{"info": {"role": "assistant", "time": {"completed": 3}}, "parts": [
{"type": "reasoning", "text": "reason"}, {"type": "text", "text": "claim now"},
{"type": "tool", "callID": "c1", "tool": "claim", "state": {
"status": "completed", "input": {"terminal": "PASS"}, "output": "recorded"}}]}]}
(agent / "raw-session.json").write_text(json.dumps(raw))
return raw
def tool_definitions(self):
return [{"name": name, "description": f"fixture {name}",
"parameters": {"type": "object", "properties": {"value": {"type": "string"}},
"required": ["value"], "additionalProperties": False}}
for name in (*opencode.ALLOWED_TOOLS, "webfetch", "websearch", "task", "todowrite", "question", "unknown-future-tool")]
def finish_fixture(self, definitions):
raw = self.session()
workspace = self.root / "workspace"
workspace.mkdir()
raw_tools = json.dumps(definitions, indent=1).encode() + b"\n"
(workspace / "opencode-tools.json").write_bytes(raw_tools)
(workspace / "opencode-system.json").write_text('["actual system"]')
(self.root / "agent/events.jsonl").write_text('{"sessionID":"ses_test","type":"step_finish"}\n')
self.stack.enter_context(patch.object(opencode, "environment", return_value={}))
self.stack.enter_context(patch.object(opencode, "binary", return_value="never-executed"))
self.stack.enter_context(patch("harness.rollout.jail_prefix", return_value=[]))
self.stack.enter_context(patch.object(opencode.subprocess, "run", return_value=SimpleNamespace(stdout=json.dumps(raw))))
return raw_tools
def test_finish_preserves_raw_registry_and_exports_only_actual_allowed_tools(self):
definitions = self.tool_definitions()
raw_tools = self.finish_fixture(definitions)
cfg = {"endpoints": {"teacher": {"model": "fixture", "base_url": "http://unused"}}}
with patch.object(paths, "config", return_value=cfg):
opencode.prepare(self.root, "teacher")
permission = json.loads((self.root / "agent/opencode.json").read_text())["permission"]
self.assertEqual({k for k, value in permission.items() if value == "allow"}, set(opencode.ALLOWED_TOOLS))
self.assertEqual(permission["*"], "deny")
opencode.finish(self.root)
self.assertEqual((self.root / "agent/raw-tools.json").read_bytes(), raw_tools)
exported = json.loads((self.root / "agent/tools.json").read_text())
self.assertEqual(len(exported), len(opencode.ALLOWED_TOOLS))
self.assertEqual(exported, [d for d in definitions if d["name"] in opencode.ALLOWED_TOOLS])
self.assertNotIn("unknown-future-tool", {d["name"] for d in exported})
def test_duplicate_registered_tool_fails_closed_but_preserves_raw_evidence(self):
definitions = self.tool_definitions()
definitions.append(copy.deepcopy(definitions[0]))
raw_tools = self.finish_fixture(definitions)
with self.assertRaisesRegex(ValueError, "duplicate tool definition"):
opencode.finish(self.root)
self.assertEqual((self.root / "agent/raw-tools.json").read_bytes(), raw_tools)
self.assertFalse((self.root / "agent/tools.json").exists())
def test_missing_or_malformed_allowed_definition_fails_closed(self):
definitions = self.tool_definitions()
with self.assertRaisesRegex(ValueError, "missing allowed tool"):
opencode._allowed_tool_definitions(definitions[1:])
definitions[0]["parameters"] = None
with self.assertRaisesRegex(ValueError, "no JSON schema"):
opencode._allowed_tool_definitions(definitions)
def test_opencode_raw_parts_and_tool_pairing_are_preserved(self):
self.session()
messages, metadata = opencode.trajectory(self.root)
self.assertEqual([part["type"] for part in messages[1]["content"]], ["thinking", "text", "toolCall"])
self.assertEqual(messages[2]["toolCallId"], "c1")
self.assertFalse(metadata["partial"])
def test_opencode_incomplete_tool_is_partial(self):
raw = self.session()
raw["messages"][1]["parts"][-1]["state"]["status"] = "running"
(self.root / "agent/raw-session.json").write_text(json.dumps(raw))
self.assertTrue(opencode.trajectory(self.root)[1]["partial"])
def test_wrong_session_is_never_exported(self):
raw = self.session(); raw["info"]["id"] = "ses_other"
(self.root / "agent/raw-session.json").write_text(json.dumps(raw))
with self.assertRaisesRegex(ValueError, "identity"):
opencode.trajectory(self.root)
def test_cli_progress_is_not_training_transcript(self):
(self.root / "agent").mkdir()
(self.root / "agent/events.jsonl").write_text(json.dumps({"type": "text", "part": {"text": "partial"}}))
messages, meta = opencode.trajectory(self.root)
self.assertEqual(messages, [])
self.assertTrue(meta["partial"])
if __name__ == "__main__":
unittest.main()