data-gen-bpu/scripts/smoke_opencode.py
bench 5fb89d66b1 opencode: terminal_exec and the quota broker, like Pi
terminal_exec's body moves to agent/extensions/terminal-core.js, shared by
Pi's terminal.ts (unchanged schema/behaviour) and OpenCode's bench.ts.
OpenCode rollouts now get a managed Execution and a broker-scoped token
(the upstream key no longer enters its environment); terminal requires both.
No checkpoint/resume for OpenCode yet: the resume scan skips it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 12:46:53 +08:00

129 lines
7.4 KiB
Python

"""Real pinned CLI + local fake model + real jail. Never compiles or contacts boards.
Run only in a disposable source copy: refuses existing state/runs or .env.local.
This validates runner plumbing, not model quality or physical board acceptance.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import yaml
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--binary", required=True)
ap.add_argument("--budget-probe", action="store_true", help="repeat claim against fake API to exercise the real budget stop")
args = ap.parse_args()
root = Path(__file__).resolve().parents[1]
if any((root / name).exists() or (root / name).is_symlink() for name in ("state", "runs", ".env.local")):
raise SystemExit("refusing nonempty runtime state; use a fresh disposable source copy")
requests = []
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_):
pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
requests.append(body)
tools = {t["function"]["name"] for t in body.get("tools", [])}
have_result = any(m.get("role") == "tool" for m in body.get("messages", []))
delta = {"role": "assistant", "content": "Offline plumbing smoke complete."}
finish = "stop"
if "claim" in tools and (not have_result or args.budget_probe):
delta = {"role": "assistant", "tool_calls": [{"index": 0, "id": "offline_claim_1", "type": "function",
"function": {"name": "claim", "arguments": json.dumps({"terminal": "BLOCKED",
"summary": "Offline fixture. No compiler, board, or real model was called."})}}]}
finish = "tool_calls"
usage = {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}
self.send_response(200)
if body.get("stream"):
self.send_header("Content-Type", "text/event-stream"); self.end_headers()
for d, f, u in ((delta, None, None), ({}, finish, usage)):
chunk = {"id": "offline_response", "object": "chat.completion.chunk", "created": 1,
"model": "offline", "choices": [{"index": 0, "delta": d, "finish_reason": f}]}
if u:
chunk["usage"] = u
self.wfile.write(("data: " + json.dumps(chunk) + "\n\n").encode())
self.wfile.write(b"data: [DONE]\n\n"); self.wfile.flush()
else:
self.send_header("Content-Type", "application/json"); self.end_headers()
self.wfile.write(json.dumps({"id": "offline_response", "object": "chat.completion", "created": 1,
"model": "offline", "choices": [{"index": 0, "message": delta, "finish_reason": finish}], "usage": usage}).encode())
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
config = yaml.safe_load((root / "config.yaml").read_text())
config["paths"].update(work_root=str(root / "work"), node_bin="/usr/bin/node", venv={})
config["runners"] = {"opencode": {"enabled": True, "bin": str(Path(args.binary).resolve())}}
config["endpoints"]["teacher"] = {"model": "offline", "base_url": f"http://127.0.0.1:{server.server_port}/v1", "api_key_env": "OFFLINE_UNUSED_KEY"}
config["rollouts"].update(max_turns=2 if args.budget_probe else 4, max_tokens=1000, hard_ceiling_s=120, jail=True, private_home=True)
# OpenCode reaches the model only through the quota broker; give it a private ledger and the teacher's group.
v1 = config["collection_v1"]
group = v1["credential_groups"]["teacher"]
v1.update(quota_ledger=str(root / "state/quota.sqlite"), credential_groups={"teacher": group},
quota_groups={group: v1["quota_groups"][group]})
os.environ["OFFLINE_UNUSED_KEY"] = "offline"
cp = root / "smoke-config.yaml"; cp.write_text(yaml.safe_dump(config))
os.environ.update(BENCH_ROOT=str(root), BENCH_CONFIG=str(cp), BENCH_STATE_DIR=str(root / "state"))
from harness import db, paths
from harness.rollout import run_rollout
from harness.runners import opencode
rid, jid = "offline", "cls--resnet18--nash-m--ptq--int8"
jd = paths.RUNS / rid / jid; jd.mkdir(parents=True)
job = {"run_id": rid, "job_id": jid, "model": "resnet18", "task": "cls", "march": "nash-m", "backend": "ptq", "qcfg": "int8",
"evaluator_version": "0.5.0", "model_spec": {"input_spec": "1x3x224x224"},
"data": {"dataset": "imagenet1k", "t1": {"path": str(root / "absent.tar"), "n": 1, "manifest_sha": None},
"calib": {"path": str(root / "absent-calib"), "n": 1}},
"gates": {"required": ["G1", "G3"], "G1": {}, "G2": {}, "G3": {}, "G4": {}}}
(jd / "job.json").write_text(json.dumps(job))
with db.connect() as con:
db.create_run(con, rid, {})
db.rollout_upsert(con, rid, jid, 0, "e2e", "teacher", runner="opencode")
try:
result = run_rollout(rid, jid, 0, "e2e", "teacher", runner="opencode")
rd = jd / "rollouts/00"
messages, trajectory = opencode.trajectory(rd)
claim = json.loads((rd / "claim.json").read_text())
tools = json.loads((rd / "agent/tools.json").read_text())
assert any(m.get("role") == "toolResult" and m.get("toolName") == "claim" for m in messages), messages
assert claim["terminal"] == "BLOCKED", claim
if args.budget_probe:
assert json.loads((rd / "rollout.json").read_text())["state"] == "TIMEOUT"
assert len(requests) <= 2, len(requests)
else:
assert not trajectory["partial"], trajectory
assert any(t["name"] == "claim" and t["parameters"].get("type") == "object" for t in tools), tools
recorded = {t["name"]: t["parameters"] for t in tools}
assert requests, "runner made no API requests"
assert set(recorded) == set(opencode.ALLOWED_TOOLS), recorded
assert len(recorded) == len(tools), "duplicate exported tool names"
for request in requests:
requested = request.get("tools", [])
names = [item["function"]["name"] for item in requested]
assert len(set(names)) == len(names), "duplicate requested tool names"
assert set(names) == set(recorded), {"requested": sorted(names), "exported": sorted(recorded)}
for item in requested:
assert item["function"]["parameters"] == recorded[item["function"]["name"]]
report = {"result": result, "trajectory": trajectory, "api_requests": len(requests),
"cost": opencode.cost(rd), "binary_sha256": hashlib.sha256(Path(args.binary).read_bytes()).hexdigest(),
"budget_probe": args.budget_probe, "request_tools_match_export": True,
"exported_tool_names": sorted(recorded),
"scope": "real OpenCode CLI and namespace; fake local model; no board or compiler"}
(root / "smoke-report.json").write_text(json.dumps(report, indent=2))
print(json.dumps(report, indent=2))
finally:
(root / "smoke-api-requests.json").write_text(json.dumps(requests, indent=2))
server.shutdown()
if __name__ == "__main__":
main()