terminal_exec's body moves to agent/extensions/terminal-core.js, shared by Pi's terminal.ts (unchanged schema/behaviour) and OpenCode's bench.ts. OpenCode rollouts now get a managed Execution and a broker-scoped token (the upstream key no longer enters its environment); terminal requires both. No checkpoint/resume for OpenCode yet: the resume scan skips it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
129 lines
7.4 KiB
Python
129 lines
7.4 KiB
Python
"""Real pinned CLI + local fake model + real jail. Never compiles or contacts boards.
|
|
|
|
Run only in a disposable source copy: refuses existing state/runs or .env.local.
|
|
This validates runner plumbing, not model quality or physical board acceptance.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--binary", required=True)
|
|
ap.add_argument("--budget-probe", action="store_true", help="repeat claim against fake API to exercise the real budget stop")
|
|
args = ap.parse_args()
|
|
root = Path(__file__).resolve().parents[1]
|
|
if any((root / name).exists() or (root / name).is_symlink() for name in ("state", "runs", ".env.local")):
|
|
raise SystemExit("refusing nonempty runtime state; use a fresh disposable source copy")
|
|
requests = []
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *_):
|
|
pass
|
|
|
|
def do_POST(self):
|
|
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
|
|
requests.append(body)
|
|
tools = {t["function"]["name"] for t in body.get("tools", [])}
|
|
have_result = any(m.get("role") == "tool" for m in body.get("messages", []))
|
|
delta = {"role": "assistant", "content": "Offline plumbing smoke complete."}
|
|
finish = "stop"
|
|
if "claim" in tools and (not have_result or args.budget_probe):
|
|
delta = {"role": "assistant", "tool_calls": [{"index": 0, "id": "offline_claim_1", "type": "function",
|
|
"function": {"name": "claim", "arguments": json.dumps({"terminal": "BLOCKED",
|
|
"summary": "Offline fixture. No compiler, board, or real model was called."})}}]}
|
|
finish = "tool_calls"
|
|
usage = {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}
|
|
self.send_response(200)
|
|
if body.get("stream"):
|
|
self.send_header("Content-Type", "text/event-stream"); self.end_headers()
|
|
for d, f, u in ((delta, None, None), ({}, finish, usage)):
|
|
chunk = {"id": "offline_response", "object": "chat.completion.chunk", "created": 1,
|
|
"model": "offline", "choices": [{"index": 0, "delta": d, "finish_reason": f}]}
|
|
if u:
|
|
chunk["usage"] = u
|
|
self.wfile.write(("data: " + json.dumps(chunk) + "\n\n").encode())
|
|
self.wfile.write(b"data: [DONE]\n\n"); self.wfile.flush()
|
|
else:
|
|
self.send_header("Content-Type", "application/json"); self.end_headers()
|
|
self.wfile.write(json.dumps({"id": "offline_response", "object": "chat.completion", "created": 1,
|
|
"model": "offline", "choices": [{"index": 0, "message": delta, "finish_reason": finish}], "usage": usage}).encode())
|
|
|
|
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
config = yaml.safe_load((root / "config.yaml").read_text())
|
|
config["paths"].update(work_root=str(root / "work"), node_bin="/usr/bin/node", venv={})
|
|
config["runners"] = {"opencode": {"enabled": True, "bin": str(Path(args.binary).resolve())}}
|
|
config["endpoints"]["teacher"] = {"model": "offline", "base_url": f"http://127.0.0.1:{server.server_port}/v1", "api_key_env": "OFFLINE_UNUSED_KEY"}
|
|
config["rollouts"].update(max_turns=2 if args.budget_probe else 4, max_tokens=1000, hard_ceiling_s=120, jail=True, private_home=True)
|
|
# OpenCode reaches the model only through the quota broker; give it a private ledger and the teacher's group.
|
|
v1 = config["collection_v1"]
|
|
group = v1["credential_groups"]["teacher"]
|
|
v1.update(quota_ledger=str(root / "state/quota.sqlite"), credential_groups={"teacher": group},
|
|
quota_groups={group: v1["quota_groups"][group]})
|
|
os.environ["OFFLINE_UNUSED_KEY"] = "offline"
|
|
cp = root / "smoke-config.yaml"; cp.write_text(yaml.safe_dump(config))
|
|
os.environ.update(BENCH_ROOT=str(root), BENCH_CONFIG=str(cp), BENCH_STATE_DIR=str(root / "state"))
|
|
from harness import db, paths
|
|
from harness.rollout import run_rollout
|
|
from harness.runners import opencode
|
|
rid, jid = "offline", "cls--resnet18--nash-m--ptq--int8"
|
|
jd = paths.RUNS / rid / jid; jd.mkdir(parents=True)
|
|
job = {"run_id": rid, "job_id": jid, "model": "resnet18", "task": "cls", "march": "nash-m", "backend": "ptq", "qcfg": "int8",
|
|
"evaluator_version": "0.5.0", "model_spec": {"input_spec": "1x3x224x224"},
|
|
"data": {"dataset": "imagenet1k", "t1": {"path": str(root / "absent.tar"), "n": 1, "manifest_sha": None},
|
|
"calib": {"path": str(root / "absent-calib"), "n": 1}},
|
|
"gates": {"required": ["G1", "G3"], "G1": {}, "G2": {}, "G3": {}, "G4": {}}}
|
|
(jd / "job.json").write_text(json.dumps(job))
|
|
with db.connect() as con:
|
|
db.create_run(con, rid, {})
|
|
db.rollout_upsert(con, rid, jid, 0, "e2e", "teacher", runner="opencode")
|
|
try:
|
|
result = run_rollout(rid, jid, 0, "e2e", "teacher", runner="opencode")
|
|
rd = jd / "rollouts/00"
|
|
messages, trajectory = opencode.trajectory(rd)
|
|
claim = json.loads((rd / "claim.json").read_text())
|
|
tools = json.loads((rd / "agent/tools.json").read_text())
|
|
assert any(m.get("role") == "toolResult" and m.get("toolName") == "claim" for m in messages), messages
|
|
assert claim["terminal"] == "BLOCKED", claim
|
|
if args.budget_probe:
|
|
assert json.loads((rd / "rollout.json").read_text())["state"] == "TIMEOUT"
|
|
assert len(requests) <= 2, len(requests)
|
|
else:
|
|
assert not trajectory["partial"], trajectory
|
|
assert any(t["name"] == "claim" and t["parameters"].get("type") == "object" for t in tools), tools
|
|
recorded = {t["name"]: t["parameters"] for t in tools}
|
|
assert requests, "runner made no API requests"
|
|
assert set(recorded) == set(opencode.ALLOWED_TOOLS), recorded
|
|
assert len(recorded) == len(tools), "duplicate exported tool names"
|
|
for request in requests:
|
|
requested = request.get("tools", [])
|
|
names = [item["function"]["name"] for item in requested]
|
|
assert len(set(names)) == len(names), "duplicate requested tool names"
|
|
assert set(names) == set(recorded), {"requested": sorted(names), "exported": sorted(recorded)}
|
|
for item in requested:
|
|
assert item["function"]["parameters"] == recorded[item["function"]["name"]]
|
|
report = {"result": result, "trajectory": trajectory, "api_requests": len(requests),
|
|
"cost": opencode.cost(rd), "binary_sha256": hashlib.sha256(Path(args.binary).read_bytes()).hexdigest(),
|
|
"budget_probe": args.budget_probe, "request_tools_match_export": True,
|
|
"exported_tool_names": sorted(recorded),
|
|
"scope": "real OpenCode CLI and namespace; fake local model; no board or compiler"}
|
|
(root / "smoke-report.json").write_text(json.dumps(report, indent=2))
|
|
print(json.dumps(report, indent=2))
|
|
finally:
|
|
(root / "smoke-api-requests.json").write_text(json.dumps(requests, indent=2))
|
|
server.shutdown()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|