"""Real pinned CLI + local fake model + real jail. Never compiles or contacts boards. Run only in a disposable source copy: refuses existing state/runs or .env.local. This validates runner plumbing, not model quality or physical board acceptance. """ from __future__ import annotations import argparse import hashlib import json import os import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import yaml def main(): ap = argparse.ArgumentParser() ap.add_argument("--binary", required=True) ap.add_argument("--budget-probe", action="store_true", help="repeat claim against fake API to exercise the real budget stop") args = ap.parse_args() root = Path(__file__).resolve().parents[1] if any((root / name).exists() or (root / name).is_symlink() for name in ("state", "runs", ".env.local")): raise SystemExit("refusing nonempty runtime state; use a fresh disposable source copy") requests = [] class Handler(BaseHTTPRequestHandler): def log_message(self, *_): pass def do_POST(self): body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) requests.append(body) tools = {t["function"]["name"] for t in body.get("tools", [])} results = sum(m.get("role") == "tool" for m in body.get("messages", [])) delta = {"role": "assistant", "content": "Offline plumbing smoke complete."} finish = "stop" if "read" in tools and not results and not args.budget_probe: # The path guard must refuse platform code, as Pi's does. delta = {"role": "assistant", "tool_calls": [{"index": 0, "id": "offline_read_1", "type": "function", "function": {"name": "read", "arguments": json.dumps({"filePath": str(root / "harness/rollout.py")})}}]} finish = "tool_calls" elif "claim" in tools and (results == 1 or args.budget_probe): delta = {"role": "assistant", "tool_calls": [{"index": 0, "id": "offline_claim_1", "type": "function", "function": {"name": "claim", "arguments": json.dumps({"terminal": "BLOCKED", "summary": "Offline fixture. No compiler, board, or real model was called."})}}]} finish = "tool_calls" usage = {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30} self.send_response(200) if body.get("stream"): self.send_header("Content-Type", "text/event-stream"); self.end_headers() for d, f, u in ((delta, None, None), ({}, finish, usage)): chunk = {"id": "offline_response", "object": "chat.completion.chunk", "created": 1, "model": "offline", "choices": [{"index": 0, "delta": d, "finish_reason": f}]} if u: chunk["usage"] = u self.wfile.write(("data: " + json.dumps(chunk) + "\n\n").encode()) self.wfile.write(b"data: [DONE]\n\n"); self.wfile.flush() else: self.send_header("Content-Type", "application/json"); self.end_headers() self.wfile.write(json.dumps({"id": "offline_response", "object": "chat.completion", "created": 1, "model": "offline", "choices": [{"index": 0, "message": delta, "finish_reason": finish}], "usage": usage}).encode()) server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) threading.Thread(target=server.serve_forever, daemon=True).start() config = yaml.safe_load((root / "config.yaml").read_text()) config["paths"].update(work_root=str(root / "work"), node_bin="/usr/bin/node", venv={}) config["runners"] = {"opencode": {"enabled": True, "bin": str(Path(args.binary).resolve())}} config["endpoints"]["teacher"] = {"model": "offline", "base_url": f"http://127.0.0.1:{server.server_port}/v1", "api_key_env": "OFFLINE_UNUSED_KEY"} config["rollouts"].update(max_turns=2 if args.budget_probe else 4, max_tokens=1000, hard_ceiling_s=120, jail=True, private_home=True) # OpenCode reaches the model only through the quota broker; give it a private ledger and the teacher's group. v1 = config["collection_v1"] group = v1["credential_groups"]["teacher"] v1.update(quota_ledger=str(root / "state/quota.sqlite"), credential_groups={"teacher": group}, quota_groups={group: v1["quota_groups"][group]}) os.environ["OFFLINE_UNUSED_KEY"] = "offline" cp = root / "smoke-config.yaml"; cp.write_text(yaml.safe_dump(config)) os.environ.update(BENCH_ROOT=str(root), BENCH_CONFIG=str(cp), BENCH_STATE_DIR=str(root / "state")) from harness import db, paths from harness.rollout import run_rollout from harness.runners import opencode rid, jid = "offline", "cls--resnet18--nash-m--ptq--int8" jd = paths.RUNS / rid / jid; jd.mkdir(parents=True) job = {"run_id": rid, "job_id": jid, "model": "resnet18", "task": "cls", "march": "nash-m", "backend": "ptq", "qcfg": "int8", "evaluator_version": "0.5.0", "model_spec": {"input_spec": "1x3x224x224"}, "data": {"dataset": "imagenet1k", "t1": {"path": str(root / "absent.tar"), "n": 1, "manifest_sha": None}, "calib": {"path": str(root / "absent-calib"), "n": 1}}, "gates": {"required": ["G1", "G3"], "G1": {}, "G2": {}, "G3": {}, "G4": {}}} (jd / "job.json").write_text(json.dumps(job)) with db.connect() as con: db.create_run(con, rid, {}) db.rollout_upsert(con, rid, jid, 0, "e2e", "teacher", runner="opencode") try: result = run_rollout(rid, jid, 0, "e2e", "teacher", runner="opencode") rd = jd / "rollouts/00" messages, trajectory = opencode.trajectory(rd) claim = json.loads((rd / "claim.json").read_text()) tools = json.loads((rd / "agent/tools.json").read_text()) assert any(m.get("role") == "toolResult" and m.get("toolName") == "claim" for m in messages), messages assert claim["terminal"] == "BLOCKED", claim if not args.budget_probe: guarded = [m for m in messages if m.get("role") == "toolResult" and m.get("toolName") == "read"] assert guarded and "path-guard" in guarded[0]["content"][0]["text"], guarded assert "deny-read" in (rd / "agent/guard.log").read_text() if args.budget_probe: assert json.loads((rd / "rollout.json").read_text())["state"] == "TIMEOUT" assert len(requests) <= 2, len(requests) else: assert not trajectory["partial"], trajectory assert any(t["name"] == "claim" and t["parameters"].get("type") == "object" for t in tools), tools recorded = {t["name"]: t["parameters"] for t in tools} assert requests, "runner made no API requests" assert set(recorded) == set(opencode.ALLOWED_TOOLS), recorded assert len(recorded) == len(tools), "duplicate exported tool names" for request in requests: requested = request.get("tools", []) names = [item["function"]["name"] for item in requested] assert len(set(names)) == len(names), "duplicate requested tool names" assert set(names) == set(recorded), {"requested": sorted(names), "exported": sorted(recorded)} for item in requested: assert item["function"]["parameters"] == recorded[item["function"]["name"]] report = {"result": result, "trajectory": trajectory, "api_requests": len(requests), "cost": opencode.cost(rd), "binary_sha256": hashlib.sha256(Path(args.binary).read_bytes()).hexdigest(), "budget_probe": args.budget_probe, "request_tools_match_export": True, "exported_tool_names": sorted(recorded), "scope": "real OpenCode CLI and namespace; fake local model; no board or compiler"} (root / "smoke-report.json").write_text(json.dumps(report, indent=2)) print(json.dumps(report, indent=2)) finally: (root / "smoke-api-requests.json").write_text(json.dumps(requests, indent=2)) server.shutdown() if __name__ == "__main__": main()