bench a595d49c05 opencode: budget counts fresh input and output, not cached context, like Pi
A staging rollout (mobilenet.v2) hit 2M tokens at turn 63: 1.95M were cache reads
of the same context. Pi on the same task counted 120k for 96 turns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 19:40:51 +08:00

77 lines
5.0 KiB
TypeScript

import { tool } from "@opencode-ai/plugin";
import { Schema } from "effect";
import fs from "node:fs";
import path from "node:path";
import { execFileSync } from "node:child_process";
import { createRequire } from "node:module";
const terminal = createRequire(import.meta.url)("../extensions/terminal-core.js"); // same core as Pi's terminal.ts
// Only attempt-local evidence is writable; the outer worker owns stages/verdicts.
export default async () => {
const workspace = path.join(process.env.BENCH_ROLLOUT_DIR!, "workspace");
const save = (name: string, value: unknown) => fs.writeFileSync(path.join(workspace, name), JSON.stringify(value));
const definitions: Record<string, unknown> = {};
let turns = 0, tokens = 0;
const seen = new Set<string>();
const exhausted = () => turns >= Number(process.env.BENCH_MAX_TURNS) || tokens >= Number(process.env.BENCH_MAX_TOKENS);
return {
"tool.definition": async (input: any, output: any) => {
// 1.18.31 registry passes Effect/Zod validators separately from JSON Schema.
if (!output.jsonSchema) {
const doc = Schema.toJsonSchemaDocument(output.parameters);
output.jsonSchema = {...doc.schema, ...(Object.keys(doc.definitions).length ? {$defs:doc.definitions} : {})};
}
if (typeof output.jsonSchema !== "object") throw new Error("bench requires actual tool JSON Schema");
definitions[input.toolID] = {name: input.toolID, description: output.description, parameters: output.jsonSchema};
save("opencode-tools.json", Object.values(definitions));
},
"experimental.chat.system.transform": async (_input: any, output: any) => save("opencode-system.json", output.system),
"chat.params": async () => {
if (exhausted()) throw new Error("bench budget exhausted; no further model request");
},
"tool.execute.before": async (input: any) => {
if (exhausted() && !["claim", "oe_doc"].includes(input.tool)) throw new Error("bench budget exhausted; finish with a natural final answer");
},
event: async ({event}: any) => {
const part = event.properties?.part;
if (part?.type === "step-finish" && !seen.has(part.id)) {
seen.add(part.id); turns++;
const usage = part.tokens || {};
tokens += (usage.input || 0) + (usage.output || 0) + (usage.reasoning || 0); // cache reads excluded, as in Pi
}
},
tool: {
claim: tool({description:"Record your terminal claim; the harness verifies it independently.",
args:{terminal:tool.schema.enum(["PASS","FAIL","BLOCKED"]), summary:tool.schema.string(),
stage:tool.schema.string().optional(), error_class:tool.schema.string().optional(),
artifacts:tool.schema.array(tool.schema.string()).optional()},
execute:async (args) => {
const record={...args,schema:"bench.claim/1",claimed_at:Date.now()/1000};
save("claim.json",record);
fs.appendFileSync(path.join(workspace,"claims.jsonl"),JSON.stringify(record)+"\n");
return `claim recorded: ${args.terminal}`;
}}),
oe_doc: tool({description:"Read OpenExplorer documentation.",args:{query:tool.schema.string()},
execute:async ({query}) => execFileSync(path.join(process.env.BENCH_ROOT!,"agent/oe-doc.sh"),[query],
{encoding:"utf8",timeout:60000,maxBuffer:4<<20}).slice(0,8000)}),
terminal_exec: (() => {
// Same description and parameters as Pi's terminal.ts; a thrown receipt is OpenCode's tool error.
const {commands} = terminal.context(process.env.BENCH_ROLLOUT_DIR);
const z = tool.schema, wait = z.string().regex(/^[a-zA-Z0-9_-]+$/).describe("Existing stage handle from this rollout; no new submission");
return tool({
description: "Wait for an existing stage with wait_handle, without model polling or resubmission. Returns the protected Worker receipt under the original phase/execution deadlines." + (commands ? " Alternatively run a command in the task development environment or assigned board. Paths: /workspace, /out, /input. Formal evaluation uses evaluate=true and predictions JSONL sample_id/top5. Select only an authorized target; CPU tasks use development. Unknown HBM provenance remains UNVERIFIED." : " This task only permits waiting; command execution is disabled."),
args: commands ? {wait_handle: wait.optional(), target: z.enum(["development", "board"]).optional(),
command: z.string().optional(), cwd: z.string().optional(), timeout: z.number().min(1).optional(),
evaluate: z.boolean().optional(), predictions: z.string().optional(),
source: z.string().describe("Workspace subdirectory to freeze for board/formal execution; defaults to .").optional()}
: {wait_handle: wait},
execute: async (args: any, context: any) => {
const params = Object.fromEntries(Object.entries(args).filter(([, v]) => v !== undefined));
return JSON.stringify(await terminal.run(process.env.BENCH_ROLLOUT_DIR, params, context?.abort));
}});
})()
}
};
};