58 lines
3.2 KiB
TypeScript
58 lines
3.2 KiB
TypeScript
import { tool } from "@opencode-ai/plugin";
|
|
import { Schema } from "effect";
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { execFileSync } from "node:child_process";
|
|
|
|
// Only attempt-local evidence is writable; the outer worker owns stages/verdicts.
|
|
export default async () => {
|
|
const workspace = path.join(process.env.BENCH_ROLLOUT_DIR!, "workspace");
|
|
const save = (name: string, value: unknown) => fs.writeFileSync(path.join(workspace, name), JSON.stringify(value));
|
|
const definitions: Record<string, unknown> = {};
|
|
let turns = 0, tokens = 0;
|
|
const seen = new Set<string>();
|
|
const exhausted = () => turns >= Number(process.env.BENCH_MAX_TURNS) || tokens >= Number(process.env.BENCH_MAX_TOKENS);
|
|
return {
|
|
"tool.definition": async (input: any, output: any) => {
|
|
// 1.18.31 registry passes Effect/Zod validators separately from JSON Schema.
|
|
if (!output.jsonSchema) {
|
|
const doc = Schema.toJsonSchemaDocument(output.parameters);
|
|
output.jsonSchema = {...doc.schema, ...(Object.keys(doc.definitions).length ? {$defs:doc.definitions} : {})};
|
|
}
|
|
if (typeof output.jsonSchema !== "object") throw new Error("bench requires actual tool JSON Schema");
|
|
definitions[input.toolID] = {name: input.toolID, description: output.description, parameters: output.jsonSchema};
|
|
save("opencode-tools.json", Object.values(definitions));
|
|
},
|
|
"experimental.chat.system.transform": async (_input: any, output: any) => save("opencode-system.json", output.system),
|
|
"chat.params": async () => {
|
|
if (exhausted()) throw new Error("bench budget exhausted; no further model request");
|
|
},
|
|
"tool.execute.before": async (input: any) => {
|
|
if (exhausted() && !["claim", "oe_doc"].includes(input.tool)) throw new Error("bench budget exhausted; finish with a natural final answer");
|
|
},
|
|
event: async ({event}: any) => {
|
|
const part = event.properties?.part;
|
|
if (part?.type === "step-finish" && !seen.has(part.id)) {
|
|
seen.add(part.id); turns++;
|
|
const usage = part.tokens || {};
|
|
tokens += (usage.input || 0) + (usage.output || 0) + (usage.reasoning || 0) + (usage.cache?.read || 0) + (usage.cache?.write || 0);
|
|
}
|
|
},
|
|
tool: {
|
|
claim: tool({description:"Record your terminal claim; the harness verifies it independently.",
|
|
args:{terminal:tool.schema.enum(["PASS","FAIL","BLOCKED"]), summary:tool.schema.string(),
|
|
stage:tool.schema.string().optional(), error_class:tool.schema.string().optional(),
|
|
artifacts:tool.schema.array(tool.schema.string()).optional()},
|
|
execute:async (args) => {
|
|
const record={...args,schema:"bench.claim/1",claimed_at:Date.now()/1000};
|
|
save("claim.json",record);
|
|
fs.appendFileSync(path.join(workspace,"claims.jsonl"),JSON.stringify(record)+"\n");
|
|
return `claim recorded: ${args.terminal}`;
|
|
}}),
|
|
oe_doc: tool({description:"Read OpenExplorer documentation.",args:{query:tool.schema.string()},
|
|
execute:async ({query}) => execFileSync(path.join(process.env.BENCH_ROOT!,"agent/oe-doc.sh"),[query],
|
|
{encoding:"utf8",timeout:60000,maxBuffer:4<<20}).slice(0,8000)})
|
|
}
|
|
};
|
|
};
|