#!/usr/bin/env python3 # -*- coding: utf-8 -*- """RDK 知识库 agent 评测 runner(准确性 + 运行轨迹采集)。 对 evalset.jsonl 中每道题,用 `codex exec` 跑一次挂载 RDK 知识库的 agent: - agent 的最终回答写入 results//answers/.txt(供 judge 评分) - 本地轨迹保存: 1. `codex exec --json` 事件流(含 item.completed 的 command_execution / mcp_tool_call 等) 2. ~/.codex/sessions 下的 session rollout 文件(完整轨迹副本) - 远端轨迹:codex 启动时加载 ~/.codex/config.toml 的 [otel] 配置(已按 ~/Downloads/codex-config.toml 合并),把 trace/logs 推送到 https://agent-otlp.d-robotics.cc ,可在观测后台按 conversation_id 检索。 用法: python3 run_agent_eval.py # 跑剩余全部题目(断点续跑) python3 run_agent_eval.py --limit 3 # 只跑前 3 题(试跑) python3 run_agent_eval.py --qid 5 6 # 只跑指定题 """ import argparse, json, os, re, subprocess, sys, time, datetime BASE = "/Users/d-robotics/agent/知识检索/rdk_eval/agent_eval" EVALSET = os.path.join(BASE, "evalset.jsonl") PROMPT_TPL = open(os.path.join(BASE, "prompts/agent_prompt.md")).read() SESSIONS_DIR = os.path.expanduser("~/.codex/sessions") CODEX = os.environ.get("CODEX_BIN", "codex") # 真实挂载的 RDK 知识库根目录(通过 -c projects..trust_level 挂载进沙箱会话) RDK_KB_ROOT = os.environ.get("RDK_KB_ROOT", "/Users/d-robotics/agent/知识检索") # 判定「调用 RDK 知识库」的路径模式(analyze_traces 也用同一套规则统计) KB_PATH_RE = __import__("re").compile(r"知识检索|rdk_qa_dataset|rdk_x_doc|d-wiki|graph-wiki|sag_wiki|/rdk_eval/images", __import__("re").I) def run_one(item, run_dir, model=None, timeout=900): qid = item["qid"] prompt = PROMPT_TPL.replace("{{QUESTION}}", item["question"]) ans_file = os.path.join(run_dir, "answers", f"{qid:04d}.txt") ev_file = os.path.join(run_dir, "events", f"{qid:04d}.jsonl") ses_dir = os.path.join(run_dir, "sessions") traj_file = os.path.join(run_dir, "traces", f"{qid:04d}.json") # 记录启动前已有的 session 文件,用于事后定位本次会话 before = set() for root, _, files in os.walk(SESSIONS_DIR): for fn in files: if fn.endswith(".jsonl"): before.add(os.path.join(root, fn)) cmd = [ CODEX, "exec", "--json", "--skip-git-repo-check", "--dangerously-bypass-approvals-and-sandbox", # 挂载 RDK 知识库目录(trusted,agent 可读/检索该知识库) "-c", f'projects."{RDK_KB_ROOT}".trust_level="trusted"', "-C", os.path.join(run_dir, "sandbox"), "-o", ans_file, ] if model: cmd += ["-m", model] use_stdin = False if item.get("image") and os.path.exists(item["image"]): cmd += ["-i", item["image"], "-"] # 图片题:-i 触发 stdin 读取,prompt 改走 stdin use_stdin = True else: cmd.append(prompt) t0 = time.time() started = datetime.datetime.now().isoformat(timespec="seconds") ret, err, session_id, conv_id, raw = -1, "", None, None, "" try: proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, input=prompt if use_stdin else None) ret, raw, err = proc.returncode, proc.stdout, proc.stderr except subprocess.TimeoutExpired as e: err = f"TIMEOUT after {timeout}s" raw = (e.stdout or b"" if isinstance(e.stdout, bytes) else e.stdout) or "" if isinstance(raw, bytes): raw = raw.decode("utf-8", "replace") dt = time.time() - t0 with open(ev_file, "w") as f: f.write(raw if isinstance(raw, str) else raw.decode("utf-8", "replace")) # 解析事件流:thread.started / item.completed / 工具调用 + RDK 知识库调用统计 items, tool_calls, kb_calls = [], [], [] for line in (raw if isinstance(raw, str) else "").splitlines(): line = line.strip() if not line.startswith("{"): continue try: ev = json.loads(line) except json.JSONDecodeError: continue t = ev.get("type") if t == "thread.started": session_id = ev.get("thread_id") or session_id conv_id = ev.get("conversation_id") or ev.get("thread_id") or conv_id if t == "item.completed": it = ev.get("item", {}) items.append({"type": it.get("type"), "id": it.get("id")}) name, argtxt = None, "" if it.get("type") in ("mcp_tool_call", "tool_call"): name = it.get("tool") or it.get("name") or "unknown" argtxt = json.dumps(it, ensure_ascii=False) elif it.get("type") == "command_execution": name = "shell" argtxt = it.get("command") or json.dumps(it, ensure_ascii=False) if name: tool_calls.append(name) # 命令/工具参数里触及 RDK 知识库路径 = 一次知识库调用 if KB_PATH_RE.search(str(argtxt)): kb_calls.append({"tool": name, "arg": str(argtxt)[:200]}) # 定位并复制 session 文件 session_file = "" time.sleep(1.5) # 等 codex 落盘 cands = [] for root, _, files in os.walk(SESSIONS_DIR): for fn in files: p = os.path.join(root, fn) if fn.endswith(".jsonl") and p not in before: cands.append(p) if cands: cands.sort(key=os.path.getmtime) src = cands[-1] session_file = os.path.join(ses_dir, os.path.basename(src)) try: with open(src) as f: content = f.read() with open(session_file, "w") as f: f.write(content) except OSError: session_file = "" answer = "" if os.path.exists(ans_file): answer = open(ans_file).read().strip() traj = { "qid": qid, "sid": item.get("sid"), "model": item.get("model"), "type": item.get("type"), "started_at": started, "latency_s": round(dt, 1), "exit_code": ret, "error": err.strip()[-2000:] if err.strip() else None, "session_id": session_id, "conversation_id": conv_id or session_id, # 用这个在 otel 后台检索轨迹 "session_file": session_file, "events_file": ev_file, "answer_file": ans_file if answer else None, "answer_chars": len(answer), "items_completed": len(items), "tool_calls": tool_calls, "tool_call_count": len(tool_calls), "rdk_kb_call_count": len(kb_calls), "rdk_kb_calls": kb_calls, "rdk_kb_root": RDK_KB_ROOT, "otel": { "traces_url": "https://agent-otlp.d-robotics.cc/v1/traces", "note": "远端轨迹已随 codex [otel] 配置自动上报;按 conversation_id 检索", }, } with open(traj_file, "w") as f: json.dump(traj, f, ensure_ascii=False, indent=1) return traj def main(): ap = argparse.ArgumentParser() ap.add_argument("--run-id", default=None) ap.add_argument("--limit", type=int, default=None) ap.add_argument("--qid", type=int, nargs="*", default=None) ap.add_argument("--model", default=None, help="覆盖 codex 默认模型") ap.add_argument("--timeout", type=int, default=900) args = ap.parse_args() run_id = args.run_id or datetime.datetime.now().strftime("run%Y%m%d_%H%M%S") run_dir = os.path.join(BASE, "results", run_id) for sub in ("answers", "events", "sessions", "traces", "sandbox"): os.makedirs(os.path.join(run_dir, sub), exist_ok=True) items = [json.loads(l) for l in open(EVALSET)] done = set() sum_path = os.path.join(run_dir, "summary.jsonl") if os.path.exists(sum_path): # 断点续跑 for l in open(sum_path): try: done.add(json.loads(l)["qid"]) except json.JSONDecodeError: pass todo = [it for it in items if it["qid"] not in done] if args.qid: todo = [it for it in todo if it["qid"] in set(args.qid)] if args.limit: todo = todo[: args.limit] print(f"[{run_id}] 待跑 {len(todo)} 题(已完成 {len(done)})", flush=True) with open(sum_path, "a") as sf: for i, it in enumerate(todo): t = run_one(it, run_dir, model=args.model, timeout=args.timeout) sf.write(json.dumps(t, ensure_ascii=False) + "\n") sf.flush() ok = "OK " if t["exit_code"] == 0 and t["answer_chars"] > 0 else "ERR" print(f" [{i+1}/{len(todo)}] qid={it['qid']:>3} {ok} " f"{t['latency_s']:>6.1f}s tools={t['tool_call_count']:>2} " f"ans={t['answer_chars']:>5}ch conv={str(t['conversation_id'])[:8]}", flush=True) print(f"完成。结果目录: {run_dir}") if __name__ == "__main__": main()