d-wiki-eval/run_agent_eval.py
dingbin.wang 922aea1872 去除已删除 text/ 目录的引用 + 补全 agent_prompt 图片题说明
- prompts/agent_prompt.md: 删除 rdk_eval/text/*.txt 引用(该目录为抓取污染草稿、
  已删除),改为 检索/ 下现存 wiki/benchmark;补充图片识别题作答说明
- README.md: 知识库挂载说明同步更新
- run_agent_eval.py / analyze_traces.py: KB_PATH_RE 知识库路径匹配更新为现存目录
  (d-wiki/graph-wiki/sag_wiki 等),移除已删除的 text/dom
2026-08-17 11:13:28 +08:00

214 lines
8.9 KiB
Python
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""RDK 知识库 agent 评测 runner(准确性 + 运行轨迹采集)。
对 evalset.jsonl 中每道题,用 `codex exec` 跑一次挂载 RDK 知识库的 agent:
- agent 的最终回答写入 results/<run_id>/answers/<qid>.txt(供 judge 评分)
- 本地轨迹保存:
1. `codex exec --json` 事件流(含 item.completed 的 command_execution / mcp_tool_call 等)
2. ~/.codex/sessions 下的 session rollout 文件(完整轨迹副本)
- 远端轨迹:codex 启动时加载 ~/.codex/config.toml 的 [otel] 配置(已按
~/Downloads/codex-config.toml 合并),把 trace/logs 推送到
https://agent-otlp.d-robotics.cc ,可在观测后台按 conversation_id 检索。
用法:
python3 run_agent_eval.py # 跑剩余全部题目(断点续跑)
python3 run_agent_eval.py --limit 3 # 只跑前 3 题(试跑)
python3 run_agent_eval.py --qid 5 6 # 只跑指定题
"""
import argparse, json, os, re, subprocess, sys, time, datetime
BASE = "/Users/d-robotics/agent/知识检索/rdk_eval/agent_eval"
EVALSET = os.path.join(BASE, "evalset.jsonl")
PROMPT_TPL = open(os.path.join(BASE, "prompts/agent_prompt.md")).read()
SESSIONS_DIR = os.path.expanduser("~/.codex/sessions")
CODEX = os.environ.get("CODEX_BIN", "codex")
# 真实挂载的 RDK 知识库根目录(通过 -c projects.<dir>.trust_level 挂载进沙箱会话)
RDK_KB_ROOT = os.environ.get("RDK_KB_ROOT", "/Users/d-robotics/agent/知识检索")
# 判定「调用 RDK 知识库」的路径模式(analyze_traces 也用同一套规则统计)
KB_PATH_RE = __import__("re").compile(r"知识检索|rdk_qa_dataset|rdk_x_doc|d-wiki|graph-wiki|sag_wiki|/rdk_eval/images", __import__("re").I)
def run_one(item, run_dir, model=None, timeout=900):
qid = item["qid"]
prompt = PROMPT_TPL.replace("{{QUESTION}}", item["question"])
ans_file = os.path.join(run_dir, "answers", f"{qid:04d}.txt")
ev_file = os.path.join(run_dir, "events", f"{qid:04d}.jsonl")
ses_dir = os.path.join(run_dir, "sessions")
traj_file = os.path.join(run_dir, "traces", f"{qid:04d}.json")
# 记录启动前已有的 session 文件,用于事后定位本次会话
before = set()
for root, _, files in os.walk(SESSIONS_DIR):
for fn in files:
if fn.endswith(".jsonl"):
before.add(os.path.join(root, fn))
cmd = [
CODEX, "exec", "--json",
"--skip-git-repo-check",
"--dangerously-bypass-approvals-and-sandbox",
# 挂载 RDK 知识库目录(trusted,agent 可读/检索该知识库)
"-c", f'projects."{RDK_KB_ROOT}".trust_level="trusted"',
"-C", os.path.join(run_dir, "sandbox"),
"-o", ans_file,
]
if model:
cmd += ["-m", model]
use_stdin = False
if item.get("image") and os.path.exists(item["image"]):
cmd += ["-i", item["image"], "-"] # 图片题:-i 触发 stdin 读取,prompt 改走 stdin
use_stdin = True
else:
cmd.append(prompt)
t0 = time.time()
started = datetime.datetime.now().isoformat(timespec="seconds")
ret, err, session_id, conv_id, raw = -1, "", None, None, ""
try:
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout,
input=prompt if use_stdin else None)
ret, raw, err = proc.returncode, proc.stdout, proc.stderr
except subprocess.TimeoutExpired as e:
err = f"TIMEOUT after {timeout}s"
raw = (e.stdout or b"" if isinstance(e.stdout, bytes) else e.stdout) or ""
if isinstance(raw, bytes):
raw = raw.decode("utf-8", "replace")
dt = time.time() - t0
with open(ev_file, "w") as f:
f.write(raw if isinstance(raw, str) else raw.decode("utf-8", "replace"))
# 解析事件流:thread.started / item.completed / 工具调用 + RDK 知识库调用统计
items, tool_calls, kb_calls = [], [], []
for line in (raw if isinstance(raw, str) else "").splitlines():
line = line.strip()
if not line.startswith("{"):
continue
try:
ev = json.loads(line)
except json.JSONDecodeError:
continue
t = ev.get("type")
if t == "thread.started":
session_id = ev.get("thread_id") or session_id
conv_id = ev.get("conversation_id") or ev.get("thread_id") or conv_id
if t == "item.completed":
it = ev.get("item", {})
items.append({"type": it.get("type"), "id": it.get("id")})
name, argtxt = None, ""
if it.get("type") in ("mcp_tool_call", "tool_call"):
name = it.get("tool") or it.get("name") or "unknown"
argtxt = json.dumps(it, ensure_ascii=False)
elif it.get("type") == "command_execution":
name = "shell"
argtxt = it.get("command") or json.dumps(it, ensure_ascii=False)
if name:
tool_calls.append(name)
# 命令/工具参数里触及 RDK 知识库路径 = 一次知识库调用
if KB_PATH_RE.search(str(argtxt)):
kb_calls.append({"tool": name, "arg": str(argtxt)[:200]})
# 定位并复制 session 文件
session_file = ""
time.sleep(1.5) # 等 codex 落盘
cands = []
for root, _, files in os.walk(SESSIONS_DIR):
for fn in files:
p = os.path.join(root, fn)
if fn.endswith(".jsonl") and p not in before:
cands.append(p)
if cands:
cands.sort(key=os.path.getmtime)
src = cands[-1]
session_file = os.path.join(ses_dir, os.path.basename(src))
try:
with open(src) as f:
content = f.read()
with open(session_file, "w") as f:
f.write(content)
except OSError:
session_file = ""
answer = ""
if os.path.exists(ans_file):
answer = open(ans_file).read().strip()
traj = {
"qid": qid,
"sid": item.get("sid"),
"model": item.get("model"),
"type": item.get("type"),
"started_at": started,
"latency_s": round(dt, 1),
"exit_code": ret,
"error": err.strip()[-2000:] if err.strip() else None,
"session_id": session_id,
"conversation_id": conv_id or session_id, # 用这个在 otel 后台检索轨迹
"session_file": session_file,
"events_file": ev_file,
"answer_file": ans_file if answer else None,
"answer_chars": len(answer),
"items_completed": len(items),
"tool_calls": tool_calls,
"tool_call_count": len(tool_calls),
"rdk_kb_call_count": len(kb_calls),
"rdk_kb_calls": kb_calls,
"rdk_kb_root": RDK_KB_ROOT,
"otel": {
"traces_url": "https://agent-otlp.d-robotics.cc/v1/traces",
"note": "远端轨迹已随 codex [otel] 配置自动上报;按 conversation_id 检索",
},
}
with open(traj_file, "w") as f:
json.dump(traj, f, ensure_ascii=False, indent=1)
return traj
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--run-id", default=None)
ap.add_argument("--limit", type=int, default=None)
ap.add_argument("--qid", type=int, nargs="*", default=None)
ap.add_argument("--model", default=None, help="覆盖 codex 默认模型")
ap.add_argument("--timeout", type=int, default=900)
args = ap.parse_args()
run_id = args.run_id or datetime.datetime.now().strftime("run%Y%m%d_%H%M%S")
run_dir = os.path.join(BASE, "results", run_id)
for sub in ("answers", "events", "sessions", "traces", "sandbox"):
os.makedirs(os.path.join(run_dir, sub), exist_ok=True)
items = [json.loads(l) for l in open(EVALSET)]
done = set()
sum_path = os.path.join(run_dir, "summary.jsonl")
if os.path.exists(sum_path): # 断点续跑
for l in open(sum_path):
try:
done.add(json.loads(l)["qid"])
except json.JSONDecodeError:
pass
todo = [it for it in items if it["qid"] not in done]
if args.qid:
todo = [it for it in todo if it["qid"] in set(args.qid)]
if args.limit:
todo = todo[: args.limit]
print(f"[{run_id}] 待跑 {len(todo)} 题(已完成 {len(done)})", flush=True)
with open(sum_path, "a") as sf:
for i, it in enumerate(todo):
t = run_one(it, run_dir, model=args.model, timeout=args.timeout)
sf.write(json.dumps(t, ensure_ascii=False) + "\n")
sf.flush()
ok = "OK " if t["exit_code"] == 0 and t["answer_chars"] > 0 else "ERR"
print(f" [{i+1}/{len(todo)}] qid={it['qid']:>3} {ok} "
f"{t['latency_s']:>6.1f}s tools={t['tool_call_count']:>2} "
f"ans={t['answer_chars']:>5}ch conv={str(t['conversation_id'])[:8]}",
flush=True)
print(f"完成。结果目录: {run_dir}")
if __name__ == "__main__":
main()