- prompts/agent_prompt.md: 删除 rdk_eval/text/*.txt 引用(该目录为抓取污染草稿、 已删除),改为 检索/ 下现存 wiki/benchmark;补充图片识别题作答说明 - README.md: 知识库挂载说明同步更新 - run_agent_eval.py / analyze_traces.py: KB_PATH_RE 知识库路径匹配更新为现存目录 (d-wiki/graph-wiki/sag_wiki 等),移除已删除的 text/dom
214 lines
8.9 KiB
Python
Executable File
214 lines
8.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""RDK 知识库 agent 评测 runner(准确性 + 运行轨迹采集)。
|
||
|
||
对 evalset.jsonl 中每道题,用 `codex exec` 跑一次挂载 RDK 知识库的 agent:
|
||
- agent 的最终回答写入 results/<run_id>/answers/<qid>.txt(供 judge 评分)
|
||
- 本地轨迹保存:
|
||
1. `codex exec --json` 事件流(含 item.completed 的 command_execution / mcp_tool_call 等)
|
||
2. ~/.codex/sessions 下的 session rollout 文件(完整轨迹副本)
|
||
- 远端轨迹:codex 启动时加载 ~/.codex/config.toml 的 [otel] 配置(已按
|
||
~/Downloads/codex-config.toml 合并),把 trace/logs 推送到
|
||
https://agent-otlp.d-robotics.cc ,可在观测后台按 conversation_id 检索。
|
||
|
||
用法:
|
||
python3 run_agent_eval.py # 跑剩余全部题目(断点续跑)
|
||
python3 run_agent_eval.py --limit 3 # 只跑前 3 题(试跑)
|
||
python3 run_agent_eval.py --qid 5 6 # 只跑指定题
|
||
"""
|
||
import argparse, json, os, re, subprocess, sys, time, datetime
|
||
|
||
BASE = "/Users/d-robotics/agent/知识检索/rdk_eval/agent_eval"
|
||
EVALSET = os.path.join(BASE, "evalset.jsonl")
|
||
PROMPT_TPL = open(os.path.join(BASE, "prompts/agent_prompt.md")).read()
|
||
SESSIONS_DIR = os.path.expanduser("~/.codex/sessions")
|
||
CODEX = os.environ.get("CODEX_BIN", "codex")
|
||
# 真实挂载的 RDK 知识库根目录(通过 -c projects.<dir>.trust_level 挂载进沙箱会话)
|
||
RDK_KB_ROOT = os.environ.get("RDK_KB_ROOT", "/Users/d-robotics/agent/知识检索")
|
||
# 判定「调用 RDK 知识库」的路径模式(analyze_traces 也用同一套规则统计)
|
||
KB_PATH_RE = __import__("re").compile(r"知识检索|rdk_qa_dataset|rdk_x_doc|d-wiki|graph-wiki|sag_wiki|/rdk_eval/images", __import__("re").I)
|
||
|
||
|
||
def run_one(item, run_dir, model=None, timeout=900):
|
||
qid = item["qid"]
|
||
prompt = PROMPT_TPL.replace("{{QUESTION}}", item["question"])
|
||
ans_file = os.path.join(run_dir, "answers", f"{qid:04d}.txt")
|
||
ev_file = os.path.join(run_dir, "events", f"{qid:04d}.jsonl")
|
||
ses_dir = os.path.join(run_dir, "sessions")
|
||
traj_file = os.path.join(run_dir, "traces", f"{qid:04d}.json")
|
||
|
||
# 记录启动前已有的 session 文件,用于事后定位本次会话
|
||
before = set()
|
||
for root, _, files in os.walk(SESSIONS_DIR):
|
||
for fn in files:
|
||
if fn.endswith(".jsonl"):
|
||
before.add(os.path.join(root, fn))
|
||
|
||
cmd = [
|
||
CODEX, "exec", "--json",
|
||
"--skip-git-repo-check",
|
||
"--dangerously-bypass-approvals-and-sandbox",
|
||
# 挂载 RDK 知识库目录(trusted,agent 可读/检索该知识库)
|
||
"-c", f'projects."{RDK_KB_ROOT}".trust_level="trusted"',
|
||
"-C", os.path.join(run_dir, "sandbox"),
|
||
"-o", ans_file,
|
||
]
|
||
if model:
|
||
cmd += ["-m", model]
|
||
use_stdin = False
|
||
if item.get("image") and os.path.exists(item["image"]):
|
||
cmd += ["-i", item["image"], "-"] # 图片题:-i 触发 stdin 读取,prompt 改走 stdin
|
||
use_stdin = True
|
||
else:
|
||
cmd.append(prompt)
|
||
|
||
t0 = time.time()
|
||
started = datetime.datetime.now().isoformat(timespec="seconds")
|
||
ret, err, session_id, conv_id, raw = -1, "", None, None, ""
|
||
try:
|
||
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout,
|
||
input=prompt if use_stdin else None)
|
||
ret, raw, err = proc.returncode, proc.stdout, proc.stderr
|
||
except subprocess.TimeoutExpired as e:
|
||
err = f"TIMEOUT after {timeout}s"
|
||
raw = (e.stdout or b"" if isinstance(e.stdout, bytes) else e.stdout) or ""
|
||
if isinstance(raw, bytes):
|
||
raw = raw.decode("utf-8", "replace")
|
||
|
||
dt = time.time() - t0
|
||
with open(ev_file, "w") as f:
|
||
f.write(raw if isinstance(raw, str) else raw.decode("utf-8", "replace"))
|
||
|
||
# 解析事件流:thread.started / item.completed / 工具调用 + RDK 知识库调用统计
|
||
items, tool_calls, kb_calls = [], [], []
|
||
for line in (raw if isinstance(raw, str) else "").splitlines():
|
||
line = line.strip()
|
||
if not line.startswith("{"):
|
||
continue
|
||
try:
|
||
ev = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
t = ev.get("type")
|
||
if t == "thread.started":
|
||
session_id = ev.get("thread_id") or session_id
|
||
conv_id = ev.get("conversation_id") or ev.get("thread_id") or conv_id
|
||
if t == "item.completed":
|
||
it = ev.get("item", {})
|
||
items.append({"type": it.get("type"), "id": it.get("id")})
|
||
name, argtxt = None, ""
|
||
if it.get("type") in ("mcp_tool_call", "tool_call"):
|
||
name = it.get("tool") or it.get("name") or "unknown"
|
||
argtxt = json.dumps(it, ensure_ascii=False)
|
||
elif it.get("type") == "command_execution":
|
||
name = "shell"
|
||
argtxt = it.get("command") or json.dumps(it, ensure_ascii=False)
|
||
if name:
|
||
tool_calls.append(name)
|
||
# 命令/工具参数里触及 RDK 知识库路径 = 一次知识库调用
|
||
if KB_PATH_RE.search(str(argtxt)):
|
||
kb_calls.append({"tool": name, "arg": str(argtxt)[:200]})
|
||
|
||
# 定位并复制 session 文件
|
||
session_file = ""
|
||
time.sleep(1.5) # 等 codex 落盘
|
||
cands = []
|
||
for root, _, files in os.walk(SESSIONS_DIR):
|
||
for fn in files:
|
||
p = os.path.join(root, fn)
|
||
if fn.endswith(".jsonl") and p not in before:
|
||
cands.append(p)
|
||
if cands:
|
||
cands.sort(key=os.path.getmtime)
|
||
src = cands[-1]
|
||
session_file = os.path.join(ses_dir, os.path.basename(src))
|
||
try:
|
||
with open(src) as f:
|
||
content = f.read()
|
||
with open(session_file, "w") as f:
|
||
f.write(content)
|
||
except OSError:
|
||
session_file = ""
|
||
|
||
answer = ""
|
||
if os.path.exists(ans_file):
|
||
answer = open(ans_file).read().strip()
|
||
|
||
traj = {
|
||
"qid": qid,
|
||
"sid": item.get("sid"),
|
||
"model": item.get("model"),
|
||
"type": item.get("type"),
|
||
"started_at": started,
|
||
"latency_s": round(dt, 1),
|
||
"exit_code": ret,
|
||
"error": err.strip()[-2000:] if err.strip() else None,
|
||
"session_id": session_id,
|
||
"conversation_id": conv_id or session_id, # 用这个在 otel 后台检索轨迹
|
||
"session_file": session_file,
|
||
"events_file": ev_file,
|
||
"answer_file": ans_file if answer else None,
|
||
"answer_chars": len(answer),
|
||
"items_completed": len(items),
|
||
"tool_calls": tool_calls,
|
||
"tool_call_count": len(tool_calls),
|
||
"rdk_kb_call_count": len(kb_calls),
|
||
"rdk_kb_calls": kb_calls,
|
||
"rdk_kb_root": RDK_KB_ROOT,
|
||
"otel": {
|
||
"traces_url": "https://agent-otlp.d-robotics.cc/v1/traces",
|
||
"note": "远端轨迹已随 codex [otel] 配置自动上报;按 conversation_id 检索",
|
||
},
|
||
}
|
||
with open(traj_file, "w") as f:
|
||
json.dump(traj, f, ensure_ascii=False, indent=1)
|
||
return traj
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--run-id", default=None)
|
||
ap.add_argument("--limit", type=int, default=None)
|
||
ap.add_argument("--qid", type=int, nargs="*", default=None)
|
||
ap.add_argument("--model", default=None, help="覆盖 codex 默认模型")
|
||
ap.add_argument("--timeout", type=int, default=900)
|
||
args = ap.parse_args()
|
||
|
||
run_id = args.run_id or datetime.datetime.now().strftime("run%Y%m%d_%H%M%S")
|
||
run_dir = os.path.join(BASE, "results", run_id)
|
||
for sub in ("answers", "events", "sessions", "traces", "sandbox"):
|
||
os.makedirs(os.path.join(run_dir, sub), exist_ok=True)
|
||
|
||
items = [json.loads(l) for l in open(EVALSET)]
|
||
done = set()
|
||
sum_path = os.path.join(run_dir, "summary.jsonl")
|
||
if os.path.exists(sum_path): # 断点续跑
|
||
for l in open(sum_path):
|
||
try:
|
||
done.add(json.loads(l)["qid"])
|
||
except json.JSONDecodeError:
|
||
pass
|
||
|
||
todo = [it for it in items if it["qid"] not in done]
|
||
if args.qid:
|
||
todo = [it for it in todo if it["qid"] in set(args.qid)]
|
||
if args.limit:
|
||
todo = todo[: args.limit]
|
||
|
||
print(f"[{run_id}] 待跑 {len(todo)} 题(已完成 {len(done)})", flush=True)
|
||
with open(sum_path, "a") as sf:
|
||
for i, it in enumerate(todo):
|
||
t = run_one(it, run_dir, model=args.model, timeout=args.timeout)
|
||
sf.write(json.dumps(t, ensure_ascii=False) + "\n")
|
||
sf.flush()
|
||
ok = "OK " if t["exit_code"] == 0 and t["answer_chars"] > 0 else "ERR"
|
||
print(f" [{i+1}/{len(todo)}] qid={it['qid']:>3} {ok} "
|
||
f"{t['latency_s']:>6.1f}s tools={t['tool_call_count']:>2} "
|
||
f"ans={t['answer_chars']:>5}ch conv={str(t['conversation_id'])[:8]}",
|
||
flush=True)
|
||
print(f"完成。结果目录: {run_dir}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|