- prompts/agent_prompt.md: 删除 rdk_eval/text/*.txt 引用(该目录为抓取污染草稿、 已删除),改为 检索/ 下现存 wiki/benchmark;补充图片识别题作答说明 - README.md: 知识库挂载说明同步更新 - run_agent_eval.py / analyze_traces.py: KB_PATH_RE 知识库路径匹配更新为现存目录 (d-wiki/graph-wiki/sag_wiki 等),移除已删除的 text/dom
143 lines
6.5 KiB
Python
143 lines
6.5 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""轨迹分析:知识库调用次数 + 检索速率 + 整体耗时。
|
||
|
||
数据源:results/<run_id>/sessions/*.jsonl(codex session 文件,每行带 timestamp)
|
||
- function_call 与其 function_call_output 按时间戳配对,得到每次工具调用的耗时
|
||
- 判定「RDK 知识库调用」:工具名匹配 --kb-pattern(MCP 检索类工具),
|
||
或 shell 命令参数触及知识库路径(grep/python 检索 qa.jsonl、官方文档等)
|
||
|
||
速率指标(每题 + 全局汇总):
|
||
- kb_calls: 知识库调用次数;kb_time_s: 检索总耗时;avg_kb_call_s: 单次检索均耗时
|
||
- time_to_first_kb_s: 首个知识库调用前的思考耗时
|
||
- kb_time_ratio: 检索耗时占整题耗时比例
|
||
- calls_per_min: 每分钟知识库调用次数(检索节奏)
|
||
|
||
用法:
|
||
python3 analyze_traces.py --run-id smoke --list-tools # 先看实际工具名
|
||
python3 analyze_traces.py --run-id smoke
|
||
"""
|
||
import argparse, collections, datetime, glob, json, os, re
|
||
|
||
BASE = "/Users/d-robotics/agent/知识检索/rdk_eval/agent_eval"
|
||
CALL_TYPES = ("function_call", "local_shell_call", "custom_tool_call", "tool_call")
|
||
KB_PATH_RE = re.compile(r"知识检索|rdk_qa_dataset|rdk_x_doc|d-wiki|graph-wiki|sag_wiki|/rdk_eval/images", re.I)
|
||
# 交互式长命令的轮询/发键调用(write_stdin 等),不计为独立检索,只把等待时间计入耗时
|
||
POLL_RE = re.compile(r"write_stdin|toolu_write", re.I)
|
||
|
||
|
||
def ts(s):
|
||
return datetime.datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()
|
||
|
||
|
||
def scan_session(path, kb_re):
|
||
"""返回 (calls, counter)。calls = [{name, args, dur_s, t_start, is_kb}]"""
|
||
calls, pending = [], {}
|
||
counter = collections.Counter()
|
||
for line in open(path, errors="replace"):
|
||
try:
|
||
rec = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
t0 = rec.get("timestamp")
|
||
p = rec.get("payload", {})
|
||
ptype = p.get("type")
|
||
if ptype == "response_item" and isinstance(p.get("item"), dict):
|
||
p = p["item"]
|
||
ptype = p.get("type")
|
||
if ptype in CALL_TYPES:
|
||
name = p.get("name") or p.get("tool_name") or p.get("tool")
|
||
args = p.get("arguments") or ""
|
||
cid = p.get("call_id") or p.get("id") or f"auto{len(calls)}"
|
||
if name:
|
||
counter[name] += 1
|
||
item = {"name": name, "args": str(args), "t_start": ts(t0) if t0 else None,
|
||
"dur_s": None, "cid": cid}
|
||
calls.append(item)
|
||
pending[cid] = item
|
||
elif ptype in ("function_call_output", "local_shell_call_output", "tool_call_output"):
|
||
cid = p.get("call_id") or p.get("id")
|
||
it = pending.pop(cid, None)
|
||
if it and t0 and it["t_start"]:
|
||
it["dur_s"] = max(0.0, ts(t0) - it["t_start"])
|
||
# 标记知识库调用
|
||
for c in calls:
|
||
c["is_kb"] = bool(kb_re.search(c["name"]) or KB_PATH_RE.search(c["args"]))
|
||
return calls, counter
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--run-id", required=True)
|
||
ap.add_argument("--kb-pattern", default=r"memorylake|memory|rdk|kb|knowledge|search|retriev|rag")
|
||
ap.add_argument("--list-tools", action="store_true")
|
||
args = ap.parse_args()
|
||
|
||
run_dir = os.path.join(BASE, "results", args.run_id)
|
||
kb_re = re.compile(args.kb_pattern, re.I)
|
||
|
||
traces = {}
|
||
for p in glob.glob(os.path.join(run_dir, "traces", "*.json")):
|
||
t = json.load(open(p))
|
||
if t.get("session_file") and os.path.exists(t["session_file"]):
|
||
traces[t["qid"]] = t
|
||
|
||
all_names = collections.Counter()
|
||
rows = []
|
||
for qid, t in sorted(traces.items()):
|
||
calls, counter = scan_session(t["session_file"], kb_re)
|
||
all_names.update(counter)
|
||
kb = [c for c in calls if c["is_kb"] and not POLL_RE.search(c["name"])]
|
||
total = len(calls)
|
||
# 检索耗时:未配对到 output 的调用用 runner 事件流里的事件数兜底(一般为 0)
|
||
kb_times = [c["dur_s"] for c in kb if c["dur_s"] is not None]
|
||
kb_time = sum(kb_times)
|
||
first_kb = min((c["t_start"] for c in kb if c["t_start"]), default=None)
|
||
t_start = min((c["t_start"] for c in calls if c["t_start"]), default=None)
|
||
ttf = round(first_kb - t_start, 1) if first_kb and t_start else None
|
||
lat = t.get("latency_s") or 0
|
||
rows.append({
|
||
"qid": qid,
|
||
"tool_calls_total": total,
|
||
"kb_calls": len(kb),
|
||
"kb_time_s": round(kb_time, 1),
|
||
"avg_kb_call_s": round(kb_time / len(kb_times), 2) if kb_times else None,
|
||
"max_kb_call_s": round(max(kb_times), 1) if kb_times else None,
|
||
"time_to_first_kb_s": ttf,
|
||
"latency_s": lat,
|
||
"kb_time_ratio": round(kb_time / lat, 2) if lat else None,
|
||
"calls_per_min": round(len(kb) / lat * 60, 2) if lat else None,
|
||
"kb_tools": sorted({c["name"] for c in kb}),
|
||
"conversation_id": t.get("conversation_id"),
|
||
})
|
||
|
||
if args.list_tools:
|
||
print("出现过的工具/函数名(用于校准 --kb-pattern):")
|
||
for name, n in all_names.most_common():
|
||
print(f" {n:>4}x {name}")
|
||
return
|
||
|
||
with open(os.path.join(run_dir, "trace_report.json"), "w") as f:
|
||
json.dump(rows, f, ensure_ascii=False, indent=1)
|
||
|
||
kb_list = [r["kb_calls"] for r in rows]
|
||
lat = [r["latency_s"] for r in rows if r.get("latency_s")]
|
||
avg_call = [r["avg_kb_call_s"] for r in rows if r.get("avg_kb_call_s") is not None]
|
||
ratio = [r["kb_time_ratio"] for r in rows if r.get("kb_time_ratio") is not None]
|
||
print(f"== 轨迹统计({args.run_id},{len(rows)} 题)==")
|
||
print(f"工具总调用: {sum(r['tool_calls_total'] for r in rows)},RDK 知识库调用: {sum(kb_list)}")
|
||
if kb_list:
|
||
z = sum(1 for k in kb_list if k == 0)
|
||
print(f"每题知识库调用: 平均 {sum(kb_list)/len(kb_list):.2f} 次,最多 {max(kb_list)},0 次题数 {z}")
|
||
if avg_call:
|
||
print(f"检索速率: 单次检索平均 {sum(avg_call)/len(avg_call):.2f}s(最慢 {max(r['max_kb_call_s'] for r in rows if r['max_kb_call_s'])}s)")
|
||
if ratio:
|
||
print(f"检索耗时占比: 平均 {sum(ratio)/len(ratio)*100:.0f}%")
|
||
if lat:
|
||
print(f"每题总耗时: 平均 {sum(lat)/len(lat):.1f}s,p50 {sorted(lat)[len(lat)//2]}s")
|
||
print("报告: " + os.path.join(run_dir, "trace_report.json"))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|