d-wiki-eval/analyze_traces.py
dingbin.wang 922aea1872 去除已删除 text/ 目录的引用 + 补全 agent_prompt 图片题说明
- prompts/agent_prompt.md: 删除 rdk_eval/text/*.txt 引用(该目录为抓取污染草稿、
  已删除),改为 检索/ 下现存 wiki/benchmark;补充图片识别题作答说明
- README.md: 知识库挂载说明同步更新
- run_agent_eval.py / analyze_traces.py: KB_PATH_RE 知识库路径匹配更新为现存目录
  (d-wiki/graph-wiki/sag_wiki 等),移除已删除的 text/dom
2026-08-17 11:13:28 +08:00

143 lines
6.5 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""轨迹分析:知识库调用次数 + 检索速率 + 整体耗时。
数据源:results/<run_id>/sessions/*.jsonl(codex session 文件,每行带 timestamp)
- function_call 与其 function_call_output 按时间戳配对,得到每次工具调用的耗时
- 判定「RDK 知识库调用」:工具名匹配 --kb-pattern(MCP 检索类工具),
或 shell 命令参数触及知识库路径(grep/python 检索 qa.jsonl、官方文档等)
速率指标(每题 + 全局汇总):
- kb_calls: 知识库调用次数;kb_time_s: 检索总耗时;avg_kb_call_s: 单次检索均耗时
- time_to_first_kb_s: 首个知识库调用前的思考耗时
- kb_time_ratio: 检索耗时占整题耗时比例
- calls_per_min: 每分钟知识库调用次数(检索节奏)
用法:
python3 analyze_traces.py --run-id smoke --list-tools # 先看实际工具名
python3 analyze_traces.py --run-id smoke
"""
import argparse, collections, datetime, glob, json, os, re
BASE = "/Users/d-robotics/agent/知识检索/rdk_eval/agent_eval"
CALL_TYPES = ("function_call", "local_shell_call", "custom_tool_call", "tool_call")
KB_PATH_RE = re.compile(r"知识检索|rdk_qa_dataset|rdk_x_doc|d-wiki|graph-wiki|sag_wiki|/rdk_eval/images", re.I)
# 交互式长命令的轮询/发键调用(write_stdin 等),不计为独立检索,只把等待时间计入耗时
POLL_RE = re.compile(r"write_stdin|toolu_write", re.I)
def ts(s):
return datetime.datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()
def scan_session(path, kb_re):
"""返回 (calls, counter)。calls = [{name, args, dur_s, t_start, is_kb}]"""
calls, pending = [], {}
counter = collections.Counter()
for line in open(path, errors="replace"):
try:
rec = json.loads(line)
except json.JSONDecodeError:
continue
t0 = rec.get("timestamp")
p = rec.get("payload", {})
ptype = p.get("type")
if ptype == "response_item" and isinstance(p.get("item"), dict):
p = p["item"]
ptype = p.get("type")
if ptype in CALL_TYPES:
name = p.get("name") or p.get("tool_name") or p.get("tool")
args = p.get("arguments") or ""
cid = p.get("call_id") or p.get("id") or f"auto{len(calls)}"
if name:
counter[name] += 1
item = {"name": name, "args": str(args), "t_start": ts(t0) if t0 else None,
"dur_s": None, "cid": cid}
calls.append(item)
pending[cid] = item
elif ptype in ("function_call_output", "local_shell_call_output", "tool_call_output"):
cid = p.get("call_id") or p.get("id")
it = pending.pop(cid, None)
if it and t0 and it["t_start"]:
it["dur_s"] = max(0.0, ts(t0) - it["t_start"])
# 标记知识库调用
for c in calls:
c["is_kb"] = bool(kb_re.search(c["name"]) or KB_PATH_RE.search(c["args"]))
return calls, counter
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--run-id", required=True)
ap.add_argument("--kb-pattern", default=r"memorylake|memory|rdk|kb|knowledge|search|retriev|rag")
ap.add_argument("--list-tools", action="store_true")
args = ap.parse_args()
run_dir = os.path.join(BASE, "results", args.run_id)
kb_re = re.compile(args.kb_pattern, re.I)
traces = {}
for p in glob.glob(os.path.join(run_dir, "traces", "*.json")):
t = json.load(open(p))
if t.get("session_file") and os.path.exists(t["session_file"]):
traces[t["qid"]] = t
all_names = collections.Counter()
rows = []
for qid, t in sorted(traces.items()):
calls, counter = scan_session(t["session_file"], kb_re)
all_names.update(counter)
kb = [c for c in calls if c["is_kb"] and not POLL_RE.search(c["name"])]
total = len(calls)
# 检索耗时:未配对到 output 的调用用 runner 事件流里的事件数兜底(一般为 0)
kb_times = [c["dur_s"] for c in kb if c["dur_s"] is not None]
kb_time = sum(kb_times)
first_kb = min((c["t_start"] for c in kb if c["t_start"]), default=None)
t_start = min((c["t_start"] for c in calls if c["t_start"]), default=None)
ttf = round(first_kb - t_start, 1) if first_kb and t_start else None
lat = t.get("latency_s") or 0
rows.append({
"qid": qid,
"tool_calls_total": total,
"kb_calls": len(kb),
"kb_time_s": round(kb_time, 1),
"avg_kb_call_s": round(kb_time / len(kb_times), 2) if kb_times else None,
"max_kb_call_s": round(max(kb_times), 1) if kb_times else None,
"time_to_first_kb_s": ttf,
"latency_s": lat,
"kb_time_ratio": round(kb_time / lat, 2) if lat else None,
"calls_per_min": round(len(kb) / lat * 60, 2) if lat else None,
"kb_tools": sorted({c["name"] for c in kb}),
"conversation_id": t.get("conversation_id"),
})
if args.list_tools:
print("出现过的工具/函数名(用于校准 --kb-pattern):")
for name, n in all_names.most_common():
print(f" {n:>4}x {name}")
return
with open(os.path.join(run_dir, "trace_report.json"), "w") as f:
json.dump(rows, f, ensure_ascii=False, indent=1)
kb_list = [r["kb_calls"] for r in rows]
lat = [r["latency_s"] for r in rows if r.get("latency_s")]
avg_call = [r["avg_kb_call_s"] for r in rows if r.get("avg_kb_call_s") is not None]
ratio = [r["kb_time_ratio"] for r in rows if r.get("kb_time_ratio") is not None]
print(f"== 轨迹统计({args.run_id},{len(rows)} 题)==")
print(f"工具总调用: {sum(r['tool_calls_total'] for r in rows)},RDK 知识库调用: {sum(kb_list)}")
if kb_list:
z = sum(1 for k in kb_list if k == 0)
print(f"每题知识库调用: 平均 {sum(kb_list)/len(kb_list):.2f} 次,最多 {max(kb_list)},0 次题数 {z}")
if avg_call:
print(f"检索速率: 单次检索平均 {sum(avg_call)/len(avg_call):.2f}s(最慢 {max(r['max_kb_call_s'] for r in rows if r['max_kb_call_s'])}s)")
if ratio:
print(f"检索耗时占比: 平均 {sum(ratio)/len(ratio)*100:.0f}%")
if lat:
print(f"每题总耗时: 平均 {sum(lat)/len(lat):.1f}s,p50 {sorted(lat)[len(lat)//2]}s")
print("报告: " + os.path.join(run_dir, "trace_report.json"))
if __name__ == "__main__":
main()