compare_runner: jev r41 vs original chain (70 rdk_eval questions, chunk/page metrics + RRF fusion)

This commit is contained in:
dingbin.wang 2026-11-06 17:39:13 +08:00
parent f3cabcce28
commit 06a3c27375

187
compare_runner.py Normal file
View File

@ -0,0 +1,187 @@
#!/usr/bin/env python3
"""compare_runner.py — jev r41 vs 原始检索链路 同题对比 (只读)
Arm A: search_knowledge(question, top_k=10) → chunk 排名
Arm B: 同批 chunks 由 jev r41 重排
gold 判定: reference_answer 实体词面匹配 (两臂同尺; LLM 判官可后插)
输出: compare_detail.jsonl + compare_report.md
用法: python3 compare_runner.py [--limit N] [--full]
"""
import json, os, re, sys, time, math, bisect, subprocess
MCP_URL = "https://knowledge-dev.d-robotics.cc/mcp"
MCP_TOKEN = "58fdb53f68bfb5206260ff671e92bbbc63e20264afb963030a6770a2db1897d8"
DATASET = "/Users/d-robotics/agent/知识检索/rdk_eval/rdk_eval_dataset.json"
OUT_DIR = "/tmp/jev_compare"
TOP_K = 10
def log(m):
print(f"[cmp] {m}", flush=True)
# ---------- 数据集 ----------
def load_questions():
d = json.load(open(DATASET, encoding="utf-8"))
qs = []
for m in d["models"]:
for q in m.get("text_questions", []):
qs.append(q)
return qs
# ---------- Arm A: 原始链路 ----------
def mcp_search(query, top_k=TOP_K):
payload = {"jsonrpc": "2.0", "id": 1, "method": "tools/call",
"params": {"name": "search_knowledge",
"arguments": {"query": query, "top_k": top_k}}}
t0 = time.time()
p = subprocess.run(["curl", "-s", "-m", "40", "-X", "POST", MCP_URL,
"-H", f"Authorization: Bearer {MCP_TOKEN}",
"-H", "Content-Type: application/json",
"-H", "Accept: application/json, text/event-stream",
"-d", json.dumps(payload)], capture_output=True, text=True)
dt = time.time() - t0
try:
r = json.loads(p.stdout)
content = r["result"]["content"]
text = "".join(c.get("text", "") for c in content if c.get("type") == "text")
data = json.loads(text)
return data.get("results", []), dt
except Exception as e:
log(f" search parse fail: {str(e)[:60]} raw={p.stdout[:120]}")
return [], dt
# ---------- gold 判官: 答案实体词面匹配 ----------
STOP_ZH = set("路个是否支持不采用什么多少几路的的了和与及或为是")
def answer_tokens(ans):
"""返回 (plain_tokens, norm_tokens): norm 类需在去空白文本中匹配(数字+量词)"""
plain, norm = set(), set()
ans = str(ans)
for m in re.finditer(r"\d+(?:\.\d+)?[A-Za-z%°℃Ω]*", ans):
t = m.group(0)
if len(t) >= 2 or (t.isdigit() and len(t) == 1):
plain.add(t.lower())
# 数字+中文量词组合 ("1 路" -> "1路"): 在规范化(去空白)文本中匹配
for m in re.finditer(r"\d+(?:\.\d+)?\s*[个路位针条次项]", ans):
norm.add(re.sub(r"\s+", "", m.group(0)).lower())
for m in re.finditer(r"[A-Za-z][A-Za-z0-9\-]{1,}", ans):
t = m.group(0)
if len(t) >= 2:
plain.add(t.lower())
zh = re.sub(r"[,。!?;:、\sA-Za-z0-9\.\-/()()]+", "|", ans)
for seg in zh.split("|"):
seg2 = "".join(ch for ch in seg if ch not in STOP_ZH)
if len(seg2) >= 3:
plain.add(seg2[:4].lower())
return ({t for t in plain if t}, norm)
def chunk_hit(chunk_text, toks):
plain, norm = toks
t = str(chunk_text).lower()
if any(tok in t for tok in plain):
return True
if norm:
tn = re.sub(r"\s+", "", t)
if any(tok in tn for tok in norm):
return True
return False
# ---------- Arm B: jev r41 本地打分 ----------
_jev = None
def jev_score(query, text):
global _jev
if _jev is None:
sys.path.insert(0, "/tmp/laya_src")
os.environ["MODEL_DIR"] = "/tmp/serve_test"
import serve as _serve
_serve.load_model()
_jev = _serve
return _jev.score_one(query, text)
# ---------- 指标 ----------
def rank_metrics(ranks):
"""ranks: 每题第一个 hit 的 1-based 位置 (None=未命中)"""
n = len(ranks)
def at_k(k):
return sum(1 for r in ranks if r is not None and r <= k) / n
mrr = sum(1.0 / r for r in ranks if r is not None) / n
return {"answer@1": round(at_k(1), 4), "answer@3": round(at_k(3), 4),
"answer@5": round(at_k(5), 4), "answer@10": round(at_k(10), 4),
"mrr": round(mrr, 4), "n": n}
def main():
limit = None
if "--limit" in sys.argv:
limit = int(sys.argv[sys.argv.index("--limit") + 1])
os.makedirs(OUT_DIR, exist_ok=True)
qs = load_questions()
if limit:
qs = qs[:limit]
log(f"questions={len(qs)}")
detail_path = os.path.join(OUT_DIR, "compare_detail.jsonl")
fout = open(detail_path, "w", encoding="utf-8")
for i, q in enumerate(qs):
qid = q["id"]
toks = answer_tokens(q["reference_answer"])
tp, tn = toks
results, lat = mcp_search(q["question"])
chunks = []
for r in results:
c = r.get("content") or ""
chunks.append({"filename": r.get("filename"), "section": r.get("section"),
"content": c, "hit": chunk_hit(c, toks)})
# Arm A 排名: 第一个 hit chunk
a_rank = next((k + 1 for k, c in enumerate(chunks) if c["hit"]), None)
# Arm B: jev 打分重排
for c in chunks:
s = jev_score(q["question"], c["content"])
c["jev_pfinal"] = s["p_final"]
c["jev_pmean"] = s["p_mean"]
order = sorted(range(len(chunks)), key=lambda k: -chunks[k]["jev_pfinal"])
b_rank = next((pos + 1 for pos, k in enumerate(order) if chunks[k]["hit"]), None)
# 页面级 (filename 去重取最高分/最先命中)
def page_ranks(chs, order_hint=None):
seen, ranks = set(), []
seq = order_hint if order_hint else list(range(len(chs)))
for k in seq:
f = chs[k]["filename"]
if f in seen:
continue
seen.add(f)
if chs[k]["hit"]:
return len(seen)
return None
a_page = page_ranks(chunks)
b_page = page_ranks(chunks, order)
fout.write(json.dumps({
"qid": qid, "question": q["question"], "answer": q["reference_answer"],
"tokens": {"plain": sorted(tp), "norm": sorted(tn)}, "a_rank": a_rank, "b_rank": b_rank,
"a_page": a_page, "b_page": b_page, "search_latency_s": round(lat, 2),
"chunks": [{"f": c["filename"], "sec": (c.get("section") or "")[:40],
"hit": c["hit"], "pfinal": c["jev_pfinal"]} for c in chunks]},
ensure_ascii=False) + "\n")
fout.flush()
log(f"{i+1}/{len(qs)} {qid} A={a_rank} B={b_rank} chunks={len(chunks)} lat={lat:.1f}s")
fout.close()
# 汇总
rows = [json.loads(l) for l in open(detail_path, encoding="utf-8")]
a_ranks = [r["a_rank"] for r in rows]
b_ranks = [r["b_rank"] for r in rows]
a_pages = [r["a_page"] for r in rows]
b_pages = [r["b_page"] for r in rows]
report = {
"chunk_level": {"original_chain": rank_metrics(a_ranks),
"jev_rerank": rank_metrics(b_ranks)},
"page_level": {"original_chain": rank_metrics(a_pages),
"jev_rerank": rank_metrics(b_pages)},
}
wins = sum(1 for r in rows if (r["b_rank"] or 99) < (r["a_rank"] or 99))
ties = sum(1 for r in rows if r["b_rank"] == r["a_rank"])
losses = sum(1 for r in rows if (r["b_rank"] or 99) > (r["a_rank"] or 99))
report["per_question"] = {"jev_better": wins, "tie": ties, "jev_worse": losses}
json.dump(report, open(os.path.join(OUT_DIR, "compare_report.json"), "w"),
ensure_ascii=False, indent=1)
log("REPORT " + json.dumps(report["chunk_level"]) + " " + json.dumps(report["per_question"]))
if __name__ == "__main__":
main()