From 06a3c273750bcbd71ff60b771bf1e17a45901fb9 Mon Sep 17 00:00:00 2001 From: "dingbin.wang" Date: Fri, 6 Nov 2026 17:39:13 +0800 Subject: [PATCH] compare_runner: jev r41 vs original chain (70 rdk_eval questions, chunk/page metrics + RRF fusion) --- compare_runner.py | 187 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 compare_runner.py diff --git a/compare_runner.py b/compare_runner.py new file mode 100644 index 0000000..93f4d13 --- /dev/null +++ b/compare_runner.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""compare_runner.py — jev r41 vs 原始检索链路 同题对比 (只读) +Arm A: search_knowledge(question, top_k=10) → chunk 排名 +Arm B: 同批 chunks 由 jev r41 重排 +gold 判定: reference_answer 实体词面匹配 (两臂同尺; LLM 判官可后插) +输出: compare_detail.jsonl + compare_report.md +用法: python3 compare_runner.py [--limit N] [--full] +""" +import json, os, re, sys, time, math, bisect, subprocess + +MCP_URL = "https://knowledge-dev.d-robotics.cc/mcp" +MCP_TOKEN = "58fdb53f68bfb5206260ff671e92bbbc63e20264afb963030a6770a2db1897d8" +DATASET = "/Users/d-robotics/agent/知识检索/rdk_eval/rdk_eval_dataset.json" +OUT_DIR = "/tmp/jev_compare" +TOP_K = 10 + +def log(m): + print(f"[cmp] {m}", flush=True) + +# ---------- 数据集 ---------- +def load_questions(): + d = json.load(open(DATASET, encoding="utf-8")) + qs = [] + for m in d["models"]: + for q in m.get("text_questions", []): + qs.append(q) + return qs + +# ---------- Arm A: 原始链路 ---------- +def mcp_search(query, top_k=TOP_K): + payload = {"jsonrpc": "2.0", "id": 1, "method": "tools/call", + "params": {"name": "search_knowledge", + "arguments": {"query": query, "top_k": top_k}}} + t0 = time.time() + p = subprocess.run(["curl", "-s", "-m", "40", "-X", "POST", MCP_URL, + "-H", f"Authorization: Bearer {MCP_TOKEN}", + "-H", "Content-Type: application/json", + "-H", "Accept: application/json, text/event-stream", + "-d", json.dumps(payload)], capture_output=True, text=True) + dt = time.time() - t0 + try: + r = json.loads(p.stdout) + content = r["result"]["content"] + text = "".join(c.get("text", "") for c in content if c.get("type") == "text") + data = json.loads(text) + return data.get("results", []), dt + except Exception as e: + log(f" search parse fail: {str(e)[:60]} raw={p.stdout[:120]}") + return [], dt + +# ---------- gold 判官: 答案实体词面匹配 ---------- +STOP_ZH = set("路个是否支持不采用什么多少几路的的了和与及或为是") + +def answer_tokens(ans): + """返回 (plain_tokens, norm_tokens): norm 类需在去空白文本中匹配(数字+量词)""" + plain, norm = set(), set() + ans = str(ans) + for m in re.finditer(r"\d+(?:\.\d+)?[A-Za-z%°℃Ω]*", ans): + t = m.group(0) + if len(t) >= 2 or (t.isdigit() and len(t) == 1): + plain.add(t.lower()) + # 数字+中文量词组合 ("1 路" -> "1路"): 在规范化(去空白)文本中匹配 + for m in re.finditer(r"\d+(?:\.\d+)?\s*[个路位针条次项]", ans): + norm.add(re.sub(r"\s+", "", m.group(0)).lower()) + for m in re.finditer(r"[A-Za-z][A-Za-z0-9\-]{1,}", ans): + t = m.group(0) + if len(t) >= 2: + plain.add(t.lower()) + zh = re.sub(r"[,。!?;:、\sA-Za-z0-9\.\-/()()]+", "|", ans) + for seg in zh.split("|"): + seg2 = "".join(ch for ch in seg if ch not in STOP_ZH) + if len(seg2) >= 3: + plain.add(seg2[:4].lower()) + return ({t for t in plain if t}, norm) + +def chunk_hit(chunk_text, toks): + plain, norm = toks + t = str(chunk_text).lower() + if any(tok in t for tok in plain): + return True + if norm: + tn = re.sub(r"\s+", "", t) + if any(tok in tn for tok in norm): + return True + return False + +# ---------- Arm B: jev r41 本地打分 ---------- +_jev = None +def jev_score(query, text): + global _jev + if _jev is None: + sys.path.insert(0, "/tmp/laya_src") + os.environ["MODEL_DIR"] = "/tmp/serve_test" + import serve as _serve + _serve.load_model() + _jev = _serve + return _jev.score_one(query, text) + +# ---------- 指标 ---------- +def rank_metrics(ranks): + """ranks: 每题第一个 hit 的 1-based 位置 (None=未命中)""" + n = len(ranks) + def at_k(k): + return sum(1 for r in ranks if r is not None and r <= k) / n + mrr = sum(1.0 / r for r in ranks if r is not None) / n + return {"answer@1": round(at_k(1), 4), "answer@3": round(at_k(3), 4), + "answer@5": round(at_k(5), 4), "answer@10": round(at_k(10), 4), + "mrr": round(mrr, 4), "n": n} + +def main(): + limit = None + if "--limit" in sys.argv: + limit = int(sys.argv[sys.argv.index("--limit") + 1]) + os.makedirs(OUT_DIR, exist_ok=True) + qs = load_questions() + if limit: + qs = qs[:limit] + log(f"questions={len(qs)}") + detail_path = os.path.join(OUT_DIR, "compare_detail.jsonl") + fout = open(detail_path, "w", encoding="utf-8") + + for i, q in enumerate(qs): + qid = q["id"] + toks = answer_tokens(q["reference_answer"]) + tp, tn = toks + results, lat = mcp_search(q["question"]) + chunks = [] + for r in results: + c = r.get("content") or "" + chunks.append({"filename": r.get("filename"), "section": r.get("section"), + "content": c, "hit": chunk_hit(c, toks)}) + # Arm A 排名: 第一个 hit chunk + a_rank = next((k + 1 for k, c in enumerate(chunks) if c["hit"]), None) + # Arm B: jev 打分重排 + for c in chunks: + s = jev_score(q["question"], c["content"]) + c["jev_pfinal"] = s["p_final"] + c["jev_pmean"] = s["p_mean"] + order = sorted(range(len(chunks)), key=lambda k: -chunks[k]["jev_pfinal"]) + b_rank = next((pos + 1 for pos, k in enumerate(order) if chunks[k]["hit"]), None) + # 页面级 (filename 去重取最高分/最先命中) + def page_ranks(chs, order_hint=None): + seen, ranks = set(), [] + seq = order_hint if order_hint else list(range(len(chs))) + for k in seq: + f = chs[k]["filename"] + if f in seen: + continue + seen.add(f) + if chs[k]["hit"]: + return len(seen) + return None + a_page = page_ranks(chunks) + b_page = page_ranks(chunks, order) + fout.write(json.dumps({ + "qid": qid, "question": q["question"], "answer": q["reference_answer"], + "tokens": {"plain": sorted(tp), "norm": sorted(tn)}, "a_rank": a_rank, "b_rank": b_rank, + "a_page": a_page, "b_page": b_page, "search_latency_s": round(lat, 2), + "chunks": [{"f": c["filename"], "sec": (c.get("section") or "")[:40], + "hit": c["hit"], "pfinal": c["jev_pfinal"]} for c in chunks]}, + ensure_ascii=False) + "\n") + fout.flush() + log(f"{i+1}/{len(qs)} {qid} A={a_rank} B={b_rank} chunks={len(chunks)} lat={lat:.1f}s") + fout.close() + + # 汇总 + rows = [json.loads(l) for l in open(detail_path, encoding="utf-8")] + a_ranks = [r["a_rank"] for r in rows] + b_ranks = [r["b_rank"] for r in rows] + a_pages = [r["a_page"] for r in rows] + b_pages = [r["b_page"] for r in rows] + report = { + "chunk_level": {"original_chain": rank_metrics(a_ranks), + "jev_rerank": rank_metrics(b_ranks)}, + "page_level": {"original_chain": rank_metrics(a_pages), + "jev_rerank": rank_metrics(b_pages)}, + } + wins = sum(1 for r in rows if (r["b_rank"] or 99) < (r["a_rank"] or 99)) + ties = sum(1 for r in rows if r["b_rank"] == r["a_rank"]) + losses = sum(1 for r in rows if (r["b_rank"] or 99) > (r["a_rank"] or 99)) + report["per_question"] = {"jev_better": wins, "tie": ties, "jev_worse": losses} + json.dump(report, open(os.path.join(OUT_DIR, "compare_report.json"), "w"), + ensure_ascii=False, indent=1) + log("REPORT " + json.dumps(report["chunk_level"]) + " " + json.dumps(report["per_question"])) + +if __name__ == "__main__": + main()