#!/usr/bin/env python3 """compare_runner.py — jev r41 vs 原始检索链路 同题对比 (只读) Arm A: search_knowledge(question, top_k=10) → chunk 排名 Arm B: 同批 chunks 由 jev r41 重排 gold 判定: reference_answer 实体词面匹配 (两臂同尺; LLM 判官可后插) 输出: compare_detail.jsonl + compare_report.md 用法: python3 compare_runner.py [--limit N] [--full] """ import json, os, re, sys, time, math, bisect, subprocess MCP_URL = "https://knowledge-dev.d-robotics.cc/mcp" MCP_TOKEN = "KB_MCP_SERVICE_KEY_见飞书交接文档08" DATASET = "/Users/d-robotics/agent/知识检索/rdk_eval/rdk_eval_dataset.json" OUT_DIR = "/tmp/jev_compare" TOP_K = 10 def log(m): print(f"[cmp] {m}", flush=True) # ---------- 数据集 ---------- def load_questions(): d = json.load(open(DATASET, encoding="utf-8")) qs = [] for m in d["models"]: for q in m.get("text_questions", []): qs.append(q) return qs # ---------- Arm A: 原始链路 ---------- def mcp_search(query, top_k=TOP_K): payload = {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "search_knowledge", "arguments": {"query": query, "top_k": top_k}}} t0 = time.time() p = subprocess.run(["curl", "-s", "-m", "40", "-X", "POST", MCP_URL, "-H", f"Authorization: Bearer {MCP_TOKEN}", "-H", "Content-Type: application/json", "-H", "Accept: application/json, text/event-stream", "-d", json.dumps(payload)], capture_output=True, text=True) dt = time.time() - t0 try: r = json.loads(p.stdout) content = r["result"]["content"] text = "".join(c.get("text", "") for c in content if c.get("type") == "text") data = json.loads(text) return data.get("results", []), dt except Exception as e: log(f" search parse fail: {str(e)[:60]} raw={p.stdout[:120]}") return [], dt # ---------- gold 判官: 答案实体词面匹配 ---------- STOP_ZH = set("路个是否支持不采用什么多少几路的的了和与及或为是") def answer_tokens(ans): """返回 (plain_tokens, norm_tokens): norm 类需在去空白文本中匹配(数字+量词)""" plain, norm = set(), set() ans = str(ans) for m in re.finditer(r"\d+(?:\.\d+)?[A-Za-z%°℃Ω]*", ans): t = m.group(0) if len(t) >= 2 or (t.isdigit() and len(t) == 1): plain.add(t.lower()) # 数字+中文量词组合 ("1 路" -> "1路"): 在规范化(去空白)文本中匹配 for m in re.finditer(r"\d+(?:\.\d+)?\s*[个路位针条次项]", ans): norm.add(re.sub(r"\s+", "", m.group(0)).lower()) for m in re.finditer(r"[A-Za-z][A-Za-z0-9\-]{1,}", ans): t = m.group(0) if len(t) >= 2: plain.add(t.lower()) zh = re.sub(r"[,。!?;:、\sA-Za-z0-9\.\-/()()]+", "|", ans) for seg in zh.split("|"): seg2 = "".join(ch for ch in seg if ch not in STOP_ZH) if len(seg2) >= 3: plain.add(seg2[:4].lower()) return ({t for t in plain if t}, norm) def chunk_hit(chunk_text, toks): plain, norm = toks t = str(chunk_text).lower() if any(tok in t for tok in plain): return True if norm: tn = re.sub(r"\s+", "", t) if any(tok in tn for tok in norm): return True return False # ---------- Arm B: jev r41 本地打分 ---------- _jev = None def jev_score(query, text): global _jev if _jev is None: sys.path.insert(0, "/tmp/laya_src") os.environ["MODEL_DIR"] = "/tmp/serve_test" import serve as _serve _serve.load_model() _jev = _serve return _jev.score_one(query, text) # ---------- 指标 ---------- def rank_metrics(ranks): """ranks: 每题第一个 hit 的 1-based 位置 (None=未命中)""" n = len(ranks) def at_k(k): return sum(1 for r in ranks if r is not None and r <= k) / n mrr = sum(1.0 / r for r in ranks if r is not None) / n return {"answer@1": round(at_k(1), 4), "answer@3": round(at_k(3), 4), "answer@5": round(at_k(5), 4), "answer@10": round(at_k(10), 4), "mrr": round(mrr, 4), "n": n} def main(): limit = None if "--limit" in sys.argv: limit = int(sys.argv[sys.argv.index("--limit") + 1]) os.makedirs(OUT_DIR, exist_ok=True) qs = load_questions() if limit: qs = qs[:limit] log(f"questions={len(qs)}") detail_path = os.path.join(OUT_DIR, "compare_detail.jsonl") fout = open(detail_path, "w", encoding="utf-8") for i, q in enumerate(qs): qid = q["id"] toks = answer_tokens(q["reference_answer"]) tp, tn = toks results, lat = mcp_search(q["question"]) chunks = [] for r in results: c = r.get("content") or "" chunks.append({"filename": r.get("filename"), "section": r.get("section"), "content": c, "hit": chunk_hit(c, toks)}) # Arm A 排名: 第一个 hit chunk a_rank = next((k + 1 for k, c in enumerate(chunks) if c["hit"]), None) # Arm B: jev 打分重排 for c in chunks: s = jev_score(q["question"], c["content"]) c["jev_pfinal"] = s["p_final"] c["jev_pmean"] = s["p_mean"] order = sorted(range(len(chunks)), key=lambda k: -chunks[k]["jev_pfinal"]) b_rank = next((pos + 1 for pos, k in enumerate(order) if chunks[k]["hit"]), None) # 页面级 (filename 去重取最高分/最先命中) def page_ranks(chs, order_hint=None): seen, ranks = set(), [] seq = order_hint if order_hint else list(range(len(chs))) for k in seq: f = chs[k]["filename"] if f in seen: continue seen.add(f) if chs[k]["hit"]: return len(seen) return None a_page = page_ranks(chunks) b_page = page_ranks(chunks, order) fout.write(json.dumps({ "qid": qid, "question": q["question"], "answer": q["reference_answer"], "tokens": {"plain": sorted(tp), "norm": sorted(tn)}, "a_rank": a_rank, "b_rank": b_rank, "a_page": a_page, "b_page": b_page, "search_latency_s": round(lat, 2), "chunks": [{"f": c["filename"], "sec": (c.get("section") or "")[:40], "hit": c["hit"], "pfinal": c["jev_pfinal"]} for c in chunks]}, ensure_ascii=False) + "\n") fout.flush() log(f"{i+1}/{len(qs)} {qid} A={a_rank} B={b_rank} chunks={len(chunks)} lat={lat:.1f}s") fout.close() # 汇总 rows = [json.loads(l) for l in open(detail_path, encoding="utf-8")] a_ranks = [r["a_rank"] for r in rows] b_ranks = [r["b_rank"] for r in rows] a_pages = [r["a_page"] for r in rows] b_pages = [r["b_page"] for r in rows] report = { "chunk_level": {"original_chain": rank_metrics(a_ranks), "jev_rerank": rank_metrics(b_ranks)}, "page_level": {"original_chain": rank_metrics(a_pages), "jev_rerank": rank_metrics(b_pages)}, } wins = sum(1 for r in rows if (r["b_rank"] or 99) < (r["a_rank"] or 99)) ties = sum(1 for r in rows if r["b_rank"] == r["a_rank"]) losses = sum(1 for r in rows if (r["b_rank"] or 99) > (r["a_rank"] or 99)) report["per_question"] = {"jev_better": wins, "tie": ties, "jev_worse": losses} json.dump(report, open(os.path.join(OUT_DIR, "compare_report.json"), "w"), ensure_ascii=False, indent=1) log("REPORT " + json.dumps(report["chunk_level"]) + " " + json.dumps(report["per_question"])) if __name__ == "__main__": main()