188 lines
7.7 KiB
Python
188 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
||
"""compare_runner.py — jev r41 vs 原始检索链路 同题对比 (只读)
|
||
Arm A: search_knowledge(question, top_k=10) → chunk 排名
|
||
Arm B: 同批 chunks 由 jev r41 重排
|
||
gold 判定: reference_answer 实体词面匹配 (两臂同尺; LLM 判官可后插)
|
||
输出: compare_detail.jsonl + compare_report.md
|
||
用法: python3 compare_runner.py [--limit N] [--full]
|
||
"""
|
||
import json, os, re, sys, time, math, bisect, subprocess
|
||
|
||
MCP_URL = "https://knowledge-dev.d-robotics.cc/mcp"
|
||
MCP_TOKEN = "KB_MCP_SERVICE_KEY_见飞书交接文档08"
|
||
DATASET = "/Users/d-robotics/agent/知识检索/rdk_eval/rdk_eval_dataset.json"
|
||
OUT_DIR = "/tmp/jev_compare"
|
||
TOP_K = 10
|
||
|
||
def log(m):
|
||
print(f"[cmp] {m}", flush=True)
|
||
|
||
# ---------- 数据集 ----------
|
||
def load_questions():
|
||
d = json.load(open(DATASET, encoding="utf-8"))
|
||
qs = []
|
||
for m in d["models"]:
|
||
for q in m.get("text_questions", []):
|
||
qs.append(q)
|
||
return qs
|
||
|
||
# ---------- Arm A: 原始链路 ----------
|
||
def mcp_search(query, top_k=TOP_K):
|
||
payload = {"jsonrpc": "2.0", "id": 1, "method": "tools/call",
|
||
"params": {"name": "search_knowledge",
|
||
"arguments": {"query": query, "top_k": top_k}}}
|
||
t0 = time.time()
|
||
p = subprocess.run(["curl", "-s", "-m", "40", "-X", "POST", MCP_URL,
|
||
"-H", f"Authorization: Bearer {MCP_TOKEN}",
|
||
"-H", "Content-Type: application/json",
|
||
"-H", "Accept: application/json, text/event-stream",
|
||
"-d", json.dumps(payload)], capture_output=True, text=True)
|
||
dt = time.time() - t0
|
||
try:
|
||
r = json.loads(p.stdout)
|
||
content = r["result"]["content"]
|
||
text = "".join(c.get("text", "") for c in content if c.get("type") == "text")
|
||
data = json.loads(text)
|
||
return data.get("results", []), dt
|
||
except Exception as e:
|
||
log(f" search parse fail: {str(e)[:60]} raw={p.stdout[:120]}")
|
||
return [], dt
|
||
|
||
# ---------- gold 判官: 答案实体词面匹配 ----------
|
||
STOP_ZH = set("路个是否支持不采用什么多少几路的的了和与及或为是")
|
||
|
||
def answer_tokens(ans):
|
||
"""返回 (plain_tokens, norm_tokens): norm 类需在去空白文本中匹配(数字+量词)"""
|
||
plain, norm = set(), set()
|
||
ans = str(ans)
|
||
for m in re.finditer(r"\d+(?:\.\d+)?[A-Za-z%°℃Ω]*", ans):
|
||
t = m.group(0)
|
||
if len(t) >= 2 or (t.isdigit() and len(t) == 1):
|
||
plain.add(t.lower())
|
||
# 数字+中文量词组合 ("1 路" -> "1路"): 在规范化(去空白)文本中匹配
|
||
for m in re.finditer(r"\d+(?:\.\d+)?\s*[个路位针条次项]", ans):
|
||
norm.add(re.sub(r"\s+", "", m.group(0)).lower())
|
||
for m in re.finditer(r"[A-Za-z][A-Za-z0-9\-]{1,}", ans):
|
||
t = m.group(0)
|
||
if len(t) >= 2:
|
||
plain.add(t.lower())
|
||
zh = re.sub(r"[,。!?;:、\sA-Za-z0-9\.\-/()()]+", "|", ans)
|
||
for seg in zh.split("|"):
|
||
seg2 = "".join(ch for ch in seg if ch not in STOP_ZH)
|
||
if len(seg2) >= 3:
|
||
plain.add(seg2[:4].lower())
|
||
return ({t for t in plain if t}, norm)
|
||
|
||
def chunk_hit(chunk_text, toks):
|
||
plain, norm = toks
|
||
t = str(chunk_text).lower()
|
||
if any(tok in t for tok in plain):
|
||
return True
|
||
if norm:
|
||
tn = re.sub(r"\s+", "", t)
|
||
if any(tok in tn for tok in norm):
|
||
return True
|
||
return False
|
||
|
||
# ---------- Arm B: jev r41 本地打分 ----------
|
||
_jev = None
|
||
def jev_score(query, text):
|
||
global _jev
|
||
if _jev is None:
|
||
sys.path.insert(0, "/tmp/laya_src")
|
||
os.environ["MODEL_DIR"] = "/tmp/serve_test"
|
||
import serve as _serve
|
||
_serve.load_model()
|
||
_jev = _serve
|
||
return _jev.score_one(query, text)
|
||
|
||
# ---------- 指标 ----------
|
||
def rank_metrics(ranks):
|
||
"""ranks: 每题第一个 hit 的 1-based 位置 (None=未命中)"""
|
||
n = len(ranks)
|
||
def at_k(k):
|
||
return sum(1 for r in ranks if r is not None and r <= k) / n
|
||
mrr = sum(1.0 / r for r in ranks if r is not None) / n
|
||
return {"answer@1": round(at_k(1), 4), "answer@3": round(at_k(3), 4),
|
||
"answer@5": round(at_k(5), 4), "answer@10": round(at_k(10), 4),
|
||
"mrr": round(mrr, 4), "n": n}
|
||
|
||
def main():
|
||
limit = None
|
||
if "--limit" in sys.argv:
|
||
limit = int(sys.argv[sys.argv.index("--limit") + 1])
|
||
os.makedirs(OUT_DIR, exist_ok=True)
|
||
qs = load_questions()
|
||
if limit:
|
||
qs = qs[:limit]
|
||
log(f"questions={len(qs)}")
|
||
detail_path = os.path.join(OUT_DIR, "compare_detail.jsonl")
|
||
fout = open(detail_path, "w", encoding="utf-8")
|
||
|
||
for i, q in enumerate(qs):
|
||
qid = q["id"]
|
||
toks = answer_tokens(q["reference_answer"])
|
||
tp, tn = toks
|
||
results, lat = mcp_search(q["question"])
|
||
chunks = []
|
||
for r in results:
|
||
c = r.get("content") or ""
|
||
chunks.append({"filename": r.get("filename"), "section": r.get("section"),
|
||
"content": c, "hit": chunk_hit(c, toks)})
|
||
# Arm A 排名: 第一个 hit chunk
|
||
a_rank = next((k + 1 for k, c in enumerate(chunks) if c["hit"]), None)
|
||
# Arm B: jev 打分重排
|
||
for c in chunks:
|
||
s = jev_score(q["question"], c["content"])
|
||
c["jev_pfinal"] = s["p_final"]
|
||
c["jev_pmean"] = s["p_mean"]
|
||
order = sorted(range(len(chunks)), key=lambda k: -chunks[k]["jev_pfinal"])
|
||
b_rank = next((pos + 1 for pos, k in enumerate(order) if chunks[k]["hit"]), None)
|
||
# 页面级 (filename 去重取最高分/最先命中)
|
||
def page_ranks(chs, order_hint=None):
|
||
seen, ranks = set(), []
|
||
seq = order_hint if order_hint else list(range(len(chs)))
|
||
for k in seq:
|
||
f = chs[k]["filename"]
|
||
if f in seen:
|
||
continue
|
||
seen.add(f)
|
||
if chs[k]["hit"]:
|
||
return len(seen)
|
||
return None
|
||
a_page = page_ranks(chunks)
|
||
b_page = page_ranks(chunks, order)
|
||
fout.write(json.dumps({
|
||
"qid": qid, "question": q["question"], "answer": q["reference_answer"],
|
||
"tokens": {"plain": sorted(tp), "norm": sorted(tn)}, "a_rank": a_rank, "b_rank": b_rank,
|
||
"a_page": a_page, "b_page": b_page, "search_latency_s": round(lat, 2),
|
||
"chunks": [{"f": c["filename"], "sec": (c.get("section") or "")[:40],
|
||
"hit": c["hit"], "pfinal": c["jev_pfinal"]} for c in chunks]},
|
||
ensure_ascii=False) + "\n")
|
||
fout.flush()
|
||
log(f"{i+1}/{len(qs)} {qid} A={a_rank} B={b_rank} chunks={len(chunks)} lat={lat:.1f}s")
|
||
fout.close()
|
||
|
||
# 汇总
|
||
rows = [json.loads(l) for l in open(detail_path, encoding="utf-8")]
|
||
a_ranks = [r["a_rank"] for r in rows]
|
||
b_ranks = [r["b_rank"] for r in rows]
|
||
a_pages = [r["a_page"] for r in rows]
|
||
b_pages = [r["b_page"] for r in rows]
|
||
report = {
|
||
"chunk_level": {"original_chain": rank_metrics(a_ranks),
|
||
"jev_rerank": rank_metrics(b_ranks)},
|
||
"page_level": {"original_chain": rank_metrics(a_pages),
|
||
"jev_rerank": rank_metrics(b_pages)},
|
||
}
|
||
wins = sum(1 for r in rows if (r["b_rank"] or 99) < (r["a_rank"] or 99))
|
||
ties = sum(1 for r in rows if r["b_rank"] == r["a_rank"])
|
||
losses = sum(1 for r in rows if (r["b_rank"] or 99) > (r["a_rank"] or 99))
|
||
report["per_question"] = {"jev_better": wins, "tie": ties, "jev_worse": losses}
|
||
json.dump(report, open(os.path.join(OUT_DIR, "compare_report.json"), "w"),
|
||
ensure_ascii=False, indent=1)
|
||
log("REPORT " + json.dumps(report["chunk_level"]) + " " + json.dumps(report["per_question"]))
|
||
|
||
if __name__ == "__main__":
|
||
main()
|