188 lines
7.7 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""compare_runner.py — jev r41 vs 原始检索链路 同题对比 (只读)
Arm A: search_knowledge(question, top_k=10) → chunk 排名
Arm B: 同批 chunks 由 jev r41 重排
gold 判定: reference_answer 实体词面匹配 (两臂同尺; LLM 判官可后插)
输出: compare_detail.jsonl + compare_report.md
用法: python3 compare_runner.py [--limit N] [--full]
"""
import json, os, re, sys, time, math, bisect, subprocess
MCP_URL = "https://knowledge-dev.d-robotics.cc/mcp"
MCP_TOKEN = "KB_MCP_SERVICE_KEY_见飞书交接文档08"
DATASET = "/Users/d-robotics/agent/知识检索/rdk_eval/rdk_eval_dataset.json"
OUT_DIR = "/tmp/jev_compare"
TOP_K = 10
def log(m):
print(f"[cmp] {m}", flush=True)
# ---------- 数据集 ----------
def load_questions():
d = json.load(open(DATASET, encoding="utf-8"))
qs = []
for m in d["models"]:
for q in m.get("text_questions", []):
qs.append(q)
return qs
# ---------- Arm A: 原始链路 ----------
def mcp_search(query, top_k=TOP_K):
payload = {"jsonrpc": "2.0", "id": 1, "method": "tools/call",
"params": {"name": "search_knowledge",
"arguments": {"query": query, "top_k": top_k}}}
t0 = time.time()
p = subprocess.run(["curl", "-s", "-m", "40", "-X", "POST", MCP_URL,
"-H", f"Authorization: Bearer {MCP_TOKEN}",
"-H", "Content-Type: application/json",
"-H", "Accept: application/json, text/event-stream",
"-d", json.dumps(payload)], capture_output=True, text=True)
dt = time.time() - t0
try:
r = json.loads(p.stdout)
content = r["result"]["content"]
text = "".join(c.get("text", "") for c in content if c.get("type") == "text")
data = json.loads(text)
return data.get("results", []), dt
except Exception as e:
log(f" search parse fail: {str(e)[:60]} raw={p.stdout[:120]}")
return [], dt
# ---------- gold 判官: 答案实体词面匹配 ----------
STOP_ZH = set("路个是否支持不采用什么多少几路的的了和与及或为是")
def answer_tokens(ans):
"""返回 (plain_tokens, norm_tokens): norm 类需在去空白文本中匹配(数字+量词)"""
plain, norm = set(), set()
ans = str(ans)
for m in re.finditer(r"\d+(?:\.\d+)?[A-Za-z%°℃Ω]*", ans):
t = m.group(0)
if len(t) >= 2 or (t.isdigit() and len(t) == 1):
plain.add(t.lower())
# 数字+中文量词组合 ("1 路" -> "1路"): 在规范化(去空白)文本中匹配
for m in re.finditer(r"\d+(?:\.\d+)?\s*[个路位针条次项]", ans):
norm.add(re.sub(r"\s+", "", m.group(0)).lower())
for m in re.finditer(r"[A-Za-z][A-Za-z0-9\-]{1,}", ans):
t = m.group(0)
if len(t) >= 2:
plain.add(t.lower())
zh = re.sub(r"[,。!?;:、\sA-Za-z0-9\.\-/()()]+", "|", ans)
for seg in zh.split("|"):
seg2 = "".join(ch for ch in seg if ch not in STOP_ZH)
if len(seg2) >= 3:
plain.add(seg2[:4].lower())
return ({t for t in plain if t}, norm)
def chunk_hit(chunk_text, toks):
plain, norm = toks
t = str(chunk_text).lower()
if any(tok in t for tok in plain):
return True
if norm:
tn = re.sub(r"\s+", "", t)
if any(tok in tn for tok in norm):
return True
return False
# ---------- Arm B: jev r41 本地打分 ----------
_jev = None
def jev_score(query, text):
global _jev
if _jev is None:
sys.path.insert(0, "/tmp/laya_src")
os.environ["MODEL_DIR"] = "/tmp/serve_test"
import serve as _serve
_serve.load_model()
_jev = _serve
return _jev.score_one(query, text)
# ---------- 指标 ----------
def rank_metrics(ranks):
"""ranks: 每题第一个 hit 的 1-based 位置 (None=未命中)"""
n = len(ranks)
def at_k(k):
return sum(1 for r in ranks if r is not None and r <= k) / n
mrr = sum(1.0 / r for r in ranks if r is not None) / n
return {"answer@1": round(at_k(1), 4), "answer@3": round(at_k(3), 4),
"answer@5": round(at_k(5), 4), "answer@10": round(at_k(10), 4),
"mrr": round(mrr, 4), "n": n}
def main():
limit = None
if "--limit" in sys.argv:
limit = int(sys.argv[sys.argv.index("--limit") + 1])
os.makedirs(OUT_DIR, exist_ok=True)
qs = load_questions()
if limit:
qs = qs[:limit]
log(f"questions={len(qs)}")
detail_path = os.path.join(OUT_DIR, "compare_detail.jsonl")
fout = open(detail_path, "w", encoding="utf-8")
for i, q in enumerate(qs):
qid = q["id"]
toks = answer_tokens(q["reference_answer"])
tp, tn = toks
results, lat = mcp_search(q["question"])
chunks = []
for r in results:
c = r.get("content") or ""
chunks.append({"filename": r.get("filename"), "section": r.get("section"),
"content": c, "hit": chunk_hit(c, toks)})
# Arm A 排名: 第一个 hit chunk
a_rank = next((k + 1 for k, c in enumerate(chunks) if c["hit"]), None)
# Arm B: jev 打分重排
for c in chunks:
s = jev_score(q["question"], c["content"])
c["jev_pfinal"] = s["p_final"]
c["jev_pmean"] = s["p_mean"]
order = sorted(range(len(chunks)), key=lambda k: -chunks[k]["jev_pfinal"])
b_rank = next((pos + 1 for pos, k in enumerate(order) if chunks[k]["hit"]), None)
# 页面级 (filename 去重取最高分/最先命中)
def page_ranks(chs, order_hint=None):
seen, ranks = set(), []
seq = order_hint if order_hint else list(range(len(chs)))
for k in seq:
f = chs[k]["filename"]
if f in seen:
continue
seen.add(f)
if chs[k]["hit"]:
return len(seen)
return None
a_page = page_ranks(chunks)
b_page = page_ranks(chunks, order)
fout.write(json.dumps({
"qid": qid, "question": q["question"], "answer": q["reference_answer"],
"tokens": {"plain": sorted(tp), "norm": sorted(tn)}, "a_rank": a_rank, "b_rank": b_rank,
"a_page": a_page, "b_page": b_page, "search_latency_s": round(lat, 2),
"chunks": [{"f": c["filename"], "sec": (c.get("section") or "")[:40],
"hit": c["hit"], "pfinal": c["jev_pfinal"]} for c in chunks]},
ensure_ascii=False) + "\n")
fout.flush()
log(f"{i+1}/{len(qs)} {qid} A={a_rank} B={b_rank} chunks={len(chunks)} lat={lat:.1f}s")
fout.close()
# 汇总
rows = [json.loads(l) for l in open(detail_path, encoding="utf-8")]
a_ranks = [r["a_rank"] for r in rows]
b_ranks = [r["b_rank"] for r in rows]
a_pages = [r["a_page"] for r in rows]
b_pages = [r["b_page"] for r in rows]
report = {
"chunk_level": {"original_chain": rank_metrics(a_ranks),
"jev_rerank": rank_metrics(b_ranks)},
"page_level": {"original_chain": rank_metrics(a_pages),
"jev_rerank": rank_metrics(b_pages)},
}
wins = sum(1 for r in rows if (r["b_rank"] or 99) < (r["a_rank"] or 99))
ties = sum(1 for r in rows if r["b_rank"] == r["a_rank"])
losses = sum(1 for r in rows if (r["b_rank"] or 99) > (r["a_rank"] or 99))
report["per_question"] = {"jev_better": wins, "tie": ties, "jev_worse": losses}
json.dump(report, open(os.path.join(OUT_DIR, "compare_report.json"), "w"),
ensure_ascii=False, indent=1)
log("REPORT " + json.dumps(report["chunk_level"]) + " " + json.dumps(report["per_question"]))
if __name__ == "__main__":
main()