237 lines
9.8 KiB
Python
237 lines
9.8 KiB
Python
#!/usr/bin/env python3
|
||
"""judge_compare.py — LLM 判官版 jev vs 原始链路对比 (v2)
|
||
1. 重新 search_knowledge(top_k=10), 保存 chunk 全文
|
||
2. LLM 判官 (glm-5.3-flash) 判每个 chunk 是否含答案信息
|
||
3. Arm A 原始排名 / Arm B jev r41 重排 / Arm C RRF α=0.5
|
||
4. 端到端 QA 臂: top-5 context 喂 glm 答题 → 判官判正确性 (最终口径)
|
||
输出: /tmp/jev_compare_v2/
|
||
"""
|
||
import json, os, re, sys, time, math, threading, queue, subprocess, bisect
|
||
|
||
MCP_URL = "https://knowledge-dev.d-robotics.cc/mcp"
|
||
MCP_TOKEN = "KB_MCP_SERVICE_KEY_见飞书交接文档08"
|
||
GLM_URL = "https://open.bigmodel.cn/api/paas/v4/chat/completions"
|
||
GLM_KEY = "GLM_KEY_见飞书交接文档08"
|
||
DATASET = "/Users/d-robotics/agent/知识检索/rdk_eval/rdk_eval_dataset.json"
|
||
OUT_DIR = "/tmp/jev_compare_v2"
|
||
TOP_K = 10
|
||
WORKERS = 4
|
||
|
||
def log(m):
|
||
print(f"[v2] {m}", flush=True)
|
||
|
||
def load_questions():
|
||
d = json.load(open(DATASET, encoding="utf-8"))
|
||
return [q for m in d["models"] for q in m.get("text_questions", [])]
|
||
|
||
def glm(messages, max_tokens=500):
|
||
body = {"model": "glm-5.3-flash", "messages": messages,
|
||
"max_tokens": max_tokens, "thinking": {"type": "disabled"}}
|
||
for attempt in range(3):
|
||
try:
|
||
import urllib.request
|
||
req = urllib.request.Request(GLM_URL, data=json.dumps(body).encode(),
|
||
headers={"Authorization": f"Bearer {GLM_KEY}", "Content-Type": "application/json"})
|
||
with urllib.request.urlopen(req, timeout=60) as r:
|
||
out = json.loads(r.read())
|
||
return out["choices"][0]["message"]["content"].strip()
|
||
except Exception as e:
|
||
if attempt == 2:
|
||
return f"GLM-ERROR: {str(e)[:80]}"
|
||
time.sleep(3 * (attempt + 1))
|
||
|
||
# ---------- 阶段1: 检索 ----------
|
||
def stage_search(qs):
|
||
path = os.path.join(OUT_DIR, "search_results.json")
|
||
cached = {}
|
||
if os.path.exists(path):
|
||
cached = json.load(open(path, encoding="utf-8"))
|
||
log(f"search cache: {sum(1 for q in qs if cached.get(q['id']))}/{len(qs)} 命中")
|
||
results = dict(cached)
|
||
for i, q in enumerate(qs):
|
||
if results.get(q["id"]):
|
||
continue
|
||
payload = {"jsonrpc": "2.0", "id": 1, "method": "tools/call",
|
||
"params": {"name": "search_knowledge",
|
||
"arguments": {"query": q["question"], "top_k": TOP_K}}}
|
||
p = subprocess.run(["curl", "-s", "-m", "40", "-X", "POST", MCP_URL,
|
||
"-H", f"Authorization: Bearer {MCP_TOKEN}",
|
||
"-H", "Content-Type: application/json",
|
||
"-H", "Accept: application/json, text/event-stream",
|
||
"-d", json.dumps(payload)], capture_output=True, text=True)
|
||
chunks = []
|
||
try:
|
||
r = json.loads(p.stdout)
|
||
text = "".join(c.get("text", "") for c in r["result"]["content"] if c.get("type") == "text")
|
||
for item in json.loads(text).get("results", []):
|
||
chunks.append({"filename": item.get("filename"), "section": item.get("section"),
|
||
"content": (item.get("content") or "")[:3000]})
|
||
except Exception as e:
|
||
log(f" {q['id']} search fail: {str(e)[:60]}")
|
||
results[q["id"]] = chunks
|
||
json.dump(results, open(path, "w", encoding="utf-8"), ensure_ascii=False)
|
||
log(f"search {i+1}/{len(qs)} {q['id']} chunks={len(chunks)}")
|
||
return results
|
||
|
||
# ---------- 阶段2: LLM 判官 ----------
|
||
_judge_cache = {}
|
||
_jq = queue.Queue()
|
||
_jlock = threading.Lock()
|
||
|
||
def judge_chunk(question, answer, content):
|
||
key = (question[:50], content[:80])
|
||
with _jlock:
|
||
if key in _judge_cache:
|
||
return _judge_cache[key]
|
||
prompt = (f"问题: {question}\n参考答案: {answer}\n\n文档内容:\n{content[:2200]}\n\n"
|
||
f"该文档内容是否包含回答上述问题所需的信息? 只回答 yes 或 no。")
|
||
out = glm([{"role": "user", "content": prompt}], max_tokens=10).lower()
|
||
ok = "yes" in out and "GLM-ERROR" not in out
|
||
with _jlock:
|
||
_judge_cache[key] = ok
|
||
return ok
|
||
|
||
def stage_judge(qs, results):
|
||
"""results 是权威 chunk 内容 (stage_search 产物); 仅从缓存继承 llm_hit"""
|
||
path = os.path.join(OUT_DIR, "judge_results.json")
|
||
cached = {}
|
||
if os.path.exists(path):
|
||
try:
|
||
cached = json.load(open(path, encoding="utf-8"))
|
||
except Exception:
|
||
cached = {}
|
||
for q in qs:
|
||
old = cached.get(q["id"]) or []
|
||
new = results.get(q["id"]) or []
|
||
if len(old) == len(new):
|
||
for c_old, c_new in zip(old, new):
|
||
if "llm_hit" in c_old:
|
||
c_new["llm_hit"] = c_old["llm_hit"]
|
||
tasks = queue.Queue()
|
||
for q in qs:
|
||
for k, c in enumerate(results.get(q["id"], [])):
|
||
if "llm_hit" not in c:
|
||
tasks.put((q, k, c))
|
||
total = tasks.qsize()
|
||
log(f"judge 待判 {total} chunks")
|
||
done = [0]
|
||
def worker():
|
||
while True:
|
||
try:
|
||
q, k, c = tasks.get_nowait()
|
||
except queue.Empty:
|
||
return
|
||
ok = judge_chunk(q["question"], q["reference_answer"], c["content"])
|
||
c["llm_hit"] = ok
|
||
with _jlock:
|
||
done[0] += 1
|
||
if done[0] % 50 == 0:
|
||
log(f"judge {done[0]}/{total}")
|
||
tasks.task_done()
|
||
ts = [threading.Thread(target=worker, daemon=True) for _ in range(WORKERS)]
|
||
for t in ts: t.start()
|
||
for t in ts: t.join()
|
||
json.dump(results, open(path, "w", encoding="utf-8"), ensure_ascii=False)
|
||
return results
|
||
|
||
# ---------- 阶段3: jev 打分 ----------
|
||
_jev = None
|
||
def jev_scores(question, chunks):
|
||
global _jev
|
||
if _jev is None:
|
||
sys.path.insert(0, "/tmp/laya_src")
|
||
os.environ["MODEL_DIR"] = "/tmp/serve_test"
|
||
import serve as _s
|
||
_s.load_model()
|
||
_jev = _s
|
||
out = []
|
||
for c in chunks:
|
||
s = _jev.score_one(question, c["content"])
|
||
out.append(s["p_final"])
|
||
return out
|
||
|
||
# ---------- 阶段4: 端到端 QA ----------
|
||
def qa_answer(question, chunks, order):
|
||
ctx = "\n\n---\n\n".join(chunks[i]["content"][:1200] for i in order[:5])
|
||
prompt = (f"根据以下知识库内容回答问题。如果内容不足以回答,回答\"知识库中没有相关信息\"。\n\n"
|
||
f"知识库内容:\n{ctx[:6000]}\n\n问题: {question}")
|
||
return glm([{"role": "user", "content": prompt}], max_tokens=300)
|
||
|
||
def judge_qa(question, reference, model_answer):
|
||
prompt = (f"问题: {question}\n标准答案: {reference}\n模型回答: {model_answer}\n\n"
|
||
f"模型回答是否正确回答了问题(与标准答案一致)? 只回答 yes 或 no。")
|
||
out = glm([{"role": "user", "content": prompt}], max_tokens=10).lower()
|
||
return "yes" in out and "GLM-ERROR" not in out
|
||
|
||
# ---------- 指标 ----------
|
||
def metrics(ranks, n):
|
||
def at(k): return round(sum(1 for x in ranks if x and x <= k) / n, 3)
|
||
return {"@1": at(1), "@3": at(3), "@5": at(5), "@10": at(10),
|
||
"mrr": round(sum(1 / x for x in ranks if x) / n, 3)}
|
||
|
||
def main():
|
||
os.makedirs(OUT_DIR, exist_ok=True)
|
||
limit = None
|
||
if "--limit" in sys.argv:
|
||
limit = int(sys.argv[sys.argv.index("--limit") + 1])
|
||
qs = load_questions()
|
||
if limit:
|
||
qs = qs[:limit]
|
||
log(f"questions={len(qs)}")
|
||
|
||
results = stage_search(qs)
|
||
results = stage_judge(qs, results)
|
||
|
||
detail_path = os.path.join(OUT_DIR, "v2_detail.jsonl")
|
||
fout = open(detail_path, "w", encoding="utf-8")
|
||
rows = []
|
||
for i, q in enumerate(qs):
|
||
chunks = results.get(q["id"], [])
|
||
if not chunks:
|
||
continue
|
||
pf = jev_scores(q["question"], chunks)
|
||
for k, c in enumerate(chunks):
|
||
c["jev_pf"] = pf[k]
|
||
a_order = list(range(len(chunks)))
|
||
b_order = sorted(range(len(chunks)), key=lambda k: -chunks[k]["jev_pf"])
|
||
K = 60
|
||
c_order = sorted(range(len(chunks)),
|
||
key=lambda i2: -(0.5 / (K + b_order.index(i2) + 1) + 0.5 / (K + i2 + 1)))
|
||
def first_hit(order):
|
||
return next((p + 1 for p, k in enumerate(order) if chunks[k].get("llm_hit")), None)
|
||
a_rank, b_rank, c_rank = first_hit(a_order), first_hit(b_order), first_hit(c_order)
|
||
|
||
# 端到端 QA: 原始 top5 vs jev top5
|
||
qa_a = qa_answer(q["question"], chunks, a_order)
|
||
qa_b = qa_answer(q["question"], chunks, b_order)
|
||
qa_a_ok = judge_qa(q["question"], q["reference_answer"], qa_a)
|
||
qa_b_ok = judge_qa(q["question"], q["reference_answer"], qa_b)
|
||
|
||
row = {"qid": q["id"], "question": q["question"], "answer": q["reference_answer"],
|
||
"a_rank": a_rank, "b_rank": b_rank, "c_rank": c_rank,
|
||
"qa_a_ok": qa_a_ok, "qa_b_ok": qa_b_ok,
|
||
"n_rel_chunks": sum(1 for c in chunks if c.get("llm_hit")),
|
||
"qa_a_text": qa_a[:200], "qa_b_text": qa_b[:200]}
|
||
rows.append(row)
|
||
fout.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||
fout.flush()
|
||
log(f"{i+1}/{len(qs)} {q['id']} A={a_rank} B={b_rank} C={c_rank} "
|
||
f"qaA={'✓' if qa_a_ok else '✗'} qaB={'✓' if qa_b_ok else '✗'}")
|
||
fout.close()
|
||
|
||
n = len(rows)
|
||
report = {
|
||
"n": n,
|
||
"A_original": metrics([r["a_rank"] for r in rows], n),
|
||
"B_jev": metrics([r["b_rank"] for r in rows], n),
|
||
"C_rrf": metrics([r["c_rank"] for r in rows], n),
|
||
"qa_accuracy": {
|
||
"A_original_top5": round(sum(1 for r in rows if r["qa_a_ok"]) / n, 3),
|
||
"B_jev_top5": round(sum(1 for r in rows if r["qa_b_ok"]) / n, 3)},
|
||
}
|
||
json.dump(report, open(os.path.join(OUT_DIR, "v2_report.json"), "w"), ensure_ascii=False, indent=1)
|
||
log("REPORT " + json.dumps(report, ensure_ascii=False))
|
||
|
||
if __name__ == "__main__":
|
||
main()
|