jev-training-data/judge_compare.py

237 lines
9.8 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""judge_compare.py — LLM 判官版 jev vs 原始链路对比 (v2)
1. 重新 search_knowledge(top_k=10), 保存 chunk 全文
2. LLM 判官 (glm-5.3-flash) 判每个 chunk 是否含答案信息
3. Arm A 原始排名 / Arm B jev r41 重排 / Arm C RRF α=0.5
4. 端到端 QA 臂: top-5 context 喂 glm 答题 → 判官判正确性 (最终口径)
输出: /tmp/jev_compare_v2/
"""
import json, os, re, sys, time, math, threading, queue, subprocess, bisect
MCP_URL = "https://knowledge-dev.d-robotics.cc/mcp"
MCP_TOKEN = "KB_MCP_SERVICE_KEY_见飞书交接文档08"
GLM_URL = "https://open.bigmodel.cn/api/paas/v4/chat/completions"
GLM_KEY = "GLM_KEY_见飞书交接文档08"
DATASET = "/Users/d-robotics/agent/知识检索/rdk_eval/rdk_eval_dataset.json"
OUT_DIR = "/tmp/jev_compare_v2"
TOP_K = 10
WORKERS = 4
def log(m):
print(f"[v2] {m}", flush=True)
def load_questions():
d = json.load(open(DATASET, encoding="utf-8"))
return [q for m in d["models"] for q in m.get("text_questions", [])]
def glm(messages, max_tokens=500):
body = {"model": "glm-5.3-flash", "messages": messages,
"max_tokens": max_tokens, "thinking": {"type": "disabled"}}
for attempt in range(3):
try:
import urllib.request
req = urllib.request.Request(GLM_URL, data=json.dumps(body).encode(),
headers={"Authorization": f"Bearer {GLM_KEY}", "Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=60) as r:
out = json.loads(r.read())
return out["choices"][0]["message"]["content"].strip()
except Exception as e:
if attempt == 2:
return f"GLM-ERROR: {str(e)[:80]}"
time.sleep(3 * (attempt + 1))
# ---------- 阶段1: 检索 ----------
def stage_search(qs):
path = os.path.join(OUT_DIR, "search_results.json")
cached = {}
if os.path.exists(path):
cached = json.load(open(path, encoding="utf-8"))
log(f"search cache: {sum(1 for q in qs if cached.get(q['id']))}/{len(qs)} 命中")
results = dict(cached)
for i, q in enumerate(qs):
if results.get(q["id"]):
continue
payload = {"jsonrpc": "2.0", "id": 1, "method": "tools/call",
"params": {"name": "search_knowledge",
"arguments": {"query": q["question"], "top_k": TOP_K}}}
p = subprocess.run(["curl", "-s", "-m", "40", "-X", "POST", MCP_URL,
"-H", f"Authorization: Bearer {MCP_TOKEN}",
"-H", "Content-Type: application/json",
"-H", "Accept: application/json, text/event-stream",
"-d", json.dumps(payload)], capture_output=True, text=True)
chunks = []
try:
r = json.loads(p.stdout)
text = "".join(c.get("text", "") for c in r["result"]["content"] if c.get("type") == "text")
for item in json.loads(text).get("results", []):
chunks.append({"filename": item.get("filename"), "section": item.get("section"),
"content": (item.get("content") or "")[:3000]})
except Exception as e:
log(f" {q['id']} search fail: {str(e)[:60]}")
results[q["id"]] = chunks
json.dump(results, open(path, "w", encoding="utf-8"), ensure_ascii=False)
log(f"search {i+1}/{len(qs)} {q['id']} chunks={len(chunks)}")
return results
# ---------- 阶段2: LLM 判官 ----------
_judge_cache = {}
_jq = queue.Queue()
_jlock = threading.Lock()
def judge_chunk(question, answer, content):
key = (question[:50], content[:80])
with _jlock:
if key in _judge_cache:
return _judge_cache[key]
prompt = (f"问题: {question}\n参考答案: {answer}\n\n文档内容:\n{content[:2200]}\n\n"
f"该文档内容是否包含回答上述问题所需的信息? 只回答 yes 或 no。")
out = glm([{"role": "user", "content": prompt}], max_tokens=10).lower()
ok = "yes" in out and "GLM-ERROR" not in out
with _jlock:
_judge_cache[key] = ok
return ok
def stage_judge(qs, results):
"""results 是权威 chunk 内容 (stage_search 产物); 仅从缓存继承 llm_hit"""
path = os.path.join(OUT_DIR, "judge_results.json")
cached = {}
if os.path.exists(path):
try:
cached = json.load(open(path, encoding="utf-8"))
except Exception:
cached = {}
for q in qs:
old = cached.get(q["id"]) or []
new = results.get(q["id"]) or []
if len(old) == len(new):
for c_old, c_new in zip(old, new):
if "llm_hit" in c_old:
c_new["llm_hit"] = c_old["llm_hit"]
tasks = queue.Queue()
for q in qs:
for k, c in enumerate(results.get(q["id"], [])):
if "llm_hit" not in c:
tasks.put((q, k, c))
total = tasks.qsize()
log(f"judge 待判 {total} chunks")
done = [0]
def worker():
while True:
try:
q, k, c = tasks.get_nowait()
except queue.Empty:
return
ok = judge_chunk(q["question"], q["reference_answer"], c["content"])
c["llm_hit"] = ok
with _jlock:
done[0] += 1
if done[0] % 50 == 0:
log(f"judge {done[0]}/{total}")
tasks.task_done()
ts = [threading.Thread(target=worker, daemon=True) for _ in range(WORKERS)]
for t in ts: t.start()
for t in ts: t.join()
json.dump(results, open(path, "w", encoding="utf-8"), ensure_ascii=False)
return results
# ---------- 阶段3: jev 打分 ----------
_jev = None
def jev_scores(question, chunks):
global _jev
if _jev is None:
sys.path.insert(0, "/tmp/laya_src")
os.environ["MODEL_DIR"] = "/tmp/serve_test"
import serve as _s
_s.load_model()
_jev = _s
out = []
for c in chunks:
s = _jev.score_one(question, c["content"])
out.append(s["p_final"])
return out
# ---------- 阶段4: 端到端 QA ----------
def qa_answer(question, chunks, order):
ctx = "\n\n---\n\n".join(chunks[i]["content"][:1200] for i in order[:5])
prompt = (f"根据以下知识库内容回答问题。如果内容不足以回答,回答\"知识库中没有相关信息\"。\n\n"
f"知识库内容:\n{ctx[:6000]}\n\n问题: {question}")
return glm([{"role": "user", "content": prompt}], max_tokens=300)
def judge_qa(question, reference, model_answer):
prompt = (f"问题: {question}\n标准答案: {reference}\n模型回答: {model_answer}\n\n"
f"模型回答是否正确回答了问题(与标准答案一致)? 只回答 yes 或 no。")
out = glm([{"role": "user", "content": prompt}], max_tokens=10).lower()
return "yes" in out and "GLM-ERROR" not in out
# ---------- 指标 ----------
def metrics(ranks, n):
def at(k): return round(sum(1 for x in ranks if x and x <= k) / n, 3)
return {"@1": at(1), "@3": at(3), "@5": at(5), "@10": at(10),
"mrr": round(sum(1 / x for x in ranks if x) / n, 3)}
def main():
os.makedirs(OUT_DIR, exist_ok=True)
limit = None
if "--limit" in sys.argv:
limit = int(sys.argv[sys.argv.index("--limit") + 1])
qs = load_questions()
if limit:
qs = qs[:limit]
log(f"questions={len(qs)}")
results = stage_search(qs)
results = stage_judge(qs, results)
detail_path = os.path.join(OUT_DIR, "v2_detail.jsonl")
fout = open(detail_path, "w", encoding="utf-8")
rows = []
for i, q in enumerate(qs):
chunks = results.get(q["id"], [])
if not chunks:
continue
pf = jev_scores(q["question"], chunks)
for k, c in enumerate(chunks):
c["jev_pf"] = pf[k]
a_order = list(range(len(chunks)))
b_order = sorted(range(len(chunks)), key=lambda k: -chunks[k]["jev_pf"])
K = 60
c_order = sorted(range(len(chunks)),
key=lambda i2: -(0.5 / (K + b_order.index(i2) + 1) + 0.5 / (K + i2 + 1)))
def first_hit(order):
return next((p + 1 for p, k in enumerate(order) if chunks[k].get("llm_hit")), None)
a_rank, b_rank, c_rank = first_hit(a_order), first_hit(b_order), first_hit(c_order)
# 端到端 QA: 原始 top5 vs jev top5
qa_a = qa_answer(q["question"], chunks, a_order)
qa_b = qa_answer(q["question"], chunks, b_order)
qa_a_ok = judge_qa(q["question"], q["reference_answer"], qa_a)
qa_b_ok = judge_qa(q["question"], q["reference_answer"], qa_b)
row = {"qid": q["id"], "question": q["question"], "answer": q["reference_answer"],
"a_rank": a_rank, "b_rank": b_rank, "c_rank": c_rank,
"qa_a_ok": qa_a_ok, "qa_b_ok": qa_b_ok,
"n_rel_chunks": sum(1 for c in chunks if c.get("llm_hit")),
"qa_a_text": qa_a[:200], "qa_b_text": qa_b[:200]}
rows.append(row)
fout.write(json.dumps(row, ensure_ascii=False) + "\n")
fout.flush()
log(f"{i+1}/{len(qs)} {q['id']} A={a_rank} B={b_rank} C={c_rank} "
f"qaA={'✓' if qa_a_ok else '✗'} qaB={'✓' if qa_b_ok else '✗'}")
fout.close()
n = len(rows)
report = {
"n": n,
"A_original": metrics([r["a_rank"] for r in rows], n),
"B_jev": metrics([r["b_rank"] for r in rows], n),
"C_rrf": metrics([r["c_rank"] for r in rows], n),
"qa_accuracy": {
"A_original_top5": round(sum(1 for r in rows if r["qa_a_ok"]) / n, 3),
"B_jev_top5": round(sum(1 for r in rows if r["qa_b_ok"]) / n, 3)},
}
json.dump(report, open(os.path.join(OUT_DIR, "v2_report.json"), "w"), ensure_ascii=False, indent=1)
log("REPORT " + json.dumps(report, ensure_ascii=False))
if __name__ == "__main__":
main()