172 lines
8.0 KiB
Python
172 lines
8.0 KiB
Python
#!/usr/bin/env python3
|
||
"""10_review_eval.py — eval 集双裁判 LLM 复核(证据锚定式)
|
||
双裁判: glm-5.2 + glm-4.6(与打标期 5.3-flash 不同代, 降共偏)
|
||
每对输出必须带 evidence(文档原文连续片段), 程序化验证引文存在性, 对不上降级为 REVIEW
|
||
输出: data/eval_v2.jsonl(带复核元数据) + data/eval_review_report.md
|
||
"""
|
||
import json, os, sys, time, re
|
||
import urllib.request
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
CFG = json.load(open(os.path.join(HERE, "..", "config", "pipeline.json"), encoding="utf-8"))
|
||
JUDGES = ["glm-5.2", "glm-4.6"]
|
||
|
||
PROMPT = """你是检索评测集的复核员。判断"文档是否包含回答该问题所需的关键信息"。
|
||
|
||
规则:
|
||
1. 若文档能回答, 必须从文档原文中逐字摘录证据句(evidence, 15-80字的连续片段), 这是硬性要求
|
||
2. 注意产品型号匹配: 问题问某型号, 文档若明确属于其他型号且参数不同, 不算回答
|
||
3. 若文档不含答案, evidence 留空字符串, 并在 reason 说明答案缺了什么
|
||
|
||
只输出 JSON: {{"p": 0.0-1.0, "grade": "full|partial|none", "evidence": "...", "reason": "20字内"}}
|
||
|
||
问题: {query}
|
||
文档所属产品域: {products}
|
||
文档内容:
|
||
---
|
||
{text}
|
||
---"""
|
||
|
||
def chat(model, prompt):
|
||
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
|
||
"temperature": 0.0, "max_tokens": 300,
|
||
"thinking": {"type": "disabled"}}).encode()
|
||
req = urllib.request.Request(CFG["gateway"]["base_url"].rstrip("/") + "/chat/completions", data=body,
|
||
headers={"Content-Type": "application/json",
|
||
"Authorization": "Bearer " + CFG["gateway"]["api_key"]})
|
||
for i in range(4):
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=120) as r:
|
||
return json.loads(r.read())["choices"][0]["message"]["content"]
|
||
except Exception:
|
||
if i == 3:
|
||
raise
|
||
time.sleep(2 ** i)
|
||
|
||
PUNCT = re.compile(r"[\s,。!?;:、\"'“”‘’()()\[\]【】《》<>\-—·…,.:;!?|\\/]+")
|
||
def norm(s): return PUNCT.sub("", s).lower()
|
||
|
||
def parse(content):
|
||
try:
|
||
content = content.strip()
|
||
if content.startswith("```"):
|
||
content = content.split("```")[1].removeprefix("json").strip()
|
||
d = json.loads(content[content.index("{"):content.rindex("}") + 1])
|
||
p = max(0.0, min(1.0, float(d.get("p", 0))))
|
||
return {"p": p, "grade": d.get("grade"), "evidence": str(d.get("evidence", ""))[:200],
|
||
"reason": str(d.get("reason", ""))[:80]}
|
||
except Exception:
|
||
return None
|
||
|
||
def review_one(row, docs_norm):
|
||
doc = docs_norm[row["doc_id"]]
|
||
results, raw_evidences = [], []
|
||
for jm in JUDGES:
|
||
try:
|
||
r = parse(chat(jm, PROMPT.format(query=row["query"], products=",".join(doc["products"]),
|
||
text=doc["text"][:4500])))
|
||
results.append(r)
|
||
raw_evidences.append((r or {}).get("evidence", ""))
|
||
except Exception as e:
|
||
results.append(None)
|
||
raw_evidences.append("")
|
||
# 引文核验: 归一化后是文档子串才算实证
|
||
ev_ok = []
|
||
for r, ev in zip(results, raw_evidences):
|
||
ok = bool(r and ev and len(norm(ev)) >= 10 and norm(ev) in doc["norm_text"])
|
||
ev_ok.append(ok)
|
||
return results, ev_ok
|
||
|
||
def main():
|
||
corpus = {json.loads(l)["doc_id"]: json.loads(l) for l in
|
||
open(os.path.join(HERE, "..", "data", "corpus.jsonl"), encoding="utf-8")}
|
||
docs_norm = {k: {"products": v["products"], "norm_text": norm(v["text"]), "text": v["text"]}
|
||
for k, v in corpus.items()}
|
||
rows = [json.loads(l) for l in
|
||
open(os.path.join(HERE, "..", "data", "jev_dataset_v0.2", "eval.jsonl"), encoding="utf-8")]
|
||
print(f"复核 {len(rows)} 对 eval 样本, 双裁判 {JUDGES}")
|
||
|
||
out = []
|
||
def work(idx_row):
|
||
idx, row = idx_row
|
||
results, ev_ok = review_one(row, docs_norm)
|
||
return idx, row, results, ev_ok
|
||
|
||
with ThreadPoolExecutor(max_workers=CFG["gateway"].get("concurrency", 4)) as ex:
|
||
for f in as_completed([ex.submit(work, x) for x in enumerate(rows)]):
|
||
idx, row, results, ev_ok = f.result()
|
||
row = dict(row)
|
||
row["review"] = {"judges": JUDGES, "results": results, "evidence_verified": ev_ok}
|
||
out.append((idx, row))
|
||
|
||
out.sort(key=lambda x: x[0])
|
||
# 分类
|
||
stat = {"keep_pos": 0, "keep_neg": 0, "flip_to_pos": 0, "flip_to_neg": 0,
|
||
"review": 0, "evidence_unverified": 0}
|
||
report_rows = []
|
||
final_rows = []
|
||
for idx, row in out:
|
||
rv = row["review"]
|
||
rs = [r for r in rv["results"] if r]
|
||
ps = [r["p"] for r in rs]
|
||
ev_ok = any(rv["evidence_verified"])
|
||
lab = row["hard_label"]
|
||
category = "review"
|
||
if len(ps) < 2:
|
||
category = "review"
|
||
elif all(p >= 0.7 for p in ps) and ev_ok:
|
||
category = "flip_to_pos" if lab == 0 else "keep_pos"
|
||
elif all(p <= 0.3 for p in ps) and not ev_ok:
|
||
category = "flip_to_neg" if lab == 1 else "keep_neg"
|
||
elif all(p >= 0.7 for p in ps):
|
||
category = "keep_pos" if lab == 1 else "review"
|
||
elif all(p <= 0.3 for p in ps):
|
||
category = "keep_neg" if lab == 0 else "review"
|
||
else:
|
||
category = "review"
|
||
if category.startswith("keep") and (lab == 1) != (category == "keep_pos"):
|
||
category = "review"
|
||
if category == "review" and any(rv["results"]) and not ev_ok and ps and all(p <= 0.3 for p in ps) and lab == 0:
|
||
category = "keep_neg"
|
||
stat["evidence_unverified"] += 1
|
||
stat[category] = stat.get(category, 0) + 1
|
||
if category == "flip_to_pos":
|
||
row["hard_label"], row["grade"], row["soft_label"] = 1, "full", round(sum(ps) / len(ps), 3)
|
||
elif category == "flip_to_neg":
|
||
row["hard_label"], row["grade"], row["soft_label"] = 0, "none", round(sum(ps) / len(ps), 3)
|
||
row["review"]["category"] = category
|
||
final_rows.append(row)
|
||
report_rows.append((category, row, ps, ev_ok))
|
||
|
||
with open(os.path.join(HERE, "..", "data", "eval_v2.jsonl"), "w", encoding="utf-8") as f:
|
||
for row in final_rows:
|
||
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||
|
||
lines = ["# eval 集双裁判复核报告", "",
|
||
f"- 样本: {len(rows)} 对 | 裁判: {', '.join(JUDGES)} | 引文程序化核验: 开启", "",
|
||
"| 类别 | 数量 |", "|---|---|"]
|
||
names = {"keep_pos": "维持正例", "keep_neg": "维持负例",
|
||
"flip_to_pos": "翻正(挖负例误标→改正)", "flip_to_neg": "翻负(锚定假阳性→改负)",
|
||
"review": "需人工"}
|
||
for k in ["keep_pos", "keep_neg", "flip_to_pos", "flip_to_neg", "review"]:
|
||
lines.append(f"| {names[k]} | {stat.get(k, 0)} |")
|
||
if stat.get("evidence_unverified"):
|
||
lines.append(f"| (维持负例中引文未验证) | {stat['evidence_unverified']} |")
|
||
lines += ["", "## 需人工 / 翻转明细", ""]
|
||
for cat, row, ps, ev_ok in report_rows:
|
||
if cat in ("flip_to_pos", "flip_to_neg", "review"):
|
||
qid = (row.get("provenance") or {}).get("qid")
|
||
lines.append(f"- **{cat}** [{qid}] label={row['hard_label']} judges_p={ps} "
|
||
f"ev_verified={ev_ok} doc={row['doc_id']}")
|
||
lines.append(f" - Q: {row['query'][:70]}")
|
||
for jm, r, ok in zip(JUDGES, row["review"]["results"], row["review"]["evidence_verified"]):
|
||
if r:
|
||
lines.append(f" - {jm}: p={r['p']} {r['grade']} | {r['reason']} | 引文核验={'通过' if ok else '未通过/无'}")
|
||
open(os.path.join(HERE, "..", "data", "eval_review_report.md"), "w", encoding="utf-8").write("\n".join(lines))
|
||
print("统计:", {k: v for k, v in stat.items() if v})
|
||
print("→ eval_v2.jsonl / eval_review_report.md")
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|