172 lines
8.0 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""10_review_eval.py — eval 集双裁判 LLM 复核(证据锚定式)
双裁判: glm-5.2 + glm-4.6(与打标期 5.3-flash 不同代, 降共偏)
每对输出必须带 evidence(文档原文连续片段), 程序化验证引文存在性, 对不上降级为 REVIEW
输出: data/eval_v2.jsonl(带复核元数据) + data/eval_review_report.md
"""
import json, os, sys, time, re
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
HERE = os.path.dirname(os.path.abspath(__file__))
CFG = json.load(open(os.path.join(HERE, "..", "config", "pipeline.json"), encoding="utf-8"))
JUDGES = ["glm-5.2", "glm-4.6"]
PROMPT = """你是检索评测集的复核员。判断"文档是否包含回答该问题所需的关键信息"。
规则:
1. 若文档能回答, 必须从文档原文中逐字摘录证据句(evidence, 15-80字的连续片段), 这是硬性要求
2. 注意产品型号匹配: 问题问某型号, 文档若明确属于其他型号且参数不同, 不算回答
3. 若文档不含答案, evidence 留空字符串, 并在 reason 说明答案缺了什么
只输出 JSON: {{"p": 0.0-1.0, "grade": "full|partial|none", "evidence": "...", "reason": "20字内"}}
问题: {query}
文档所属产品域: {products}
文档内容:
---
{text}
---"""
def chat(model, prompt):
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
"temperature": 0.0, "max_tokens": 300,
"thinking": {"type": "disabled"}}).encode()
req = urllib.request.Request(CFG["gateway"]["base_url"].rstrip("/") + "/chat/completions", data=body,
headers={"Content-Type": "application/json",
"Authorization": "Bearer " + CFG["gateway"]["api_key"]})
for i in range(4):
try:
with urllib.request.urlopen(req, timeout=120) as r:
return json.loads(r.read())["choices"][0]["message"]["content"]
except Exception:
if i == 3:
raise
time.sleep(2 ** i)
PUNCT = re.compile(r"[\s,。!?;:、\"'“”‘’()()\[\]【】《》<>\-—·…,.:;!?|\\/]+")
def norm(s): return PUNCT.sub("", s).lower()
def parse(content):
try:
content = content.strip()
if content.startswith("```"):
content = content.split("```")[1].removeprefix("json").strip()
d = json.loads(content[content.index("{"):content.rindex("}") + 1])
p = max(0.0, min(1.0, float(d.get("p", 0))))
return {"p": p, "grade": d.get("grade"), "evidence": str(d.get("evidence", ""))[:200],
"reason": str(d.get("reason", ""))[:80]}
except Exception:
return None
def review_one(row, docs_norm):
doc = docs_norm[row["doc_id"]]
results, raw_evidences = [], []
for jm in JUDGES:
try:
r = parse(chat(jm, PROMPT.format(query=row["query"], products=",".join(doc["products"]),
text=doc["text"][:4500])))
results.append(r)
raw_evidences.append((r or {}).get("evidence", ""))
except Exception as e:
results.append(None)
raw_evidences.append("")
# 引文核验: 归一化后是文档子串才算实证
ev_ok = []
for r, ev in zip(results, raw_evidences):
ok = bool(r and ev and len(norm(ev)) >= 10 and norm(ev) in doc["norm_text"])
ev_ok.append(ok)
return results, ev_ok
def main():
corpus = {json.loads(l)["doc_id"]: json.loads(l) for l in
open(os.path.join(HERE, "..", "data", "corpus.jsonl"), encoding="utf-8")}
docs_norm = {k: {"products": v["products"], "norm_text": norm(v["text"]), "text": v["text"]}
for k, v in corpus.items()}
rows = [json.loads(l) for l in
open(os.path.join(HERE, "..", "data", "jev_dataset_v0.2", "eval.jsonl"), encoding="utf-8")]
print(f"复核 {len(rows)} 对 eval 样本, 双裁判 {JUDGES}")
out = []
def work(idx_row):
idx, row = idx_row
results, ev_ok = review_one(row, docs_norm)
return idx, row, results, ev_ok
with ThreadPoolExecutor(max_workers=CFG["gateway"].get("concurrency", 4)) as ex:
for f in as_completed([ex.submit(work, x) for x in enumerate(rows)]):
idx, row, results, ev_ok = f.result()
row = dict(row)
row["review"] = {"judges": JUDGES, "results": results, "evidence_verified": ev_ok}
out.append((idx, row))
out.sort(key=lambda x: x[0])
# 分类
stat = {"keep_pos": 0, "keep_neg": 0, "flip_to_pos": 0, "flip_to_neg": 0,
"review": 0, "evidence_unverified": 0}
report_rows = []
final_rows = []
for idx, row in out:
rv = row["review"]
rs = [r for r in rv["results"] if r]
ps = [r["p"] for r in rs]
ev_ok = any(rv["evidence_verified"])
lab = row["hard_label"]
category = "review"
if len(ps) < 2:
category = "review"
elif all(p >= 0.7 for p in ps) and ev_ok:
category = "flip_to_pos" if lab == 0 else "keep_pos"
elif all(p <= 0.3 for p in ps) and not ev_ok:
category = "flip_to_neg" if lab == 1 else "keep_neg"
elif all(p >= 0.7 for p in ps):
category = "keep_pos" if lab == 1 else "review"
elif all(p <= 0.3 for p in ps):
category = "keep_neg" if lab == 0 else "review"
else:
category = "review"
if category.startswith("keep") and (lab == 1) != (category == "keep_pos"):
category = "review"
if category == "review" and any(rv["results"]) and not ev_ok and ps and all(p <= 0.3 for p in ps) and lab == 0:
category = "keep_neg"
stat["evidence_unverified"] += 1
stat[category] = stat.get(category, 0) + 1
if category == "flip_to_pos":
row["hard_label"], row["grade"], row["soft_label"] = 1, "full", round(sum(ps) / len(ps), 3)
elif category == "flip_to_neg":
row["hard_label"], row["grade"], row["soft_label"] = 0, "none", round(sum(ps) / len(ps), 3)
row["review"]["category"] = category
final_rows.append(row)
report_rows.append((category, row, ps, ev_ok))
with open(os.path.join(HERE, "..", "data", "eval_v2.jsonl"), "w", encoding="utf-8") as f:
for row in final_rows:
f.write(json.dumps(row, ensure_ascii=False) + "\n")
lines = ["# eval 集双裁判复核报告", "",
f"- 样本: {len(rows)} 对 | 裁判: {', '.join(JUDGES)} | 引文程序化核验: 开启", "",
"| 类别 | 数量 |", "|---|---|"]
names = {"keep_pos": "维持正例", "keep_neg": "维持负例",
"flip_to_pos": "翻正(挖负例误标→改正)", "flip_to_neg": "翻负(锚定假阳性→改负)",
"review": "需人工"}
for k in ["keep_pos", "keep_neg", "flip_to_pos", "flip_to_neg", "review"]:
lines.append(f"| {names[k]} | {stat.get(k, 0)} |")
if stat.get("evidence_unverified"):
lines.append(f"| (维持负例中引文未验证) | {stat['evidence_unverified']} |")
lines += ["", "## 需人工 / 翻转明细", ""]
for cat, row, ps, ev_ok in report_rows:
if cat in ("flip_to_pos", "flip_to_neg", "review"):
qid = (row.get("provenance") or {}).get("qid")
lines.append(f"- **{cat}** [{qid}] label={row['hard_label']} judges_p={ps} "
f"ev_verified={ev_ok} doc={row['doc_id']}")
lines.append(f" - Q: {row['query'][:70]}")
for jm, r, ok in zip(JUDGES, row["review"]["results"], row["review"]["evidence_verified"]):
if r:
lines.append(f" - {jm}: p={r['p']} {r['grade']} | {r['reason']} | 引文核验={'通过' if ok else '未通过/无'}")
open(os.path.join(HERE, "..", "data", "eval_review_report.md"), "w", encoding="utf-8").write("\n".join(lines))
print("统计:", {k: v for k, v in stat.items() if v})
print("→ eval_v2.jsonl / eval_review_report.md")
if __name__ == "__main__":
sys.exit(main())