143 lines
5.9 KiB
Python
143 lines
5.9 KiB
Python
#!/usr/bin/env python3
|
||
"""11_real_query_pairs.py — 真实问法 → BM25 候选 → 双裁判证据标注 → 训练对
|
||
输入: data/real_queries.json + data/corpus.jsonl
|
||
输出: data/real_query_pairs.jsonl {query, doc_id, hard_label, soft_label, judges, evidence_ok}
|
||
"""
|
||
import json, os, sys, time, re
|
||
import urllib.request
|
||
from collections import Counter
|
||
from math import log
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
CFG = json.load(open(os.path.join(HERE, "..", "config", "pipeline.json"), encoding="utf-8"))
|
||
JUDGES = ["glm-5.2", "glm-4.6"]
|
||
PUNCT = re.compile(r"[\s,。!?;:、\"'“”‘’()()\[\]【】《》<>\-—·…,.:;!?|\\/]+")
|
||
|
||
def norm(s): return PUNCT.sub("", s).lower()
|
||
def bigrams(t):
|
||
t = norm(t)
|
||
return ([t[i:i+2] for i in range(len(t)-1)] if len(t) > 1 else ([t] if t else [])) + re.findall(r"[a-z0-9]{2,}", t)
|
||
|
||
def bm25_top(query, docs_t, idf, avgdl, k=4):
|
||
q = set(bigrams(query))
|
||
sc = []
|
||
for did, s in docs_t.items():
|
||
v = sum(idf[t] * 2.2 / (1 + 0.25 + 0.75 * len(s) / avgdl) for t in q if t in s and t in idf)
|
||
if v > 0:
|
||
sc.append((v, did))
|
||
sc.sort(reverse=True)
|
||
return [d for _, d in sc[:k]]
|
||
|
||
PROMPT = """你是检索评测的标注员。判断"文档是否包含回答该问题所需的关键信息"。
|
||
|
||
规则:
|
||
1. 若文档能回答, 必须从文档原文中逐字摘录证据句(evidence, 15-80字的连续片段), 这是硬性要求
|
||
2. 注意产品型号匹配: 问题问某型号, 文档若明确属于其他型号且参数不同, 不算回答
|
||
3. 若文档不含答案, evidence 留空字符串, 并在 reason 说明答案缺了什么
|
||
|
||
只输出 JSON: {{"p": 0.0-1.0, "grade": "full|partial|none", "evidence": "...", "reason": "20字内"}}
|
||
|
||
问题: {query}
|
||
文档所属产品域: {products}
|
||
文档内容:
|
||
---
|
||
{text}
|
||
---"""
|
||
|
||
def chat(model, prompt):
|
||
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
|
||
"temperature": 0.0, "max_tokens": 300,
|
||
"thinking": {"type": "disabled"}}).encode()
|
||
req = urllib.request.Request(CFG["gateway"]["base_url"].rstrip("/") + "/chat/completions", data=body,
|
||
headers={"Content-Type": "application/json",
|
||
"Authorization": "Bearer " + CFG["gateway"]["api_key"]})
|
||
for i in range(4):
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=120) as r:
|
||
return json.loads(r.read())["choices"][0]["message"]["content"]
|
||
except Exception:
|
||
if i == 3:
|
||
raise
|
||
time.sleep(2 ** i)
|
||
|
||
def parse(content):
|
||
try:
|
||
content = content.strip()
|
||
if content.startswith("```"):
|
||
content = content.split("```")[1].removeprefix("json").strip()
|
||
d = json.loads(content[content.index("{"):content.rindex("}") + 1])
|
||
return {"p": max(0.0, min(1.0, float(d.get("p", 0)))),
|
||
"evidence": str(d.get("evidence", ""))[:200]}
|
||
except Exception:
|
||
return None
|
||
|
||
def main():
|
||
queries = json.load(open(os.path.join(HERE, "..", "data", "real_queries.json"), encoding="utf-8"))
|
||
docs = [json.loads(l) for l in open(os.path.join(HERE, "..", "data", "corpus.jsonl"), encoding="utf-8")]
|
||
docs_t = {d["doc_id"]: set(bigrams(d["text"][:20000])) for d in docs}
|
||
df = Counter()
|
||
for s in docs_t.values():
|
||
df.update(s)
|
||
N = len(docs_t)
|
||
avgdl = sum(len(s) for s in docs_t.values()) / max(N, 1)
|
||
idf = {t: log(1 + (N - n + 0.5) / (n + 0.5)) for t, n in df.items()}
|
||
norm_texts = {d["doc_id"]: norm(d["text"]) for d in docs}
|
||
|
||
cands = []
|
||
seen_q = set()
|
||
for q in queries:
|
||
if q in seen_q:
|
||
continue
|
||
seen_q.add(q)
|
||
for did in bm25_top(q, docs_t, idf, avgdl, k=4):
|
||
cands.append((q, did))
|
||
print(f"真实问法 {len(seen_q)} 条 × BM25 top4 → {len(cands)} 候选对, 双裁判标注中...")
|
||
|
||
def work(pair):
|
||
q, did = pair
|
||
doc = next(d for d in docs if d["doc_id"] == did)
|
||
rs = []
|
||
for jm in JUDGES:
|
||
try:
|
||
rs.append(parse(chat(jm, PROMPT.format(query=q, products=",".join(doc["products"]),
|
||
text=doc["text"][:4500]))))
|
||
except Exception:
|
||
rs.append(None)
|
||
ev_ok = [bool(r and len(norm(r["evidence"])) >= 10 and norm(r["evidence"]) in norm_texts[did])
|
||
for r in rs]
|
||
return q, did, rs, ev_ok
|
||
|
||
out = []
|
||
stat = Counter()
|
||
with ThreadPoolExecutor(max_workers=CFG["gateway"].get("concurrency", 4)) as ex:
|
||
futs = [ex.submit(work, c) for c in cands]
|
||
for i, f in enumerate(as_completed(futs)):
|
||
q, did, rs, ev_ok = f.result()
|
||
ps = [r["p"] for r in rs if r]
|
||
if len(ps) < 2:
|
||
stat["judge_fail"] += 1
|
||
continue
|
||
ev_ok_any = any(ev_ok)
|
||
if all(p >= 0.7 for p in ps) and ev_ok_any:
|
||
lab, soft, cat = 1, round(sum(ps) / 2, 3), "pos"
|
||
elif all(p <= 0.3 for p in ps):
|
||
lab, soft, cat = 0, round(sum(ps) / 2, 3), "neg"
|
||
else:
|
||
stat["dropped_mid"] += 1
|
||
continue
|
||
stat[cat] += 1
|
||
out.append({"query": q, "doc_id": did, "hard_label": lab, "soft_label": soft,
|
||
"judges": rs, "evidence_ok": ev_ok_any,
|
||
"provenance": {"source": "real_query", "license": "internal"}})
|
||
if (i + 1) % 50 == 0:
|
||
print(f" {i+1}/{len(cands)} {dict(stat)}", flush=True)
|
||
|
||
with open(os.path.join(HERE, "..", "data", "real_query_pairs.jsonl"), "w", encoding="utf-8") as f:
|
||
for r in out:
|
||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||
print(f"完成: {dict(stat)} → real_query_pairs.jsonl ({len(out)} 对)")
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|