143 lines
5.9 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""11_real_query_pairs.py — 真实问法 → BM25 候选 → 双裁判证据标注 → 训练对
输入: data/real_queries.json + data/corpus.jsonl
输出: data/real_query_pairs.jsonl {query, doc_id, hard_label, soft_label, judges, evidence_ok}
"""
import json, os, sys, time, re
import urllib.request
from collections import Counter
from math import log
from concurrent.futures import ThreadPoolExecutor, as_completed
HERE = os.path.dirname(os.path.abspath(__file__))
CFG = json.load(open(os.path.join(HERE, "..", "config", "pipeline.json"), encoding="utf-8"))
JUDGES = ["glm-5.2", "glm-4.6"]
PUNCT = re.compile(r"[\s,。!?;:、\"'“”‘’()()\[\]【】《》<>\-—·…,.:;!?|\\/]+")
def norm(s): return PUNCT.sub("", s).lower()
def bigrams(t):
t = norm(t)
return ([t[i:i+2] for i in range(len(t)-1)] if len(t) > 1 else ([t] if t else [])) + re.findall(r"[a-z0-9]{2,}", t)
def bm25_top(query, docs_t, idf, avgdl, k=4):
q = set(bigrams(query))
sc = []
for did, s in docs_t.items():
v = sum(idf[t] * 2.2 / (1 + 0.25 + 0.75 * len(s) / avgdl) for t in q if t in s and t in idf)
if v > 0:
sc.append((v, did))
sc.sort(reverse=True)
return [d for _, d in sc[:k]]
PROMPT = """你是检索评测的标注员。判断"文档是否包含回答该问题所需的关键信息"。
规则:
1. 若文档能回答, 必须从文档原文中逐字摘录证据句(evidence, 15-80字的连续片段), 这是硬性要求
2. 注意产品型号匹配: 问题问某型号, 文档若明确属于其他型号且参数不同, 不算回答
3. 若文档不含答案, evidence 留空字符串, 并在 reason 说明答案缺了什么
只输出 JSON: {{"p": 0.0-1.0, "grade": "full|partial|none", "evidence": "...", "reason": "20字内"}}
问题: {query}
文档所属产品域: {products}
文档内容:
---
{text}
---"""
def chat(model, prompt):
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
"temperature": 0.0, "max_tokens": 300,
"thinking": {"type": "disabled"}}).encode()
req = urllib.request.Request(CFG["gateway"]["base_url"].rstrip("/") + "/chat/completions", data=body,
headers={"Content-Type": "application/json",
"Authorization": "Bearer " + CFG["gateway"]["api_key"]})
for i in range(4):
try:
with urllib.request.urlopen(req, timeout=120) as r:
return json.loads(r.read())["choices"][0]["message"]["content"]
except Exception:
if i == 3:
raise
time.sleep(2 ** i)
def parse(content):
try:
content = content.strip()
if content.startswith("```"):
content = content.split("```")[1].removeprefix("json").strip()
d = json.loads(content[content.index("{"):content.rindex("}") + 1])
return {"p": max(0.0, min(1.0, float(d.get("p", 0)))),
"evidence": str(d.get("evidence", ""))[:200]}
except Exception:
return None
def main():
queries = json.load(open(os.path.join(HERE, "..", "data", "real_queries.json"), encoding="utf-8"))
docs = [json.loads(l) for l in open(os.path.join(HERE, "..", "data", "corpus.jsonl"), encoding="utf-8")]
docs_t = {d["doc_id"]: set(bigrams(d["text"][:20000])) for d in docs}
df = Counter()
for s in docs_t.values():
df.update(s)
N = len(docs_t)
avgdl = sum(len(s) for s in docs_t.values()) / max(N, 1)
idf = {t: log(1 + (N - n + 0.5) / (n + 0.5)) for t, n in df.items()}
norm_texts = {d["doc_id"]: norm(d["text"]) for d in docs}
cands = []
seen_q = set()
for q in queries:
if q in seen_q:
continue
seen_q.add(q)
for did in bm25_top(q, docs_t, idf, avgdl, k=4):
cands.append((q, did))
print(f"真实问法 {len(seen_q)} 条 × BM25 top4 → {len(cands)} 候选对, 双裁判标注中...")
def work(pair):
q, did = pair
doc = next(d for d in docs if d["doc_id"] == did)
rs = []
for jm in JUDGES:
try:
rs.append(parse(chat(jm, PROMPT.format(query=q, products=",".join(doc["products"]),
text=doc["text"][:4500]))))
except Exception:
rs.append(None)
ev_ok = [bool(r and len(norm(r["evidence"])) >= 10 and norm(r["evidence"]) in norm_texts[did])
for r in rs]
return q, did, rs, ev_ok
out = []
stat = Counter()
with ThreadPoolExecutor(max_workers=CFG["gateway"].get("concurrency", 4)) as ex:
futs = [ex.submit(work, c) for c in cands]
for i, f in enumerate(as_completed(futs)):
q, did, rs, ev_ok = f.result()
ps = [r["p"] for r in rs if r]
if len(ps) < 2:
stat["judge_fail"] += 1
continue
ev_ok_any = any(ev_ok)
if all(p >= 0.7 for p in ps) and ev_ok_any:
lab, soft, cat = 1, round(sum(ps) / 2, 3), "pos"
elif all(p <= 0.3 for p in ps):
lab, soft, cat = 0, round(sum(ps) / 2, 3), "neg"
else:
stat["dropped_mid"] += 1
continue
stat[cat] += 1
out.append({"query": q, "doc_id": did, "hard_label": lab, "soft_label": soft,
"judges": rs, "evidence_ok": ev_ok_any,
"provenance": {"source": "real_query", "license": "internal"}})
if (i + 1) % 50 == 0:
print(f" {i+1}/{len(cands)} {dict(stat)}", flush=True)
with open(os.path.join(HERE, "..", "data", "real_query_pairs.jsonl"), "w", encoding="utf-8") as f:
for r in out:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
print(f"完成: {dict(stat)} → real_query_pairs.jsonl ({len(out)} 对)")
if __name__ == "__main__":
sys.exit(main())