96 lines
4.2 KiB
Python
96 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
||
"""03_mine_negatives.py — BM25(char-bigram) 检索: 难负例挖掘 + 未锚定题的 LLM 候选队列
|
||
输出:
|
||
data/negatives_eval.json 每题 {qid, hard:[], medium:[], easy:[], candidates_top10:[]}
|
||
data/llm_anchor_queue.json 未锚定/短答案题 + BM25 top10 候选, 待 LLM 挑选正例
|
||
"""
|
||
import json, os, random, re, sys
|
||
from collections import Counter
|
||
from math import log
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
CORPUS = os.path.join(HERE, "..", "data", "corpus.jsonl")
|
||
ANCHORS = os.path.join(HERE, "..", "data", "eval_anchors.json")
|
||
OUT_NEG = os.path.join(HERE, "..", "data", "negatives_eval.json")
|
||
OUT_QUEUE = os.path.join(HERE, "..", "data", "llm_anchor_queue.json")
|
||
|
||
PUNCT = re.compile(r"[\s,。!?;:、\"'“”‘’()()\[\]【】《》<>\-—·…,.:;!?|\\/]+")
|
||
def norm(s): return PUNCT.sub("", s).lower()
|
||
|
||
def bigrams(text):
|
||
t = norm(text)
|
||
if len(t) < 2: return [t] if t else []
|
||
return [t[i:i+2] for i in range(len(t)-1)] + re.findall(r"[a-z0-9]{2,}", t)
|
||
|
||
def bm25_index(docs):
|
||
docs_t = {d["doc_id"]: set(bigrams(d["text"][:20000])) for d in docs}
|
||
df = Counter()
|
||
for s in docs_t.values():
|
||
df.update(s)
|
||
N, avgdl = len(docs_t), sum(len(s) for s in docs_t.values()) / max(len(docs_t), 1)
|
||
idf = {t: log(1 + (N - n + 0.5) / (n + 0.5)) for t, n in df.items()}
|
||
return docs_t, idf, avgdl
|
||
|
||
def bm25_search(query, docs_t, idf, avgdl, k=10, ok_filter=None):
|
||
q = set(bigrams(query))
|
||
scores = []
|
||
for did, s in docs_t.items():
|
||
if ok_filter and not ok_filter(did):
|
||
continue
|
||
dl, sc = len(s), 0.0
|
||
for t in q:
|
||
if t in s and t in idf:
|
||
sc += idf[t] * (s.count(t) if False else 1) * (2.2) / (1 + 0.25 + 0.75 * dl / avgdl)
|
||
if sc > 0:
|
||
scores.append((sc, did))
|
||
scores.sort(reverse=True)
|
||
return [d for _, d in scores[:k]]
|
||
|
||
MODEL2PRODUCT = {"rdk x3": "x3", "rdk x3 module": "x3-module", "rdk x5": "x5",
|
||
"rdk x5 module": "x5-module", "rdk s100": "s100", "rdk s100p": "s100p",
|
||
"rdk s600": "s600"}
|
||
|
||
def main():
|
||
random.seed(20261014)
|
||
docs = [json.loads(l) for l in open(CORPUS, encoding="utf-8")]
|
||
anchors = json.load(open(ANCHORS, encoding="utf-8"))
|
||
print("建 BM25 索引...")
|
||
docs_t, idf, avgdl = bm25_index(docs)
|
||
by_id = {d["doc_id"]: d for d in docs}
|
||
|
||
neg_out, queue = {}, []
|
||
for q in anchors["questions"]:
|
||
qid = q["qid"]
|
||
if q["excluded_reason"]:
|
||
continue
|
||
model = (q.get("model") or "").strip().lower()
|
||
prod = MODEL2PRODUCT.get(model)
|
||
pos_ids = {m["doc_id"] for m in q["matched_docs"]}
|
||
in_scope = lambda did: (prod in (by_id[did]["products"]) or
|
||
"common" in did or "families" in did)
|
||
top = bm25_search(q["question"], docs_t, idf, avgdl, k=12)
|
||
top_inscope = [d for d in top if in_scope(d)]
|
||
if not q["matched_docs"]:
|
||
queue.append({"qid": qid, "model": q.get("model"), "question": q["question"],
|
||
"reference_answer": q.get("reference_answer"),
|
||
"candidates": top_inscope[:10] or top[:10]})
|
||
hard = [d for d in top_inscope if d not in pos_ids][:5]
|
||
pool_same = [d["doc_id"] for d in docs if in_scope(d["doc_id"]) and d["doc_id"] not in pos_ids]
|
||
pool_cross = [d["doc_id"] for d in docs if prod and prod not in d["products"] and "common" not in d["doc_id"]]
|
||
neg_out[qid] = {
|
||
"hard": hard,
|
||
"medium": random.sample([d for d in pool_same if d not in hard], min(3, len(pool_same))),
|
||
"easy": random.sample(pool_cross, min(3, len(pool_cross))),
|
||
"candidates_top10": top_inscope[:10],
|
||
}
|
||
|
||
json.dump(neg_out, open(OUT_NEG, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
||
json.dump({"meta": {"queue_size": len(queue)}, "questions": queue},
|
||
open(OUT_QUEUE, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
||
n_hard = sum(len(v["hard"]) for v in neg_out.values())
|
||
print(f"负例: {len(neg_out)} 题, 难负例 {n_hard} 条 → negatives_eval.json")
|
||
print(f"LLM 锚定队列: {len(queue)} 题 (每题附 BM10 候选) → llm_anchor_queue.json")
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|