96 lines
4.2 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""03_mine_negatives.py — BM25(char-bigram) 检索: 难负例挖掘 + 未锚定题的 LLM 候选队列
输出:
data/negatives_eval.json 每题 {qid, hard:[], medium:[], easy:[], candidates_top10:[]}
data/llm_anchor_queue.json 未锚定/短答案题 + BM25 top10 候选, 待 LLM 挑选正例
"""
import json, os, random, re, sys
from collections import Counter
from math import log
HERE = os.path.dirname(os.path.abspath(__file__))
CORPUS = os.path.join(HERE, "..", "data", "corpus.jsonl")
ANCHORS = os.path.join(HERE, "..", "data", "eval_anchors.json")
OUT_NEG = os.path.join(HERE, "..", "data", "negatives_eval.json")
OUT_QUEUE = os.path.join(HERE, "..", "data", "llm_anchor_queue.json")
PUNCT = re.compile(r"[\s,。!?;:、\"'“”‘’()()\[\]【】《》<>\-—·…,.:;!?|\\/]+")
def norm(s): return PUNCT.sub("", s).lower()
def bigrams(text):
t = norm(text)
if len(t) < 2: return [t] if t else []
return [t[i:i+2] for i in range(len(t)-1)] + re.findall(r"[a-z0-9]{2,}", t)
def bm25_index(docs):
docs_t = {d["doc_id"]: set(bigrams(d["text"][:20000])) for d in docs}
df = Counter()
for s in docs_t.values():
df.update(s)
N, avgdl = len(docs_t), sum(len(s) for s in docs_t.values()) / max(len(docs_t), 1)
idf = {t: log(1 + (N - n + 0.5) / (n + 0.5)) for t, n in df.items()}
return docs_t, idf, avgdl
def bm25_search(query, docs_t, idf, avgdl, k=10, ok_filter=None):
q = set(bigrams(query))
scores = []
for did, s in docs_t.items():
if ok_filter and not ok_filter(did):
continue
dl, sc = len(s), 0.0
for t in q:
if t in s and t in idf:
sc += idf[t] * (s.count(t) if False else 1) * (2.2) / (1 + 0.25 + 0.75 * dl / avgdl)
if sc > 0:
scores.append((sc, did))
scores.sort(reverse=True)
return [d for _, d in scores[:k]]
MODEL2PRODUCT = {"rdk x3": "x3", "rdk x3 module": "x3-module", "rdk x5": "x5",
"rdk x5 module": "x5-module", "rdk s100": "s100", "rdk s100p": "s100p",
"rdk s600": "s600"}
def main():
random.seed(20261014)
docs = [json.loads(l) for l in open(CORPUS, encoding="utf-8")]
anchors = json.load(open(ANCHORS, encoding="utf-8"))
print("建 BM25 索引...")
docs_t, idf, avgdl = bm25_index(docs)
by_id = {d["doc_id"]: d for d in docs}
neg_out, queue = {}, []
for q in anchors["questions"]:
qid = q["qid"]
if q["excluded_reason"]:
continue
model = (q.get("model") or "").strip().lower()
prod = MODEL2PRODUCT.get(model)
pos_ids = {m["doc_id"] for m in q["matched_docs"]}
in_scope = lambda did: (prod in (by_id[did]["products"]) or
"common" in did or "families" in did)
top = bm25_search(q["question"], docs_t, idf, avgdl, k=12)
top_inscope = [d for d in top if in_scope(d)]
if not q["matched_docs"]:
queue.append({"qid": qid, "model": q.get("model"), "question": q["question"],
"reference_answer": q.get("reference_answer"),
"candidates": top_inscope[:10] or top[:10]})
hard = [d for d in top_inscope if d not in pos_ids][:5]
pool_same = [d["doc_id"] for d in docs if in_scope(d["doc_id"]) and d["doc_id"] not in pos_ids]
pool_cross = [d["doc_id"] for d in docs if prod and prod not in d["products"] and "common" not in d["doc_id"]]
neg_out[qid] = {
"hard": hard,
"medium": random.sample([d for d in pool_same if d not in hard], min(3, len(pool_same))),
"easy": random.sample(pool_cross, min(3, len(pool_cross))),
"candidates_top10": top_inscope[:10],
}
json.dump(neg_out, open(OUT_NEG, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
json.dump({"meta": {"queue_size": len(queue)}, "questions": queue},
open(OUT_QUEUE, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
n_hard = sum(len(v["hard"]) for v in neg_out.values())
print(f"负例: {len(neg_out)} 题, 难负例 {n_hard} 条 → negatives_eval.json")
print(f"LLM 锚定队列: {len(queue)} 题 (每题附 BM10 候选) → llm_anchor_queue.json")
if __name__ == "__main__":
sys.exit(main())