kb_eval_docs/kb-eval/docsearch.py
peng01.guo 15b3337339 RDK 知识库数据集:论坛帖子导出 + 文档中心 Markdown + RAG 评测集(352 条)及全量生产管线
- forum-solved-export: 3539 个已解决话题(问题+全部对话+采纳回答),含索引
- rdk-docs-export: 21 个文档子站 ~3430 篇 Markdown(源码仓库/抓取/反解析三种方式)
- kb-eval: 352 条 RAG 检索评测集(答案全部锚定文档、多轮子代理交叉审计),
  含检索/出题/校验工具与子代理提示词模板,支持增量扩产
- pipeline: 论坛枚举导出、文档抓取反解析、覆盖率统计全部脚本(可重跑)
2026-09-18 17:14:40 +08:00

128 lines
4.5 KiB
Python

"""BM25 search over the rdk-docs-export corpus + exact phrase verification."""
import os, re, sys, math, pickle, json
from collections import Counter, defaultdict
CORPUS = "/Users/pptg/workspace/demo/eval/rdk-docs-export"
IDX = "/Users/pptg/workspace/demo/eval/kb-eval/.idx.pkl"
_ascii_word = re.compile(r'[A-Za-z0-9_./+-]{2,}')
def tokenize(text):
toks = []
# keep ascii words/digits (commands, paths, product names)
for m in _ascii_word.finditer(text):
w = m.group(0).lower().strip('./+-')
if len(w) >= 2: toks.append(w)
# chinese: character bigrams + unigrams over CJK runs
for run in re.findall(r'[\u4e00-\u9fff]+', text):
toks.extend(run[i:i+2] for i in range(len(run)-1))
if len(run) == 1: toks.append(run)
return toks
def iter_docs():
for dp, dn, fn in os.walk(CORPUS):
dn[:] = [d for d in dn if not d.startswith('.') and d != '_uncovered_titles']
for f in fn:
if f.endswith(('.md', '.mdx')):
p = os.path.join(dp, f)
yield os.path.relpath(p, CORPUS), p
class Index:
def __init__(self):
self.files = [] # relpath list
self.df = Counter() # token -> doc freq
self.tf = [] # per-doc Counter
self.len = [] # doc lengths
self.texts = {} # relpath -> raw text (lazy load on demand instead)
def build(self):
n = 0
for rel, p in iter_docs():
try:
text = open(p, encoding='utf-8', errors='replace').read()
except Exception:
continue
toks = tokenize(text)
c = Counter(toks)
self.files.append(rel)
self.tf.append(c)
self.len.append(len(toks))
for t in c: self.df[t] += 1
n += 1
self.N = n
self.avgdl = sum(self.len) / max(n,1)
pickle.dump({'files': self.files, 'df': self.df, 'tf': self.tf, 'len': self.len, 'N': n, 'avgdl': self.avgdl},
open(IDX, 'wb'))
print(f"indexed {n} docs", file=sys.stderr)
def load(self):
d = pickle.load(open(IDX, 'rb'))
self.__dict__.update(d)
def search(self, query, topk=8):
toks = tokenize(query)
scores = defaultdict(float)
k1, b = 1.5, 0.75
for t in toks:
df = self.df.get(t, 0)
if df == 0: continue
idf = math.log(1 + (self.N - df + 0.5) / (df + 0.5))
for i, c in enumerate(self.tf):
f = c.get(t, 0)
if f == 0: continue
denom = f + k1 * (1 - b + b * self.len[i] / self.avgdl)
scores[i] += idf * f * (k1 + 1) / denom
ranked = sorted(scores.items(), key=lambda kv: -kv[1])[:topk]
return [(self.files[i], round(s, 2)) for i, s in ranked]
def phrase_hits(phrase, max_hits=20):
"""exact substring search across corpus (case-insensitive), returns relpath:count"""
ph = phrase.lower()
out = []
for rel, p in iter_docs():
try:
t = open(p, encoding='utf-8', errors='replace').read().lower()
except Exception:
continue
c = t.count(ph)
if c: out.append((rel, c))
if len(out) >= max_hits: break
return out
def grep_doc(rel, pattern, ctx=3):
"""show pattern matches with context in one doc"""
p = os.path.join(CORPUS, rel)
if not os.path.exists(p):
print(f" (no file {rel})"); return
lines = open(p, encoding='utf-8', errors='replace').read().splitlines()
pat = re.compile(re.escape(pattern), re.I)
shown = 0
for i, ln in enumerate(lines):
if pat.search(ln):
lo, hi = max(0, i-ctx), min(len(lines), i+ctx+1)
print(f" --- {rel}:{i+1}")
for j in range(lo, hi):
print(f" {lines[j][:150]}")
shown += 1
if shown >= 3: break
if not shown: print(f" (no match of {pattern!r} in {rel})")
if __name__ == '__main__':
cmd = sys.argv[1]
idx = Index()
idx.load()
if cmd == 's':
for q in sys.argv[2:]:
print(f"### Q: {q}")
for f, s in idx.search(q, 8):
print(f" {s:>7} {f}")
elif cmd == 'g':
rel, pattern = sys.argv[2], sys.argv[3]
grep_doc(rel, pattern)
elif cmd == 'p':
for ph in sys.argv[2:]:
print(f"### P: {ph}")
hits = phrase_hits(ph)
if not hits: print(" (no hits)")
for f, c in hits: print(f" {c}x {f}")