- forum-solved-export: 3539 个已解决话题(问题+全部对话+采纳回答),含索引 - rdk-docs-export: 21 个文档子站 ~3430 篇 Markdown(源码仓库/抓取/反解析三种方式) - kb-eval: 352 条 RAG 检索评测集(答案全部锚定文档、多轮子代理交叉审计), 含检索/出题/校验工具与子代理提示词模板,支持增量扩产 - pipeline: 论坛枚举导出、文档抓取反解析、覆盖率统计全部脚本(可重跑)
128 lines
4.5 KiB
Python
128 lines
4.5 KiB
Python
"""BM25 search over the rdk-docs-export corpus + exact phrase verification."""
|
|
import os, re, sys, math, pickle, json
|
|
from collections import Counter, defaultdict
|
|
|
|
CORPUS = "/Users/pptg/workspace/demo/eval/rdk-docs-export"
|
|
IDX = "/Users/pptg/workspace/demo/eval/kb-eval/.idx.pkl"
|
|
|
|
_ascii_word = re.compile(r'[A-Za-z0-9_./+-]{2,}')
|
|
|
|
def tokenize(text):
|
|
toks = []
|
|
# keep ascii words/digits (commands, paths, product names)
|
|
for m in _ascii_word.finditer(text):
|
|
w = m.group(0).lower().strip('./+-')
|
|
if len(w) >= 2: toks.append(w)
|
|
# chinese: character bigrams + unigrams over CJK runs
|
|
for run in re.findall(r'[\u4e00-\u9fff]+', text):
|
|
toks.extend(run[i:i+2] for i in range(len(run)-1))
|
|
if len(run) == 1: toks.append(run)
|
|
return toks
|
|
|
|
def iter_docs():
|
|
for dp, dn, fn in os.walk(CORPUS):
|
|
dn[:] = [d for d in dn if not d.startswith('.') and d != '_uncovered_titles']
|
|
for f in fn:
|
|
if f.endswith(('.md', '.mdx')):
|
|
p = os.path.join(dp, f)
|
|
yield os.path.relpath(p, CORPUS), p
|
|
|
|
class Index:
|
|
def __init__(self):
|
|
self.files = [] # relpath list
|
|
self.df = Counter() # token -> doc freq
|
|
self.tf = [] # per-doc Counter
|
|
self.len = [] # doc lengths
|
|
self.texts = {} # relpath -> raw text (lazy load on demand instead)
|
|
|
|
def build(self):
|
|
n = 0
|
|
for rel, p in iter_docs():
|
|
try:
|
|
text = open(p, encoding='utf-8', errors='replace').read()
|
|
except Exception:
|
|
continue
|
|
toks = tokenize(text)
|
|
c = Counter(toks)
|
|
self.files.append(rel)
|
|
self.tf.append(c)
|
|
self.len.append(len(toks))
|
|
for t in c: self.df[t] += 1
|
|
n += 1
|
|
self.N = n
|
|
self.avgdl = sum(self.len) / max(n,1)
|
|
pickle.dump({'files': self.files, 'df': self.df, 'tf': self.tf, 'len': self.len, 'N': n, 'avgdl': self.avgdl},
|
|
open(IDX, 'wb'))
|
|
print(f"indexed {n} docs", file=sys.stderr)
|
|
|
|
def load(self):
|
|
d = pickle.load(open(IDX, 'rb'))
|
|
self.__dict__.update(d)
|
|
|
|
def search(self, query, topk=8):
|
|
toks = tokenize(query)
|
|
scores = defaultdict(float)
|
|
k1, b = 1.5, 0.75
|
|
for t in toks:
|
|
df = self.df.get(t, 0)
|
|
if df == 0: continue
|
|
idf = math.log(1 + (self.N - df + 0.5) / (df + 0.5))
|
|
for i, c in enumerate(self.tf):
|
|
f = c.get(t, 0)
|
|
if f == 0: continue
|
|
denom = f + k1 * (1 - b + b * self.len[i] / self.avgdl)
|
|
scores[i] += idf * f * (k1 + 1) / denom
|
|
ranked = sorted(scores.items(), key=lambda kv: -kv[1])[:topk]
|
|
return [(self.files[i], round(s, 2)) for i, s in ranked]
|
|
|
|
def phrase_hits(phrase, max_hits=20):
|
|
"""exact substring search across corpus (case-insensitive), returns relpath:count"""
|
|
ph = phrase.lower()
|
|
out = []
|
|
for rel, p in iter_docs():
|
|
try:
|
|
t = open(p, encoding='utf-8', errors='replace').read().lower()
|
|
except Exception:
|
|
continue
|
|
c = t.count(ph)
|
|
if c: out.append((rel, c))
|
|
if len(out) >= max_hits: break
|
|
return out
|
|
|
|
def grep_doc(rel, pattern, ctx=3):
|
|
"""show pattern matches with context in one doc"""
|
|
p = os.path.join(CORPUS, rel)
|
|
if not os.path.exists(p):
|
|
print(f" (no file {rel})"); return
|
|
lines = open(p, encoding='utf-8', errors='replace').read().splitlines()
|
|
pat = re.compile(re.escape(pattern), re.I)
|
|
shown = 0
|
|
for i, ln in enumerate(lines):
|
|
if pat.search(ln):
|
|
lo, hi = max(0, i-ctx), min(len(lines), i+ctx+1)
|
|
print(f" --- {rel}:{i+1}")
|
|
for j in range(lo, hi):
|
|
print(f" {lines[j][:150]}")
|
|
shown += 1
|
|
if shown >= 3: break
|
|
if not shown: print(f" (no match of {pattern!r} in {rel})")
|
|
|
|
if __name__ == '__main__':
|
|
cmd = sys.argv[1]
|
|
idx = Index()
|
|
idx.load()
|
|
if cmd == 's':
|
|
for q in sys.argv[2:]:
|
|
print(f"### Q: {q}")
|
|
for f, s in idx.search(q, 8):
|
|
print(f" {s:>7} {f}")
|
|
elif cmd == 'g':
|
|
rel, pattern = sys.argv[2], sys.argv[3]
|
|
grep_doc(rel, pattern)
|
|
elif cmd == 'p':
|
|
for ph in sys.argv[2:]:
|
|
print(f"### P: {ph}")
|
|
hits = phrase_hits(ph)
|
|
if not hits: print(" (no hits)")
|
|
for f, c in hits: print(f" {c}x {f}")
|