kb_eval_docs/kb-eval/make_digests.py
peng01.guo 15b3337339 RDK 知识库数据集:论坛帖子导出 + 文档中心 Markdown + RAG 评测集(352 条)及全量生产管线
- forum-solved-export: 3539 个已解决话题(问题+全部对话+采纳回答),含索引
- rdk-docs-export: 21 个文档子站 ~3430 篇 Markdown(源码仓库/抓取/反解析三种方式)
- kb-eval: 352 条 RAG 检索评测集(答案全部锚定文档、多轮子代理交叉审计),
  含检索/出题/校验工具与子代理提示词模板,支持增量扩产
- pipeline: 论坛枚举导出、文档抓取反解析、覆盖率统计全部脚本(可重跑)
2026-09-18 17:14:40 +08:00

92 lines
4.1 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""从论坛已解决帖生成出题候选摘要(按波次)。
用法:
python3 make_digests.py --out digestsN --mode answer|question [--top N] [--batch 30]
两种筛选模式:
answer —— 用「采纳回答 ↔ 文档」BM25 匹配度排序(常规波次)
question —— 用「标题+提问 ↔ 文档」匹配度排序(用于采纳回答过短/无效的存量帖)
自动排除:已出题的帖子(items.jsonl 的 source_topic)与已生成过摘要的帖子
(digests*/ 目录中 [id] 标记)。产出摘要文件可直接交给出题子代理
(提示词模板见 AGENT_PROMPT_TEMPLATE.md)。
"""
import os, re, sys, json, tarfile, html, argparse
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from docsearch import Index
ROOT = os.environ.get('EVAL_ROOT', os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
ARCHIVE = os.path.join(ROOT, 'forum-solved-export', 'raw.jsons.tar.gz')
def strip_html(h):
t = re.sub(r'<br\s*/?>', ' ', h or '')
t = re.sub(r'<[^>]+>', ' ', t)
return re.sub(r'\s+', ' ', html.unescape(t)).strip()
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--out', required=True, help='摘要输出目录,如 digests6')
ap.add_argument('--mode', choices=['answer', 'question'], default='answer')
ap.add_argument('--top', type=int, default=1000)
ap.add_argument('--batch', type=int, default=30)
ap.add_argument('--min-answer', type=int, default=60)
args = ap.parse_args()
used = set()
items_path = os.path.join(os.path.dirname(__file__), 'items.jsonl')
if os.path.exists(items_path):
used = {json.loads(l)['source_topic'] for l in open(items_path)}
digested = set()
here = os.path.dirname(os.path.abspath(__file__))
for d in [x for x in os.listdir(here) if x.startswith('digests') and os.path.isdir(os.path.join(here, x))]:
for f in os.listdir(os.path.join(here, d)):
for line in open(os.path.join(here, d, f)):
m = re.match(r'===== \[(\d+)\]', line)
if m: digested.add(int(m.group(1)))
exclude = used | digested
print(f'排除:已出题 {len(used)},已摘要 {len(digested)}')
posts = []
with tarfile.open(ARCHIVE, 'r:gz') as tf:
for m in tf:
base = m.name.rsplit('/', 1)[-1]
if base.startswith('._') or not m.name.endswith('.json'): continue
rec = json.loads(tf.extractfile(m).read().decode('utf-8'))
if rec['id'] in exclude: continue
acc = rec.get('accepted_answer') or {}
pn = acc.get('post_number')
ans = next((p for p in rec['posts'] if p.get('post_number') == pn), None)
q = rec['posts'][0] if rec['posts'] else {}
posts.append({'id': rec['id'], 'title': rec.get('title', ''), 'tags': rec.get('tags') or [],
'question': strip_html(q.get('cooked', ''))[:450],
'answer': strip_html(ans.get('cooked', ''))[:450] if ans else ''})
idx = Index(); idx.load()
scored = []
for p in posts:
text = p['answer'][:600] if args.mode == 'answer' else (p['title'] + ' ' + p['question'])[:600]
if args.mode == 'answer' and len(p['answer']) < args.min_answer:
continue
hits = idx.search(text, 2)
if hits:
p['score'] = hits[0][1]
scored.append(p)
scored.sort(key=lambda x: -x['score'])
sel = scored[:args.top]
print(f'池 {len(scored)} → 取前 {len(sel)}')
outdir = os.path.join(here, args.out)
os.makedirs(outdir, exist_ok=True)
for i in range(0, len(sel), args.batch):
with open(os.path.join(outdir, f'{args.out}_{i//args.batch:02d}.txt'), 'w') as f:
for p in sel[i:i+args.batch]:
f.write(f"===== [{p['id']}] {p['title']}\n")
f.write(f" tags: {','.join(p['tags'])}\n")
f.write(f" Q: {p['question'][:420]}\n")
f.write(f" A: {p['answer'][:420] or '(采纳回答过短)'}\n\n")
print(f'批次: {(len(sel)+args.batch-1)//args.batch},目录: {outdir}')
if __name__ == '__main__':
main()