- forum-solved-export: 3539 个已解决话题(问题+全部对话+采纳回答),含索引 - rdk-docs-export: 21 个文档子站 ~3430 篇 Markdown(源码仓库/抓取/反解析三种方式) - kb-eval: 352 条 RAG 检索评测集(答案全部锚定文档、多轮子代理交叉审计), 含检索/出题/校验工具与子代理提示词模板,支持增量扩产 - pipeline: 论坛枚举导出、文档抓取反解析、覆盖率统计全部脚本(可重跑)
92 lines
4.1 KiB
Python
92 lines
4.1 KiB
Python
"""从论坛已解决帖生成出题候选摘要(按波次)。
|
||
|
||
用法:
|
||
python3 make_digests.py --out digestsN --mode answer|question [--top N] [--batch 30]
|
||
|
||
两种筛选模式:
|
||
answer —— 用「采纳回答 ↔ 文档」BM25 匹配度排序(常规波次)
|
||
question —— 用「标题+提问 ↔ 文档」匹配度排序(用于采纳回答过短/无效的存量帖)
|
||
|
||
自动排除:已出题的帖子(items.jsonl 的 source_topic)与已生成过摘要的帖子
|
||
(digests*/ 目录中 [id] 标记)。产出摘要文件可直接交给出题子代理
|
||
(提示词模板见 AGENT_PROMPT_TEMPLATE.md)。
|
||
"""
|
||
import os, re, sys, json, tarfile, html, argparse
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
from docsearch import Index
|
||
|
||
ROOT = os.environ.get('EVAL_ROOT', os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||
ARCHIVE = os.path.join(ROOT, 'forum-solved-export', 'raw.jsons.tar.gz')
|
||
|
||
def strip_html(h):
|
||
t = re.sub(r'<br\s*/?>', ' ', h or '')
|
||
t = re.sub(r'<[^>]+>', ' ', t)
|
||
return re.sub(r'\s+', ' ', html.unescape(t)).strip()
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument('--out', required=True, help='摘要输出目录,如 digests6')
|
||
ap.add_argument('--mode', choices=['answer', 'question'], default='answer')
|
||
ap.add_argument('--top', type=int, default=1000)
|
||
ap.add_argument('--batch', type=int, default=30)
|
||
ap.add_argument('--min-answer', type=int, default=60)
|
||
args = ap.parse_args()
|
||
|
||
used = set()
|
||
items_path = os.path.join(os.path.dirname(__file__), 'items.jsonl')
|
||
if os.path.exists(items_path):
|
||
used = {json.loads(l)['source_topic'] for l in open(items_path)}
|
||
digested = set()
|
||
here = os.path.dirname(os.path.abspath(__file__))
|
||
for d in [x for x in os.listdir(here) if x.startswith('digests') and os.path.isdir(os.path.join(here, x))]:
|
||
for f in os.listdir(os.path.join(here, d)):
|
||
for line in open(os.path.join(here, d, f)):
|
||
m = re.match(r'===== \[(\d+)\]', line)
|
||
if m: digested.add(int(m.group(1)))
|
||
exclude = used | digested
|
||
print(f'排除:已出题 {len(used)},已摘要 {len(digested)}')
|
||
|
||
posts = []
|
||
with tarfile.open(ARCHIVE, 'r:gz') as tf:
|
||
for m in tf:
|
||
base = m.name.rsplit('/', 1)[-1]
|
||
if base.startswith('._') or not m.name.endswith('.json'): continue
|
||
rec = json.loads(tf.extractfile(m).read().decode('utf-8'))
|
||
if rec['id'] in exclude: continue
|
||
acc = rec.get('accepted_answer') or {}
|
||
pn = acc.get('post_number')
|
||
ans = next((p for p in rec['posts'] if p.get('post_number') == pn), None)
|
||
q = rec['posts'][0] if rec['posts'] else {}
|
||
posts.append({'id': rec['id'], 'title': rec.get('title', ''), 'tags': rec.get('tags') or [],
|
||
'question': strip_html(q.get('cooked', ''))[:450],
|
||
'answer': strip_html(ans.get('cooked', ''))[:450] if ans else ''})
|
||
|
||
idx = Index(); idx.load()
|
||
scored = []
|
||
for p in posts:
|
||
text = p['answer'][:600] if args.mode == 'answer' else (p['title'] + ' ' + p['question'])[:600]
|
||
if args.mode == 'answer' and len(p['answer']) < args.min_answer:
|
||
continue
|
||
hits = idx.search(text, 2)
|
||
if hits:
|
||
p['score'] = hits[0][1]
|
||
scored.append(p)
|
||
scored.sort(key=lambda x: -x['score'])
|
||
sel = scored[:args.top]
|
||
print(f'池 {len(scored)} → 取前 {len(sel)}')
|
||
|
||
outdir = os.path.join(here, args.out)
|
||
os.makedirs(outdir, exist_ok=True)
|
||
for i in range(0, len(sel), args.batch):
|
||
with open(os.path.join(outdir, f'{args.out}_{i//args.batch:02d}.txt'), 'w') as f:
|
||
for p in sel[i:i+args.batch]:
|
||
f.write(f"===== [{p['id']}] {p['title']}\n")
|
||
f.write(f" tags: {','.join(p['tags'])}\n")
|
||
f.write(f" Q: {p['question'][:420]}\n")
|
||
f.write(f" A: {p['answer'][:420] or '(采纳回答过短)'}\n\n")
|
||
print(f'批次: {(len(sel)+args.batch-1)//args.batch},目录: {outdir}')
|
||
|
||
if __name__ == '__main__':
|
||
main()
|