- forum-solved-export: 3539 个已解决话题(问题+全部对话+采纳回答),含索引 - rdk-docs-export: 21 个文档子站 ~3430 篇 Markdown(源码仓库/抓取/反解析三种方式) - kb-eval: 352 条 RAG 检索评测集(答案全部锚定文档、多轮子代理交叉审计), 含检索/出题/校验工具与子代理提示词模板,支持增量扩产 - pipeline: 论坛枚举导出、文档抓取反解析、覆盖率统计全部脚本(可重跑)
35 lines
1.3 KiB
Python
35 lines
1.3 KiB
Python
"""Append items with mandatory verification: verify_phrases must hit expected_docs."""
|
|
import json, sys, os, re
|
|
|
|
ITEMS = '/Users/pptg/workspace/demo/eval/kb-eval/items.jsonl'
|
|
CORPUS = '/Users/pptg/workspace/demo/eval/rdk-docs-export'
|
|
|
|
raw = sys.stdin.read()
|
|
items = json.loads(raw)
|
|
ok = fail = 0
|
|
for it in items:
|
|
phrases = it.pop('verify_phrases', [])
|
|
good = True
|
|
for ph in phrases:
|
|
hit = False
|
|
for doc in it['expected_docs']:
|
|
p = os.path.join(CORPUS, doc)
|
|
if os.path.exists(p):
|
|
t = open(p, encoding='utf-8', errors='replace').read().lower()
|
|
if ph.lower() in t: hit = True; break
|
|
if not hit:
|
|
print(f"VERIFY-FAIL [{it.get('source_topic')}] phrase not in docs: {ph!r}", file=sys.stderr)
|
|
good = False
|
|
if not good:
|
|
fail += 1; continue
|
|
for doc in it['expected_docs']:
|
|
if not os.path.exists(os.path.join(CORPUS, doc)):
|
|
print(f"DOC-MISS [{it.get('source_topic')}]: {doc}", file=sys.stderr)
|
|
fail += 1; good = False; break
|
|
if not good: continue
|
|
with open(ITEMS, 'a') as f:
|
|
f.write(json.dumps(it, ensure_ascii=False) + '\n')
|
|
ok += 1
|
|
n = sum(1 for _ in open(ITEMS)) if os.path.exists(ITEMS) else 0
|
|
print(f"appended {ok}, rejected {fail}, total now {n}")
|