- forum-solved-export: 3539 个已解决话题(问题+全部对话+采纳回答),含索引 - rdk-docs-export: 21 个文档子站 ~3430 篇 Markdown(源码仓库/抓取/反解析三种方式) - kb-eval: 352 条 RAG 检索评测集(答案全部锚定文档、多轮子代理交叉审计), 含检索/出题/校验工具与子代理提示词模板,支持增量扩产 - pipeline: 论坛枚举导出、文档抓取反解析、覆盖率统计全部脚本(可重跑)
126 lines
6.1 KiB
Python
126 lines
6.1 KiB
Python
import re, os, json, csv
|
|
import os as _os
|
|
ROOT = _os.environ.get('EVAL_ROOT', _os.abspath(_os.path.join(_os.path.dirname(__file__), '..', '..')))
|
|
|
|
SITES = ["rdk_x_doc","rdk_s_doc","rdk_studio_doc","tros_doc","model_zoo_doc","case_doc",
|
|
"magicbox_doc","xburn_doc","accessories_doc","accessories_bmi088_doc",
|
|
"accessories_stereo_camera_doc","accessories_audio_kit_doc","x5_cases_doc",
|
|
"tros_vims_doc","rdk_doc"]
|
|
EXPORT = _os.path.join(ROOT, 'rdk-docs-export')
|
|
NUM = re.compile(r'^(\d+)(?:[-_.]|\s)+(.*)$')
|
|
def strip_num(s):
|
|
m = NUM.match(s)
|
|
return m.group(2) if m else s
|
|
|
|
def parse_fm(path):
|
|
try: txt = open(path, encoding='utf-8', errors='replace').read(4000)
|
|
except Exception: return {}
|
|
if not txt.startswith('---'): return {}
|
|
end = txt.find('\n---', 3)
|
|
if end < 0: return {}
|
|
fm = {}
|
|
for line in txt[3:end].splitlines():
|
|
m = re.match(r'^(\w[\w-]*):\s*(.+?)\s*$', line)
|
|
if m: fm[m.group(1)] = m.group(2).strip('"\'')
|
|
return fm
|
|
|
|
def load_sitemap(site):
|
|
f = f_os.path.join(ROOT, "'_work'", 'sitemaps', '{site}.xml')
|
|
if not os.path.exists(f): return []
|
|
raw = open(f, encoding='utf-8', errors='replace').read()
|
|
return [u.replace('https://developer.d-robotics.cc','') for u in re.findall(r'<loc>([^<]+)</loc>', raw)]
|
|
|
|
def build_indexes(root):
|
|
"""file path -> url candidates; plus lookup structures"""
|
|
file_urls = {} # relpath -> list of urls
|
|
for sub, en in (('docs', False), (os.path.join('i18n','en','docusaurus-plugin-content-docs','current'), True)):
|
|
droot = os.path.join(root, sub)
|
|
if not os.path.isdir(droot): continue
|
|
for dp, dn, fn in os.walk(droot):
|
|
rel_dir = os.path.relpath(dp, droot)
|
|
parts = [] if rel_dir=='.' else rel_dir.split(os.sep)
|
|
for f in fn:
|
|
if not f.endswith(('.md','.mdx')): continue
|
|
rel = os.path.join(sub, os.path.relpath(os.path.join(dp,f), root)) if False else os.path.relpath(os.path.join(dp,f), root)
|
|
name = f.rsplit('.',1)[0]
|
|
fm = parse_fm(os.path.join(dp,f))
|
|
slug = fm.get('slug')
|
|
# candidate URLs: with prefixes (raw) and stripped
|
|
raw_segs = parts + ([] if name in ('index','_index') else [name])
|
|
strip_segs = [strip_num(s) for s in raw_segs]
|
|
cands = []
|
|
for segs in (raw_segs, strip_segs):
|
|
u = '/'.join(segs)
|
|
if u: cands.append(u)
|
|
if slug:
|
|
s = slug.strip('/')
|
|
cands.append(s)
|
|
# absolute slug replaces tail
|
|
if slug.startswith('/') and s:
|
|
cands.append(s)
|
|
elif segs and s:
|
|
cands.append('/'.join(raw_segs[:-1] + s.split('/')))
|
|
file_urls[rel] = cands
|
|
return file_urls
|
|
|
|
results = {}
|
|
for site in SITES:
|
|
root = os.path.join(EXPORT, site)
|
|
urls = load_sitemap(site)
|
|
norm = []
|
|
for u in urls:
|
|
u = u.rstrip('/') or f'/{site}'
|
|
norm.append(u)
|
|
file_urls = build_indexes(root)
|
|
# url -> files
|
|
url2file = {}
|
|
for rel, cands in file_urls.items():
|
|
for c in cands:
|
|
for u in norm:
|
|
rest = u[len('/'+site):].strip('/')
|
|
if rest.lower() == c.lower():
|
|
url2file[u] = rel
|
|
# fallback fuzzy: last-segment match
|
|
fuzzy = {}
|
|
files_by_lastseg = {}
|
|
for rel, cands in file_urls.items():
|
|
base = rel.rsplit('/',1)[-1].rsplit('.',1)[0]
|
|
files_by_lastseg.setdefault(strip_num(base).lower(), []).append(rel)
|
|
dirs_by_lastseg = {}
|
|
for dp, dn, fn in os.walk(root):
|
|
for d in dn:
|
|
if d.startswith('.') or d in ('node_modules',): continue
|
|
dirs_by_lastseg.setdefault(strip_num(d).lower(), []).append(os.path.relpath(os.path.join(dp,d), root))
|
|
stats = {"live": len(norm), "sourced": 0, "alias": 0, "category": 0, "generated": 0, "unmatched": []}
|
|
for u in norm:
|
|
if u in url2file: stats['sourced'] += 1; continue
|
|
rest = u[len('/'+site):].strip('/')
|
|
if not rest or rest in ('search','markdown-page') or rest.startswith('blog') or rest.startswith('en/blog'):
|
|
stats['generated'] += 1; continue
|
|
last = strip_num(rest.split('/')[-1]).lower()
|
|
en_rest = rest[3:] if rest.startswith('en/') else rest
|
|
last_en = strip_num(en_rest.split('/')[-1]).lower()
|
|
if last_en in files_by_lastseg or last in files_by_lastseg:
|
|
stats['alias'] += 1
|
|
elif last_en in dirs_by_lastseg or last in dirs_by_lastseg:
|
|
stats['category'] += 1
|
|
else:
|
|
stats['unmatched'].append(u)
|
|
n_src = sum(1 for _ in file_urls)
|
|
results[site] = (stats, n_src)
|
|
|
|
rows = []
|
|
for site,(s,nsrc) in results.items():
|
|
rows.append({"site": site, "live_sitemap_urls": s['live'], "urls_backed_by_source_md": s['sourced']+s['alias'],
|
|
"direct": s['sourced'], "alias_route": s['alias'], "category_index_urls": s['category'],
|
|
"generated_urls": s['generated'], "unmatched_urls": len(s['unmatched']),
|
|
"source_md_files_in_repo": nsrc})
|
|
with open(os.path.join(EXPORT,'coverage.csv'),'w',newline='') as f:
|
|
w = csv.DictWriter(f, fieldnames=list(rows[0].keys())); w.writeheader(); w.writerows(rows)
|
|
print(f"{'site':32s} {'live':>5s} {'backed':>7s} {'direct':>7s} {'alias':>6s} {'catIdx':>7s} {'gen':>4s} {'unmat':>6s} {'srcFiles':>8s}")
|
|
for r in rows:
|
|
print(f"{r['site']:32s} {r['live_sitemap_urls']:>5d} {r['urls_backed_by_source_md']:>7d} {r['direct']:>7d} {r['alias_route']:>6d} {r['category_index_urls']:>7d} {r['generated_urls']:>4d} {r['unmatched_urls']:>6d} {r['source_md_files_in_repo']:>8d}")
|
|
t = lambda k: sum(r[k] for r in rows)
|
|
print(f"\nTOTAL live={t('live_sitemap_urls')} backed={t('urls_backed_by_source_md')} catIdx={t('category_index_urls')} gen={t('generated_urls')} unmatched={t('unmatched_urls')} srcFiles={t('source_md_files_in_repo')}")
|
|
json.dump({k: v[0]['unmatched'] for k,v in results.items() if v[0]['unmatched']}, open('/tmp/unmatched.json','w'), ensure_ascii=False, indent=1)
|