kb_eval_docs/pipeline/docs/parse_sidebar.py
peng01.guo 15b3337339 RDK 知识库数据集:论坛帖子导出 + 文档中心 Markdown + RAG 评测集(352 条)及全量生产管线
- forum-solved-export: 3539 个已解决话题(问题+全部对话+采纳回答),含索引
- rdk-docs-export: 21 个文档子站 ~3430 篇 Markdown(源码仓库/抓取/反解析三种方式)
- kb-eval: 352 条 RAG 检索评测集(答案全部锚定文档、多轮子代理交叉审计),
  含检索/出题/校验工具与子代理提示词模板,支持增量扩产
- pipeline: 论坛枚举导出、文档抓取反解析、覆盖率统计全部脚本(可重跑)
2026-09-18 17:14:40 +08:00

65 lines
2.2 KiB
Python

import re, json, codecs
class P:
def __init__(self, s):
self.s = s; self.i = 0
def ws(self):
while self.i < len(self.s) and self.s[self.i] in ' \t\n\r': self.i += 1
def parse(self):
self.ws()
c = self.s[self.i]
if c == '{': return self.obj()
if c == '[': return self.arr()
if c == '"': return self.str()
# bare token (true/false/null/number/identifier)
m = re.match(r'[^,}\]\s:]+', self.s[self.i:])
self.i += len(m.group(0)); return m.group(0)
def obj(self):
self.i += 1; o = {}
while True:
self.ws()
if self.s[self.i] == '}': self.i += 1; return o
# key
if self.s[self.i] == '"':
k = self.str()
else:
m = re.match(r'[^:\s,{}]+', self.s[self.i:]); k = m.group(0); self.i += len(k)
self.ws()
assert self.s[self.i] == ':', f"expected : at {self.i}: {self.s[self.i:self.i+40]!r}"
self.i += 1
v = self.parse()
o[k] = v
self.ws()
if self.s[self.i] == ',': self.i += 1
def arr(self):
self.i += 1; a = []
while True:
self.ws()
if self.s[self.i] == ']': self.i += 1; return a
a.append(self.parse())
self.ws()
if self.s[self.i] == ',': self.i += 1
def str(self):
# JS string with escapes
assert self.s[self.i] == '"'
self.i += 1; out = []
while True:
c = self.s[self.i]
if c == '\\':
n = self.s[self.i+1]
if n == 'u':
out.append(chr(int(self.s[self.i+2:self.i+6], 16))); self.i += 6
else:
out.append({'n':'\n','t':'\t','r':'\r','"':'"','\\':'\\','/':'/'}.get(n, n)); self.i += 2
elif c == '"':
self.i += 1; return ''.join(out)
else:
out.append(c); self.i += 1
def extract_sidebar(js):
i = js.find('sidebar:{')
if i < 0: return None
p = P(js)
p.i = i + len('sidebar')
return p.parse()['sidebar'] # value after key... wait: parse expects ':'. handle manually: