新增 kb-update skill:不定期间歇增量更新全流程
- skills/kb-update/SKILL.md:七阶段编排(检测→导出→出题→审计→合并→推送) - skills/kb-update/scripts/status.py:线上增量检测(论坛已解决话题枚举比对 + 文档站 sitemap 比对 + 本地数据集状态) - kb-eval/merge_agent_items.py:中央合并器(schema/锚点/路径/冲突强校验、近重复警告、--check 终检) - 挂载 .zcode/skills/kb-update 软链,供 agent 工具按需触发
This commit is contained in:
parent
15b3337339
commit
7f8fd745b1
1
.zcode/skills/kb-update
Symbolic link
1
.zcode/skills/kb-update
Symbolic link
@ -0,0 +1 @@
|
||||
../../skills/kb-update
|
||||
@ -26,7 +26,11 @@ kb-eval/
|
||||
├── AGENT_PROMPT_TEMPLATE.md 出题子代理提示词模板(含历轮审计教训)
|
||||
├── AUDIT_PROMPT_TEMPLATE.md 交叉审计提示词模板
|
||||
├── agent_items/ 各出题代理原始产出(含 verify_phrases,审计溯源用)
|
||||
├── merge_agent_items.py 中央合并器(schema/锚点/冲突强校验 + 近重复警告)
|
||||
└── README.md 评测集构建方法、难度定义、三轮审计修复记录
|
||||
skills/
|
||||
└── kb-update/ 增量更新 skill(SKILL.md 编排手册 + status.py 在线检测)
|
||||
挂载于 .zcode/skills/ 供 agent 工具触发
|
||||
```
|
||||
|
||||
## 快速开始
|
||||
@ -46,6 +50,11 @@ python3 docsearch.py s "PWM 频率 范围" # 检索示例
|
||||
|
||||
## 增量更新
|
||||
|
||||
> **推荐入口:skill `kb-update`**(见 `skills/kb-update/SKILL.md`)——一条命令检测线上新增,
|
||||
> 后续按阶段编排:增量导出 → 候选生成 → 子代理出题(`AGENT_PROMPT_TEMPLATE.md`)→ 交叉审计
|
||||
> (`AUDIT_PROMPT_TEMPLATE.md`)→ 中央合并校验(`kb-eval/merge_agent_items.py`)→ 提交推送。
|
||||
> 适合不定期触发 agent 工具做全流程更新。以下为各环节手动执行方式:
|
||||
|
||||
**论坛有新帖时**:
|
||||
```bash
|
||||
cd pipeline/forum
|
||||
|
||||
122
kb-eval/merge_agent_items.py
Normal file
122
kb-eval/merge_agent_items.py
Normal file
@ -0,0 +1,122 @@
|
||||
"""中央合并出题代理产物:强校验 + 去重检查 + 写入数据集。
|
||||
|
||||
用法:
|
||||
python3 merge_agent_items.py agent_items/agent17.jsonl [agent18.jsonl ...] [--dry-run] [--allow-dup-topic]
|
||||
python3 merge_agent_items.py --check # 只校验现有数据集完整性
|
||||
|
||||
校验项(任一失败整条拒绝):
|
||||
1. schema:question/expected_answer/expected_docs(1~3)/difficulty(简单|中等|困难)/source_topic(int)/verify_phrases(1~2)
|
||||
2. expected_docs 路径存在且不含 i18n/en、无重复项
|
||||
3. verify_phrases 逐字(大小写不敏感)命中至少一篇金标
|
||||
4. question 与现有+本批完全重复
|
||||
5. source_topic 与现有冲突(--allow-dup-topic 放行,供"一帖多题"场景)
|
||||
|
||||
警告项(不拒绝,打印供人工复核):
|
||||
- 问题近重复(token 重叠 ≥0.6):注意"子问包含式"重复——宽题答案包含窄题答案
|
||||
- 同 topic 多题
|
||||
写入:合并后同时更新 items.jsonl 与 eval_dataset.json(去掉 verify_phrases 字段)。
|
||||
"""
|
||||
import os, sys, json, re, argparse
|
||||
from collections import Counter
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.environ.get('EVAL_ROOT', os.path.abspath(os.path.join(HERE, '..')))
|
||||
CORPUS = os.path.join(ROOT, 'rdk-docs-export')
|
||||
FIELDS = {'question', 'expected_answer', 'expected_docs', 'difficulty', 'source_topic', 'verify_phrases'}
|
||||
|
||||
def norm(q):
|
||||
return set(re.findall(r'[\u4e00-\u9fff]+|[a-z0-9_.]+', q.lower()))
|
||||
|
||||
def verify_item(it):
|
||||
errs = []
|
||||
if set(it.keys()) != FIELDS: errs.append(f'schema 字段异常: {sorted(set(it.keys()) ^ FIELDS)}')
|
||||
if not isinstance(it.get('expected_docs'), list) or not (1 <= len(it['expected_docs']) <= 3):
|
||||
errs.append('expected_docs 须为 1~3 篇')
|
||||
if it.get('difficulty') not in ('简单', '中等', '困难'): errs.append('difficulty 非法')
|
||||
if not it.get('verify_phrases'): errs.append('verify_phrases 缺失')
|
||||
if errs: return errs
|
||||
for d in it['expected_docs']:
|
||||
p = os.path.join(CORPUS, d)
|
||||
if not os.path.exists(p): errs.append(f'文档不存在: {d}'); continue
|
||||
if '/i18n/' in f'/{d}': errs.append(f'英文镜像: {d}')
|
||||
if len(set(it['expected_docs'])) != len(it['expected_docs']): errs.append('expected_docs 内部重复')
|
||||
for ph in it['verify_phrases']:
|
||||
hit = any(ph.lower() in open(os.path.join(CORPUS, d), encoding='utf-8', errors='replace').read().lower()
|
||||
for d in it['expected_docs'] if os.path.exists(os.path.join(CORPUS, d)))
|
||||
if not hit: errs.append(f'锚点未命中: {ph[:40]!r}')
|
||||
return errs
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('files', nargs='*', help='agent 产物 jsonl 文件')
|
||||
ap.add_argument('--dry-run', action='store_true')
|
||||
ap.add_argument('--allow-dup-topic', action='store_true')
|
||||
ap.add_argument('--check', action='store_true', help='只校验现有数据集')
|
||||
args = ap.parse_args()
|
||||
|
||||
items_path = os.path.join(HERE, 'items.jsonl')
|
||||
existing = [json.loads(l) for l in open(items_path)]
|
||||
|
||||
if args.check:
|
||||
bad = 0
|
||||
for it in existing:
|
||||
if set(it.keys()) != {f for f in FIELDS if f != 'verify_phrases'}:
|
||||
print('SCHEMA', it.get('source_topic')); bad += 1
|
||||
for d in it['expected_docs']:
|
||||
if not os.path.exists(os.path.join(CORPUS, d)):
|
||||
print('DOC-MISS', it['source_topic'], d); bad += 1
|
||||
qs = [it['question'] for it in existing]
|
||||
print(f'现有 {len(existing)} 条,问题重复 {len(qs)-len(set(qs))},异常 {bad}')
|
||||
sys.exit(1 if bad else 0)
|
||||
|
||||
incoming = []
|
||||
for fp in args.files:
|
||||
for l in open(fp):
|
||||
l = l.strip()
|
||||
if l: incoming.append(json.loads(l))
|
||||
print(f'待合并 {len(incoming)} 条(来自 {len(args.files)} 个文件)')
|
||||
|
||||
ex_topics = Counter(it['source_topic'] for it in existing)
|
||||
kept, rejected = [], 0
|
||||
for it in incoming:
|
||||
errs = verify_item(it)
|
||||
if not errs and it['question'] in [x['question'] for x in existing] + [x['question'] for x in kept]:
|
||||
errs.append('question 完全重复')
|
||||
if not errs and not args.allow_dup_topic and ex_topics.get(it['source_topic']):
|
||||
errs.append(f"source_topic={it['source_topic']} 已存在(--allow-dup-topic 放行)")
|
||||
if errs:
|
||||
rejected += 1
|
||||
print(f' REJECT [{it.get("source_topic")}]', '; '.join(errs))
|
||||
else:
|
||||
kept.append(it)
|
||||
|
||||
# 近重复警告
|
||||
warn = 0
|
||||
for a in range(len(kept)):
|
||||
for b in range(a + 1, len(kept)):
|
||||
s1, s2 = norm(kept[a]['question']), norm(kept[b]['question'])
|
||||
if s1 and s2 and len(s1 & s2) / min(len(s1), len(s2)) >= 0.6:
|
||||
print(f" WARN 近重复: [{kept[a]['source_topic']}] {kept[a]['question'][:30]} <-> "
|
||||
f"[{kept[b]['source_topic']}] {kept[b]['question'][:30]}"); warn += 1
|
||||
for e in existing:
|
||||
s1, s2 = norm(kept[a]['question']), norm(e['question'])
|
||||
if s1 and s2 and len(s1 & s2) / min(len(s1), len(s2)) >= 0.6:
|
||||
print(f" WARN 与存量近重复: [{kept[a]['source_topic']}] {kept[a]['question'][:30]} <-> "
|
||||
f"[{e['source_topic']}] {e['question'][:30]}"); warn += 1
|
||||
|
||||
final = existing + kept
|
||||
print(f'\n通过 {len(kept)} / 拒绝 {rejected} / 近重复警告 {warn} → 合并后共 {len(final)} 条'
|
||||
f'({"、".join(f"{k}{v}" for k, v in Counter(i["difficulty"] for i in final).items())})')
|
||||
if args.dry_run:
|
||||
print('dry-run,未写入'); return
|
||||
if kept:
|
||||
with open(items_path, 'w') as f:
|
||||
for it in final:
|
||||
it2 = {k: it[k] for k in ('question', 'expected_answer', 'expected_docs', 'difficulty', 'source_topic')}
|
||||
f.write(json.dumps(it2, ensure_ascii=False) + '\n')
|
||||
json.dump([{k: it[k] for k in ('question', 'expected_answer', 'expected_docs', 'difficulty', 'source_topic')}
|
||||
for it in final], open(os.path.join(HERE, 'eval_dataset.json'), 'w'), ensure_ascii=False, indent=2)
|
||||
print('已写入 items.jsonl 与 eval_dataset.json')
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
88
skills/kb-update/SKILL.md
Normal file
88
skills/kb-update/SKILL.md
Normal file
@ -0,0 +1,88 @@
|
||||
---
|
||||
name: kb-update
|
||||
description: RDK 知识库数据集的定期增量更新。不定期间歇触发时,检测论坛新帖与文档站更新、增量导出数据、用子代理从新帖中提取新的评测问题(答案锚定文档、多代理交叉审计、中央合并校验),最后提交并推送到 Gitea。适用于"更新一下知识库数据""看看论坛文档有没有新内容""扩充评测集"等场景。
|
||||
---
|
||||
|
||||
# RDK 知识库增量更新
|
||||
|
||||
本 skill 在仓库 `EVAL_ROOT`(默认为本 skill 所在目录往上三级)下工作,依赖 `pipeline/`(数据导出)、`rdk-docs-export/`(文档语料)、`kb-eval/`(评测工具与数据集)。
|
||||
外部依赖:Python 3.9+、`pip3 install --user html2text`。
|
||||
|
||||
按顺序执行以下阶段,**每个阶段有产出再进入下一阶段;没有新增就跳过对应阶段**。
|
||||
|
||||
## 阶段 1:检测有没有新东西
|
||||
|
||||
```bash
|
||||
python3 skills/kb-update/scripts/status.py --forum-scan
|
||||
```
|
||||
|
||||
输出三类信息:① 线上已解决话题 vs 本地导出(新增 ID 列表写入 `_work/new_topic_ids.json`);② 各文档站 sitemap 在线页数 vs 本地篇数(差异大标 `!!`);③ 本地数据集规模。
|
||||
若论坛与文档均无变化,流程到此结束。
|
||||
|
||||
## 阶段 2:增量导出数据
|
||||
|
||||
**论坛有新增**(`pipeline/forum/`,export 支持断点续跑,已导出话题自动跳过):
|
||||
```bash
|
||||
python3 pipeline/forum/export_topics.py # 拉取新话题 → forum-solved-export/
|
||||
python3 pipeline/forum/build_index.py # 重建 index.md / CSV
|
||||
```
|
||||
|
||||
**文档有变化**(`pipeline/docs/`,按 status.py 提示选跑):
|
||||
```bash
|
||||
bash pipeline/docs/clone_repos.sh # 15 个源码仓库站(重新稀疏克隆+覆盖)
|
||||
python3 pipeline/docs/scrape_sphinx_x5sdk.py && python3 pipeline/docs/scrape_sphinx_oe.py
|
||||
python3 pipeline/docs/fetch_rspress_chunks.py # Rspress 三站(oe_s_doc / oe_llm_*)
|
||||
python3 pipeline/docs/extract_rspress_oe_s.py && python3 pipeline/docs/extract_rspress_llm.py
|
||||
python3 pipeline/docs/coverage.py # 刷新覆盖率
|
||||
```
|
||||
|
||||
**无论动了哪个,都重建检索索引**:
|
||||
```bash
|
||||
cd kb-eval && python3 -c "import docsearch; docsearch.Index().build()"
|
||||
```
|
||||
|
||||
## 阶段 3:生成出题候选
|
||||
|
||||
```bash
|
||||
cd kb-eval
|
||||
python3 make_digests.py --out digestsN --mode answer --top 500 --batch 30
|
||||
```
|
||||
`digestsN` 的 N 取当前最大波次号 +1(目录名即波次号)。`--mode question` 用于打捞"采纳回答过短但问题本身可文档回答"的帖子(存量耗尽时用)。输出摘要批次文件,每个约 30 帖。
|
||||
|
||||
## 阶段 4:子代理并行出题
|
||||
|
||||
用 `kb-eval/AGENT_PROMPT_TEMPLATE.md` 模板,每 6~8 个摘要批次派一个出题子代理(并行),替换模板中的 `{批次文件}`(如 `digests6/digests6_00.txt ~ digests6_06.txt`)、`{输出文件}`(`agent_items/agentN.jsonl`,N 接续现有最大编号)、`{现有条数}`(eval_dataset.json 当前条数)。
|
||||
产出为 JSONL,每条含 verify_phrases 锚点短语,代理会自校验。
|
||||
|
||||
## 阶段 5:交叉审计
|
||||
|
||||
用 `kb-eval/AUDIT_PROMPT_TEMPLATE.md` 模板派 2~3 个审计子代理(并行),分别查:① 事实准确性(锚点之外的事实断言);② 金标文档合理性(承载性/平台一致/目录页);③ 问题质量与重复(**特别注意"子问包含式"重复——宽题的答案包含窄题的答案**,以及题干泄答案、难度虚高)。
|
||||
审计代理只出报告不改文件。
|
||||
|
||||
## 阶段 6:中央合并与修复
|
||||
|
||||
```bash
|
||||
cd kb-eval
|
||||
python3 merge_agent_items.py agent_items/agentN.jsonl [更多文件] --dry-run # 先干跑看拒绝/警告
|
||||
# 按审计报告修复(事实错误改文本、坏金标删或换、强重复整条剔除、难度重标),修复后重跑 dry-run
|
||||
python3 merge_agent_items.py agent_items/agentN.jsonl ... # 正式写入
|
||||
python3 merge_agent_items.py --check # 终检
|
||||
```
|
||||
若审计发现 ERROR 级问题,修复后必须让受影响条目的 verify_phrases 重新命中(merge 会自动复验)。
|
||||
|
||||
## 阶段 7:提交推送
|
||||
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "增量更新:新增 X 帖 / 文档刷新 / 评测集 +Y 条(细节)"
|
||||
git push
|
||||
```
|
||||
|
||||
## 注意事项(历轮审计教训)
|
||||
|
||||
- 论坛搜索接口有 500 条上限,枚举必须翻 `latest.json`;
|
||||
- Rspress 站无公开源码仓库,只能 chunk 反解析;其 3 个页面线上即坏,属永久缺口;
|
||||
- 出题规则核心:**答案必须锚定文档原文**,论坛说法与文档冲突时以文档为准;
|
||||
- 题干不得泄露答案(不把答案中的型号/参数名写进题干);
|
||||
- 难度标准从严:查参数/规格/工具名的单跳题就是"简单",勿虚标"中等";
|
||||
- tar 归档解析要过滤 macOS 的 `._*` AppleDouble 条目。
|
||||
113
skills/kb-update/scripts/status.py
Normal file
113
skills/kb-update/scripts/status.py
Normal file
@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""检测线上有没有新东西:论坛新增已解决话题 / 文档站页面变化 / 本地数据集状态。
|
||||
|
||||
用法:python3 status.py [--forum-scan]
|
||||
--forum-scan 实际翻页枚举线上已解决话题(约 1-2 分钟,230+ 页请求);
|
||||
不加则只统计本地并给出线上快速计数(about.json 的 topics_count 仅作参考)。
|
||||
"""
|
||||
import os, sys, json, glob, re, argparse, urllib.request
|
||||
|
||||
ROOT = os.environ.get('EVAL_ROOT', os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')))
|
||||
FORUM = "https://forum.d-robotics.cc"
|
||||
DOCS = os.path.join(ROOT, 'rdk-docs-export')
|
||||
KBEVAL = os.path.join(ROOT, 'kb-eval')
|
||||
|
||||
def get_json(url, retries=3):
|
||||
for i in range(retries):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "kb-update/1.0", "Accept": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return json.load(r)
|
||||
except Exception:
|
||||
if i == retries - 1: raise
|
||||
import time; time.sleep(3)
|
||||
|
||||
def scan_forum_solved():
|
||||
"""翻 latest.json 全量枚举已解决话题 id -> title(与 pipeline/forum/enumerate_topics.py 同口径)"""
|
||||
solved, page = {}, 0
|
||||
while True:
|
||||
d = get_json(f"{FORUM}/latest.json?no_definitions=true&page={page}")
|
||||
topics = d.get("topic_list", {}).get("topics", [])
|
||||
if not topics: break
|
||||
new = 0
|
||||
for t in topics:
|
||||
if t.get("has_accepted_answer") and t["id"] not in solved:
|
||||
solved[t["id"]] = t.get("title", ""); new += 1
|
||||
if new == 0 and page > 0: break
|
||||
page += 1
|
||||
if page > 400: break
|
||||
return solved
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--forum-scan', action='store_true')
|
||||
args = ap.parse_args()
|
||||
|
||||
print("=" * 60)
|
||||
print("RDK 知识库 更新状态检测")
|
||||
print("=" * 60)
|
||||
|
||||
# ---- 本地状态 ----
|
||||
local_topics = {int(f[:-3]) for f in os.listdir(os.path.join(ROOT, 'forum-solved-export', 'topics')) if f.endswith('.md')}
|
||||
doc_files = [os.path.relpath(f, DOCS) for f in glob.glob(os.path.join(DOCS, '**', '*.m*'), recursive=True)]
|
||||
doc_files = [f for f in doc_files if not f.startswith('i18n') and '/i18n/' not in f and '_uncovered' not in f]
|
||||
items = [json.loads(l) for l in open(os.path.join(KBEVAL, 'items.jsonl'))]
|
||||
from collections import Counter
|
||||
diff = Counter(i['difficulty'] for i in items)
|
||||
waves = sorted({d for d in os.listdir(KBEVAL) if d.startswith('digests') and os.path.isdir(os.path.join(KBEVAL, d))})
|
||||
print(f"\n[本地] 论坛话题 {len(local_topics)} | 中文文档 {len(doc_files)} 篇 | 评测集 {len(items)} 条"
|
||||
f"({'/'.join(f'{k}{v}' for k,v in diff.items())})| 已处理波次: {len(waves)}")
|
||||
|
||||
# ---- 论坛增量 ----
|
||||
if args.forum_scan:
|
||||
print("\n[论坛] 在线枚举已解决话题中……")
|
||||
online = scan_forum_solved()
|
||||
new_ids = sorted(set(online) - local_topics)
|
||||
print(f" 线上已解决 {len(online)},本地 {len(local_topics)},新增 {len(new_ids)}")
|
||||
for i in new_ids[:20]:
|
||||
print(f" + [{i}] {online[i][:50]}")
|
||||
if len(new_ids) > 20: print(f" ... 共 {len(new_ids)} 条")
|
||||
if new_ids:
|
||||
out = os.path.join(ROOT, '_work', 'new_topic_ids.json')
|
||||
os.makedirs(os.path.dirname(out), exist_ok=True)
|
||||
json.dump(new_ids, open(out, 'w'))
|
||||
print(f" 新增 ID 已写入 {out}")
|
||||
else:
|
||||
print(" 论坛无新增,可跳过 forum 导出步骤")
|
||||
else:
|
||||
try:
|
||||
about = get_json(f"{FORUM}/about.json")["about"]["stats"]
|
||||
print(f"\n[论坛] 线上话题总数 {about['topics_count']}(快查,非已解决口径;"
|
||||
f"加 --forum-scan 做全量已解决枚举比对)")
|
||||
except Exception as e:
|
||||
print(f"\n[论坛] 快查失败: {e}")
|
||||
|
||||
# ---- 文档站状态 ----
|
||||
print("\n[文档] 各站本地篇数(对比记忆值判断是否需要更新;源码仓库站用 git pull 检测更准):")
|
||||
sites = {}
|
||||
for f in doc_files:
|
||||
sites[f.split('/')[0]] = sites.get(f.split('/')[0], 0) + 1
|
||||
for s, n in sorted(sites.items(), key=lambda x: -x[1]):
|
||||
print(f" {s:36s} {n}")
|
||||
sitemap_sites = ["rdk_x_doc", "rdk_s_doc", "rdk_studio_doc", "tros_doc", "model_zoo_doc",
|
||||
"case_doc", "magicbox_doc", "xburn_doc", "accessories_doc",
|
||||
"accessories_bmi088_doc", "accessories_stereo_camera_doc", "rdk_doc"]
|
||||
print("\n[文档] sitemap 在线页数比对(有 sitemap 的站):")
|
||||
for site in sitemap_sites:
|
||||
try:
|
||||
req = urllib.request.Request(f"https://developer.d-robotics.cc/{site}/sitemap.xml",
|
||||
headers={"User-Agent": "kb-update/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=20) as r:
|
||||
n = r.read().decode('utf-8', errors='replace').count('<loc>')
|
||||
local = sites.get(site, 0)
|
||||
mark = "OK" if abs(n - local) < 30 else "!! 差异较大,建议更新"
|
||||
print(f" {site:36s} 在线 {n:4d} / 本地 {local:4d} {mark}")
|
||||
except Exception as e:
|
||||
print(f" {site:36s} 获取失败: {str(e)[:40]}")
|
||||
|
||||
print("\n[建议] 论坛有新增 → pipeline/forum/export_topics.py;文档有变化 → pipeline/docs/ 各脚本;"
|
||||
"\n 然后重建索引(kb-eval: python3 -c 'import docsearch; docsearch.Index().build()')"
|
||||
"\n 再按 skills/kb-update/SKILL.md 的出题流程扩产评测集。")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user