55 lines
2.3 KiB
Python
55 lines
2.3 KiB
Python
#!/usr/bin/env python3
|
|
"""01_dump_corpus.py — 从 rdk_data_clean 导出 jev 训练语料
|
|
输出: data/corpus.jsonl 每行: {doc_id, path, products, content_hash, text, truncated, orig_chars}
|
|
截断策略: 正文 >6000 字符时保留 头部5600 + 全部 markdown 标题行(拼接在尾部), truncated=true
|
|
"""
|
|
import json, os, re, sys, hashlib
|
|
|
|
ROOT = os.path.expanduser("~/agent/rdk_data_clean")
|
|
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "corpus.jsonl")
|
|
MAX_CHARS = 6000
|
|
HEAD_CHARS = 5600
|
|
|
|
def truncate(text: str):
|
|
if len(text) <= MAX_CHARS:
|
|
return text, False
|
|
head = text[:HEAD_CHARS]
|
|
heads = [ln.strip() for ln in text.splitlines() if re.match(r"^#{1,6}\s", ln.strip())]
|
|
tail = "\n\n[本文档其余章节标题]\n" + "\n".join(heads[:120])
|
|
return head + tail, True
|
|
|
|
def main():
|
|
n_ok, n_skip = 0, 0
|
|
with open(OUT, "w", encoding="utf-8") as fout:
|
|
for line in open(os.path.join(ROOT, "manifest.jsonl"), encoding="utf-8"):
|
|
d = json.loads(line)
|
|
if not d.get("indexable") or d.get("decision") != "routed":
|
|
continue
|
|
path = os.path.join(ROOT, d["output"])
|
|
if not os.path.exists(path):
|
|
n_skip += 1
|
|
continue
|
|
raw = open(path, encoding="utf-8", errors="replace").read()
|
|
# 轻清洗: 去掉 HTML 注释与连续空行
|
|
raw = re.sub(r"<!--.*?-->", "", raw, flags=re.S)
|
|
raw = re.sub(r"\n{3,}", "\n\n", raw).strip()
|
|
text, truncated = truncate(raw)
|
|
rel = d["output"].replace("documents/", "").removesuffix(".md")
|
|
rec = {
|
|
"doc_id": rel,
|
|
"path": d["output"],
|
|
"source": d.get("source"),
|
|
"products": d.get("products", []),
|
|
"content_hash": d.get("content_hash"),
|
|
"text_sha256": hashlib.sha256(text.encode()).hexdigest(),
|
|
"orig_chars": len(raw),
|
|
"truncated": truncated,
|
|
"text": text,
|
|
}
|
|
fout.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
n_ok += 1
|
|
print(f"导出 {n_ok} 篇 → {os.path.normpath(OUT)} (缺失跳过 {n_skip})")
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|