55 lines
2.3 KiB
Python

#!/usr/bin/env python3
"""01_dump_corpus.py — 从 rdk_data_clean 导出 jev 训练语料
输出: data/corpus.jsonl 每行: {doc_id, path, products, content_hash, text, truncated, orig_chars}
截断策略: 正文 >6000 字符时保留 头部5600 + 全部 markdown 标题行(拼接在尾部), truncated=true
"""
import json, os, re, sys, hashlib
ROOT = os.path.expanduser("~/agent/rdk_data_clean")
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "corpus.jsonl")
MAX_CHARS = 6000
HEAD_CHARS = 5600
def truncate(text: str):
if len(text) <= MAX_CHARS:
return text, False
head = text[:HEAD_CHARS]
heads = [ln.strip() for ln in text.splitlines() if re.match(r"^#{1,6}\s", ln.strip())]
tail = "\n\n[本文档其余章节标题]\n" + "\n".join(heads[:120])
return head + tail, True
def main():
n_ok, n_skip = 0, 0
with open(OUT, "w", encoding="utf-8") as fout:
for line in open(os.path.join(ROOT, "manifest.jsonl"), encoding="utf-8"):
d = json.loads(line)
if not d.get("indexable") or d.get("decision") != "routed":
continue
path = os.path.join(ROOT, d["output"])
if not os.path.exists(path):
n_skip += 1
continue
raw = open(path, encoding="utf-8", errors="replace").read()
# 轻清洗: 去掉 HTML 注释与连续空行
raw = re.sub(r"<!--.*?-->", "", raw, flags=re.S)
raw = re.sub(r"\n{3,}", "\n\n", raw).strip()
text, truncated = truncate(raw)
rel = d["output"].replace("documents/", "").removesuffix(".md")
rec = {
"doc_id": rel,
"path": d["output"],
"source": d.get("source"),
"products": d.get("products", []),
"content_hash": d.get("content_hash"),
"text_sha256": hashlib.sha256(text.encode()).hexdigest(),
"orig_chars": len(raw),
"truncated": truncated,
"text": text,
}
fout.write(json.dumps(rec, ensure_ascii=False) + "\n")
n_ok += 1
print(f"导出 {n_ok} 篇 → {os.path.normpath(OUT)} (缺失跳过 {n_skip})")
if __name__ == "__main__":
sys.exit(main())