knowledge-platform/tests/unit/test_document_parsers.py
2026-09-01 14:17:20 +08:00

138 lines
4.3 KiB
Python

"""Unit tests for lightweight parser providers and basic cleanup."""
from io import BytesIO
from typing import cast
from uuid import uuid4
import pymupdf
import pytest
from docx import Document as DocxDocument
from ekp_core.document_cleaner import clean_parsed_document
from ekp_domain import ParsedBlock, ParsedBlockType, ParsedDocument
from ekp_parser import (
DocumentParseError,
DocxParser,
MarkdownParser,
ParserRouter,
PdfParser,
TextParser,
UnsupportedParserTypeError,
)
def _pdf_bytes(*page_texts: str) -> bytes:
document: pymupdf.Document = pymupdf.open() # type: ignore[no-untyped-call]
for text in page_texts:
page = document.new_page()
page.insert_text((72, 72), text)
try:
return cast(bytes, document.tobytes()) # type: ignore[no-untyped-call]
finally:
document.close() # type: ignore[no-untyped-call]
def _docx_bytes() -> bytes:
document = DocxDocument()
document.add_heading("产品手册", level=1)
document.add_paragraph("产品正文")
document.add_heading("退款", level=2)
document.add_paragraph("退款周期为七天")
payload = BytesIO()
document.save(payload)
return payload.getvalue()
def test_pdf_parser_preserves_one_based_page_numbers() -> None:
document_id = uuid4()
parsed = PdfParser().parse(
document_id,
"manual.pdf",
BytesIO(_pdf_bytes("First page", "Second page")),
)
assert parsed.document_id == document_id
assert [(block.text.strip(), block.page_number) for block in parsed.blocks] == [
("First page", 1),
("Second page", 2),
]
def test_pdf_parser_wraps_malformed_file_error() -> None:
with pytest.raises(DocumentParseError, match="unable to parse PDF"):
PdfParser().parse(uuid4(), "broken.pdf", BytesIO(b"not-a-pdf"))
def test_docx_parser_preserves_headings_and_section_hierarchy() -> None:
parsed = DocxParser().parse(uuid4(), "manual.docx", BytesIO(_docx_bytes()))
assert [block.block_type for block in parsed.blocks] == [
ParsedBlockType.HEADING,
ParsedBlockType.PARAGRAPH,
ParsedBlockType.HEADING,
ParsedBlockType.PARAGRAPH,
]
assert parsed.blocks[-1].section_path == ("产品手册", "退款")
def test_text_parser_splits_paragraphs_and_accepts_utf8_bom() -> None:
parsed = TextParser().parse(
uuid4(),
"policy.txt",
BytesIO(b"\xef\xbb\xbfFirst line\ncontinued\n\nSecond paragraph"),
)
assert [block.text for block in parsed.blocks] == [
"First line\ncontinued",
"Second paragraph",
]
def test_text_parser_rejects_non_utf8_content() -> None:
with pytest.raises(DocumentParseError, match="not valid UTF-8"):
TextParser().parse(uuid4(), "policy.txt", BytesIO(b"\xff\xfe"))
def test_markdown_parser_retains_heading_levels_and_paragraphs() -> None:
parsed = MarkdownParser().parse(
uuid4(),
"guide.md",
BytesIO("# 产品\n简介第一行\n简介第二行\n\n## 退款\n七天。".encode()),
)
assert [(block.block_type, block.text) for block in parsed.blocks] == [
(ParsedBlockType.HEADING, "产品"),
(ParsedBlockType.PARAGRAPH, "简介第一行\n简介第二行"),
(ParsedBlockType.HEADING, "退款"),
(ParsedBlockType.PARAGRAPH, "七天。"),
]
assert parsed.blocks[-1].section_path == ("产品", "退款")
def test_cleaner_normalizes_whitespace_and_removes_empty_blocks() -> None:
document_id = uuid4()
raw = ParsedDocument(
document_id=document_id,
filename="policy.txt",
blocks=(
ParsedBlock(" 第一行\n 异常换行 ", ParsedBlockType.PARAGRAPH),
ParsedBlock("\n\t", ParsedBlockType.PARAGRAPH),
ParsedBlock(
" 标题 ",
ParsedBlockType.HEADING,
section_path=(" 第一章 ", "\t"),
),
),
)
cleaned = clean_parsed_document(raw)
assert [block.text for block in cleaned.blocks] == ["第一行 异常换行", "标题"]
assert cleaned.blocks[-1].section_path == ("第一章",)
def test_parser_router_rejects_unregistered_extension() -> None:
with pytest.raises(UnsupportedParserTypeError):
ParserRouter().parse(uuid4(), "slides.pptx", BytesIO(b"unsupported"))