138 lines
4.3 KiB
Python
138 lines
4.3 KiB
Python
"""Unit tests for lightweight parser providers and basic cleanup."""
|
|
|
|
from io import BytesIO
|
|
from typing import cast
|
|
from uuid import uuid4
|
|
|
|
import pymupdf
|
|
import pytest
|
|
from docx import Document as DocxDocument
|
|
|
|
from ekp_core.document_cleaner import clean_parsed_document
|
|
from ekp_domain import ParsedBlock, ParsedBlockType, ParsedDocument
|
|
from ekp_parser import (
|
|
DocumentParseError,
|
|
DocxParser,
|
|
MarkdownParser,
|
|
ParserRouter,
|
|
PdfParser,
|
|
TextParser,
|
|
UnsupportedParserTypeError,
|
|
)
|
|
|
|
|
|
def _pdf_bytes(*page_texts: str) -> bytes:
|
|
document: pymupdf.Document = pymupdf.open() # type: ignore[no-untyped-call]
|
|
for text in page_texts:
|
|
page = document.new_page()
|
|
page.insert_text((72, 72), text)
|
|
try:
|
|
return cast(bytes, document.tobytes()) # type: ignore[no-untyped-call]
|
|
finally:
|
|
document.close() # type: ignore[no-untyped-call]
|
|
|
|
|
|
def _docx_bytes() -> bytes:
|
|
document = DocxDocument()
|
|
document.add_heading("产品手册", level=1)
|
|
document.add_paragraph("产品正文")
|
|
document.add_heading("退款", level=2)
|
|
document.add_paragraph("退款周期为七天")
|
|
payload = BytesIO()
|
|
document.save(payload)
|
|
return payload.getvalue()
|
|
|
|
|
|
def test_pdf_parser_preserves_one_based_page_numbers() -> None:
|
|
document_id = uuid4()
|
|
|
|
parsed = PdfParser().parse(
|
|
document_id,
|
|
"manual.pdf",
|
|
BytesIO(_pdf_bytes("First page", "Second page")),
|
|
)
|
|
|
|
assert parsed.document_id == document_id
|
|
assert [(block.text.strip(), block.page_number) for block in parsed.blocks] == [
|
|
("First page", 1),
|
|
("Second page", 2),
|
|
]
|
|
|
|
|
|
def test_pdf_parser_wraps_malformed_file_error() -> None:
|
|
with pytest.raises(DocumentParseError, match="unable to parse PDF"):
|
|
PdfParser().parse(uuid4(), "broken.pdf", BytesIO(b"not-a-pdf"))
|
|
|
|
|
|
def test_docx_parser_preserves_headings_and_section_hierarchy() -> None:
|
|
parsed = DocxParser().parse(uuid4(), "manual.docx", BytesIO(_docx_bytes()))
|
|
|
|
assert [block.block_type for block in parsed.blocks] == [
|
|
ParsedBlockType.HEADING,
|
|
ParsedBlockType.PARAGRAPH,
|
|
ParsedBlockType.HEADING,
|
|
ParsedBlockType.PARAGRAPH,
|
|
]
|
|
assert parsed.blocks[-1].section_path == ("产品手册", "退款")
|
|
|
|
|
|
def test_text_parser_splits_paragraphs_and_accepts_utf8_bom() -> None:
|
|
parsed = TextParser().parse(
|
|
uuid4(),
|
|
"policy.txt",
|
|
BytesIO(b"\xef\xbb\xbfFirst line\ncontinued\n\nSecond paragraph"),
|
|
)
|
|
|
|
assert [block.text for block in parsed.blocks] == [
|
|
"First line\ncontinued",
|
|
"Second paragraph",
|
|
]
|
|
|
|
|
|
def test_text_parser_rejects_non_utf8_content() -> None:
|
|
with pytest.raises(DocumentParseError, match="not valid UTF-8"):
|
|
TextParser().parse(uuid4(), "policy.txt", BytesIO(b"\xff\xfe"))
|
|
|
|
|
|
def test_markdown_parser_retains_heading_levels_and_paragraphs() -> None:
|
|
parsed = MarkdownParser().parse(
|
|
uuid4(),
|
|
"guide.md",
|
|
BytesIO("# 产品\n简介第一行\n简介第二行\n\n## 退款\n七天。".encode()),
|
|
)
|
|
|
|
assert [(block.block_type, block.text) for block in parsed.blocks] == [
|
|
(ParsedBlockType.HEADING, "产品"),
|
|
(ParsedBlockType.PARAGRAPH, "简介第一行\n简介第二行"),
|
|
(ParsedBlockType.HEADING, "退款"),
|
|
(ParsedBlockType.PARAGRAPH, "七天。"),
|
|
]
|
|
assert parsed.blocks[-1].section_path == ("产品", "退款")
|
|
|
|
|
|
def test_cleaner_normalizes_whitespace_and_removes_empty_blocks() -> None:
|
|
document_id = uuid4()
|
|
raw = ParsedDocument(
|
|
document_id=document_id,
|
|
filename="policy.txt",
|
|
blocks=(
|
|
ParsedBlock(" 第一行\n 异常换行 ", ParsedBlockType.PARAGRAPH),
|
|
ParsedBlock("\n\t", ParsedBlockType.PARAGRAPH),
|
|
ParsedBlock(
|
|
" 标题 ",
|
|
ParsedBlockType.HEADING,
|
|
section_path=(" 第一章 ", "\t"),
|
|
),
|
|
),
|
|
)
|
|
|
|
cleaned = clean_parsed_document(raw)
|
|
|
|
assert [block.text for block in cleaned.blocks] == ["第一行 异常换行", "标题"]
|
|
assert cleaned.blocks[-1].section_path == ("第一章",)
|
|
|
|
|
|
def test_parser_router_rejects_unregistered_extension() -> None:
|
|
with pytest.raises(UnsupportedParserTypeError):
|
|
ParserRouter().parse(uuid4(), "slides.pptx", BytesIO(b"unsupported"))
|