knowledge-platform/tests/integration/test_document_parsing.py
2026-09-01 14:17:20 +08:00

103 lines
3.6 KiB
Python

"""Integration tests for upload-triggered parsing and derived persistence."""
import json
from io import BytesIO
from pathlib import Path
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.database import create_session_factory
from ekp_core.document_parsing import (
DocumentAlreadyParsedError,
DocumentIngestionService,
DocumentParsingService,
EmptyParsedDocumentError,
)
from ekp_core.document_upload import DocumentUploadService
from ekp_domain import Base, Document, DocumentStatus
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
def _build_services(
tmp_path: Path,
) -> tuple[
DocumentIngestionService,
DocumentUploadService,
DocumentParsingService,
LocalObjectStorage,
sessionmaker[Session],
]:
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
return DocumentIngestionService(upload, parsing), upload, parsing, storage, session_factory
def test_ingestion_triggers_cleaning_and_persists_versioned_parsed_json(tmp_path: Path) -> None:
ingestion, _, _, storage, _ = _build_services(tmp_path)
document, parsed = ingestion.ingest(
"制度.txt",
"text/plain",
BytesIO(" 第一行\n异常换行\n\n\n 第二段 ".encode()),
)
assert document.status is DocumentStatus.PROCESSING
assert document.parsed_storage_key == f"parsed/{document.id}/document.json"
assert [block.text for block in parsed.blocks] == ["第一行 异常换行", "第二段"]
with storage.open(document.parsed_storage_key) as parsed_source:
stored_result = json.load(parsed_source)
assert stored_result == {
"schema_version": 1,
"document_id": str(document.id),
"filename": "制度.txt",
"blocks": [
{
"text": "第一行 异常换行",
"block_type": "paragraph",
"page_number": None,
"section_path": [],
},
{
"text": "第二段",
"block_type": "paragraph",
"page_number": None,
"section_path": [],
},
],
}
def test_parse_failure_sets_failed_status_and_keeps_original_source(tmp_path: Path) -> None:
_, upload, parsing, storage, session_factory = _build_services(tmp_path)
document = upload.upload("empty-after-cleaning.txt", "text/plain", BytesIO(b" \n\t"))
with pytest.raises(EmptyParsedDocumentError):
parsing.process(document.id)
with session_factory() as session:
stored = session.get(Document, document.id)
assert stored is not None
assert stored.status is DocumentStatus.FAILED
assert stored.parsed_storage_key is None
assert "no extractable text" in (stored.error_message or "")
assert storage.exists(document.storage_key)
assert not storage.exists(f"parsed/{document.id}/document.json")
def test_parsed_result_is_not_silently_overwritten(tmp_path: Path) -> None:
ingestion, _, parsing, storage, _ = _build_services(tmp_path)
document, _ = ingestion.ingest("policy.md", "text/markdown", BytesIO(b"# Policy"))
with pytest.raises(DocumentAlreadyParsedError):
parsing.process(document.id)
assert document.parsed_storage_key is not None
assert storage.exists(document.parsed_storage_key)