103 lines
3.6 KiB
Python
103 lines
3.6 KiB
Python
"""Integration tests for upload-triggered parsing and derived persistence."""
|
|
|
|
import json
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
|
|
from ekp_core.database import create_session_factory
|
|
from ekp_core.document_parsing import (
|
|
DocumentAlreadyParsedError,
|
|
DocumentIngestionService,
|
|
DocumentParsingService,
|
|
EmptyParsedDocumentError,
|
|
)
|
|
from ekp_core.document_upload import DocumentUploadService
|
|
from ekp_domain import Base, Document, DocumentStatus
|
|
from ekp_object_storage import LocalObjectStorage
|
|
from ekp_parser import ParserRouter
|
|
|
|
|
|
def _build_services(
|
|
tmp_path: Path,
|
|
) -> tuple[
|
|
DocumentIngestionService,
|
|
DocumentUploadService,
|
|
DocumentParsingService,
|
|
LocalObjectStorage,
|
|
sessionmaker[Session],
|
|
]:
|
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(engine)
|
|
session_factory = create_session_factory(engine)
|
|
storage = LocalObjectStorage(tmp_path / "storage")
|
|
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
|
|
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
|
|
return DocumentIngestionService(upload, parsing), upload, parsing, storage, session_factory
|
|
|
|
|
|
def test_ingestion_triggers_cleaning_and_persists_versioned_parsed_json(tmp_path: Path) -> None:
|
|
ingestion, _, _, storage, _ = _build_services(tmp_path)
|
|
|
|
document, parsed = ingestion.ingest(
|
|
"制度.txt",
|
|
"text/plain",
|
|
BytesIO(" 第一行\n异常换行\n\n\n 第二段 ".encode()),
|
|
)
|
|
|
|
assert document.status is DocumentStatus.PROCESSING
|
|
assert document.parsed_storage_key == f"parsed/{document.id}/document.json"
|
|
assert [block.text for block in parsed.blocks] == ["第一行 异常换行", "第二段"]
|
|
with storage.open(document.parsed_storage_key) as parsed_source:
|
|
stored_result = json.load(parsed_source)
|
|
assert stored_result == {
|
|
"schema_version": 1,
|
|
"document_id": str(document.id),
|
|
"filename": "制度.txt",
|
|
"blocks": [
|
|
{
|
|
"text": "第一行 异常换行",
|
|
"block_type": "paragraph",
|
|
"page_number": None,
|
|
"section_path": [],
|
|
},
|
|
{
|
|
"text": "第二段",
|
|
"block_type": "paragraph",
|
|
"page_number": None,
|
|
"section_path": [],
|
|
},
|
|
],
|
|
}
|
|
|
|
|
|
def test_parse_failure_sets_failed_status_and_keeps_original_source(tmp_path: Path) -> None:
|
|
_, upload, parsing, storage, session_factory = _build_services(tmp_path)
|
|
document = upload.upload("empty-after-cleaning.txt", "text/plain", BytesIO(b" \n\t"))
|
|
|
|
with pytest.raises(EmptyParsedDocumentError):
|
|
parsing.process(document.id)
|
|
|
|
with session_factory() as session:
|
|
stored = session.get(Document, document.id)
|
|
assert stored is not None
|
|
assert stored.status is DocumentStatus.FAILED
|
|
assert stored.parsed_storage_key is None
|
|
assert "no extractable text" in (stored.error_message or "")
|
|
assert storage.exists(document.storage_key)
|
|
assert not storage.exists(f"parsed/{document.id}/document.json")
|
|
|
|
|
|
def test_parsed_result_is_not_silently_overwritten(tmp_path: Path) -> None:
|
|
ingestion, _, parsing, storage, _ = _build_services(tmp_path)
|
|
document, _ = ingestion.ingest("policy.md", "text/markdown", BytesIO(b"# Policy"))
|
|
|
|
with pytest.raises(DocumentAlreadyParsedError):
|
|
parsing.process(document.id)
|
|
|
|
assert document.parsed_storage_key is not None
|
|
assert storage.exists(document.parsed_storage_key)
|