113 lines
4.4 KiB
Python
113 lines
4.4 KiB
Python
"""Integration tests for upload-to-chunk processing and canonical metadata."""
|
|
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, select
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
|
|
from ekp_core.chunking import StructureAwareChunker
|
|
from ekp_core.database import create_session_factory
|
|
from ekp_core.document_chunking import (
|
|
DocumentAlreadyChunkedError,
|
|
DocumentChunkingService,
|
|
DocumentPipelineService,
|
|
)
|
|
from ekp_core.document_parsing import DocumentParsingService
|
|
from ekp_core.document_upload import DocumentUploadService
|
|
from ekp_domain import Base, Chunk, Document, DocumentStatus
|
|
from ekp_object_storage import LocalObjectStorage
|
|
from ekp_parser import ParserRouter
|
|
|
|
|
|
def _build_services(
|
|
tmp_path: Path,
|
|
*,
|
|
max_tokens: int = 450,
|
|
overlap_tokens: int = 75,
|
|
) -> tuple[
|
|
DocumentPipelineService,
|
|
DocumentUploadService,
|
|
DocumentParsingService,
|
|
DocumentChunkingService,
|
|
LocalObjectStorage,
|
|
sessionmaker[Session],
|
|
]:
|
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(engine)
|
|
session_factory = create_session_factory(engine)
|
|
storage = LocalObjectStorage(tmp_path / "storage")
|
|
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
|
|
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
|
|
chunking = DocumentChunkingService(
|
|
session_factory,
|
|
storage,
|
|
StructureAwareChunker(max_tokens=max_tokens, overlap_tokens=overlap_tokens),
|
|
)
|
|
pipeline = DocumentPipelineService(upload, parsing, chunking)
|
|
return pipeline, upload, parsing, chunking, storage, session_factory
|
|
|
|
|
|
def test_pipeline_persists_traceable_chunks_without_marking_document_ready(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
pipeline, _, _, _, _, session_factory = _build_services(tmp_path)
|
|
|
|
result = pipeline.ingest(
|
|
"policy.md",
|
|
"text/markdown",
|
|
BytesIO("# 产品\n产品介绍\n\n## 退款\n退款周期为七天。".encode()),
|
|
)
|
|
|
|
assert result.document.status is DocumentStatus.PROCESSING
|
|
assert len(result.chunks) == 2
|
|
assert result.chunks[1].section == "产品 / 退款"
|
|
assert result.chunks[1].filename == "policy.md"
|
|
assert result.chunks[1].strategy_version == "structure_v1"
|
|
with session_factory() as session:
|
|
stored_chunks = session.scalars(select(Chunk).order_by(Chunk.chunk_index)).all()
|
|
stored_document = session.get(Document, result.document.id)
|
|
assert [chunk.chunk_id for chunk in stored_chunks] == [
|
|
chunk.chunk_id for chunk in result.chunks
|
|
]
|
|
assert stored_document is not None
|
|
assert stored_document.status is DocumentStatus.PROCESSING
|
|
|
|
|
|
def test_chunking_uses_overlap_and_never_silently_overwrites(tmp_path: Path) -> None:
|
|
pipeline, _, _, chunking, _, session_factory = _build_services(
|
|
tmp_path,
|
|
max_tokens=6,
|
|
overlap_tokens=2,
|
|
)
|
|
result = pipeline.ingest("numbers.txt", "text/plain", BytesIO("一二三四五六七八九十".encode()))
|
|
|
|
assert [chunk.content for chunk in result.chunks] == ["一二三四五六", "五六七八九十"]
|
|
with pytest.raises(DocumentAlreadyChunkedError):
|
|
chunking.process(result.document.id)
|
|
with session_factory() as session:
|
|
assert len(session.scalars(select(Chunk)).all()) == 2
|
|
|
|
|
|
def test_invalid_parsed_artifact_sets_failed_without_partial_chunks(tmp_path: Path) -> None:
|
|
_, upload, parsing, chunking, storage, session_factory = _build_services(tmp_path)
|
|
document = upload.upload("policy.txt", "text/plain", BytesIO("有效内容".encode()))
|
|
parsing.process(document.id)
|
|
assert document.parsed_storage_key is None # Detached upload object is not mutated by parsing.
|
|
parsed_key = f"parsed/{document.id}/document.json"
|
|
storage.delete(parsed_key)
|
|
storage.put(parsed_key, BytesIO(b"invalid-json"))
|
|
|
|
with pytest.raises(ValueError, match="invalid parsed document artifact"):
|
|
chunking.process(document.id)
|
|
|
|
with session_factory() as session:
|
|
stored_document = session.get(Document, document.id)
|
|
chunks = session.scalars(select(Chunk)).all()
|
|
assert stored_document is not None
|
|
assert stored_document.status is DocumentStatus.FAILED
|
|
assert "invalid parsed document artifact" in (stored_document.error_message or "")
|
|
assert chunks == []
|
|
assert storage.exists(document.storage_key)
|