knowledge-platform/tests/integration/test_document_chunking.py
2026-09-01 14:31:30 +08:00

113 lines
4.4 KiB
Python

"""Integration tests for upload-to-chunk processing and canonical metadata."""
from io import BytesIO
from pathlib import Path
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.chunking import StructureAwareChunker
from ekp_core.database import create_session_factory
from ekp_core.document_chunking import (
DocumentAlreadyChunkedError,
DocumentChunkingService,
DocumentPipelineService,
)
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_domain import Base, Chunk, Document, DocumentStatus
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
def _build_services(
tmp_path: Path,
*,
max_tokens: int = 450,
overlap_tokens: int = 75,
) -> tuple[
DocumentPipelineService,
DocumentUploadService,
DocumentParsingService,
DocumentChunkingService,
LocalObjectStorage,
sessionmaker[Session],
]:
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(max_tokens=max_tokens, overlap_tokens=overlap_tokens),
)
pipeline = DocumentPipelineService(upload, parsing, chunking)
return pipeline, upload, parsing, chunking, storage, session_factory
def test_pipeline_persists_traceable_chunks_without_marking_document_ready(
tmp_path: Path,
) -> None:
pipeline, _, _, _, _, session_factory = _build_services(tmp_path)
result = pipeline.ingest(
"policy.md",
"text/markdown",
BytesIO("# 产品\n产品介绍\n\n## 退款\n退款周期为七天。".encode()),
)
assert result.document.status is DocumentStatus.PROCESSING
assert len(result.chunks) == 2
assert result.chunks[1].section == "产品 / 退款"
assert result.chunks[1].filename == "policy.md"
assert result.chunks[1].strategy_version == "structure_v1"
with session_factory() as session:
stored_chunks = session.scalars(select(Chunk).order_by(Chunk.chunk_index)).all()
stored_document = session.get(Document, result.document.id)
assert [chunk.chunk_id for chunk in stored_chunks] == [
chunk.chunk_id for chunk in result.chunks
]
assert stored_document is not None
assert stored_document.status is DocumentStatus.PROCESSING
def test_chunking_uses_overlap_and_never_silently_overwrites(tmp_path: Path) -> None:
pipeline, _, _, chunking, _, session_factory = _build_services(
tmp_path,
max_tokens=6,
overlap_tokens=2,
)
result = pipeline.ingest("numbers.txt", "text/plain", BytesIO("一二三四五六七八九十".encode()))
assert [chunk.content for chunk in result.chunks] == ["一二三四五六", "五六七八九十"]
with pytest.raises(DocumentAlreadyChunkedError):
chunking.process(result.document.id)
with session_factory() as session:
assert len(session.scalars(select(Chunk)).all()) == 2
def test_invalid_parsed_artifact_sets_failed_without_partial_chunks(tmp_path: Path) -> None:
_, upload, parsing, chunking, storage, session_factory = _build_services(tmp_path)
document = upload.upload("policy.txt", "text/plain", BytesIO("有效内容".encode()))
parsing.process(document.id)
assert document.parsed_storage_key is None # Detached upload object is not mutated by parsing.
parsed_key = f"parsed/{document.id}/document.json"
storage.delete(parsed_key)
storage.put(parsed_key, BytesIO(b"invalid-json"))
with pytest.raises(ValueError, match="invalid parsed document artifact"):
chunking.process(document.id)
with session_factory() as session:
stored_document = session.get(Document, document.id)
chunks = session.scalars(select(Chunk)).all()
assert stored_document is not None
assert stored_document.status is DocumentStatus.FAILED
assert "invalid parsed document artifact" in (stored_document.error_message or "")
assert chunks == []
assert storage.exists(document.storage_key)