knowledge-platform/tests/integration/test_document_indexing.py
2026-09-02 11:28:13 +08:00

147 lines
5.2 KiB
Python

"""Integration tests for chunk embedding and rebuildable vector indexing."""
from io import BytesIO
from pathlib import Path
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.chunking import StructureAwareChunker
from ekp_core.database import create_session_factory
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
from ekp_core.document_indexing import (
DocumentAlreadyIndexedError,
DocumentIndexingPipelineService,
DocumentIndexingService,
)
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_domain import Base, ChunkIndexState, Document, DocumentStatus, IndexProfile
from ekp_embedding import FakeEmbeddingProvider
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_vector_store import FakeVectorStore, VectorRecord
class PartiallyFailingVectorStore(FakeVectorStore):
"""Expose one partial upsert before simulating a provider batch failure."""
def upsert(self, records: tuple[VectorRecord, ...]) -> None:
super().upsert(records[:1])
raise RuntimeError("simulated vector provider failure")
def _build_services(
tmp_path: Path,
*,
vector_store: FakeVectorStore | None = None,
) -> tuple[
DocumentIndexingPipelineService,
DocumentIndexingService,
FakeVectorStore,
sessionmaker[Session],
]:
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
)
document_pipeline = DocumentPipelineService(upload, parsing, chunking)
embedding = FakeEmbeddingProvider(
model_name="fake-bge-large-zh",
dimensions=4,
max_batch_size=2,
)
profile = IndexProfile(
profile_id="test-profile-v1",
chunk_strategy_version="structure_v1",
embedding_model="fake-bge-large-zh",
vector_dimension=4,
)
selected_store = vector_store or FakeVectorStore()
indexing = DocumentIndexingService(
session_factory,
embedding,
selected_store,
profile,
)
return (
DocumentIndexingPipelineService(document_pipeline, indexing),
indexing,
selected_store,
session_factory,
)
def test_pipeline_marks_ready_only_after_all_chunks_are_indexed(tmp_path: Path) -> None:
pipeline, indexing, vector_store, session_factory = _build_services(tmp_path)
result = pipeline.ingest(
"policy.md",
"text/markdown",
BytesIO("# 产品\n产品介绍\n\n## 退款\n退款周期为七天。".encode()),
)
document_id = result.ingestion.document.id
assert result.ingestion.document.status is DocumentStatus.READY
assert result.indexing.indexed_chunks == 2
assert len(vector_store.records) == 2
assert {record.document_id for record in vector_store.records.values()} == {document_id}
with session_factory() as session:
stored_document = session.get(Document, document_id)
states = session.scalars(select(ChunkIndexState)).all()
assert stored_document is not None
assert stored_document.status is DocumentStatus.READY
assert len(states) == 2
assert all(state.vector_dimension == 4 for state in states)
with pytest.raises(DocumentAlreadyIndexedError):
indexing.process(document_id)
assert vector_store.upsert_calls == 1
shadow_indexing = DocumentIndexingService(
session_factory,
FakeEmbeddingProvider("fake-bge-large-zh", dimensions=4),
vector_store,
IndexProfile(
"test-profile-v2",
"structure_v1",
"fake-bge-large-zh",
4,
),
)
shadow_indexing.process(document_id)
assert len(vector_store.records) == 4
assert {record.index_profile_id for record in vector_store.records.values()} == {
"test-profile-v1",
"test-profile-v2",
}
def test_partial_vector_write_is_compensated_and_document_fails(tmp_path: Path) -> None:
failing_store = PartiallyFailingVectorStore()
pipeline, _, _, session_factory = _build_services(
tmp_path,
vector_store=failing_store,
)
with pytest.raises(RuntimeError, match="simulated vector provider failure"):
pipeline.ingest("policy.txt", "text/plain", BytesIO("退款周期为七天。".encode()))
with session_factory() as session:
stored_document = session.scalar(select(Document))
states = session.scalars(select(ChunkIndexState)).all()
assert stored_document is not None
assert stored_document.status is DocumentStatus.FAILED
assert "simulated vector provider failure" in (stored_document.error_message or "")
assert states == []
assert failing_store.records == {}
assert failing_store.delete_calls == 1