"""Integration tests for chunk embedding and rebuildable vector indexing.""" from io import BytesIO from pathlib import Path import pytest from sqlalchemy import create_engine, select from sqlalchemy.orm import Session, sessionmaker from ekp_core.chunking import StructureAwareChunker from ekp_core.database import create_session_factory from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService from ekp_core.document_indexing import ( DocumentAlreadyIndexedError, DocumentIndexingPipelineService, DocumentIndexingService, ) from ekp_core.document_parsing import DocumentParsingService from ekp_core.document_upload import DocumentUploadService from ekp_domain import Base, ChunkIndexState, Document, DocumentStatus, IndexProfile from ekp_embedding import FakeEmbeddingProvider from ekp_object_storage import LocalObjectStorage from ekp_parser import ParserRouter from ekp_vector_store import FakeVectorStore, VectorRecord class PartiallyFailingVectorStore(FakeVectorStore): """Expose one partial upsert before simulating a provider batch failure.""" def upsert(self, records: tuple[VectorRecord, ...]) -> None: super().upsert(records[:1]) raise RuntimeError("simulated vector provider failure") def _build_services( tmp_path: Path, *, vector_store: FakeVectorStore | None = None, ) -> tuple[ DocumentIndexingPipelineService, DocumentIndexingService, FakeVectorStore, sessionmaker[Session], ]: engine = create_engine("sqlite+pysqlite:///:memory:") Base.metadata.create_all(engine) session_factory = create_session_factory(engine) storage = LocalObjectStorage(tmp_path / "storage") upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024) parsing = DocumentParsingService(session_factory, storage, ParserRouter()) chunking = DocumentChunkingService( session_factory, storage, StructureAwareChunker(max_tokens=450, overlap_tokens=75), ) document_pipeline = DocumentPipelineService(upload, parsing, chunking) embedding = FakeEmbeddingProvider( model_name="fake-bge-large-zh", dimensions=4, max_batch_size=2, ) profile = IndexProfile( profile_id="test-profile-v1", chunk_strategy_version="structure_v1", embedding_model="fake-bge-large-zh", vector_dimension=4, ) selected_store = vector_store or FakeVectorStore() indexing = DocumentIndexingService( session_factory, embedding, selected_store, profile, ) return ( DocumentIndexingPipelineService(document_pipeline, indexing), indexing, selected_store, session_factory, ) def test_pipeline_marks_ready_only_after_all_chunks_are_indexed(tmp_path: Path) -> None: pipeline, indexing, vector_store, session_factory = _build_services(tmp_path) result = pipeline.ingest( "policy.md", "text/markdown", BytesIO("# 产品\n产品介绍\n\n## 退款\n退款周期为七天。".encode()), ) document_id = result.ingestion.document.id assert result.ingestion.document.status is DocumentStatus.READY assert result.indexing.indexed_chunks == 2 assert len(vector_store.records) == 2 assert {record.document_id for record in vector_store.records.values()} == {document_id} with session_factory() as session: stored_document = session.get(Document, document_id) states = session.scalars(select(ChunkIndexState)).all() assert stored_document is not None assert stored_document.status is DocumentStatus.READY assert len(states) == 2 assert all(state.vector_dimension == 4 for state in states) with pytest.raises(DocumentAlreadyIndexedError): indexing.process(document_id) assert vector_store.upsert_calls == 1 shadow_indexing = DocumentIndexingService( session_factory, FakeEmbeddingProvider("fake-bge-large-zh", dimensions=4), vector_store, IndexProfile( "test-profile-v2", "structure_v1", "fake-bge-large-zh", 4, ), ) shadow_indexing.process(document_id) assert len(vector_store.records) == 4 assert {record.index_profile_id for record in vector_store.records.values()} == { "test-profile-v1", "test-profile-v2", } def test_partial_vector_write_is_compensated_and_document_fails(tmp_path: Path) -> None: failing_store = PartiallyFailingVectorStore() pipeline, _, _, session_factory = _build_services( tmp_path, vector_store=failing_store, ) with pytest.raises(RuntimeError, match="simulated vector provider failure"): pipeline.ingest("policy.txt", "text/plain", BytesIO("退款周期为七天。".encode())) with session_factory() as session: stored_document = session.scalar(select(Document)) states = session.scalars(select(ChunkIndexState)).all() assert stored_document is not None assert stored_document.status is DocumentStatus.FAILED assert "simulated vector provider failure" in (stored_document.error_message or "") assert states == [] assert failing_store.records == {} assert failing_store.delete_calls == 1