147 lines
5.2 KiB
Python
147 lines
5.2 KiB
Python
"""Integration tests for chunk embedding and rebuildable vector indexing."""
|
|
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, select
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
|
|
from ekp_core.chunking import StructureAwareChunker
|
|
from ekp_core.database import create_session_factory
|
|
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
|
|
from ekp_core.document_indexing import (
|
|
DocumentAlreadyIndexedError,
|
|
DocumentIndexingPipelineService,
|
|
DocumentIndexingService,
|
|
)
|
|
from ekp_core.document_parsing import DocumentParsingService
|
|
from ekp_core.document_upload import DocumentUploadService
|
|
from ekp_domain import Base, ChunkIndexState, Document, DocumentStatus, IndexProfile
|
|
from ekp_embedding import FakeEmbeddingProvider
|
|
from ekp_object_storage import LocalObjectStorage
|
|
from ekp_parser import ParserRouter
|
|
from ekp_vector_store import FakeVectorStore, VectorRecord
|
|
|
|
|
|
class PartiallyFailingVectorStore(FakeVectorStore):
|
|
"""Expose one partial upsert before simulating a Mochow batch failure."""
|
|
|
|
def upsert(self, records: tuple[VectorRecord, ...]) -> None:
|
|
super().upsert(records[:1])
|
|
raise RuntimeError("simulated Mochow failure")
|
|
|
|
|
|
def _build_services(
|
|
tmp_path: Path,
|
|
*,
|
|
vector_store: FakeVectorStore | None = None,
|
|
) -> tuple[
|
|
DocumentIndexingPipelineService,
|
|
DocumentIndexingService,
|
|
FakeVectorStore,
|
|
sessionmaker[Session],
|
|
]:
|
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(engine)
|
|
session_factory = create_session_factory(engine)
|
|
storage = LocalObjectStorage(tmp_path / "storage")
|
|
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
|
|
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
|
|
chunking = DocumentChunkingService(
|
|
session_factory,
|
|
storage,
|
|
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
|
|
)
|
|
document_pipeline = DocumentPipelineService(upload, parsing, chunking)
|
|
embedding = FakeEmbeddingProvider(
|
|
model_name="fake-bge-large-zh",
|
|
dimensions=4,
|
|
max_batch_size=2,
|
|
)
|
|
profile = IndexProfile(
|
|
profile_id="test-profile-v1",
|
|
chunk_strategy_version="structure_v1",
|
|
embedding_model="fake-bge-large-zh",
|
|
vector_dimension=4,
|
|
)
|
|
selected_store = vector_store or FakeVectorStore()
|
|
indexing = DocumentIndexingService(
|
|
session_factory,
|
|
embedding,
|
|
selected_store,
|
|
profile,
|
|
)
|
|
return (
|
|
DocumentIndexingPipelineService(document_pipeline, indexing),
|
|
indexing,
|
|
selected_store,
|
|
session_factory,
|
|
)
|
|
|
|
|
|
def test_pipeline_marks_ready_only_after_all_chunks_are_indexed(tmp_path: Path) -> None:
|
|
pipeline, indexing, vector_store, session_factory = _build_services(tmp_path)
|
|
|
|
result = pipeline.ingest(
|
|
"policy.md",
|
|
"text/markdown",
|
|
BytesIO("# 产品\n产品介绍\n\n## 退款\n退款周期为七天。".encode()),
|
|
)
|
|
|
|
document_id = result.ingestion.document.id
|
|
assert result.ingestion.document.status is DocumentStatus.READY
|
|
assert result.indexing.indexed_chunks == 2
|
|
assert len(vector_store.records) == 2
|
|
assert {record.document_id for record in vector_store.records.values()} == {document_id}
|
|
with session_factory() as session:
|
|
stored_document = session.get(Document, document_id)
|
|
states = session.scalars(select(ChunkIndexState)).all()
|
|
assert stored_document is not None
|
|
assert stored_document.status is DocumentStatus.READY
|
|
assert len(states) == 2
|
|
assert all(state.vector_dimension == 4 for state in states)
|
|
|
|
with pytest.raises(DocumentAlreadyIndexedError):
|
|
indexing.process(document_id)
|
|
assert vector_store.upsert_calls == 1
|
|
|
|
shadow_indexing = DocumentIndexingService(
|
|
session_factory,
|
|
FakeEmbeddingProvider("fake-bge-large-zh", dimensions=4),
|
|
vector_store,
|
|
IndexProfile(
|
|
"test-profile-v2",
|
|
"structure_v1",
|
|
"fake-bge-large-zh",
|
|
4,
|
|
),
|
|
)
|
|
shadow_indexing.process(document_id)
|
|
assert len(vector_store.records) == 4
|
|
assert {record.index_profile_id for record in vector_store.records.values()} == {
|
|
"test-profile-v1",
|
|
"test-profile-v2",
|
|
}
|
|
|
|
|
|
def test_partial_vector_write_is_compensated_and_document_fails(tmp_path: Path) -> None:
|
|
failing_store = PartiallyFailingVectorStore()
|
|
pipeline, _, _, session_factory = _build_services(
|
|
tmp_path,
|
|
vector_store=failing_store,
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="simulated Mochow failure"):
|
|
pipeline.ingest("policy.txt", "text/plain", BytesIO("退款周期为七天。".encode()))
|
|
|
|
with session_factory() as session:
|
|
stored_document = session.scalar(select(Document))
|
|
states = session.scalars(select(ChunkIndexState)).all()
|
|
assert stored_document is not None
|
|
assert stored_document.status is DocumentStatus.FAILED
|
|
assert "simulated Mochow failure" in (stored_document.error_message or "")
|
|
assert states == []
|
|
assert failing_store.records == {}
|
|
assert failing_store.delete_calls == 1
|