knowledge-platform/tests/evaluation/test_indexing_regression.py

68 lines
2.8 KiB
Python

"""Golden Dataset source-trace regression through the completed index pipeline."""
import json
from io import BytesIO
from pathlib import Path
from sqlalchemy import create_engine
from ekp_core.chunking import StructureAwareChunker
from ekp_core.database import create_session_factory
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_domain import Base, IndexProfile
from ekp_embedding import FakeEmbeddingProvider
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_vector_store import FakeVectorStore
def test_golden_sources_retain_filename_and_section_in_vector_records(tmp_path: Path) -> None:
"""Run upload-to-index offline and verify retrieval metadata survives intact."""
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(),
)
document_pipeline = DocumentPipelineService(upload, parsing, chunking)
embedding = FakeEmbeddingProvider(model_name="golden-fake", dimensions=8)
vector_store = FakeVectorStore()
indexing = DocumentIndexingService(
session_factory,
embedding,
vector_store,
IndexProfile("golden-v1", "structure_v1", "golden-fake", 8),
)
pipeline = DocumentIndexingPipelineService(document_pipeline, indexing)
records = [
json.loads(line)
for line in Path("fixtures/evaluation/golden.jsonl").read_text().splitlines()
if line
]
for golden_record in records:
for filename in golden_record["expected_documents"]:
payload = (Path("fixtures/documents") / filename).read_bytes()
pipeline.ingest(filename, "text/markdown", BytesIO(payload))
indexed_records = tuple(vector_store.records.values())
for golden_record in records:
for filename in golden_record["expected_documents"]:
matching = [record for record in indexed_records if record.filename == filename]
assert matching
available_sections = {
section
for record in matching
if record.section is not None
for section in record.section.split(" / ")
}
assert set(golden_record["expected_sections"]).issubset(available_sections)