knowledge-platform/tests/evaluation/test_dense_retrieval_regression.py

94 lines
3.9 KiB
Python

"""Offline Golden Dataset regression for DEMO-007 Dense Retrieval."""
import json
from collections.abc import Sequence
from io import BytesIO
from pathlib import Path
from sqlalchemy import create_engine
from ekp_core.chunking import StructureAwareChunker
from ekp_core.database import create_session_factory
from ekp_core.dense_retrieval import DenseRetrievalService
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_domain import Base, IndexProfile
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_vector_store import FakeVectorStore
class GoldenKeywordEmbeddingProvider:
"""Small semantic fixture that keeps retrieval evaluation offline and stable."""
model_name = "golden-keyword-v1"
dimensions = 4
max_batch_size = 16
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
vectors: list[tuple[float, ...]] = []
for text in texts:
if "报销" in text or "行程单" in text or "票据" in text:
vectors.append((0.0, 0.0, 0.0, 1.0))
elif "住宿" in text or "出差" in text or "酒店" in text:
vectors.append((1.0, 0.0, 0.0, 0.0))
elif "支持" in text or "问题" in text or "联系方式" in text:
vectors.append((0.0, 0.0, 1.0, 0.0))
elif "退款" in text or "产品A" in text or "产品 A" in text:
vectors.append((0.0, 1.0, 0.0, 0.0))
else:
vectors.append((0.1, 0.1, 0.1, 0.1))
return tuple(vectors)
def test_dense_top_twenty_recalls_each_golden_source(tmp_path: Path) -> None:
"""Run ingestion and query embedding, then require the expected source at rank one."""
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(),
)
embedding = GoldenKeywordEmbeddingProvider()
vector_store = FakeVectorStore()
profile = IndexProfile("golden-dense-v1", "structure_v1", embedding.model_name, 4)
indexing = DocumentIndexingService(session_factory, embedding, vector_store, profile)
pipeline = DocumentIndexingPipelineService(
DocumentPipelineService(upload, parsing, chunking),
indexing,
)
retrieval = DenseRetrievalService(embedding, vector_store, profile)
golden_records = [
json.loads(line)
for line in Path("fixtures/evaluation/golden.jsonl").read_text().splitlines()
if line
]
filenames = {filename for record in golden_records for filename in record["expected_documents"]}
for filename in sorted(filenames):
payload = (Path("fixtures/documents") / filename).read_bytes()
pipeline.ingest(filename, "text/markdown", BytesIO(payload))
for record in golden_records:
result = retrieval.search(record["query"])
assert result.candidates
best = result.candidates[0]
assert best.filename in record["expected_documents"]
matching_sections = [
candidate.section
for candidate in result.candidates
if candidate.filename in record["expected_documents"] and candidate.section is not None
]
assert any(
expected_section in section
for section in matching_sections
for expected_section in record["expected_sections"]
)