knowledge-platform/tests/evaluation/test_reranked_search_regression.py

95 lines
4.1 KiB
Python

"""Golden Dataset regression for reranking and canonical source results."""
import json
from collections.abc import Sequence
from io import BytesIO
from pathlib import Path
from sqlalchemy import create_engine
from ekp_core.chunking import StructureAwareChunker
from ekp_core.database import create_session_factory
from ekp_core.dense_retrieval import DenseRetrievalService
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.search import SearchService
from ekp_domain import Base, IndexProfile
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_reranker import FakeRerankerProvider
from ekp_vector_store import FakeVectorStore
class GoldenKeywordEmbeddingProvider:
"""Small semantic fixture matching the Dense regression without external calls."""
model_name = "golden-keyword-v1"
dimensions = 4
max_batch_size = 16
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
vectors: list[tuple[float, ...]] = []
for text in texts:
if "报销" in text or "行程单" in text or "票据" in text:
vectors.append((0.0, 0.0, 0.0, 1.0))
elif "住宿" in text or "出差" in text or "酒店" in text:
vectors.append((1.0, 0.0, 0.0, 0.0))
elif "支持" in text or "问题" in text or "联系方式" in text:
vectors.append((0.0, 0.0, 1.0, 0.0))
elif "退款" in text or "产品A" in text or "产品 A" in text:
vectors.append((0.0, 1.0, 0.0, 0.0))
else:
vectors.append((0.1, 0.1, 0.1, 0.1))
return tuple(vectors)
def test_reranker_promotes_expected_canonical_source_to_top_one(tmp_path: Path) -> None:
"""Run the complete Demo pipeline through Dense Top20, Rerank, and Source Result."""
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(),
)
embedding = GoldenKeywordEmbeddingProvider()
vector_store = FakeVectorStore()
profile = IndexProfile("golden-rerank-v1", "structure_v1", embedding.model_name, 4)
indexing = DocumentIndexingService(session_factory, embedding, vector_store, profile)
pipeline = DocumentIndexingPipelineService(
DocumentPipelineService(upload, parsing, chunking),
indexing,
)
dense = DenseRetrievalService(embedding, vector_store, profile)
# The fake keeps Golden Dataset evaluation deterministic while pinning the
# active runtime model identifier selected by ARCH-004.
reranker = FakeRerankerProvider(model_name="bce-reranker-base")
search = SearchService(session_factory, dense, reranker)
assert reranker.model_name == "bce-reranker-base"
golden_records = [
json.loads(line)
for line in Path("fixtures/evaluation/golden.jsonl").read_text().splitlines()
if line
]
filenames = {filename for record in golden_records for filename in record["expected_documents"]}
for filename in sorted(filenames):
payload = (Path("fixtures/documents") / filename).read_bytes()
pipeline.ingest(filename, "text/markdown", BytesIO(payload))
for record in golden_records:
result = search.search(record["query"])
assert result.results
best = result.results[0]
assert best.filename in record["expected_documents"]
assert best.section is not None
assert any(section in best.section for section in record["expected_sections"]), record["id"]
assert best.content
assert best.rerank_score >= 0