94 lines
3.9 KiB
Python
94 lines
3.9 KiB
Python
"""Offline Golden Dataset regression for DEMO-007 Dense Retrieval."""
|
|
|
|
import json
|
|
from collections.abc import Sequence
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import create_engine
|
|
|
|
from ekp_core.chunking import StructureAwareChunker
|
|
from ekp_core.database import create_session_factory
|
|
from ekp_core.dense_retrieval import DenseRetrievalService
|
|
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
|
|
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
|
|
from ekp_core.document_parsing import DocumentParsingService
|
|
from ekp_core.document_upload import DocumentUploadService
|
|
from ekp_domain import Base, IndexProfile
|
|
from ekp_object_storage import LocalObjectStorage
|
|
from ekp_parser import ParserRouter
|
|
from ekp_vector_store import FakeVectorStore
|
|
|
|
|
|
class GoldenKeywordEmbeddingProvider:
|
|
"""Small semantic fixture that keeps retrieval evaluation offline and stable."""
|
|
|
|
model_name = "golden-keyword-v1"
|
|
dimensions = 4
|
|
max_batch_size = 16
|
|
|
|
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
|
|
vectors: list[tuple[float, ...]] = []
|
|
for text in texts:
|
|
if "报销" in text or "行程单" in text or "票据" in text:
|
|
vectors.append((0.0, 0.0, 0.0, 1.0))
|
|
elif "住宿" in text or "出差" in text or "酒店" in text:
|
|
vectors.append((1.0, 0.0, 0.0, 0.0))
|
|
elif "支持" in text or "问题" in text or "联系方式" in text:
|
|
vectors.append((0.0, 0.0, 1.0, 0.0))
|
|
elif "退款" in text or "产品A" in text or "产品 A" in text:
|
|
vectors.append((0.0, 1.0, 0.0, 0.0))
|
|
else:
|
|
vectors.append((0.1, 0.1, 0.1, 0.1))
|
|
return tuple(vectors)
|
|
|
|
|
|
def test_dense_top_twenty_recalls_each_golden_source(tmp_path: Path) -> None:
|
|
"""Run ingestion and query embedding, then require the expected source at rank one."""
|
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(engine)
|
|
session_factory = create_session_factory(engine)
|
|
storage = LocalObjectStorage(tmp_path / "storage")
|
|
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
|
|
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
|
|
chunking = DocumentChunkingService(
|
|
session_factory,
|
|
storage,
|
|
StructureAwareChunker(),
|
|
)
|
|
embedding = GoldenKeywordEmbeddingProvider()
|
|
vector_store = FakeVectorStore()
|
|
profile = IndexProfile("golden-dense-v1", "structure_v1", embedding.model_name, 4)
|
|
indexing = DocumentIndexingService(session_factory, embedding, vector_store, profile)
|
|
pipeline = DocumentIndexingPipelineService(
|
|
DocumentPipelineService(upload, parsing, chunking),
|
|
indexing,
|
|
)
|
|
retrieval = DenseRetrievalService(embedding, vector_store, profile)
|
|
golden_records = [
|
|
json.loads(line)
|
|
for line in Path("fixtures/evaluation/golden.jsonl").read_text().splitlines()
|
|
if line
|
|
]
|
|
|
|
filenames = {filename for record in golden_records for filename in record["expected_documents"]}
|
|
for filename in sorted(filenames):
|
|
payload = (Path("fixtures/documents") / filename).read_bytes()
|
|
pipeline.ingest(filename, "text/markdown", BytesIO(payload))
|
|
|
|
for record in golden_records:
|
|
result = retrieval.search(record["query"])
|
|
assert result.candidates
|
|
best = result.candidates[0]
|
|
assert best.filename in record["expected_documents"]
|
|
matching_sections = [
|
|
candidate.section
|
|
for candidate in result.candidates
|
|
if candidate.filename in record["expected_documents"] and candidate.section is not None
|
|
]
|
|
assert any(
|
|
expected_section in section
|
|
for section in matching_sections
|
|
for expected_section in record["expected_sections"]
|
|
)
|