95 lines
4.1 KiB
Python
95 lines
4.1 KiB
Python
"""Golden Dataset regression for reranking and canonical source results."""
|
|
|
|
import json
|
|
from collections.abc import Sequence
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import create_engine
|
|
|
|
from ekp_core.chunking import StructureAwareChunker
|
|
from ekp_core.database import create_session_factory
|
|
from ekp_core.dense_retrieval import DenseRetrievalService
|
|
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
|
|
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
|
|
from ekp_core.document_parsing import DocumentParsingService
|
|
from ekp_core.document_upload import DocumentUploadService
|
|
from ekp_core.search import SearchService
|
|
from ekp_domain import Base, IndexProfile
|
|
from ekp_object_storage import LocalObjectStorage
|
|
from ekp_parser import ParserRouter
|
|
from ekp_reranker import FakeRerankerProvider
|
|
from ekp_vector_store import FakeVectorStore
|
|
|
|
|
|
class GoldenKeywordEmbeddingProvider:
|
|
"""Small semantic fixture matching the Dense regression without external calls."""
|
|
|
|
model_name = "golden-keyword-v1"
|
|
dimensions = 4
|
|
max_batch_size = 16
|
|
|
|
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
|
|
vectors: list[tuple[float, ...]] = []
|
|
for text in texts:
|
|
if "报销" in text or "行程单" in text or "票据" in text:
|
|
vectors.append((0.0, 0.0, 0.0, 1.0))
|
|
elif "住宿" in text or "出差" in text or "酒店" in text:
|
|
vectors.append((1.0, 0.0, 0.0, 0.0))
|
|
elif "支持" in text or "问题" in text or "联系方式" in text:
|
|
vectors.append((0.0, 0.0, 1.0, 0.0))
|
|
elif "退款" in text or "产品A" in text or "产品 A" in text:
|
|
vectors.append((0.0, 1.0, 0.0, 0.0))
|
|
else:
|
|
vectors.append((0.1, 0.1, 0.1, 0.1))
|
|
return tuple(vectors)
|
|
|
|
|
|
def test_reranker_promotes_expected_canonical_source_to_top_one(tmp_path: Path) -> None:
|
|
"""Run the complete Demo pipeline through Dense Top20, Rerank, and Source Result."""
|
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(engine)
|
|
session_factory = create_session_factory(engine)
|
|
storage = LocalObjectStorage(tmp_path / "storage")
|
|
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
|
|
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
|
|
chunking = DocumentChunkingService(
|
|
session_factory,
|
|
storage,
|
|
StructureAwareChunker(),
|
|
)
|
|
embedding = GoldenKeywordEmbeddingProvider()
|
|
vector_store = FakeVectorStore()
|
|
profile = IndexProfile("golden-rerank-v1", "structure_v1", embedding.model_name, 4)
|
|
indexing = DocumentIndexingService(session_factory, embedding, vector_store, profile)
|
|
pipeline = DocumentIndexingPipelineService(
|
|
DocumentPipelineService(upload, parsing, chunking),
|
|
indexing,
|
|
)
|
|
dense = DenseRetrievalService(embedding, vector_store, profile)
|
|
# The fake keeps Golden Dataset evaluation deterministic while pinning the
|
|
# active runtime model identifier selected by ARCH-004.
|
|
reranker = FakeRerankerProvider(model_name="bce-reranker-base")
|
|
search = SearchService(session_factory, dense, reranker)
|
|
assert reranker.model_name == "bce-reranker-base"
|
|
golden_records = [
|
|
json.loads(line)
|
|
for line in Path("fixtures/evaluation/golden.jsonl").read_text().splitlines()
|
|
if line
|
|
]
|
|
|
|
filenames = {filename for record in golden_records for filename in record["expected_documents"]}
|
|
for filename in sorted(filenames):
|
|
payload = (Path("fixtures/documents") / filename).read_bytes()
|
|
pipeline.ingest(filename, "text/markdown", BytesIO(payload))
|
|
|
|
for record in golden_records:
|
|
result = search.search(record["query"])
|
|
assert result.results
|
|
best = result.results[0]
|
|
assert best.filename in record["expected_documents"]
|
|
assert best.section is not None
|
|
assert any(section in best.section for section in record["expected_sections"]), record["id"]
|
|
assert best.content
|
|
assert best.rerank_score >= 0
|