feat: implement DEMO-008 reranking and source results
This commit is contained in:
parent
a0aa30c41c
commit
f696aeec2b
@ -41,9 +41,11 @@ INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1
|
||||
DENSE_TOP_N=20
|
||||
|
||||
RERANKER_PROVIDER=qianfan
|
||||
RERANKER_MODEL=
|
||||
RERANKER_ENDPOINT=
|
||||
RERANKER_MODEL=qwen3-reranker-0.6b
|
||||
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
|
||||
RERANKER_API_KEY=
|
||||
RERANKER_TIMEOUT_SECONDS=30
|
||||
SEARCH_TOP_K=5
|
||||
|
||||
LLM_PROVIDER=qianfan
|
||||
LLM_MODEL=
|
||||
|
||||
10
README.md
10
README.md
@ -93,5 +93,11 @@ pre-provisioned table schema.
|
||||
DEMO-007 adds query embedding and profile-scoped Dense Top-N retrieval. The
|
||||
default is Top 20 with a configurable Mochow HNSW `ef`; the in-memory fake uses
|
||||
exact cosine search for deterministic offline regression. Returned objects are
|
||||
raw dense candidates only. Reranking and final source-result assembly remain in
|
||||
DEMO-008, while public search REST endpoints remain in DEMO-009.
|
||||
raw dense candidates only.
|
||||
|
||||
DEMO-008 adds a provider-neutral reranker, a validated Qianfan
|
||||
`qwen3-reranker-0.6b` adapter, and the final Top20-to-Top5 search pipeline. Final
|
||||
source results are rehydrated from canonical PostgreSQL chunks and retain both
|
||||
Dense and reranker scores; stale vector content fails closed. See
|
||||
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
|
||||
Public search REST endpoints remain in DEMO-009.
|
||||
|
||||
@ -1,9 +1,13 @@
|
||||
"""Runtime provider composition from secret-safe application settings."""
|
||||
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.config import Settings
|
||||
from ekp_core.dense_retrieval import DenseRetrievalService
|
||||
from ekp_core.search import SearchService
|
||||
from ekp_domain import IndexProfile
|
||||
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
|
||||
from ekp_reranker import QianfanRerankerProvider, RerankerProvider
|
||||
from ekp_vector_store import MochowVectorStore, VectorStore, create_mochow_vector_store
|
||||
|
||||
BGE_LARGE_ZH_DIMENSIONS = 1024
|
||||
@ -68,3 +72,31 @@ def create_dense_retrieval_service(
|
||||
index_profile=index_profile,
|
||||
default_top_n=settings.dense_top_n,
|
||||
)
|
||||
|
||||
|
||||
def create_reranker_provider(settings: Settings) -> QianfanRerankerProvider:
|
||||
"""Create the Demo Qwen3 reranker using an explicit or shared Qianfan key."""
|
||||
secret = settings.reranker_api_key or settings.baidu_qianfan_api_key
|
||||
if secret is None:
|
||||
raise ProviderConfigurationError("RERANKER_API_KEY or BAIDU_QIANFAN_API_KEY is required")
|
||||
return QianfanRerankerProvider(
|
||||
api_key=secret.get_secret_value(),
|
||||
endpoint=settings.reranker_endpoint,
|
||||
model_name=settings.reranker_model,
|
||||
timeout_seconds=settings.reranker_timeout_seconds,
|
||||
)
|
||||
|
||||
|
||||
def create_search_service(
|
||||
settings: Settings,
|
||||
session_factory: sessionmaker[Session],
|
||||
dense_retriever: DenseRetrievalService,
|
||||
reranker_provider: RerankerProvider,
|
||||
) -> SearchService:
|
||||
"""Compose DEMO-008 search while leaving REST serialization to DEMO-009."""
|
||||
return SearchService(
|
||||
session_factory=session_factory,
|
||||
dense_retriever=dense_retriever,
|
||||
reranker_provider=reranker_provider,
|
||||
default_top_k=settings.search_top_k,
|
||||
)
|
||||
|
||||
@ -77,8 +77,8 @@ DEMO-007 使用同一 `EmbeddingProvider` 将 Query 转换为 1024 维向量,
|
||||
- 返回值是尚未 Rerank 的候选集,`score` 原样保留 Mochow 返回值,不在此阶段设置阈值。
|
||||
|
||||
Demo 明确不实现 RBAC/Document ACL;正式 V1 引入权限后,允许范围仍必须在召回阶段下推,
|
||||
不得先做全局 Top-K 再过滤。Reranker 与最终 Source Result 属于 DEMO-008,REST API 属于
|
||||
DEMO-009。
|
||||
不得先做全局 Top-K 再过滤。Reranker 与最终 Source Result 的配置参见
|
||||
`docs/development/RERANKER_SETUP.md`;REST API 属于 DEMO-009。
|
||||
|
||||
## 验证边界
|
||||
|
||||
|
||||
69
docs/development/RERANKER_SETUP.md
Normal file
69
docs/development/RERANKER_SETUP.md
Normal file
@ -0,0 +1,69 @@
|
||||
# DEMO-008 千帆 Reranker 与 Source Result 配置
|
||||
|
||||
本文记录 Knowledge Platform Demo 的重排序连接方式、请求边界和 canonical Source Result
|
||||
契约。真实 API Key 只允许由运行环境注入,不得写入代码、示例配置、日志或提交记录。
|
||||
|
||||
## 运行配置
|
||||
|
||||
```dotenv
|
||||
RERANKER_PROVIDER=qianfan
|
||||
RERANKER_MODEL=qwen3-reranker-0.6b
|
||||
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
|
||||
RERANKER_API_KEY=<运行环境注入,可空>
|
||||
RERANKER_TIMEOUT_SECONDS=30
|
||||
DENSE_TOP_N=20
|
||||
SEARCH_TOP_K=5
|
||||
```
|
||||
|
||||
`RERANKER_API_KEY` 未单独配置时,Demo 会复用 `BAIDU_QIANFAN_API_KEY`。两者都不存在时
|
||||
Provider 创建失败。凭证使用 `SecretStr`,HTTP/JSON 异常不会包含请求头、请求正文或
|
||||
响应正文。
|
||||
|
||||
## Reranker 请求契约
|
||||
|
||||
DEMO-008 固定使用千帆 `POST /v2/rerank`,模型基线为
|
||||
`qwen3-reranker-0.6b`:
|
||||
|
||||
- Query 去除首尾空白后不能为空,最多 1600 字符;
|
||||
- documents 不能为空,最多 64 条,每条最多 4096 字符;
|
||||
- 当前标准链路发送 Dense Top20,并请求 Reranker Top5;
|
||||
- 响应必须恰好返回请求的 Top-N,`index` 唯一且指向原输入位置;
|
||||
- `relevance_score` 必须是有限数值,并保持降序;
|
||||
- 响应若回显 document,必须与对应输入文本完全一致。
|
||||
|
||||
Adapter 只向业务层返回原候选索引和 Reranker 分数,不暴露千帆响应结构。默认测试使用
|
||||
`FakeRerankerProvider` 的确定性字符覆盖率排序,不发起网络请求。
|
||||
|
||||
## Source Result 契约
|
||||
|
||||
Reranker 对 Mochow Dense Candidate 的 `content` 评分;选出 Top5 后,`SearchService` 使用
|
||||
`chunk_id` 一次性回查 PostgreSQL canonical `chunks` 表,并由 canonical row 生成:
|
||||
|
||||
```text
|
||||
document_id
|
||||
filename
|
||||
page
|
||||
section
|
||||
chunk_id
|
||||
content
|
||||
score
|
||||
rerank_score
|
||||
```
|
||||
|
||||
其中 `score` 保留 Dense 分数,`rerank_score` 保存精排分数。若 Chunk 缺失、document_id
|
||||
不一致或 Mochow content 已与 canonical content 不一致,搜索会失败关闭,避免返回来源与
|
||||
评分文本不一致的结果。
|
||||
|
||||
DEMO-008 不创建公开 REST Endpoint;`POST /api/search` 的序列化和错误映射属于
|
||||
DEMO-009。Demo 仍不实现 RBAC/Document ACL,正式 V1 权限范围必须在 Dense 召回阶段
|
||||
下推,不能在 Reranker 后过滤。
|
||||
|
||||
## 验证边界
|
||||
|
||||
默认测试离线覆盖 Dense Top20 → Reranker → Top5 → canonical Source Result,并使用合成
|
||||
Golden Dataset 验证预期文档和章节升至首位。真实千帆联调前还需在受控运行环境确认:
|
||||
|
||||
- API Key 已开通目标 Reranker 模型权限;
|
||||
- Endpoint 网络可达;
|
||||
- 模型 API 名称与控制台服务一致;
|
||||
- 真实中文 Query 的延迟、配额和 Top5 质量满足 Demo 演示需求。
|
||||
@ -2,11 +2,11 @@
|
||||
|
||||
当前阶段:Knowledge Retrieval Demo v0.1
|
||||
|
||||
已完成:DEMO-001、DEMO-002、DEMO-003、DEMO-004、DEMO-005、DEMO-006、DEMO-007
|
||||
已完成:DEMO-001、DEMO-002、DEMO-003、DEMO-004、DEMO-005、DEMO-006、DEMO-007、DEMO-008
|
||||
进行中:无
|
||||
阻塞:无
|
||||
|
||||
下一任务:DEMO-008
|
||||
下一任务:DEMO-009
|
||||
|
||||
完整 V1 待办将在 Demo Scope 完成后继续执行。当前任务顺序以
|
||||
`docs/demo/DEMO_SCOPE.md` 为准。
|
||||
@ -101,3 +101,19 @@
|
||||
章节。
|
||||
- 默认自动化测试不连接真实千帆/Mochow;真实环境仍需按
|
||||
`docs/development/MOCHOW_SETUP.md` 注入凭证并验证网络、表 Schema 与索引状态。
|
||||
|
||||
### DEMO-008
|
||||
|
||||
- 新增与厂商无关的 `RerankerProvider` / `RerankResult` 契约及确定性 Fake,使用原始
|
||||
candidate index 映射精排结果,业务服务不依赖千帆响应结构。
|
||||
- 新增千帆 v2 `qwen3-reranker-0.6b` Adapter,调用 `/v2/rerank`,严格限制 Query、
|
||||
documents 和 Top-N,校验响应数量、唯一索引、回显文本、有限分数与降序关系;错误不会
|
||||
回显请求内容或凭证。
|
||||
- 新增 `SearchService`,完成 Dense Top20 → Reranker → Top5。Reranker 选中候选后按
|
||||
`chunk_id` 回查 PostgreSQL canonical Chunk,再生成包含 document、filename、page、
|
||||
section、chunk、content、Dense score 和 rerank score 的最终 Source Result。
|
||||
- Source Enrichment 对缺失 Chunk、document_id 不一致和 stale vector content 采用失败
|
||||
关闭,防止 Mochow 副本与 canonical 来源不一致时返回错误证据。
|
||||
- 新增 Provider、配置脱敏、排序映射、canonical 回查、一致性失败和 Golden Dataset
|
||||
全链路回归。默认测试不调用真实千帆;真实联调边界记录在
|
||||
`docs/development/RERANKER_SETUP.md`。公开 Search REST API 留在 DEMO-009。
|
||||
|
||||
@ -8,6 +8,7 @@ from ekp_core.document_indexing import (
|
||||
DocumentIndexingService,
|
||||
)
|
||||
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
|
||||
from ekp_core.search import SearchResult, SearchService, SourceResult
|
||||
|
||||
__all__ = [
|
||||
"DocumentChunkingService",
|
||||
@ -18,5 +19,8 @@ __all__ = [
|
||||
"DocumentIndexingService",
|
||||
"DocumentPipelineService",
|
||||
"DocumentParsingService",
|
||||
"SearchResult",
|
||||
"SearchService",
|
||||
"SourceResult",
|
||||
"clean_parsed_document",
|
||||
]
|
||||
|
||||
@ -46,13 +46,21 @@ class Settings(BaseSettings):
|
||||
mochow_search_ef: int = Field(default=200, ge=1, le=10000)
|
||||
mochow_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
index_profile_id: str = "demo-bge-large-zh-structure-v1"
|
||||
dense_top_n: int = Field(default=20, ge=1, le=1000)
|
||||
dense_top_n: int = Field(default=20, ge=1, le=64)
|
||||
reranker_provider: Literal["qianfan"] = "qianfan"
|
||||
reranker_model: Literal["qwen3-reranker-0.6b"] = "qwen3-reranker-0.6b"
|
||||
reranker_endpoint: str = "https://qianfan.baidubce.com/v2/rerank"
|
||||
reranker_api_key: SecretStr | None = None
|
||||
reranker_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
search_top_k: int = Field(default=5, ge=1, le=64)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_chunk_window(self) -> "Settings":
|
||||
"""Require forward progress for every overlapping chunk window."""
|
||||
if self.chunk_overlap_tokens >= self.chunk_max_tokens:
|
||||
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
|
||||
if self.search_top_k > self.dense_top_n:
|
||||
raise ValueError("search_top_k must not exceed dense_top_n")
|
||||
return self
|
||||
|
||||
|
||||
|
||||
192
packages/core/src/ekp_core/search.py
Normal file
192
packages/core/src/ekp_core/search.py
Normal file
@ -0,0 +1,192 @@
|
||||
"""Reranked search orchestration and canonical source-result assembly."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from math import isfinite
|
||||
from typing import Protocol
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.dense_retrieval import DenseRetrievalResult
|
||||
from ekp_domain import Chunk
|
||||
from ekp_reranker import RerankerProvider
|
||||
from ekp_vector_store import DenseSearchCandidate
|
||||
|
||||
DEFAULT_SEARCH_TOP_K = 5
|
||||
|
||||
|
||||
class SearchConfigurationError(ValueError):
|
||||
"""Raised when candidate and final result limits cannot form a valid search."""
|
||||
|
||||
|
||||
class RerankerContractError(RuntimeError):
|
||||
"""Raised when reranker output cannot map safely to dense candidates."""
|
||||
|
||||
|
||||
class CanonicalSourceError(RuntimeError):
|
||||
"""Raised when a selected vector candidate cannot be traced to canonical Chunk data."""
|
||||
|
||||
|
||||
class DenseRetriever(Protocol):
|
||||
"""Minimal dense-retrieval boundary consumed by the final search pipeline."""
|
||||
|
||||
def search(self, query: str, top_n: int | None = None) -> DenseRetrievalResult:
|
||||
"""Return normalized query and dense candidates."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SourceResult:
|
||||
"""One canonical, traceable result after reranking."""
|
||||
|
||||
document_id: UUID
|
||||
filename: str
|
||||
page: int | None
|
||||
section: str | None
|
||||
chunk_id: UUID
|
||||
content: str
|
||||
score: float
|
||||
rerank_score: float
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SearchResult:
|
||||
"""Final Top-K result set ready for DEMO-009 REST serialization."""
|
||||
|
||||
query: str
|
||||
index_profile_id: str
|
||||
results: tuple[SourceResult, ...]
|
||||
|
||||
|
||||
class SearchService:
|
||||
"""Compose Dense Top-N, reranking, and PostgreSQL source enrichment."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker[Session],
|
||||
dense_retriever: DenseRetriever,
|
||||
reranker_provider: RerankerProvider,
|
||||
default_top_k: int = DEFAULT_SEARCH_TOP_K,
|
||||
) -> None:
|
||||
if (
|
||||
isinstance(default_top_k, bool)
|
||||
or not 1 <= default_top_k <= reranker_provider.max_documents
|
||||
):
|
||||
raise SearchConfigurationError(
|
||||
"default_top_k must fit within the reranker document limit"
|
||||
)
|
||||
self._session_factory = session_factory
|
||||
self._dense_retriever = dense_retriever
|
||||
self._reranker_provider = reranker_provider
|
||||
self._default_top_k = default_top_k
|
||||
|
||||
def search(self, query: str, top_k: int | None = None) -> SearchResult:
|
||||
"""Return Top-K sources while preserving both dense and reranker scores."""
|
||||
requested_top_k = self._default_top_k if top_k is None else top_k
|
||||
if (
|
||||
isinstance(requested_top_k, bool)
|
||||
or not 1 <= requested_top_k <= self._reranker_provider.max_documents
|
||||
):
|
||||
raise SearchConfigurationError("top_k must fit within the reranker document limit")
|
||||
|
||||
dense_result = self._dense_retriever.search(query)
|
||||
candidates = dense_result.candidates
|
||||
if not candidates:
|
||||
return SearchResult(
|
||||
query=dense_result.query,
|
||||
index_profile_id=dense_result.index_profile_id,
|
||||
results=(),
|
||||
)
|
||||
if len(candidates) > self._reranker_provider.max_documents:
|
||||
raise SearchConfigurationError("dense candidate count exceeds reranker limit")
|
||||
if len({candidate.chunk_id for candidate in candidates}) != len(candidates):
|
||||
raise RerankerContractError("dense retrieval returned duplicate chunk candidates")
|
||||
if any(
|
||||
candidate.index_profile_id != dense_result.index_profile_id for candidate in candidates
|
||||
):
|
||||
raise RerankerContractError("dense candidate crossed the selected index profile")
|
||||
if any(not isfinite(candidate.score) for candidate in candidates):
|
||||
raise RerankerContractError("dense candidate returned a non-finite score")
|
||||
if any(not candidate.content for candidate in candidates):
|
||||
raise RerankerContractError("dense candidate content must not be empty")
|
||||
|
||||
final_count = min(requested_top_k, len(candidates))
|
||||
ranked = self._reranker_provider.rerank(
|
||||
dense_result.query,
|
||||
tuple(candidate.content for candidate in candidates),
|
||||
final_count,
|
||||
)
|
||||
self._validate_ranked_results(ranked, candidates, final_count)
|
||||
selected = tuple((candidates[result.candidate_index], result.score) for result in ranked)
|
||||
return SearchResult(
|
||||
query=dense_result.query,
|
||||
index_profile_id=dense_result.index_profile_id,
|
||||
results=self._load_sources(selected),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_ranked_results(
|
||||
ranked: tuple[object, ...],
|
||||
candidates: tuple[DenseSearchCandidate, ...],
|
||||
expected_count: int,
|
||||
) -> None:
|
||||
"""Fail closed if a provider returns ambiguous or reordered mappings."""
|
||||
if len(ranked) != expected_count:
|
||||
raise RerankerContractError("reranker returned the wrong result count")
|
||||
indexes: list[int] = []
|
||||
previous_score: float | None = None
|
||||
for result in ranked:
|
||||
candidate_index = getattr(result, "candidate_index", None)
|
||||
score = getattr(result, "score", None)
|
||||
if isinstance(candidate_index, bool) or not isinstance(candidate_index, int):
|
||||
raise RerankerContractError("reranker returned an invalid candidate index")
|
||||
if not 0 <= candidate_index < len(candidates):
|
||||
raise RerankerContractError("reranker candidate index is out of range")
|
||||
if isinstance(score, bool) or not isinstance(score, (int, float)):
|
||||
raise RerankerContractError("reranker returned an invalid score")
|
||||
converted_score = float(score)
|
||||
if not isfinite(converted_score):
|
||||
raise RerankerContractError("reranker returned a non-finite score")
|
||||
if previous_score is not None and previous_score < converted_score:
|
||||
raise RerankerContractError("reranker results are not score-descending")
|
||||
indexes.append(candidate_index)
|
||||
previous_score = converted_score
|
||||
if len(indexes) != len(set(indexes)):
|
||||
raise RerankerContractError("reranker returned duplicate candidate indexes")
|
||||
|
||||
def _load_sources(
|
||||
self,
|
||||
selected: tuple[tuple[DenseSearchCandidate, float], ...],
|
||||
) -> tuple[SourceResult, ...]:
|
||||
"""Rehydrate final content and provenance from canonical PostgreSQL rows."""
|
||||
chunk_ids = [candidate.chunk_id for candidate, _ in selected]
|
||||
with self._session_factory() as session:
|
||||
chunks = {
|
||||
chunk.chunk_id: chunk
|
||||
for chunk in session.scalars(
|
||||
select(Chunk).where(Chunk.chunk_id.in_(chunk_ids))
|
||||
).all()
|
||||
}
|
||||
if set(chunks) != set(chunk_ids):
|
||||
raise CanonicalSourceError("a reranked chunk is missing from canonical storage")
|
||||
|
||||
results: list[SourceResult] = []
|
||||
for candidate, rerank_score in selected:
|
||||
chunk = chunks[candidate.chunk_id]
|
||||
if chunk.document_id != candidate.document_id:
|
||||
raise CanonicalSourceError("vector candidate document does not match its chunk")
|
||||
if chunk.content != candidate.content:
|
||||
raise CanonicalSourceError("vector candidate content is stale")
|
||||
results.append(
|
||||
SourceResult(
|
||||
document_id=chunk.document_id,
|
||||
filename=chunk.filename,
|
||||
page=chunk.page,
|
||||
section=chunk.section,
|
||||
chunk_id=chunk.chunk_id,
|
||||
content=chunk.content,
|
||||
score=candidate.score,
|
||||
rerank_score=rerank_score,
|
||||
)
|
||||
)
|
||||
return tuple(results)
|
||||
13
packages/providers/reranker/src/ekp_reranker/__init__.py
Normal file
13
packages/providers/reranker/src/ekp_reranker/__init__.py
Normal file
@ -0,0 +1,13 @@
|
||||
"""Reranker provider abstraction and Demo implementations."""
|
||||
|
||||
from ekp_reranker.base import RerankerProvider, RerankerProviderError, RerankResult
|
||||
from ekp_reranker.fake import FakeRerankerProvider
|
||||
from ekp_reranker.qianfan import QianfanRerankerProvider
|
||||
|
||||
__all__ = [
|
||||
"FakeRerankerProvider",
|
||||
"QianfanRerankerProvider",
|
||||
"RerankResult",
|
||||
"RerankerProvider",
|
||||
"RerankerProviderError",
|
||||
]
|
||||
49
packages/providers/reranker/src/ekp_reranker/base.py
Normal file
49
packages/providers/reranker/src/ekp_reranker/base.py
Normal file
@ -0,0 +1,49 @@
|
||||
"""Provider-neutral reranking contract for retrieval candidates."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from math import isfinite
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
class RerankerProviderError(RuntimeError):
|
||||
"""Raised when a reranker rejects or cannot serve a request."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RerankResult:
|
||||
"""One ranked candidate reference using its original input position."""
|
||||
|
||||
candidate_index: int
|
||||
score: float
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Reject invalid mappings before core search consumes them."""
|
||||
if (
|
||||
isinstance(self.candidate_index, bool)
|
||||
or not isinstance(self.candidate_index, int)
|
||||
or self.candidate_index < 0
|
||||
):
|
||||
raise ValueError("rerank candidate_index must be a non-negative integer")
|
||||
if not isfinite(self.score):
|
||||
raise ValueError("rerank score must be finite")
|
||||
|
||||
|
||||
class RerankerProvider(Protocol):
|
||||
"""Rank text candidates without exposing a vendor-specific response."""
|
||||
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
"""Return the stable provider model identifier."""
|
||||
|
||||
@property
|
||||
def max_documents(self) -> int:
|
||||
"""Return the largest candidate batch accepted by this provider."""
|
||||
|
||||
def rerank(
|
||||
self,
|
||||
query: str,
|
||||
documents: Sequence[str],
|
||||
top_n: int,
|
||||
) -> tuple[RerankResult, ...]:
|
||||
"""Return Top-N references in descending relevance order."""
|
||||
65
packages/providers/reranker/src/ekp_reranker/fake.py
Normal file
65
packages/providers/reranker/src/ekp_reranker/fake.py
Normal file
@ -0,0 +1,65 @@
|
||||
"""Deterministic lexical reranker for offline tests and evaluation."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from ekp_reranker.base import RerankerProviderError, RerankResult
|
||||
|
||||
FAKE_MAX_DOCUMENTS = 64
|
||||
|
||||
|
||||
class FakeRerankerProvider:
|
||||
"""Rank by query-character coverage with stable input-index tie-breaking."""
|
||||
|
||||
def __init__(self, model_name: str = "fake-reranker") -> None:
|
||||
if not model_name:
|
||||
raise ValueError("fake reranker model_name must not be empty")
|
||||
self._model_name = model_name
|
||||
self.calls: list[tuple[str, tuple[str, ...], int]] = []
|
||||
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
return self._model_name
|
||||
|
||||
@property
|
||||
def max_documents(self) -> int:
|
||||
return FAKE_MAX_DOCUMENTS
|
||||
|
||||
def rerank(
|
||||
self,
|
||||
query: str,
|
||||
documents: Sequence[str],
|
||||
top_n: int,
|
||||
) -> tuple[RerankResult, ...]:
|
||||
"""Produce repeatable relevance scores without model or network access."""
|
||||
normalized_query = query.strip()
|
||||
if not normalized_query:
|
||||
raise RerankerProviderError("rerank query must not be empty")
|
||||
if not documents:
|
||||
raise RerankerProviderError("rerank documents must not be empty")
|
||||
if len(documents) > self.max_documents:
|
||||
raise RerankerProviderError(
|
||||
f"fake reranker accepts at most {self.max_documents} documents"
|
||||
)
|
||||
if any(not document for document in documents):
|
||||
raise RerankerProviderError("rerank document must not be empty")
|
||||
if isinstance(top_n, bool) or not 1 <= top_n <= len(documents):
|
||||
raise RerankerProviderError("rerank top_n must be within the candidate batch")
|
||||
|
||||
captured_documents = tuple(documents)
|
||||
self.calls.append((normalized_query, captured_documents, top_n))
|
||||
query_units = {character.lower() for character in normalized_query if character.isalnum()}
|
||||
if not query_units:
|
||||
raise RerankerProviderError("rerank query must contain searchable characters")
|
||||
ranked = [
|
||||
RerankResult(
|
||||
candidate_index=index,
|
||||
score=len(
|
||||
query_units
|
||||
& {character.lower() for character in document if character.isalnum()}
|
||||
)
|
||||
/ len(query_units),
|
||||
)
|
||||
for index, document in enumerate(captured_documents)
|
||||
]
|
||||
ranked.sort(key=lambda result: (-result.score, result.candidate_index))
|
||||
return tuple(ranked[:top_n])
|
||||
180
packages/providers/reranker/src/ekp_reranker/qianfan.py
Normal file
180
packages/providers/reranker/src/ekp_reranker/qianfan.py
Normal file
@ -0,0 +1,180 @@
|
||||
"""Baidu Qianfan v2 adapter for the Demo Qwen3 reranker baseline."""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from math import isfinite
|
||||
from numbers import Real
|
||||
from typing import Protocol, cast
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from ekp_reranker.base import RerankerProviderError, RerankResult
|
||||
|
||||
QWEN3_RERANKER_MODEL = "qwen3-reranker-0.6b"
|
||||
QIANFAN_RERANK_ENDPOINT = "https://qianfan.baidubce.com/v2/rerank"
|
||||
QIANFAN_MAX_DOCUMENTS = 64
|
||||
QIANFAN_MAX_QUERY_CHARACTERS = 1600
|
||||
QIANFAN_MAX_DOCUMENT_CHARACTERS = 4096
|
||||
|
||||
|
||||
class JsonHttpClient(Protocol):
|
||||
"""Injectable synchronous JSON boundary for the Qianfan rerank API."""
|
||||
|
||||
def post_json(
|
||||
self,
|
||||
url: str,
|
||||
headers: Mapping[str, str],
|
||||
payload: Mapping[str, object],
|
||||
timeout_seconds: float,
|
||||
) -> dict[str, object]:
|
||||
"""POST JSON and return a decoded object."""
|
||||
|
||||
|
||||
class UrllibJsonHttpClient:
|
||||
"""Dependency-free client with bounded waits and sanitized failures."""
|
||||
|
||||
def post_json(
|
||||
self,
|
||||
url: str,
|
||||
headers: Mapping[str, str],
|
||||
payload: Mapping[str, object],
|
||||
timeout_seconds: float,
|
||||
) -> dict[str, object]:
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
request = Request(url=url, data=body, headers=dict(headers), method="POST")
|
||||
try:
|
||||
with urlopen(request, timeout=timeout_seconds) as response: # noqa: S310
|
||||
decoded: object = json.loads(response.read())
|
||||
except HTTPError as exc:
|
||||
raise RerankerProviderError(
|
||||
f"Qianfan rerank request failed with HTTP {exc.code}"
|
||||
) from exc
|
||||
except (URLError, TimeoutError) as exc:
|
||||
raise RerankerProviderError("Qianfan rerank request could not complete") from exc
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RerankerProviderError("Qianfan reranker returned invalid JSON") from exc
|
||||
if not isinstance(decoded, dict):
|
||||
raise RerankerProviderError("Qianfan reranker returned a non-object response")
|
||||
return cast(dict[str, object], decoded)
|
||||
|
||||
|
||||
class QianfanRerankerProvider:
|
||||
"""Call Qianfan rerank and map ranked indexes back to input candidates."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str,
|
||||
endpoint: str = QIANFAN_RERANK_ENDPOINT,
|
||||
model_name: str = QWEN3_RERANKER_MODEL,
|
||||
timeout_seconds: float = 30.0,
|
||||
http_client: JsonHttpClient | None = None,
|
||||
) -> None:
|
||||
if not api_key:
|
||||
raise ValueError("Qianfan API key must not be empty")
|
||||
if not endpoint.startswith("https://"):
|
||||
raise ValueError("Qianfan rerank endpoint must use HTTPS")
|
||||
if not model_name:
|
||||
raise ValueError("Qianfan reranker model_name must not be empty")
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError("timeout_seconds must be positive")
|
||||
self._api_key = api_key
|
||||
self._endpoint = endpoint
|
||||
self._model_name = model_name
|
||||
self._timeout_seconds = timeout_seconds
|
||||
self._http_client = http_client or UrllibJsonHttpClient()
|
||||
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
return self._model_name
|
||||
|
||||
@property
|
||||
def max_documents(self) -> int:
|
||||
return QIANFAN_MAX_DOCUMENTS
|
||||
|
||||
def rerank(
|
||||
self,
|
||||
query: str,
|
||||
documents: Sequence[str],
|
||||
top_n: int,
|
||||
) -> tuple[RerankResult, ...]:
|
||||
"""Validate service limits and preserve Qianfan's ranked result order."""
|
||||
normalized_query = query.strip()
|
||||
candidate_documents = tuple(documents)
|
||||
self._validate_request(normalized_query, candidate_documents, top_n)
|
||||
response = self._http_client.post_json(
|
||||
self._endpoint,
|
||||
{
|
||||
"Authorization": f"Bearer {self._api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
{
|
||||
"model": self.model_name,
|
||||
"query": normalized_query,
|
||||
"documents": list(candidate_documents),
|
||||
"top_n": top_n,
|
||||
},
|
||||
self._timeout_seconds,
|
||||
)
|
||||
raw_results = response.get("results")
|
||||
if not isinstance(raw_results, list) or len(raw_results) != top_n:
|
||||
raise RerankerProviderError("Qianfan returned an unexpected rerank result count")
|
||||
|
||||
results = tuple(self._validated_result(item, candidate_documents) for item in raw_results)
|
||||
indexes = [result.candidate_index for result in results]
|
||||
if len(indexes) != len(set(indexes)):
|
||||
raise RerankerProviderError("Qianfan returned duplicate rerank indexes")
|
||||
if any(
|
||||
previous.score < current.score
|
||||
for previous, current in zip(results, results[1:], strict=False)
|
||||
):
|
||||
raise RerankerProviderError("Qianfan rerank results are not score-descending")
|
||||
return results
|
||||
|
||||
def _validate_request(
|
||||
self,
|
||||
query: str,
|
||||
documents: tuple[str, ...],
|
||||
top_n: int,
|
||||
) -> None:
|
||||
"""Enforce current documented Qianfan request bounds before network I/O."""
|
||||
if not query:
|
||||
raise RerankerProviderError("rerank query must not be empty")
|
||||
if len(query) > QIANFAN_MAX_QUERY_CHARACTERS:
|
||||
raise RerankerProviderError(
|
||||
f"rerank query exceeds {QIANFAN_MAX_QUERY_CHARACTERS} characters"
|
||||
)
|
||||
if not documents:
|
||||
raise RerankerProviderError("rerank documents must not be empty")
|
||||
if len(documents) > self.max_documents:
|
||||
raise RerankerProviderError(
|
||||
f"Qianfan accepts at most {self.max_documents} rerank documents"
|
||||
)
|
||||
if any(not document for document in documents):
|
||||
raise RerankerProviderError("rerank document must not be empty")
|
||||
if any(len(document) > QIANFAN_MAX_DOCUMENT_CHARACTERS for document in documents):
|
||||
raise RerankerProviderError(
|
||||
f"rerank document exceeds {QIANFAN_MAX_DOCUMENT_CHARACTERS} characters"
|
||||
)
|
||||
if isinstance(top_n, bool) or not 1 <= top_n <= len(documents):
|
||||
raise RerankerProviderError("rerank top_n must be within the candidate batch")
|
||||
|
||||
@staticmethod
|
||||
def _validated_result(
|
||||
item: object,
|
||||
documents: tuple[str, ...],
|
||||
) -> RerankResult:
|
||||
"""Require a unique in-range index, finite score, and matching echoed text."""
|
||||
if not isinstance(item, dict):
|
||||
raise RerankerProviderError("Qianfan returned an invalid rerank item")
|
||||
index = item.get("index")
|
||||
score = item.get("relevance_score")
|
||||
if isinstance(index, bool) or not isinstance(index, int):
|
||||
raise RerankerProviderError("Qianfan rerank item has an invalid index")
|
||||
if not 0 <= index < len(documents):
|
||||
raise RerankerProviderError("Qianfan rerank index is out of range")
|
||||
if isinstance(score, bool) or not isinstance(score, Real) or not isfinite(float(score)):
|
||||
raise RerankerProviderError("Qianfan rerank item has an invalid score")
|
||||
document = item.get("document")
|
||||
if document is not None and document != documents[index]:
|
||||
raise RerankerProviderError("Qianfan rerank item does not match its index")
|
||||
return RerankResult(candidate_index=index, score=float(score))
|
||||
@ -37,6 +37,7 @@ where = [
|
||||
"packages/providers/object_storage/src",
|
||||
"packages/providers/parser/src",
|
||||
"packages/providers/embedding/src",
|
||||
"packages/providers/reranker/src",
|
||||
"packages/providers/vector_store/src",
|
||||
]
|
||||
|
||||
@ -55,6 +56,7 @@ known-first-party = [
|
||||
"ekp_embedding",
|
||||
"ekp_object_storage",
|
||||
"ekp_parser",
|
||||
"ekp_reranker",
|
||||
"ekp_vector_store",
|
||||
]
|
||||
|
||||
@ -68,6 +70,7 @@ mypy_path = [
|
||||
"packages/providers/object_storage/src",
|
||||
"packages/providers/parser/src",
|
||||
"packages/providers/embedding/src",
|
||||
"packages/providers/reranker/src",
|
||||
"packages/providers/vector_store/src",
|
||||
]
|
||||
|
||||
|
||||
86
tests/evaluation/test_reranked_search_regression.py
Normal file
86
tests/evaluation/test_reranked_search_regression.py
Normal file
@ -0,0 +1,86 @@
|
||||
"""Golden Dataset regression for reranking and canonical source results."""
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
from ekp_core.chunking import StructureAwareChunker
|
||||
from ekp_core.database import create_session_factory
|
||||
from ekp_core.dense_retrieval import DenseRetrievalService
|
||||
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
|
||||
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
|
||||
from ekp_core.document_parsing import DocumentParsingService
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.search import SearchService
|
||||
from ekp_domain import Base, IndexProfile
|
||||
from ekp_object_storage import LocalObjectStorage
|
||||
from ekp_parser import ParserRouter
|
||||
from ekp_reranker import FakeRerankerProvider
|
||||
from ekp_vector_store import FakeVectorStore
|
||||
|
||||
|
||||
class GoldenKeywordEmbeddingProvider:
|
||||
"""Small semantic fixture matching the Dense regression without external calls."""
|
||||
|
||||
model_name = "golden-keyword-v1"
|
||||
dimensions = 2
|
||||
max_batch_size = 16
|
||||
|
||||
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
|
||||
vectors: list[tuple[float, ...]] = []
|
||||
for text in texts:
|
||||
if "住宿" in text or "出差" in text:
|
||||
vectors.append((1.0, 0.0))
|
||||
elif "退款" in text or "产品A" in text or "产品 A" in text:
|
||||
vectors.append((0.0, 1.0))
|
||||
else:
|
||||
vectors.append((0.1, 0.1))
|
||||
return tuple(vectors)
|
||||
|
||||
|
||||
def test_reranker_promotes_expected_canonical_source_to_top_one(tmp_path: Path) -> None:
|
||||
"""Run the complete Demo pipeline through Dense Top20, Rerank, and Source Result."""
|
||||
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
session_factory = create_session_factory(engine)
|
||||
storage = LocalObjectStorage(tmp_path / "storage")
|
||||
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
|
||||
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
|
||||
chunking = DocumentChunkingService(
|
||||
session_factory,
|
||||
storage,
|
||||
StructureAwareChunker(),
|
||||
)
|
||||
embedding = GoldenKeywordEmbeddingProvider()
|
||||
vector_store = FakeVectorStore()
|
||||
profile = IndexProfile("golden-rerank-v1", "structure_v1", embedding.model_name, 2)
|
||||
indexing = DocumentIndexingService(session_factory, embedding, vector_store, profile)
|
||||
pipeline = DocumentIndexingPipelineService(
|
||||
DocumentPipelineService(upload, parsing, chunking),
|
||||
indexing,
|
||||
)
|
||||
dense = DenseRetrievalService(embedding, vector_store, profile)
|
||||
search = SearchService(session_factory, dense, FakeRerankerProvider())
|
||||
golden_records = [
|
||||
json.loads(line)
|
||||
for line in Path("fixtures/evaluation/golden.jsonl").read_text().splitlines()
|
||||
if line
|
||||
]
|
||||
|
||||
filenames = {filename for record in golden_records for filename in record["expected_documents"]}
|
||||
for filename in sorted(filenames):
|
||||
payload = (Path("fixtures/documents") / filename).read_bytes()
|
||||
pipeline.ingest(filename, "text/markdown", BytesIO(payload))
|
||||
|
||||
for record in golden_records:
|
||||
result = search.search(record["query"])
|
||||
assert result.results
|
||||
best = result.results[0]
|
||||
assert best.filename in record["expected_documents"]
|
||||
assert best.section is not None
|
||||
assert any(section in best.section for section in record["expected_sections"])
|
||||
assert best.content
|
||||
assert best.rerank_score >= 0
|
||||
@ -32,6 +32,11 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
|
||||
assert settings.mochow_timeout_seconds == 30
|
||||
assert settings.index_profile_id == "demo-bge-large-zh-structure-v1"
|
||||
assert settings.dense_top_n == 20
|
||||
assert settings.reranker_provider == "qianfan"
|
||||
assert settings.reranker_model == "qwen3-reranker-0.6b"
|
||||
assert settings.reranker_endpoint.endswith("/v2/rerank")
|
||||
assert settings.reranker_api_key is None
|
||||
assert settings.search_top_k == 5
|
||||
|
||||
|
||||
def test_settings_reject_invalid_port(isolated_settings_environment: None) -> None:
|
||||
@ -46,12 +51,21 @@ def test_settings_reject_chunk_overlap_without_forward_progress(
|
||||
Settings(chunk_max_tokens=100, chunk_overlap_tokens=100)
|
||||
|
||||
|
||||
def test_settings_reject_final_top_k_larger_than_dense_candidates(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match="search_top_k"):
|
||||
Settings(dense_top_n=5, search_top_k=6)
|
||||
|
||||
|
||||
def test_settings_redact_provider_secrets(isolated_settings_environment: None) -> None:
|
||||
settings = Settings(
|
||||
baidu_qianfan_api_key=SecretStr("qianfan-secret"),
|
||||
baidu_mochow_api_key=SecretStr("mochow-secret"),
|
||||
reranker_api_key=SecretStr("reranker-secret"),
|
||||
)
|
||||
|
||||
rendered = repr(settings)
|
||||
assert "qianfan-secret" not in rendered
|
||||
assert "mochow-secret" not in rendered
|
||||
assert "reranker-secret" not in rendered
|
||||
|
||||
@ -8,6 +8,7 @@ from ekp_api.providers import (
|
||||
create_dense_retrieval_service,
|
||||
create_embedding_provider,
|
||||
create_index_profile,
|
||||
create_reranker_provider,
|
||||
create_vector_store,
|
||||
)
|
||||
from ekp_core.config import Settings
|
||||
@ -56,3 +57,14 @@ def test_provider_wiring_builds_dense_retrieval_without_network(
|
||||
)
|
||||
|
||||
assert service.search("测试查询").candidates == ()
|
||||
|
||||
|
||||
def test_provider_wiring_reuses_qianfan_key_for_demo_reranker(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
settings = Settings(baidu_qianfan_api_key=SecretStr("shared-test-key"))
|
||||
|
||||
reranker = create_reranker_provider(settings)
|
||||
|
||||
assert reranker.model_name == "qwen3-reranker-0.6b"
|
||||
assert reranker.max_documents == 64
|
||||
|
||||
112
tests/unit/test_qianfan_reranker.py
Normal file
112
tests/unit/test_qianfan_reranker.py
Normal file
@ -0,0 +1,112 @@
|
||||
"""Unit tests for the Qianfan Qwen3 reranker request/response contract."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
|
||||
from ekp_reranker import QianfanRerankerProvider, RerankerProviderError
|
||||
|
||||
|
||||
class FakeJsonHttpClient:
|
||||
"""Capture rerank JSON requests without network access."""
|
||||
|
||||
def __init__(self, response: dict[str, object]) -> None:
|
||||
self.response = response
|
||||
self.calls: list[tuple[str, Mapping[str, str], Mapping[str, object], float]] = []
|
||||
|
||||
def post_json(
|
||||
self,
|
||||
url: str,
|
||||
headers: Mapping[str, str],
|
||||
payload: Mapping[str, object],
|
||||
timeout_seconds: float,
|
||||
) -> dict[str, object]:
|
||||
self.calls.append((url, headers, payload, timeout_seconds))
|
||||
return self.response
|
||||
|
||||
|
||||
def test_qianfan_reranker_sends_top_n_and_preserves_ranked_indexes() -> None:
|
||||
client = FakeJsonHttpClient(
|
||||
{
|
||||
"results": [
|
||||
{"document": "退款周期为七天", "relevance_score": 0.93, "index": 1},
|
||||
{"document": "住宿标准说明", "relevance_score": 0.21, "index": 0},
|
||||
]
|
||||
}
|
||||
)
|
||||
provider = QianfanRerankerProvider(
|
||||
api_key="test-key",
|
||||
endpoint="https://qianfan.example/v2/rerank",
|
||||
timeout_seconds=12,
|
||||
http_client=client,
|
||||
)
|
||||
|
||||
results = provider.rerank(" 产品A退款周期 ", ("住宿标准说明", "退款周期为七天"), 2)
|
||||
|
||||
assert provider.model_name == "qwen3-reranker-0.6b"
|
||||
assert provider.max_documents == 64
|
||||
assert [result.candidate_index for result in results] == [1, 0]
|
||||
url, headers, payload, timeout = client.calls[0]
|
||||
assert url.endswith("/v2/rerank")
|
||||
assert headers["Authorization"] == "Bearer test-key"
|
||||
assert payload == {
|
||||
"model": "qwen3-reranker-0.6b",
|
||||
"query": "产品A退款周期",
|
||||
"documents": ["住宿标准说明", "退款周期为七天"],
|
||||
"top_n": 2,
|
||||
}
|
||||
assert timeout == 12
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response",
|
||||
[
|
||||
{"results": []},
|
||||
{
|
||||
"results": [
|
||||
{"relevance_score": 0.9, "index": 0},
|
||||
{"relevance_score": 0.8, "index": 0},
|
||||
]
|
||||
},
|
||||
{
|
||||
"results": [
|
||||
{"relevance_score": 0.3, "index": 0},
|
||||
{"relevance_score": 0.8, "index": 1},
|
||||
]
|
||||
},
|
||||
{
|
||||
"results": [
|
||||
{"document": "错误内容", "relevance_score": 0.9, "index": 0},
|
||||
{"relevance_score": 0.8, "index": 1},
|
||||
]
|
||||
},
|
||||
{
|
||||
"results": [
|
||||
{"relevance_score": float("nan"), "index": 0},
|
||||
{"relevance_score": 0.8, "index": 1},
|
||||
]
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_qianfan_reranker_rejects_invalid_responses(response: dict[str, object]) -> None:
|
||||
provider = QianfanRerankerProvider(
|
||||
api_key="test-key",
|
||||
http_client=FakeJsonHttpClient(response),
|
||||
)
|
||||
|
||||
with pytest.raises(RerankerProviderError):
|
||||
provider.rerank("查询", ("文档一", "文档二"), 2)
|
||||
|
||||
|
||||
def test_qianfan_reranker_enforces_document_and_character_limits() -> None:
|
||||
provider = QianfanRerankerProvider(
|
||||
api_key="test-key",
|
||||
http_client=FakeJsonHttpClient({"results": []}),
|
||||
)
|
||||
|
||||
with pytest.raises(RerankerProviderError, match="1600"):
|
||||
provider.rerank("问" * 1601, ("内容",), 1)
|
||||
with pytest.raises(RerankerProviderError, match="64"):
|
||||
provider.rerank("查询", ("内容",) * 65, 5)
|
||||
with pytest.raises(RerankerProviderError, match="4096"):
|
||||
provider.rerank("查询", ("内" * 4097,), 1)
|
||||
41
tests/unit/test_reranker.py
Normal file
41
tests/unit/test_reranker.py
Normal file
@ -0,0 +1,41 @@
|
||||
"""Unit tests for provider-neutral reranking behavior."""
|
||||
|
||||
import pytest
|
||||
|
||||
from ekp_reranker import FakeRerankerProvider, RerankerProviderError, RerankResult
|
||||
|
||||
|
||||
def test_fake_reranker_orders_by_query_coverage_and_keeps_source_index() -> None:
|
||||
provider = FakeRerankerProvider()
|
||||
|
||||
results = provider.rerank(
|
||||
"员工出差住宿标准",
|
||||
("提交报销材料", "员工出差住宿标准说明", "其他制度"),
|
||||
top_n=2,
|
||||
)
|
||||
|
||||
assert [result.candidate_index for result in results] == [1, 0]
|
||||
assert results[0].score > results[1].score
|
||||
assert provider.calls == [
|
||||
(
|
||||
"员工出差住宿标准",
|
||||
("提交报销材料", "员工出差住宿标准说明", "其他制度"),
|
||||
2,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_fake_reranker_rejects_invalid_batch_and_top_n() -> None:
|
||||
provider = FakeRerankerProvider()
|
||||
|
||||
with pytest.raises(RerankerProviderError, match="documents"):
|
||||
provider.rerank("查询", (), 1)
|
||||
with pytest.raises(RerankerProviderError, match="top_n"):
|
||||
provider.rerank("查询", ("内容",), 2)
|
||||
|
||||
|
||||
def test_rerank_result_rejects_invalid_index_and_score() -> None:
|
||||
with pytest.raises(ValueError, match="candidate_index"):
|
||||
RerankResult(candidate_index=True, score=0.5)
|
||||
with pytest.raises(ValueError, match="finite"):
|
||||
RerankResult(candidate_index=0, score=float("nan"))
|
||||
167
tests/unit/test_search.py
Normal file
167
tests/unit/test_search.py
Normal file
@ -0,0 +1,167 @@
|
||||
"""Unit tests for reranked search and canonical source enrichment."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.database import create_session_factory
|
||||
from ekp_core.dense_retrieval import DenseRetrievalResult
|
||||
from ekp_core.search import CanonicalSourceError, RerankerContractError, SearchService
|
||||
from ekp_domain import Base, Chunk, Document, DocumentStatus
|
||||
from ekp_reranker import FakeRerankerProvider, RerankResult
|
||||
from ekp_vector_store import DenseSearchCandidate
|
||||
|
||||
|
||||
class StubDenseRetriever:
|
||||
"""Return prebuilt candidates while recording the query."""
|
||||
|
||||
def __init__(self, candidates: tuple[DenseSearchCandidate, ...]) -> None:
|
||||
self.candidates = candidates
|
||||
self.queries: list[str] = []
|
||||
|
||||
def search(self, query: str, top_n: int | None = None) -> DenseRetrievalResult:
|
||||
self.queries.append(query)
|
||||
return DenseRetrievalResult(query.strip(), "profile-v1", self.candidates)
|
||||
|
||||
|
||||
def _candidate(
|
||||
chunk_id: UUID,
|
||||
document_id: UUID,
|
||||
content: str,
|
||||
*,
|
||||
dense_score: float,
|
||||
) -> DenseSearchCandidate:
|
||||
return DenseSearchCandidate(
|
||||
index_record_id=uuid4(),
|
||||
chunk_id=chunk_id,
|
||||
document_id=document_id,
|
||||
filename="stale-index-name.md",
|
||||
page=None,
|
||||
page_end=None,
|
||||
section="stale-index-section",
|
||||
chunk_index=0,
|
||||
content=content,
|
||||
score=dense_score,
|
||||
index_profile_id="profile-v1",
|
||||
)
|
||||
|
||||
|
||||
def _canonical_source_session() -> tuple[sessionmaker[Session], UUID, UUID, UUID]:
|
||||
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
session_factory = create_session_factory(engine)
|
||||
document_id = uuid4()
|
||||
first_chunk_id = uuid4()
|
||||
second_chunk_id = uuid4()
|
||||
with session_factory() as session:
|
||||
session.add(
|
||||
Document(
|
||||
id=document_id,
|
||||
filename="policy.md",
|
||||
media_type="text/markdown",
|
||||
size_bytes=100,
|
||||
storage_key=f"documents/{document_id}/policy.md",
|
||||
status=DocumentStatus.READY,
|
||||
)
|
||||
)
|
||||
session.add_all(
|
||||
[
|
||||
Chunk(
|
||||
chunk_id=first_chunk_id,
|
||||
document_id=document_id,
|
||||
filename="policy.md",
|
||||
page=1,
|
||||
page_end=1,
|
||||
section="报销材料",
|
||||
section_path=["报销材料"],
|
||||
chunk_index=0,
|
||||
content="提交报销材料",
|
||||
token_count=6,
|
||||
strategy_version="structure_v1",
|
||||
),
|
||||
Chunk(
|
||||
chunk_id=second_chunk_id,
|
||||
document_id=document_id,
|
||||
filename="policy.md",
|
||||
page=2,
|
||||
page_end=2,
|
||||
section="住宿标准",
|
||||
section_path=["住宿标准"],
|
||||
chunk_index=1,
|
||||
content="员工出差住宿标准说明",
|
||||
token_count=9,
|
||||
strategy_version="structure_v1",
|
||||
),
|
||||
]
|
||||
)
|
||||
session.commit()
|
||||
return session_factory, document_id, first_chunk_id, second_chunk_id
|
||||
|
||||
|
||||
def test_search_reranks_and_rehydrates_source_from_canonical_chunk() -> None:
|
||||
session_factory, document_id, first_chunk_id, second_chunk_id = _canonical_source_session()
|
||||
dense = StubDenseRetriever(
|
||||
(
|
||||
_candidate(first_chunk_id, document_id, "提交报销材料", dense_score=0.91),
|
||||
_candidate(second_chunk_id, document_id, "员工出差住宿标准说明", dense_score=0.72),
|
||||
)
|
||||
)
|
||||
reranker = FakeRerankerProvider()
|
||||
service = SearchService(session_factory, dense, reranker, default_top_k=1)
|
||||
|
||||
result = service.search("员工出差住宿标准")
|
||||
|
||||
assert len(result.results) == 1
|
||||
source = result.results[0]
|
||||
assert source.chunk_id == second_chunk_id
|
||||
assert source.filename == "policy.md"
|
||||
assert source.page == 2
|
||||
assert source.section == "住宿标准"
|
||||
assert source.content == "员工出差住宿标准说明"
|
||||
assert source.score == 0.72
|
||||
assert source.rerank_score == 1.0
|
||||
|
||||
|
||||
def test_search_rejects_stale_vector_content() -> None:
|
||||
session_factory, document_id, first_chunk_id, _ = _canonical_source_session()
|
||||
dense = StubDenseRetriever(
|
||||
(_candidate(first_chunk_id, document_id, "索引中的旧内容", dense_score=0.8),)
|
||||
)
|
||||
service = SearchService(
|
||||
session_factory,
|
||||
dense,
|
||||
FakeRerankerProvider(),
|
||||
)
|
||||
|
||||
with pytest.raises(CanonicalSourceError, match="stale"):
|
||||
service.search("旧内容", top_k=1)
|
||||
|
||||
|
||||
class DuplicateReranker:
|
||||
model_name = "bad"
|
||||
max_documents = 64
|
||||
|
||||
def rerank(
|
||||
self,
|
||||
query: str,
|
||||
documents: Sequence[str],
|
||||
top_n: int,
|
||||
) -> tuple[RerankResult, ...]:
|
||||
return (RerankResult(0, 0.9), RerankResult(0, 0.8))
|
||||
|
||||
|
||||
def test_search_rejects_duplicate_reranker_indexes() -> None:
|
||||
session_factory, document_id, first_chunk_id, second_chunk_id = _canonical_source_session()
|
||||
dense = StubDenseRetriever(
|
||||
(
|
||||
_candidate(first_chunk_id, document_id, "提交报销材料", dense_score=0.9),
|
||||
_candidate(second_chunk_id, document_id, "员工出差住宿标准说明", dense_score=0.8),
|
||||
)
|
||||
)
|
||||
service = SearchService(session_factory, dense, DuplicateReranker(), default_top_k=2)
|
||||
|
||||
with pytest.raises(RerankerContractError, match="duplicate"):
|
||||
service.search("住宿标准")
|
||||
Loading…
x
Reference in New Issue
Block a user