feat: implement DEMO-008 reranking and source results

This commit is contained in:
d-robotics 2026-09-01 15:48:34 +08:00
parent a0aa30c41c
commit f696aeec2b
20 changed files with 1080 additions and 9 deletions

View File

@ -41,9 +41,11 @@ INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1
DENSE_TOP_N=20
RERANKER_PROVIDER=qianfan
RERANKER_MODEL=
RERANKER_ENDPOINT=
RERANKER_MODEL=qwen3-reranker-0.6b
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
RERANKER_API_KEY=
RERANKER_TIMEOUT_SECONDS=30
SEARCH_TOP_K=5
LLM_PROVIDER=qianfan
LLM_MODEL=

View File

@ -93,5 +93,11 @@ pre-provisioned table schema.
DEMO-007 adds query embedding and profile-scoped Dense Top-N retrieval. The
default is Top 20 with a configurable Mochow HNSW `ef`; the in-memory fake uses
exact cosine search for deterministic offline regression. Returned objects are
raw dense candidates only. Reranking and final source-result assembly remain in
DEMO-008, while public search REST endpoints remain in DEMO-009.
raw dense candidates only.
DEMO-008 adds a provider-neutral reranker, a validated Qianfan
`qwen3-reranker-0.6b` adapter, and the final Top20-to-Top5 search pipeline. Final
source results are rehydrated from canonical PostgreSQL chunks and retain both
Dense and reranker scores; stale vector content fails closed. See
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
Public search REST endpoints remain in DEMO-009.

View File

@ -1,9 +1,13 @@
"""Runtime provider composition from secret-safe application settings."""
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.config import Settings
from ekp_core.dense_retrieval import DenseRetrievalService
from ekp_core.search import SearchService
from ekp_domain import IndexProfile
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
from ekp_reranker import QianfanRerankerProvider, RerankerProvider
from ekp_vector_store import MochowVectorStore, VectorStore, create_mochow_vector_store
BGE_LARGE_ZH_DIMENSIONS = 1024
@ -68,3 +72,31 @@ def create_dense_retrieval_service(
index_profile=index_profile,
default_top_n=settings.dense_top_n,
)
def create_reranker_provider(settings: Settings) -> QianfanRerankerProvider:
"""Create the Demo Qwen3 reranker using an explicit or shared Qianfan key."""
secret = settings.reranker_api_key or settings.baidu_qianfan_api_key
if secret is None:
raise ProviderConfigurationError("RERANKER_API_KEY or BAIDU_QIANFAN_API_KEY is required")
return QianfanRerankerProvider(
api_key=secret.get_secret_value(),
endpoint=settings.reranker_endpoint,
model_name=settings.reranker_model,
timeout_seconds=settings.reranker_timeout_seconds,
)
def create_search_service(
settings: Settings,
session_factory: sessionmaker[Session],
dense_retriever: DenseRetrievalService,
reranker_provider: RerankerProvider,
) -> SearchService:
"""Compose DEMO-008 search while leaving REST serialization to DEMO-009."""
return SearchService(
session_factory=session_factory,
dense_retriever=dense_retriever,
reranker_provider=reranker_provider,
default_top_k=settings.search_top_k,
)

View File

@ -77,8 +77,8 @@ DEMO-007 使用同一 `EmbeddingProvider` 将 Query 转换为 1024 维向量,
- 返回值是尚未 Rerank 的候选集,`score` 原样保留 Mochow 返回值,不在此阶段设置阈值。
Demo 明确不实现 RBAC/Document ACL;正式 V1 引入权限后,允许范围仍必须在召回阶段下推,
不得先做全局 Top-K 再过滤。Reranker 与最终 Source Result 属于 DEMO-008,REST API 属于
DEMO-009。
不得先做全局 Top-K 再过滤。Reranker 与最终 Source Result 的配置参见
`docs/development/RERANKER_SETUP.md`;REST API 属于 DEMO-009。
## 验证边界

View File

@ -0,0 +1,69 @@
# DEMO-008 千帆 Reranker 与 Source Result 配置
本文记录 Knowledge Platform Demo 的重排序连接方式、请求边界和 canonical Source Result
契约。真实 API Key 只允许由运行环境注入,不得写入代码、示例配置、日志或提交记录。
## 运行配置
```dotenv
RERANKER_PROVIDER=qianfan
RERANKER_MODEL=qwen3-reranker-0.6b
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
RERANKER_API_KEY=<运行环境注入,可空>
RERANKER_TIMEOUT_SECONDS=30
DENSE_TOP_N=20
SEARCH_TOP_K=5
```
`RERANKER_API_KEY` 未单独配置时,Demo 会复用 `BAIDU_QIANFAN_API_KEY`。两者都不存在时
Provider 创建失败。凭证使用 `SecretStr`,HTTP/JSON 异常不会包含请求头、请求正文或
响应正文。
## Reranker 请求契约
DEMO-008 固定使用千帆 `POST /v2/rerank`,模型基线为
`qwen3-reranker-0.6b`:
- Query 去除首尾空白后不能为空,最多 1600 字符;
- documents 不能为空,最多 64 条,每条最多 4096 字符;
- 当前标准链路发送 Dense Top20,并请求 Reranker Top5;
- 响应必须恰好返回请求的 Top-N,`index` 唯一且指向原输入位置;
- `relevance_score` 必须是有限数值,并保持降序;
- 响应若回显 document,必须与对应输入文本完全一致。
Adapter 只向业务层返回原候选索引和 Reranker 分数,不暴露千帆响应结构。默认测试使用
`FakeRerankerProvider` 的确定性字符覆盖率排序,不发起网络请求。
## Source Result 契约
Reranker 对 Mochow Dense Candidate 的 `content` 评分;选出 Top5 后,`SearchService` 使用
`chunk_id` 一次性回查 PostgreSQL canonical `chunks` 表,并由 canonical row 生成:
```text
document_id
filename
page
section
chunk_id
content
score
rerank_score
```
其中 `score` 保留 Dense 分数,`rerank_score` 保存精排分数。若 Chunk 缺失、document_id
不一致或 Mochow content 已与 canonical content 不一致,搜索会失败关闭,避免返回来源与
评分文本不一致的结果。
DEMO-008 不创建公开 REST Endpoint;`POST /api/search` 的序列化和错误映射属于
DEMO-009。Demo 仍不实现 RBAC/Document ACL,正式 V1 权限范围必须在 Dense 召回阶段
下推,不能在 Reranker 后过滤。
## 验证边界
默认测试离线覆盖 Dense Top20 → Reranker → Top5 → canonical Source Result,并使用合成
Golden Dataset 验证预期文档和章节升至首位。真实千帆联调前还需在受控运行环境确认:
- API Key 已开通目标 Reranker 模型权限;
- Endpoint 网络可达;
- 模型 API 名称与控制台服务一致;
- 真实中文 Query 的延迟、配额和 Top5 质量满足 Demo 演示需求。

View File

@ -2,11 +2,11 @@
当前阶段:Knowledge Retrieval Demo v0.1
已完成:DEMO-001、DEMO-002、DEMO-003、DEMO-004、DEMO-005、DEMO-006、DEMO-007
已完成:DEMO-001、DEMO-002、DEMO-003、DEMO-004、DEMO-005、DEMO-006、DEMO-007、DEMO-008
进行中:无
阻塞:无
下一任务:DEMO-008
下一任务:DEMO-009
完整 V1 待办将在 Demo Scope 完成后继续执行。当前任务顺序以
`docs/demo/DEMO_SCOPE.md` 为准。
@ -101,3 +101,19 @@
章节。
- 默认自动化测试不连接真实千帆/Mochow;真实环境仍需按
`docs/development/MOCHOW_SETUP.md` 注入凭证并验证网络、表 Schema 与索引状态。
### DEMO-008
- 新增与厂商无关的 `RerankerProvider` / `RerankResult` 契约及确定性 Fake,使用原始
candidate index 映射精排结果,业务服务不依赖千帆响应结构。
- 新增千帆 v2 `qwen3-reranker-0.6b` Adapter,调用 `/v2/rerank`,严格限制 Query、
documents 和 Top-N,校验响应数量、唯一索引、回显文本、有限分数与降序关系;错误不会
回显请求内容或凭证。
- 新增 `SearchService`,完成 Dense Top20 → Reranker → Top5。Reranker 选中候选后按
`chunk_id` 回查 PostgreSQL canonical Chunk,再生成包含 document、filename、page、
section、chunk、content、Dense score 和 rerank score 的最终 Source Result。
- Source Enrichment 对缺失 Chunk、document_id 不一致和 stale vector content 采用失败
关闭,防止 Mochow 副本与 canonical 来源不一致时返回错误证据。
- 新增 Provider、配置脱敏、排序映射、canonical 回查、一致性失败和 Golden Dataset
全链路回归。默认测试不调用真实千帆;真实联调边界记录在
`docs/development/RERANKER_SETUP.md`。公开 Search REST API 留在 DEMO-009。

View File

@ -8,6 +8,7 @@ from ekp_core.document_indexing import (
DocumentIndexingService,
)
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
from ekp_core.search import SearchResult, SearchService, SourceResult
__all__ = [
"DocumentChunkingService",
@ -18,5 +19,8 @@ __all__ = [
"DocumentIndexingService",
"DocumentPipelineService",
"DocumentParsingService",
"SearchResult",
"SearchService",
"SourceResult",
"clean_parsed_document",
]

View File

@ -46,13 +46,21 @@ class Settings(BaseSettings):
mochow_search_ef: int = Field(default=200, ge=1, le=10000)
mochow_timeout_seconds: float = Field(default=30.0, gt=0)
index_profile_id: str = "demo-bge-large-zh-structure-v1"
dense_top_n: int = Field(default=20, ge=1, le=1000)
dense_top_n: int = Field(default=20, ge=1, le=64)
reranker_provider: Literal["qianfan"] = "qianfan"
reranker_model: Literal["qwen3-reranker-0.6b"] = "qwen3-reranker-0.6b"
reranker_endpoint: str = "https://qianfan.baidubce.com/v2/rerank"
reranker_api_key: SecretStr | None = None
reranker_timeout_seconds: float = Field(default=30.0, gt=0)
search_top_k: int = Field(default=5, ge=1, le=64)
@model_validator(mode="after")
def validate_chunk_window(self) -> "Settings":
"""Require forward progress for every overlapping chunk window."""
if self.chunk_overlap_tokens >= self.chunk_max_tokens:
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
if self.search_top_k > self.dense_top_n:
raise ValueError("search_top_k must not exceed dense_top_n")
return self

View File

@ -0,0 +1,192 @@
"""Reranked search orchestration and canonical source-result assembly."""
from dataclasses import dataclass
from math import isfinite
from typing import Protocol
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.dense_retrieval import DenseRetrievalResult
from ekp_domain import Chunk
from ekp_reranker import RerankerProvider
from ekp_vector_store import DenseSearchCandidate
DEFAULT_SEARCH_TOP_K = 5
class SearchConfigurationError(ValueError):
"""Raised when candidate and final result limits cannot form a valid search."""
class RerankerContractError(RuntimeError):
"""Raised when reranker output cannot map safely to dense candidates."""
class CanonicalSourceError(RuntimeError):
"""Raised when a selected vector candidate cannot be traced to canonical Chunk data."""
class DenseRetriever(Protocol):
"""Minimal dense-retrieval boundary consumed by the final search pipeline."""
def search(self, query: str, top_n: int | None = None) -> DenseRetrievalResult:
"""Return normalized query and dense candidates."""
@dataclass(frozen=True, slots=True)
class SourceResult:
"""One canonical, traceable result after reranking."""
document_id: UUID
filename: str
page: int | None
section: str | None
chunk_id: UUID
content: str
score: float
rerank_score: float
@dataclass(frozen=True, slots=True)
class SearchResult:
"""Final Top-K result set ready for DEMO-009 REST serialization."""
query: str
index_profile_id: str
results: tuple[SourceResult, ...]
class SearchService:
"""Compose Dense Top-N, reranking, and PostgreSQL source enrichment."""
def __init__(
self,
session_factory: sessionmaker[Session],
dense_retriever: DenseRetriever,
reranker_provider: RerankerProvider,
default_top_k: int = DEFAULT_SEARCH_TOP_K,
) -> None:
if (
isinstance(default_top_k, bool)
or not 1 <= default_top_k <= reranker_provider.max_documents
):
raise SearchConfigurationError(
"default_top_k must fit within the reranker document limit"
)
self._session_factory = session_factory
self._dense_retriever = dense_retriever
self._reranker_provider = reranker_provider
self._default_top_k = default_top_k
def search(self, query: str, top_k: int | None = None) -> SearchResult:
"""Return Top-K sources while preserving both dense and reranker scores."""
requested_top_k = self._default_top_k if top_k is None else top_k
if (
isinstance(requested_top_k, bool)
or not 1 <= requested_top_k <= self._reranker_provider.max_documents
):
raise SearchConfigurationError("top_k must fit within the reranker document limit")
dense_result = self._dense_retriever.search(query)
candidates = dense_result.candidates
if not candidates:
return SearchResult(
query=dense_result.query,
index_profile_id=dense_result.index_profile_id,
results=(),
)
if len(candidates) > self._reranker_provider.max_documents:
raise SearchConfigurationError("dense candidate count exceeds reranker limit")
if len({candidate.chunk_id for candidate in candidates}) != len(candidates):
raise RerankerContractError("dense retrieval returned duplicate chunk candidates")
if any(
candidate.index_profile_id != dense_result.index_profile_id for candidate in candidates
):
raise RerankerContractError("dense candidate crossed the selected index profile")
if any(not isfinite(candidate.score) for candidate in candidates):
raise RerankerContractError("dense candidate returned a non-finite score")
if any(not candidate.content for candidate in candidates):
raise RerankerContractError("dense candidate content must not be empty")
final_count = min(requested_top_k, len(candidates))
ranked = self._reranker_provider.rerank(
dense_result.query,
tuple(candidate.content for candidate in candidates),
final_count,
)
self._validate_ranked_results(ranked, candidates, final_count)
selected = tuple((candidates[result.candidate_index], result.score) for result in ranked)
return SearchResult(
query=dense_result.query,
index_profile_id=dense_result.index_profile_id,
results=self._load_sources(selected),
)
@staticmethod
def _validate_ranked_results(
ranked: tuple[object, ...],
candidates: tuple[DenseSearchCandidate, ...],
expected_count: int,
) -> None:
"""Fail closed if a provider returns ambiguous or reordered mappings."""
if len(ranked) != expected_count:
raise RerankerContractError("reranker returned the wrong result count")
indexes: list[int] = []
previous_score: float | None = None
for result in ranked:
candidate_index = getattr(result, "candidate_index", None)
score = getattr(result, "score", None)
if isinstance(candidate_index, bool) or not isinstance(candidate_index, int):
raise RerankerContractError("reranker returned an invalid candidate index")
if not 0 <= candidate_index < len(candidates):
raise RerankerContractError("reranker candidate index is out of range")
if isinstance(score, bool) or not isinstance(score, (int, float)):
raise RerankerContractError("reranker returned an invalid score")
converted_score = float(score)
if not isfinite(converted_score):
raise RerankerContractError("reranker returned a non-finite score")
if previous_score is not None and previous_score < converted_score:
raise RerankerContractError("reranker results are not score-descending")
indexes.append(candidate_index)
previous_score = converted_score
if len(indexes) != len(set(indexes)):
raise RerankerContractError("reranker returned duplicate candidate indexes")
def _load_sources(
self,
selected: tuple[tuple[DenseSearchCandidate, float], ...],
) -> tuple[SourceResult, ...]:
"""Rehydrate final content and provenance from canonical PostgreSQL rows."""
chunk_ids = [candidate.chunk_id for candidate, _ in selected]
with self._session_factory() as session:
chunks = {
chunk.chunk_id: chunk
for chunk in session.scalars(
select(Chunk).where(Chunk.chunk_id.in_(chunk_ids))
).all()
}
if set(chunks) != set(chunk_ids):
raise CanonicalSourceError("a reranked chunk is missing from canonical storage")
results: list[SourceResult] = []
for candidate, rerank_score in selected:
chunk = chunks[candidate.chunk_id]
if chunk.document_id != candidate.document_id:
raise CanonicalSourceError("vector candidate document does not match its chunk")
if chunk.content != candidate.content:
raise CanonicalSourceError("vector candidate content is stale")
results.append(
SourceResult(
document_id=chunk.document_id,
filename=chunk.filename,
page=chunk.page,
section=chunk.section,
chunk_id=chunk.chunk_id,
content=chunk.content,
score=candidate.score,
rerank_score=rerank_score,
)
)
return tuple(results)

View File

@ -0,0 +1,13 @@
"""Reranker provider abstraction and Demo implementations."""
from ekp_reranker.base import RerankerProvider, RerankerProviderError, RerankResult
from ekp_reranker.fake import FakeRerankerProvider
from ekp_reranker.qianfan import QianfanRerankerProvider
__all__ = [
"FakeRerankerProvider",
"QianfanRerankerProvider",
"RerankResult",
"RerankerProvider",
"RerankerProviderError",
]

View File

@ -0,0 +1,49 @@
"""Provider-neutral reranking contract for retrieval candidates."""
from collections.abc import Sequence
from dataclasses import dataclass
from math import isfinite
from typing import Protocol
class RerankerProviderError(RuntimeError):
"""Raised when a reranker rejects or cannot serve a request."""
@dataclass(frozen=True, slots=True)
class RerankResult:
"""One ranked candidate reference using its original input position."""
candidate_index: int
score: float
def __post_init__(self) -> None:
"""Reject invalid mappings before core search consumes them."""
if (
isinstance(self.candidate_index, bool)
or not isinstance(self.candidate_index, int)
or self.candidate_index < 0
):
raise ValueError("rerank candidate_index must be a non-negative integer")
if not isfinite(self.score):
raise ValueError("rerank score must be finite")
class RerankerProvider(Protocol):
"""Rank text candidates without exposing a vendor-specific response."""
@property
def model_name(self) -> str:
"""Return the stable provider model identifier."""
@property
def max_documents(self) -> int:
"""Return the largest candidate batch accepted by this provider."""
def rerank(
self,
query: str,
documents: Sequence[str],
top_n: int,
) -> tuple[RerankResult, ...]:
"""Return Top-N references in descending relevance order."""

View File

@ -0,0 +1,65 @@
"""Deterministic lexical reranker for offline tests and evaluation."""
from collections.abc import Sequence
from ekp_reranker.base import RerankerProviderError, RerankResult
FAKE_MAX_DOCUMENTS = 64
class FakeRerankerProvider:
"""Rank by query-character coverage with stable input-index tie-breaking."""
def __init__(self, model_name: str = "fake-reranker") -> None:
if not model_name:
raise ValueError("fake reranker model_name must not be empty")
self._model_name = model_name
self.calls: list[tuple[str, tuple[str, ...], int]] = []
@property
def model_name(self) -> str:
return self._model_name
@property
def max_documents(self) -> int:
return FAKE_MAX_DOCUMENTS
def rerank(
self,
query: str,
documents: Sequence[str],
top_n: int,
) -> tuple[RerankResult, ...]:
"""Produce repeatable relevance scores without model or network access."""
normalized_query = query.strip()
if not normalized_query:
raise RerankerProviderError("rerank query must not be empty")
if not documents:
raise RerankerProviderError("rerank documents must not be empty")
if len(documents) > self.max_documents:
raise RerankerProviderError(
f"fake reranker accepts at most {self.max_documents} documents"
)
if any(not document for document in documents):
raise RerankerProviderError("rerank document must not be empty")
if isinstance(top_n, bool) or not 1 <= top_n <= len(documents):
raise RerankerProviderError("rerank top_n must be within the candidate batch")
captured_documents = tuple(documents)
self.calls.append((normalized_query, captured_documents, top_n))
query_units = {character.lower() for character in normalized_query if character.isalnum()}
if not query_units:
raise RerankerProviderError("rerank query must contain searchable characters")
ranked = [
RerankResult(
candidate_index=index,
score=len(
query_units
& {character.lower() for character in document if character.isalnum()}
)
/ len(query_units),
)
for index, document in enumerate(captured_documents)
]
ranked.sort(key=lambda result: (-result.score, result.candidate_index))
return tuple(ranked[:top_n])

View File

@ -0,0 +1,180 @@
"""Baidu Qianfan v2 adapter for the Demo Qwen3 reranker baseline."""
import json
from collections.abc import Mapping, Sequence
from math import isfinite
from numbers import Real
from typing import Protocol, cast
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from ekp_reranker.base import RerankerProviderError, RerankResult
QWEN3_RERANKER_MODEL = "qwen3-reranker-0.6b"
QIANFAN_RERANK_ENDPOINT = "https://qianfan.baidubce.com/v2/rerank"
QIANFAN_MAX_DOCUMENTS = 64
QIANFAN_MAX_QUERY_CHARACTERS = 1600
QIANFAN_MAX_DOCUMENT_CHARACTERS = 4096
class JsonHttpClient(Protocol):
"""Injectable synchronous JSON boundary for the Qianfan rerank API."""
def post_json(
self,
url: str,
headers: Mapping[str, str],
payload: Mapping[str, object],
timeout_seconds: float,
) -> dict[str, object]:
"""POST JSON and return a decoded object."""
class UrllibJsonHttpClient:
"""Dependency-free client with bounded waits and sanitized failures."""
def post_json(
self,
url: str,
headers: Mapping[str, str],
payload: Mapping[str, object],
timeout_seconds: float,
) -> dict[str, object]:
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
request = Request(url=url, data=body, headers=dict(headers), method="POST")
try:
with urlopen(request, timeout=timeout_seconds) as response: # noqa: S310
decoded: object = json.loads(response.read())
except HTTPError as exc:
raise RerankerProviderError(
f"Qianfan rerank request failed with HTTP {exc.code}"
) from exc
except (URLError, TimeoutError) as exc:
raise RerankerProviderError("Qianfan rerank request could not complete") from exc
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RerankerProviderError("Qianfan reranker returned invalid JSON") from exc
if not isinstance(decoded, dict):
raise RerankerProviderError("Qianfan reranker returned a non-object response")
return cast(dict[str, object], decoded)
class QianfanRerankerProvider:
"""Call Qianfan rerank and map ranked indexes back to input candidates."""
def __init__(
self,
api_key: str,
endpoint: str = QIANFAN_RERANK_ENDPOINT,
model_name: str = QWEN3_RERANKER_MODEL,
timeout_seconds: float = 30.0,
http_client: JsonHttpClient | None = None,
) -> None:
if not api_key:
raise ValueError("Qianfan API key must not be empty")
if not endpoint.startswith("https://"):
raise ValueError("Qianfan rerank endpoint must use HTTPS")
if not model_name:
raise ValueError("Qianfan reranker model_name must not be empty")
if timeout_seconds <= 0:
raise ValueError("timeout_seconds must be positive")
self._api_key = api_key
self._endpoint = endpoint
self._model_name = model_name
self._timeout_seconds = timeout_seconds
self._http_client = http_client or UrllibJsonHttpClient()
@property
def model_name(self) -> str:
return self._model_name
@property
def max_documents(self) -> int:
return QIANFAN_MAX_DOCUMENTS
def rerank(
self,
query: str,
documents: Sequence[str],
top_n: int,
) -> tuple[RerankResult, ...]:
"""Validate service limits and preserve Qianfan's ranked result order."""
normalized_query = query.strip()
candidate_documents = tuple(documents)
self._validate_request(normalized_query, candidate_documents, top_n)
response = self._http_client.post_json(
self._endpoint,
{
"Authorization": f"Bearer {self._api_key}",
"Content-Type": "application/json",
},
{
"model": self.model_name,
"query": normalized_query,
"documents": list(candidate_documents),
"top_n": top_n,
},
self._timeout_seconds,
)
raw_results = response.get("results")
if not isinstance(raw_results, list) or len(raw_results) != top_n:
raise RerankerProviderError("Qianfan returned an unexpected rerank result count")
results = tuple(self._validated_result(item, candidate_documents) for item in raw_results)
indexes = [result.candidate_index for result in results]
if len(indexes) != len(set(indexes)):
raise RerankerProviderError("Qianfan returned duplicate rerank indexes")
if any(
previous.score < current.score
for previous, current in zip(results, results[1:], strict=False)
):
raise RerankerProviderError("Qianfan rerank results are not score-descending")
return results
def _validate_request(
self,
query: str,
documents: tuple[str, ...],
top_n: int,
) -> None:
"""Enforce current documented Qianfan request bounds before network I/O."""
if not query:
raise RerankerProviderError("rerank query must not be empty")
if len(query) > QIANFAN_MAX_QUERY_CHARACTERS:
raise RerankerProviderError(
f"rerank query exceeds {QIANFAN_MAX_QUERY_CHARACTERS} characters"
)
if not documents:
raise RerankerProviderError("rerank documents must not be empty")
if len(documents) > self.max_documents:
raise RerankerProviderError(
f"Qianfan accepts at most {self.max_documents} rerank documents"
)
if any(not document for document in documents):
raise RerankerProviderError("rerank document must not be empty")
if any(len(document) > QIANFAN_MAX_DOCUMENT_CHARACTERS for document in documents):
raise RerankerProviderError(
f"rerank document exceeds {QIANFAN_MAX_DOCUMENT_CHARACTERS} characters"
)
if isinstance(top_n, bool) or not 1 <= top_n <= len(documents):
raise RerankerProviderError("rerank top_n must be within the candidate batch")
@staticmethod
def _validated_result(
item: object,
documents: tuple[str, ...],
) -> RerankResult:
"""Require a unique in-range index, finite score, and matching echoed text."""
if not isinstance(item, dict):
raise RerankerProviderError("Qianfan returned an invalid rerank item")
index = item.get("index")
score = item.get("relevance_score")
if isinstance(index, bool) or not isinstance(index, int):
raise RerankerProviderError("Qianfan rerank item has an invalid index")
if not 0 <= index < len(documents):
raise RerankerProviderError("Qianfan rerank index is out of range")
if isinstance(score, bool) or not isinstance(score, Real) or not isfinite(float(score)):
raise RerankerProviderError("Qianfan rerank item has an invalid score")
document = item.get("document")
if document is not None and document != documents[index]:
raise RerankerProviderError("Qianfan rerank item does not match its index")
return RerankResult(candidate_index=index, score=float(score))

View File

@ -37,6 +37,7 @@ where = [
"packages/providers/object_storage/src",
"packages/providers/parser/src",
"packages/providers/embedding/src",
"packages/providers/reranker/src",
"packages/providers/vector_store/src",
]
@ -55,6 +56,7 @@ known-first-party = [
"ekp_embedding",
"ekp_object_storage",
"ekp_parser",
"ekp_reranker",
"ekp_vector_store",
]
@ -68,6 +70,7 @@ mypy_path = [
"packages/providers/object_storage/src",
"packages/providers/parser/src",
"packages/providers/embedding/src",
"packages/providers/reranker/src",
"packages/providers/vector_store/src",
]

View File

@ -0,0 +1,86 @@
"""Golden Dataset regression for reranking and canonical source results."""
import json
from collections.abc import Sequence
from io import BytesIO
from pathlib import Path
from sqlalchemy import create_engine
from ekp_core.chunking import StructureAwareChunker
from ekp_core.database import create_session_factory
from ekp_core.dense_retrieval import DenseRetrievalService
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.search import SearchService
from ekp_domain import Base, IndexProfile
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_reranker import FakeRerankerProvider
from ekp_vector_store import FakeVectorStore
class GoldenKeywordEmbeddingProvider:
"""Small semantic fixture matching the Dense regression without external calls."""
model_name = "golden-keyword-v1"
dimensions = 2
max_batch_size = 16
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
vectors: list[tuple[float, ...]] = []
for text in texts:
if "住宿" in text or "出差" in text:
vectors.append((1.0, 0.0))
elif "退款" in text or "产品A" in text or "产品 A" in text:
vectors.append((0.0, 1.0))
else:
vectors.append((0.1, 0.1))
return tuple(vectors)
def test_reranker_promotes_expected_canonical_source_to_top_one(tmp_path: Path) -> None:
"""Run the complete Demo pipeline through Dense Top20, Rerank, and Source Result."""
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(),
)
embedding = GoldenKeywordEmbeddingProvider()
vector_store = FakeVectorStore()
profile = IndexProfile("golden-rerank-v1", "structure_v1", embedding.model_name, 2)
indexing = DocumentIndexingService(session_factory, embedding, vector_store, profile)
pipeline = DocumentIndexingPipelineService(
DocumentPipelineService(upload, parsing, chunking),
indexing,
)
dense = DenseRetrievalService(embedding, vector_store, profile)
search = SearchService(session_factory, dense, FakeRerankerProvider())
golden_records = [
json.loads(line)
for line in Path("fixtures/evaluation/golden.jsonl").read_text().splitlines()
if line
]
filenames = {filename for record in golden_records for filename in record["expected_documents"]}
for filename in sorted(filenames):
payload = (Path("fixtures/documents") / filename).read_bytes()
pipeline.ingest(filename, "text/markdown", BytesIO(payload))
for record in golden_records:
result = search.search(record["query"])
assert result.results
best = result.results[0]
assert best.filename in record["expected_documents"]
assert best.section is not None
assert any(section in best.section for section in record["expected_sections"])
assert best.content
assert best.rerank_score >= 0

View File

@ -32,6 +32,11 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
assert settings.mochow_timeout_seconds == 30
assert settings.index_profile_id == "demo-bge-large-zh-structure-v1"
assert settings.dense_top_n == 20
assert settings.reranker_provider == "qianfan"
assert settings.reranker_model == "qwen3-reranker-0.6b"
assert settings.reranker_endpoint.endswith("/v2/rerank")
assert settings.reranker_api_key is None
assert settings.search_top_k == 5
def test_settings_reject_invalid_port(isolated_settings_environment: None) -> None:
@ -46,12 +51,21 @@ def test_settings_reject_chunk_overlap_without_forward_progress(
Settings(chunk_max_tokens=100, chunk_overlap_tokens=100)
def test_settings_reject_final_top_k_larger_than_dense_candidates(
isolated_settings_environment: None,
) -> None:
with pytest.raises(ValidationError, match="search_top_k"):
Settings(dense_top_n=5, search_top_k=6)
def test_settings_redact_provider_secrets(isolated_settings_environment: None) -> None:
settings = Settings(
baidu_qianfan_api_key=SecretStr("qianfan-secret"),
baidu_mochow_api_key=SecretStr("mochow-secret"),
reranker_api_key=SecretStr("reranker-secret"),
)
rendered = repr(settings)
assert "qianfan-secret" not in rendered
assert "mochow-secret" not in rendered
assert "reranker-secret" not in rendered

View File

@ -8,6 +8,7 @@ from ekp_api.providers import (
create_dense_retrieval_service,
create_embedding_provider,
create_index_profile,
create_reranker_provider,
create_vector_store,
)
from ekp_core.config import Settings
@ -56,3 +57,14 @@ def test_provider_wiring_builds_dense_retrieval_without_network(
)
assert service.search("测试查询").candidates == ()
def test_provider_wiring_reuses_qianfan_key_for_demo_reranker(
isolated_settings_environment: None,
) -> None:
settings = Settings(baidu_qianfan_api_key=SecretStr("shared-test-key"))
reranker = create_reranker_provider(settings)
assert reranker.model_name == "qwen3-reranker-0.6b"
assert reranker.max_documents == 64

View File

@ -0,0 +1,112 @@
"""Unit tests for the Qianfan Qwen3 reranker request/response contract."""
from collections.abc import Mapping
import pytest
from ekp_reranker import QianfanRerankerProvider, RerankerProviderError
class FakeJsonHttpClient:
"""Capture rerank JSON requests without network access."""
def __init__(self, response: dict[str, object]) -> None:
self.response = response
self.calls: list[tuple[str, Mapping[str, str], Mapping[str, object], float]] = []
def post_json(
self,
url: str,
headers: Mapping[str, str],
payload: Mapping[str, object],
timeout_seconds: float,
) -> dict[str, object]:
self.calls.append((url, headers, payload, timeout_seconds))
return self.response
def test_qianfan_reranker_sends_top_n_and_preserves_ranked_indexes() -> None:
client = FakeJsonHttpClient(
{
"results": [
{"document": "退款周期为七天", "relevance_score": 0.93, "index": 1},
{"document": "住宿标准说明", "relevance_score": 0.21, "index": 0},
]
}
)
provider = QianfanRerankerProvider(
api_key="test-key",
endpoint="https://qianfan.example/v2/rerank",
timeout_seconds=12,
http_client=client,
)
results = provider.rerank(" 产品A退款周期 ", ("住宿标准说明", "退款周期为七天"), 2)
assert provider.model_name == "qwen3-reranker-0.6b"
assert provider.max_documents == 64
assert [result.candidate_index for result in results] == [1, 0]
url, headers, payload, timeout = client.calls[0]
assert url.endswith("/v2/rerank")
assert headers["Authorization"] == "Bearer test-key"
assert payload == {
"model": "qwen3-reranker-0.6b",
"query": "产品A退款周期",
"documents": ["住宿标准说明", "退款周期为七天"],
"top_n": 2,
}
assert timeout == 12
@pytest.mark.parametrize(
"response",
[
{"results": []},
{
"results": [
{"relevance_score": 0.9, "index": 0},
{"relevance_score": 0.8, "index": 0},
]
},
{
"results": [
{"relevance_score": 0.3, "index": 0},
{"relevance_score": 0.8, "index": 1},
]
},
{
"results": [
{"document": "错误内容", "relevance_score": 0.9, "index": 0},
{"relevance_score": 0.8, "index": 1},
]
},
{
"results": [
{"relevance_score": float("nan"), "index": 0},
{"relevance_score": 0.8, "index": 1},
]
},
],
)
def test_qianfan_reranker_rejects_invalid_responses(response: dict[str, object]) -> None:
provider = QianfanRerankerProvider(
api_key="test-key",
http_client=FakeJsonHttpClient(response),
)
with pytest.raises(RerankerProviderError):
provider.rerank("查询", ("文档一", "文档二"), 2)
def test_qianfan_reranker_enforces_document_and_character_limits() -> None:
provider = QianfanRerankerProvider(
api_key="test-key",
http_client=FakeJsonHttpClient({"results": []}),
)
with pytest.raises(RerankerProviderError, match="1600"):
provider.rerank("问" * 1601, ("内容",), 1)
with pytest.raises(RerankerProviderError, match="64"):
provider.rerank("查询", ("内容",) * 65, 5)
with pytest.raises(RerankerProviderError, match="4096"):
provider.rerank("查询", ("内" * 4097,), 1)

View File

@ -0,0 +1,41 @@
"""Unit tests for provider-neutral reranking behavior."""
import pytest
from ekp_reranker import FakeRerankerProvider, RerankerProviderError, RerankResult
def test_fake_reranker_orders_by_query_coverage_and_keeps_source_index() -> None:
provider = FakeRerankerProvider()
results = provider.rerank(
"员工出差住宿标准",
("提交报销材料", "员工出差住宿标准说明", "其他制度"),
top_n=2,
)
assert [result.candidate_index for result in results] == [1, 0]
assert results[0].score > results[1].score
assert provider.calls == [
(
"员工出差住宿标准",
("提交报销材料", "员工出差住宿标准说明", "其他制度"),
2,
)
]
def test_fake_reranker_rejects_invalid_batch_and_top_n() -> None:
provider = FakeRerankerProvider()
with pytest.raises(RerankerProviderError, match="documents"):
provider.rerank("查询", (), 1)
with pytest.raises(RerankerProviderError, match="top_n"):
provider.rerank("查询", ("内容",), 2)
def test_rerank_result_rejects_invalid_index_and_score() -> None:
with pytest.raises(ValueError, match="candidate_index"):
RerankResult(candidate_index=True, score=0.5)
with pytest.raises(ValueError, match="finite"):
RerankResult(candidate_index=0, score=float("nan"))

167
tests/unit/test_search.py Normal file
View File

@ -0,0 +1,167 @@
"""Unit tests for reranked search and canonical source enrichment."""
from collections.abc import Sequence
from uuid import UUID, uuid4
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.database import create_session_factory
from ekp_core.dense_retrieval import DenseRetrievalResult
from ekp_core.search import CanonicalSourceError, RerankerContractError, SearchService
from ekp_domain import Base, Chunk, Document, DocumentStatus
from ekp_reranker import FakeRerankerProvider, RerankResult
from ekp_vector_store import DenseSearchCandidate
class StubDenseRetriever:
"""Return prebuilt candidates while recording the query."""
def __init__(self, candidates: tuple[DenseSearchCandidate, ...]) -> None:
self.candidates = candidates
self.queries: list[str] = []
def search(self, query: str, top_n: int | None = None) -> DenseRetrievalResult:
self.queries.append(query)
return DenseRetrievalResult(query.strip(), "profile-v1", self.candidates)
def _candidate(
chunk_id: UUID,
document_id: UUID,
content: str,
*,
dense_score: float,
) -> DenseSearchCandidate:
return DenseSearchCandidate(
index_record_id=uuid4(),
chunk_id=chunk_id,
document_id=document_id,
filename="stale-index-name.md",
page=None,
page_end=None,
section="stale-index-section",
chunk_index=0,
content=content,
score=dense_score,
index_profile_id="profile-v1",
)
def _canonical_source_session() -> tuple[sessionmaker[Session], UUID, UUID, UUID]:
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
document_id = uuid4()
first_chunk_id = uuid4()
second_chunk_id = uuid4()
with session_factory() as session:
session.add(
Document(
id=document_id,
filename="policy.md",
media_type="text/markdown",
size_bytes=100,
storage_key=f"documents/{document_id}/policy.md",
status=DocumentStatus.READY,
)
)
session.add_all(
[
Chunk(
chunk_id=first_chunk_id,
document_id=document_id,
filename="policy.md",
page=1,
page_end=1,
section="报销材料",
section_path=["报销材料"],
chunk_index=0,
content="提交报销材料",
token_count=6,
strategy_version="structure_v1",
),
Chunk(
chunk_id=second_chunk_id,
document_id=document_id,
filename="policy.md",
page=2,
page_end=2,
section="住宿标准",
section_path=["住宿标准"],
chunk_index=1,
content="员工出差住宿标准说明",
token_count=9,
strategy_version="structure_v1",
),
]
)
session.commit()
return session_factory, document_id, first_chunk_id, second_chunk_id
def test_search_reranks_and_rehydrates_source_from_canonical_chunk() -> None:
session_factory, document_id, first_chunk_id, second_chunk_id = _canonical_source_session()
dense = StubDenseRetriever(
(
_candidate(first_chunk_id, document_id, "提交报销材料", dense_score=0.91),
_candidate(second_chunk_id, document_id, "员工出差住宿标准说明", dense_score=0.72),
)
)
reranker = FakeRerankerProvider()
service = SearchService(session_factory, dense, reranker, default_top_k=1)
result = service.search("员工出差住宿标准")
assert len(result.results) == 1
source = result.results[0]
assert source.chunk_id == second_chunk_id
assert source.filename == "policy.md"
assert source.page == 2
assert source.section == "住宿标准"
assert source.content == "员工出差住宿标准说明"
assert source.score == 0.72
assert source.rerank_score == 1.0
def test_search_rejects_stale_vector_content() -> None:
session_factory, document_id, first_chunk_id, _ = _canonical_source_session()
dense = StubDenseRetriever(
(_candidate(first_chunk_id, document_id, "索引中的旧内容", dense_score=0.8),)
)
service = SearchService(
session_factory,
dense,
FakeRerankerProvider(),
)
with pytest.raises(CanonicalSourceError, match="stale"):
service.search("旧内容", top_k=1)
class DuplicateReranker:
model_name = "bad"
max_documents = 64
def rerank(
self,
query: str,
documents: Sequence[str],
top_n: int,
) -> tuple[RerankResult, ...]:
return (RerankResult(0, 0.9), RerankResult(0, 0.8))
def test_search_rejects_duplicate_reranker_indexes() -> None:
session_factory, document_id, first_chunk_id, second_chunk_id = _canonical_source_session()
dense = StubDenseRetriever(
(
_candidate(first_chunk_id, document_id, "提交报销材料", dense_score=0.9),
_candidate(second_chunk_id, document_id, "员工出差住宿标准说明", dense_score=0.8),
)
)
service = SearchService(session_factory, dense, DuplicateReranker(), default_top_k=2)
with pytest.raises(RerankerContractError, match="duplicate"):
service.search("住宿标准")