diff --git a/.env.example b/.env.example index 38fdde5..95bc3e7 100644 --- a/.env.example +++ b/.env.example @@ -41,9 +41,11 @@ INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1 DENSE_TOP_N=20 RERANKER_PROVIDER=qianfan -RERANKER_MODEL= -RERANKER_ENDPOINT= +RERANKER_MODEL=qwen3-reranker-0.6b +RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank RERANKER_API_KEY= +RERANKER_TIMEOUT_SECONDS=30 +SEARCH_TOP_K=5 LLM_PROVIDER=qianfan LLM_MODEL= diff --git a/README.md b/README.md index 539a181..968af24 100644 --- a/README.md +++ b/README.md @@ -93,5 +93,11 @@ pre-provisioned table schema. DEMO-007 adds query embedding and profile-scoped Dense Top-N retrieval. The default is Top 20 with a configurable Mochow HNSW `ef`; the in-memory fake uses exact cosine search for deterministic offline regression. Returned objects are -raw dense candidates only. Reranking and final source-result assembly remain in -DEMO-008, while public search REST endpoints remain in DEMO-009. +raw dense candidates only. + +DEMO-008 adds a provider-neutral reranker, a validated Qianfan +`qwen3-reranker-0.6b` adapter, and the final Top20-to-Top5 search pipeline. Final +source results are rehydrated from canonical PostgreSQL chunks and retain both +Dense and reranker scores; stale vector content fails closed. See +`docs/development/RERANKER_SETUP.md` for configuration and validation limits. +Public search REST endpoints remain in DEMO-009. diff --git a/apps/api/src/ekp_api/providers.py b/apps/api/src/ekp_api/providers.py index 814ecf1..ec26e45 100644 --- a/apps/api/src/ekp_api/providers.py +++ b/apps/api/src/ekp_api/providers.py @@ -1,9 +1,13 @@ """Runtime provider composition from secret-safe application settings.""" +from sqlalchemy.orm import Session, sessionmaker + from ekp_core.config import Settings from ekp_core.dense_retrieval import DenseRetrievalService +from ekp_core.search import SearchService from ekp_domain import IndexProfile from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider +from ekp_reranker import QianfanRerankerProvider, RerankerProvider from ekp_vector_store import MochowVectorStore, VectorStore, create_mochow_vector_store BGE_LARGE_ZH_DIMENSIONS = 1024 @@ -68,3 +72,31 @@ def create_dense_retrieval_service( index_profile=index_profile, default_top_n=settings.dense_top_n, ) + + +def create_reranker_provider(settings: Settings) -> QianfanRerankerProvider: + """Create the Demo Qwen3 reranker using an explicit or shared Qianfan key.""" + secret = settings.reranker_api_key or settings.baidu_qianfan_api_key + if secret is None: + raise ProviderConfigurationError("RERANKER_API_KEY or BAIDU_QIANFAN_API_KEY is required") + return QianfanRerankerProvider( + api_key=secret.get_secret_value(), + endpoint=settings.reranker_endpoint, + model_name=settings.reranker_model, + timeout_seconds=settings.reranker_timeout_seconds, + ) + + +def create_search_service( + settings: Settings, + session_factory: sessionmaker[Session], + dense_retriever: DenseRetrievalService, + reranker_provider: RerankerProvider, +) -> SearchService: + """Compose DEMO-008 search while leaving REST serialization to DEMO-009.""" + return SearchService( + session_factory=session_factory, + dense_retriever=dense_retriever, + reranker_provider=reranker_provider, + default_top_k=settings.search_top_k, + ) diff --git a/docs/development/MOCHOW_SETUP.md b/docs/development/MOCHOW_SETUP.md index ccc1a0a..0ff0e9c 100644 --- a/docs/development/MOCHOW_SETUP.md +++ b/docs/development/MOCHOW_SETUP.md @@ -77,8 +77,8 @@ DEMO-007 使用同一 `EmbeddingProvider` 将 Query 转换为 1024 维向量, - 返回值是尚未 Rerank 的候选集,`score` 原样保留 Mochow 返回值,不在此阶段设置阈值。 Demo 明确不实现 RBAC/Document ACL;正式 V1 引入权限后,允许范围仍必须在召回阶段下推, -不得先做全局 Top-K 再过滤。Reranker 与最终 Source Result 属于 DEMO-008,REST API 属于 -DEMO-009。 +不得先做全局 Top-K 再过滤。Reranker 与最终 Source Result 的配置参见 +`docs/development/RERANKER_SETUP.md`;REST API 属于 DEMO-009。 ## 验证边界 diff --git a/docs/development/RERANKER_SETUP.md b/docs/development/RERANKER_SETUP.md new file mode 100644 index 0000000..32e4e2f --- /dev/null +++ b/docs/development/RERANKER_SETUP.md @@ -0,0 +1,69 @@ +# DEMO-008 千帆 Reranker 与 Source Result 配置 + +本文记录 Knowledge Platform Demo 的重排序连接方式、请求边界和 canonical Source Result +契约。真实 API Key 只允许由运行环境注入,不得写入代码、示例配置、日志或提交记录。 + +## 运行配置 + +```dotenv +RERANKER_PROVIDER=qianfan +RERANKER_MODEL=qwen3-reranker-0.6b +RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank +RERANKER_API_KEY=<运行环境注入,可空> +RERANKER_TIMEOUT_SECONDS=30 +DENSE_TOP_N=20 +SEARCH_TOP_K=5 +``` + +`RERANKER_API_KEY` 未单独配置时,Demo 会复用 `BAIDU_QIANFAN_API_KEY`。两者都不存在时 +Provider 创建失败。凭证使用 `SecretStr`,HTTP/JSON 异常不会包含请求头、请求正文或 +响应正文。 + +## Reranker 请求契约 + +DEMO-008 固定使用千帆 `POST /v2/rerank`,模型基线为 +`qwen3-reranker-0.6b`: + +- Query 去除首尾空白后不能为空,最多 1600 字符; +- documents 不能为空,最多 64 条,每条最多 4096 字符; +- 当前标准链路发送 Dense Top20,并请求 Reranker Top5; +- 响应必须恰好返回请求的 Top-N,`index` 唯一且指向原输入位置; +- `relevance_score` 必须是有限数值,并保持降序; +- 响应若回显 document,必须与对应输入文本完全一致。 + +Adapter 只向业务层返回原候选索引和 Reranker 分数,不暴露千帆响应结构。默认测试使用 +`FakeRerankerProvider` 的确定性字符覆盖率排序,不发起网络请求。 + +## Source Result 契约 + +Reranker 对 Mochow Dense Candidate 的 `content` 评分;选出 Top5 后,`SearchService` 使用 +`chunk_id` 一次性回查 PostgreSQL canonical `chunks` 表,并由 canonical row 生成: + +```text +document_id +filename +page +section +chunk_id +content +score +rerank_score +``` + +其中 `score` 保留 Dense 分数,`rerank_score` 保存精排分数。若 Chunk 缺失、document_id +不一致或 Mochow content 已与 canonical content 不一致,搜索会失败关闭,避免返回来源与 +评分文本不一致的结果。 + +DEMO-008 不创建公开 REST Endpoint;`POST /api/search` 的序列化和错误映射属于 +DEMO-009。Demo 仍不实现 RBAC/Document ACL,正式 V1 权限范围必须在 Dense 召回阶段 +下推,不能在 Reranker 后过滤。 + +## 验证边界 + +默认测试离线覆盖 Dense Top20 → Reranker → Top5 → canonical Source Result,并使用合成 +Golden Dataset 验证预期文档和章节升至首位。真实千帆联调前还需在受控运行环境确认: + +- API Key 已开通目标 Reranker 模型权限; +- Endpoint 网络可达; +- 模型 API 名称与控制台服务一致; +- 真实中文 Query 的延迟、配额和 Top5 质量满足 Demo 演示需求。 diff --git a/docs/plans/PROGRESS.md b/docs/plans/PROGRESS.md index 876c8ed..1dc7080 100644 --- a/docs/plans/PROGRESS.md +++ b/docs/plans/PROGRESS.md @@ -2,11 +2,11 @@ 当前阶段:Knowledge Retrieval Demo v0.1 -已完成:DEMO-001、DEMO-002、DEMO-003、DEMO-004、DEMO-005、DEMO-006、DEMO-007 +已完成:DEMO-001、DEMO-002、DEMO-003、DEMO-004、DEMO-005、DEMO-006、DEMO-007、DEMO-008 进行中:无 阻塞:无 -下一任务:DEMO-008 +下一任务:DEMO-009 完整 V1 待办将在 Demo Scope 完成后继续执行。当前任务顺序以 `docs/demo/DEMO_SCOPE.md` 为准。 @@ -101,3 +101,19 @@ 章节。 - 默认自动化测试不连接真实千帆/Mochow;真实环境仍需按 `docs/development/MOCHOW_SETUP.md` 注入凭证并验证网络、表 Schema 与索引状态。 + +### DEMO-008 + +- 新增与厂商无关的 `RerankerProvider` / `RerankResult` 契约及确定性 Fake,使用原始 + candidate index 映射精排结果,业务服务不依赖千帆响应结构。 +- 新增千帆 v2 `qwen3-reranker-0.6b` Adapter,调用 `/v2/rerank`,严格限制 Query、 + documents 和 Top-N,校验响应数量、唯一索引、回显文本、有限分数与降序关系;错误不会 + 回显请求内容或凭证。 +- 新增 `SearchService`,完成 Dense Top20 → Reranker → Top5。Reranker 选中候选后按 + `chunk_id` 回查 PostgreSQL canonical Chunk,再生成包含 document、filename、page、 + section、chunk、content、Dense score 和 rerank score 的最终 Source Result。 +- Source Enrichment 对缺失 Chunk、document_id 不一致和 stale vector content 采用失败 + 关闭,防止 Mochow 副本与 canonical 来源不一致时返回错误证据。 +- 新增 Provider、配置脱敏、排序映射、canonical 回查、一致性失败和 Golden Dataset + 全链路回归。默认测试不调用真实千帆;真实联调边界记录在 + `docs/development/RERANKER_SETUP.md`。公开 Search REST API 留在 DEMO-009。 diff --git a/packages/core/src/ekp_core/__init__.py b/packages/core/src/ekp_core/__init__.py index 230aca1..1125ea2 100644 --- a/packages/core/src/ekp_core/__init__.py +++ b/packages/core/src/ekp_core/__init__.py @@ -8,6 +8,7 @@ from ekp_core.document_indexing import ( DocumentIndexingService, ) from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService +from ekp_core.search import SearchResult, SearchService, SourceResult __all__ = [ "DocumentChunkingService", @@ -18,5 +19,8 @@ __all__ = [ "DocumentIndexingService", "DocumentPipelineService", "DocumentParsingService", + "SearchResult", + "SearchService", + "SourceResult", "clean_parsed_document", ] diff --git a/packages/core/src/ekp_core/config.py b/packages/core/src/ekp_core/config.py index af22555..49feae9 100644 --- a/packages/core/src/ekp_core/config.py +++ b/packages/core/src/ekp_core/config.py @@ -46,13 +46,21 @@ class Settings(BaseSettings): mochow_search_ef: int = Field(default=200, ge=1, le=10000) mochow_timeout_seconds: float = Field(default=30.0, gt=0) index_profile_id: str = "demo-bge-large-zh-structure-v1" - dense_top_n: int = Field(default=20, ge=1, le=1000) + dense_top_n: int = Field(default=20, ge=1, le=64) + reranker_provider: Literal["qianfan"] = "qianfan" + reranker_model: Literal["qwen3-reranker-0.6b"] = "qwen3-reranker-0.6b" + reranker_endpoint: str = "https://qianfan.baidubce.com/v2/rerank" + reranker_api_key: SecretStr | None = None + reranker_timeout_seconds: float = Field(default=30.0, gt=0) + search_top_k: int = Field(default=5, ge=1, le=64) @model_validator(mode="after") def validate_chunk_window(self) -> "Settings": """Require forward progress for every overlapping chunk window.""" if self.chunk_overlap_tokens >= self.chunk_max_tokens: raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens") + if self.search_top_k > self.dense_top_n: + raise ValueError("search_top_k must not exceed dense_top_n") return self diff --git a/packages/core/src/ekp_core/search.py b/packages/core/src/ekp_core/search.py new file mode 100644 index 0000000..c6b465d --- /dev/null +++ b/packages/core/src/ekp_core/search.py @@ -0,0 +1,192 @@ +"""Reranked search orchestration and canonical source-result assembly.""" + +from dataclasses import dataclass +from math import isfinite +from typing import Protocol +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.orm import Session, sessionmaker + +from ekp_core.dense_retrieval import DenseRetrievalResult +from ekp_domain import Chunk +from ekp_reranker import RerankerProvider +from ekp_vector_store import DenseSearchCandidate + +DEFAULT_SEARCH_TOP_K = 5 + + +class SearchConfigurationError(ValueError): + """Raised when candidate and final result limits cannot form a valid search.""" + + +class RerankerContractError(RuntimeError): + """Raised when reranker output cannot map safely to dense candidates.""" + + +class CanonicalSourceError(RuntimeError): + """Raised when a selected vector candidate cannot be traced to canonical Chunk data.""" + + +class DenseRetriever(Protocol): + """Minimal dense-retrieval boundary consumed by the final search pipeline.""" + + def search(self, query: str, top_n: int | None = None) -> DenseRetrievalResult: + """Return normalized query and dense candidates.""" + + +@dataclass(frozen=True, slots=True) +class SourceResult: + """One canonical, traceable result after reranking.""" + + document_id: UUID + filename: str + page: int | None + section: str | None + chunk_id: UUID + content: str + score: float + rerank_score: float + + +@dataclass(frozen=True, slots=True) +class SearchResult: + """Final Top-K result set ready for DEMO-009 REST serialization.""" + + query: str + index_profile_id: str + results: tuple[SourceResult, ...] + + +class SearchService: + """Compose Dense Top-N, reranking, and PostgreSQL source enrichment.""" + + def __init__( + self, + session_factory: sessionmaker[Session], + dense_retriever: DenseRetriever, + reranker_provider: RerankerProvider, + default_top_k: int = DEFAULT_SEARCH_TOP_K, + ) -> None: + if ( + isinstance(default_top_k, bool) + or not 1 <= default_top_k <= reranker_provider.max_documents + ): + raise SearchConfigurationError( + "default_top_k must fit within the reranker document limit" + ) + self._session_factory = session_factory + self._dense_retriever = dense_retriever + self._reranker_provider = reranker_provider + self._default_top_k = default_top_k + + def search(self, query: str, top_k: int | None = None) -> SearchResult: + """Return Top-K sources while preserving both dense and reranker scores.""" + requested_top_k = self._default_top_k if top_k is None else top_k + if ( + isinstance(requested_top_k, bool) + or not 1 <= requested_top_k <= self._reranker_provider.max_documents + ): + raise SearchConfigurationError("top_k must fit within the reranker document limit") + + dense_result = self._dense_retriever.search(query) + candidates = dense_result.candidates + if not candidates: + return SearchResult( + query=dense_result.query, + index_profile_id=dense_result.index_profile_id, + results=(), + ) + if len(candidates) > self._reranker_provider.max_documents: + raise SearchConfigurationError("dense candidate count exceeds reranker limit") + if len({candidate.chunk_id for candidate in candidates}) != len(candidates): + raise RerankerContractError("dense retrieval returned duplicate chunk candidates") + if any( + candidate.index_profile_id != dense_result.index_profile_id for candidate in candidates + ): + raise RerankerContractError("dense candidate crossed the selected index profile") + if any(not isfinite(candidate.score) for candidate in candidates): + raise RerankerContractError("dense candidate returned a non-finite score") + if any(not candidate.content for candidate in candidates): + raise RerankerContractError("dense candidate content must not be empty") + + final_count = min(requested_top_k, len(candidates)) + ranked = self._reranker_provider.rerank( + dense_result.query, + tuple(candidate.content for candidate in candidates), + final_count, + ) + self._validate_ranked_results(ranked, candidates, final_count) + selected = tuple((candidates[result.candidate_index], result.score) for result in ranked) + return SearchResult( + query=dense_result.query, + index_profile_id=dense_result.index_profile_id, + results=self._load_sources(selected), + ) + + @staticmethod + def _validate_ranked_results( + ranked: tuple[object, ...], + candidates: tuple[DenseSearchCandidate, ...], + expected_count: int, + ) -> None: + """Fail closed if a provider returns ambiguous or reordered mappings.""" + if len(ranked) != expected_count: + raise RerankerContractError("reranker returned the wrong result count") + indexes: list[int] = [] + previous_score: float | None = None + for result in ranked: + candidate_index = getattr(result, "candidate_index", None) + score = getattr(result, "score", None) + if isinstance(candidate_index, bool) or not isinstance(candidate_index, int): + raise RerankerContractError("reranker returned an invalid candidate index") + if not 0 <= candidate_index < len(candidates): + raise RerankerContractError("reranker candidate index is out of range") + if isinstance(score, bool) or not isinstance(score, (int, float)): + raise RerankerContractError("reranker returned an invalid score") + converted_score = float(score) + if not isfinite(converted_score): + raise RerankerContractError("reranker returned a non-finite score") + if previous_score is not None and previous_score < converted_score: + raise RerankerContractError("reranker results are not score-descending") + indexes.append(candidate_index) + previous_score = converted_score + if len(indexes) != len(set(indexes)): + raise RerankerContractError("reranker returned duplicate candidate indexes") + + def _load_sources( + self, + selected: tuple[tuple[DenseSearchCandidate, float], ...], + ) -> tuple[SourceResult, ...]: + """Rehydrate final content and provenance from canonical PostgreSQL rows.""" + chunk_ids = [candidate.chunk_id for candidate, _ in selected] + with self._session_factory() as session: + chunks = { + chunk.chunk_id: chunk + for chunk in session.scalars( + select(Chunk).where(Chunk.chunk_id.in_(chunk_ids)) + ).all() + } + if set(chunks) != set(chunk_ids): + raise CanonicalSourceError("a reranked chunk is missing from canonical storage") + + results: list[SourceResult] = [] + for candidate, rerank_score in selected: + chunk = chunks[candidate.chunk_id] + if chunk.document_id != candidate.document_id: + raise CanonicalSourceError("vector candidate document does not match its chunk") + if chunk.content != candidate.content: + raise CanonicalSourceError("vector candidate content is stale") + results.append( + SourceResult( + document_id=chunk.document_id, + filename=chunk.filename, + page=chunk.page, + section=chunk.section, + chunk_id=chunk.chunk_id, + content=chunk.content, + score=candidate.score, + rerank_score=rerank_score, + ) + ) + return tuple(results) diff --git a/packages/providers/reranker/src/ekp_reranker/__init__.py b/packages/providers/reranker/src/ekp_reranker/__init__.py new file mode 100644 index 0000000..0dbe20c --- /dev/null +++ b/packages/providers/reranker/src/ekp_reranker/__init__.py @@ -0,0 +1,13 @@ +"""Reranker provider abstraction and Demo implementations.""" + +from ekp_reranker.base import RerankerProvider, RerankerProviderError, RerankResult +from ekp_reranker.fake import FakeRerankerProvider +from ekp_reranker.qianfan import QianfanRerankerProvider + +__all__ = [ + "FakeRerankerProvider", + "QianfanRerankerProvider", + "RerankResult", + "RerankerProvider", + "RerankerProviderError", +] diff --git a/packages/providers/reranker/src/ekp_reranker/base.py b/packages/providers/reranker/src/ekp_reranker/base.py new file mode 100644 index 0000000..2f9303a --- /dev/null +++ b/packages/providers/reranker/src/ekp_reranker/base.py @@ -0,0 +1,49 @@ +"""Provider-neutral reranking contract for retrieval candidates.""" + +from collections.abc import Sequence +from dataclasses import dataclass +from math import isfinite +from typing import Protocol + + +class RerankerProviderError(RuntimeError): + """Raised when a reranker rejects or cannot serve a request.""" + + +@dataclass(frozen=True, slots=True) +class RerankResult: + """One ranked candidate reference using its original input position.""" + + candidate_index: int + score: float + + def __post_init__(self) -> None: + """Reject invalid mappings before core search consumes them.""" + if ( + isinstance(self.candidate_index, bool) + or not isinstance(self.candidate_index, int) + or self.candidate_index < 0 + ): + raise ValueError("rerank candidate_index must be a non-negative integer") + if not isfinite(self.score): + raise ValueError("rerank score must be finite") + + +class RerankerProvider(Protocol): + """Rank text candidates without exposing a vendor-specific response.""" + + @property + def model_name(self) -> str: + """Return the stable provider model identifier.""" + + @property + def max_documents(self) -> int: + """Return the largest candidate batch accepted by this provider.""" + + def rerank( + self, + query: str, + documents: Sequence[str], + top_n: int, + ) -> tuple[RerankResult, ...]: + """Return Top-N references in descending relevance order.""" diff --git a/packages/providers/reranker/src/ekp_reranker/fake.py b/packages/providers/reranker/src/ekp_reranker/fake.py new file mode 100644 index 0000000..a001499 --- /dev/null +++ b/packages/providers/reranker/src/ekp_reranker/fake.py @@ -0,0 +1,65 @@ +"""Deterministic lexical reranker for offline tests and evaluation.""" + +from collections.abc import Sequence + +from ekp_reranker.base import RerankerProviderError, RerankResult + +FAKE_MAX_DOCUMENTS = 64 + + +class FakeRerankerProvider: + """Rank by query-character coverage with stable input-index tie-breaking.""" + + def __init__(self, model_name: str = "fake-reranker") -> None: + if not model_name: + raise ValueError("fake reranker model_name must not be empty") + self._model_name = model_name + self.calls: list[tuple[str, tuple[str, ...], int]] = [] + + @property + def model_name(self) -> str: + return self._model_name + + @property + def max_documents(self) -> int: + return FAKE_MAX_DOCUMENTS + + def rerank( + self, + query: str, + documents: Sequence[str], + top_n: int, + ) -> tuple[RerankResult, ...]: + """Produce repeatable relevance scores without model or network access.""" + normalized_query = query.strip() + if not normalized_query: + raise RerankerProviderError("rerank query must not be empty") + if not documents: + raise RerankerProviderError("rerank documents must not be empty") + if len(documents) > self.max_documents: + raise RerankerProviderError( + f"fake reranker accepts at most {self.max_documents} documents" + ) + if any(not document for document in documents): + raise RerankerProviderError("rerank document must not be empty") + if isinstance(top_n, bool) or not 1 <= top_n <= len(documents): + raise RerankerProviderError("rerank top_n must be within the candidate batch") + + captured_documents = tuple(documents) + self.calls.append((normalized_query, captured_documents, top_n)) + query_units = {character.lower() for character in normalized_query if character.isalnum()} + if not query_units: + raise RerankerProviderError("rerank query must contain searchable characters") + ranked = [ + RerankResult( + candidate_index=index, + score=len( + query_units + & {character.lower() for character in document if character.isalnum()} + ) + / len(query_units), + ) + for index, document in enumerate(captured_documents) + ] + ranked.sort(key=lambda result: (-result.score, result.candidate_index)) + return tuple(ranked[:top_n]) diff --git a/packages/providers/reranker/src/ekp_reranker/qianfan.py b/packages/providers/reranker/src/ekp_reranker/qianfan.py new file mode 100644 index 0000000..f2fbcf9 --- /dev/null +++ b/packages/providers/reranker/src/ekp_reranker/qianfan.py @@ -0,0 +1,180 @@ +"""Baidu Qianfan v2 adapter for the Demo Qwen3 reranker baseline.""" + +import json +from collections.abc import Mapping, Sequence +from math import isfinite +from numbers import Real +from typing import Protocol, cast +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from ekp_reranker.base import RerankerProviderError, RerankResult + +QWEN3_RERANKER_MODEL = "qwen3-reranker-0.6b" +QIANFAN_RERANK_ENDPOINT = "https://qianfan.baidubce.com/v2/rerank" +QIANFAN_MAX_DOCUMENTS = 64 +QIANFAN_MAX_QUERY_CHARACTERS = 1600 +QIANFAN_MAX_DOCUMENT_CHARACTERS = 4096 + + +class JsonHttpClient(Protocol): + """Injectable synchronous JSON boundary for the Qianfan rerank API.""" + + def post_json( + self, + url: str, + headers: Mapping[str, str], + payload: Mapping[str, object], + timeout_seconds: float, + ) -> dict[str, object]: + """POST JSON and return a decoded object.""" + + +class UrllibJsonHttpClient: + """Dependency-free client with bounded waits and sanitized failures.""" + + def post_json( + self, + url: str, + headers: Mapping[str, str], + payload: Mapping[str, object], + timeout_seconds: float, + ) -> dict[str, object]: + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + request = Request(url=url, data=body, headers=dict(headers), method="POST") + try: + with urlopen(request, timeout=timeout_seconds) as response: # noqa: S310 + decoded: object = json.loads(response.read()) + except HTTPError as exc: + raise RerankerProviderError( + f"Qianfan rerank request failed with HTTP {exc.code}" + ) from exc + except (URLError, TimeoutError) as exc: + raise RerankerProviderError("Qianfan rerank request could not complete") from exc + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RerankerProviderError("Qianfan reranker returned invalid JSON") from exc + if not isinstance(decoded, dict): + raise RerankerProviderError("Qianfan reranker returned a non-object response") + return cast(dict[str, object], decoded) + + +class QianfanRerankerProvider: + """Call Qianfan rerank and map ranked indexes back to input candidates.""" + + def __init__( + self, + api_key: str, + endpoint: str = QIANFAN_RERANK_ENDPOINT, + model_name: str = QWEN3_RERANKER_MODEL, + timeout_seconds: float = 30.0, + http_client: JsonHttpClient | None = None, + ) -> None: + if not api_key: + raise ValueError("Qianfan API key must not be empty") + if not endpoint.startswith("https://"): + raise ValueError("Qianfan rerank endpoint must use HTTPS") + if not model_name: + raise ValueError("Qianfan reranker model_name must not be empty") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + self._api_key = api_key + self._endpoint = endpoint + self._model_name = model_name + self._timeout_seconds = timeout_seconds + self._http_client = http_client or UrllibJsonHttpClient() + + @property + def model_name(self) -> str: + return self._model_name + + @property + def max_documents(self) -> int: + return QIANFAN_MAX_DOCUMENTS + + def rerank( + self, + query: str, + documents: Sequence[str], + top_n: int, + ) -> tuple[RerankResult, ...]: + """Validate service limits and preserve Qianfan's ranked result order.""" + normalized_query = query.strip() + candidate_documents = tuple(documents) + self._validate_request(normalized_query, candidate_documents, top_n) + response = self._http_client.post_json( + self._endpoint, + { + "Authorization": f"Bearer {self._api_key}", + "Content-Type": "application/json", + }, + { + "model": self.model_name, + "query": normalized_query, + "documents": list(candidate_documents), + "top_n": top_n, + }, + self._timeout_seconds, + ) + raw_results = response.get("results") + if not isinstance(raw_results, list) or len(raw_results) != top_n: + raise RerankerProviderError("Qianfan returned an unexpected rerank result count") + + results = tuple(self._validated_result(item, candidate_documents) for item in raw_results) + indexes = [result.candidate_index for result in results] + if len(indexes) != len(set(indexes)): + raise RerankerProviderError("Qianfan returned duplicate rerank indexes") + if any( + previous.score < current.score + for previous, current in zip(results, results[1:], strict=False) + ): + raise RerankerProviderError("Qianfan rerank results are not score-descending") + return results + + def _validate_request( + self, + query: str, + documents: tuple[str, ...], + top_n: int, + ) -> None: + """Enforce current documented Qianfan request bounds before network I/O.""" + if not query: + raise RerankerProviderError("rerank query must not be empty") + if len(query) > QIANFAN_MAX_QUERY_CHARACTERS: + raise RerankerProviderError( + f"rerank query exceeds {QIANFAN_MAX_QUERY_CHARACTERS} characters" + ) + if not documents: + raise RerankerProviderError("rerank documents must not be empty") + if len(documents) > self.max_documents: + raise RerankerProviderError( + f"Qianfan accepts at most {self.max_documents} rerank documents" + ) + if any(not document for document in documents): + raise RerankerProviderError("rerank document must not be empty") + if any(len(document) > QIANFAN_MAX_DOCUMENT_CHARACTERS for document in documents): + raise RerankerProviderError( + f"rerank document exceeds {QIANFAN_MAX_DOCUMENT_CHARACTERS} characters" + ) + if isinstance(top_n, bool) or not 1 <= top_n <= len(documents): + raise RerankerProviderError("rerank top_n must be within the candidate batch") + + @staticmethod + def _validated_result( + item: object, + documents: tuple[str, ...], + ) -> RerankResult: + """Require a unique in-range index, finite score, and matching echoed text.""" + if not isinstance(item, dict): + raise RerankerProviderError("Qianfan returned an invalid rerank item") + index = item.get("index") + score = item.get("relevance_score") + if isinstance(index, bool) or not isinstance(index, int): + raise RerankerProviderError("Qianfan rerank item has an invalid index") + if not 0 <= index < len(documents): + raise RerankerProviderError("Qianfan rerank index is out of range") + if isinstance(score, bool) or not isinstance(score, Real) or not isfinite(float(score)): + raise RerankerProviderError("Qianfan rerank item has an invalid score") + document = item.get("document") + if document is not None and document != documents[index]: + raise RerankerProviderError("Qianfan rerank item does not match its index") + return RerankResult(candidate_index=index, score=float(score)) diff --git a/pyproject.toml b/pyproject.toml index 45fe24d..d4cfbae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ where = [ "packages/providers/object_storage/src", "packages/providers/parser/src", "packages/providers/embedding/src", + "packages/providers/reranker/src", "packages/providers/vector_store/src", ] @@ -55,6 +56,7 @@ known-first-party = [ "ekp_embedding", "ekp_object_storage", "ekp_parser", + "ekp_reranker", "ekp_vector_store", ] @@ -68,6 +70,7 @@ mypy_path = [ "packages/providers/object_storage/src", "packages/providers/parser/src", "packages/providers/embedding/src", + "packages/providers/reranker/src", "packages/providers/vector_store/src", ] diff --git a/tests/evaluation/test_reranked_search_regression.py b/tests/evaluation/test_reranked_search_regression.py new file mode 100644 index 0000000..2cfcecc --- /dev/null +++ b/tests/evaluation/test_reranked_search_regression.py @@ -0,0 +1,86 @@ +"""Golden Dataset regression for reranking and canonical source results.""" + +import json +from collections.abc import Sequence +from io import BytesIO +from pathlib import Path + +from sqlalchemy import create_engine + +from ekp_core.chunking import StructureAwareChunker +from ekp_core.database import create_session_factory +from ekp_core.dense_retrieval import DenseRetrievalService +from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService +from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService +from ekp_core.document_parsing import DocumentParsingService +from ekp_core.document_upload import DocumentUploadService +from ekp_core.search import SearchService +from ekp_domain import Base, IndexProfile +from ekp_object_storage import LocalObjectStorage +from ekp_parser import ParserRouter +from ekp_reranker import FakeRerankerProvider +from ekp_vector_store import FakeVectorStore + + +class GoldenKeywordEmbeddingProvider: + """Small semantic fixture matching the Dense regression without external calls.""" + + model_name = "golden-keyword-v1" + dimensions = 2 + max_batch_size = 16 + + def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]: + vectors: list[tuple[float, ...]] = [] + for text in texts: + if "住宿" in text or "出差" in text: + vectors.append((1.0, 0.0)) + elif "退款" in text or "产品A" in text or "产品 A" in text: + vectors.append((0.0, 1.0)) + else: + vectors.append((0.1, 0.1)) + return tuple(vectors) + + +def test_reranker_promotes_expected_canonical_source_to_top_one(tmp_path: Path) -> None: + """Run the complete Demo pipeline through Dense Top20, Rerank, and Source Result.""" + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + storage = LocalObjectStorage(tmp_path / "storage") + upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024) + parsing = DocumentParsingService(session_factory, storage, ParserRouter()) + chunking = DocumentChunkingService( + session_factory, + storage, + StructureAwareChunker(), + ) + embedding = GoldenKeywordEmbeddingProvider() + vector_store = FakeVectorStore() + profile = IndexProfile("golden-rerank-v1", "structure_v1", embedding.model_name, 2) + indexing = DocumentIndexingService(session_factory, embedding, vector_store, profile) + pipeline = DocumentIndexingPipelineService( + DocumentPipelineService(upload, parsing, chunking), + indexing, + ) + dense = DenseRetrievalService(embedding, vector_store, profile) + search = SearchService(session_factory, dense, FakeRerankerProvider()) + golden_records = [ + json.loads(line) + for line in Path("fixtures/evaluation/golden.jsonl").read_text().splitlines() + if line + ] + + filenames = {filename for record in golden_records for filename in record["expected_documents"]} + for filename in sorted(filenames): + payload = (Path("fixtures/documents") / filename).read_bytes() + pipeline.ingest(filename, "text/markdown", BytesIO(payload)) + + for record in golden_records: + result = search.search(record["query"]) + assert result.results + best = result.results[0] + assert best.filename in record["expected_documents"] + assert best.section is not None + assert any(section in best.section for section in record["expected_sections"]) + assert best.content + assert best.rerank_score >= 0 diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 43c62f5..9c7957e 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -32,6 +32,11 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non assert settings.mochow_timeout_seconds == 30 assert settings.index_profile_id == "demo-bge-large-zh-structure-v1" assert settings.dense_top_n == 20 + assert settings.reranker_provider == "qianfan" + assert settings.reranker_model == "qwen3-reranker-0.6b" + assert settings.reranker_endpoint.endswith("/v2/rerank") + assert settings.reranker_api_key is None + assert settings.search_top_k == 5 def test_settings_reject_invalid_port(isolated_settings_environment: None) -> None: @@ -46,12 +51,21 @@ def test_settings_reject_chunk_overlap_without_forward_progress( Settings(chunk_max_tokens=100, chunk_overlap_tokens=100) +def test_settings_reject_final_top_k_larger_than_dense_candidates( + isolated_settings_environment: None, +) -> None: + with pytest.raises(ValidationError, match="search_top_k"): + Settings(dense_top_n=5, search_top_k=6) + + def test_settings_redact_provider_secrets(isolated_settings_environment: None) -> None: settings = Settings( baidu_qianfan_api_key=SecretStr("qianfan-secret"), baidu_mochow_api_key=SecretStr("mochow-secret"), + reranker_api_key=SecretStr("reranker-secret"), ) rendered = repr(settings) assert "qianfan-secret" not in rendered assert "mochow-secret" not in rendered + assert "reranker-secret" not in rendered diff --git a/tests/unit/test_provider_wiring.py b/tests/unit/test_provider_wiring.py index eab79a7..81af822 100644 --- a/tests/unit/test_provider_wiring.py +++ b/tests/unit/test_provider_wiring.py @@ -8,6 +8,7 @@ from ekp_api.providers import ( create_dense_retrieval_service, create_embedding_provider, create_index_profile, + create_reranker_provider, create_vector_store, ) from ekp_core.config import Settings @@ -56,3 +57,14 @@ def test_provider_wiring_builds_dense_retrieval_without_network( ) assert service.search("测试查询").candidates == () + + +def test_provider_wiring_reuses_qianfan_key_for_demo_reranker( + isolated_settings_environment: None, +) -> None: + settings = Settings(baidu_qianfan_api_key=SecretStr("shared-test-key")) + + reranker = create_reranker_provider(settings) + + assert reranker.model_name == "qwen3-reranker-0.6b" + assert reranker.max_documents == 64 diff --git a/tests/unit/test_qianfan_reranker.py b/tests/unit/test_qianfan_reranker.py new file mode 100644 index 0000000..ae45a6b --- /dev/null +++ b/tests/unit/test_qianfan_reranker.py @@ -0,0 +1,112 @@ +"""Unit tests for the Qianfan Qwen3 reranker request/response contract.""" + +from collections.abc import Mapping + +import pytest + +from ekp_reranker import QianfanRerankerProvider, RerankerProviderError + + +class FakeJsonHttpClient: + """Capture rerank JSON requests without network access.""" + + def __init__(self, response: dict[str, object]) -> None: + self.response = response + self.calls: list[tuple[str, Mapping[str, str], Mapping[str, object], float]] = [] + + def post_json( + self, + url: str, + headers: Mapping[str, str], + payload: Mapping[str, object], + timeout_seconds: float, + ) -> dict[str, object]: + self.calls.append((url, headers, payload, timeout_seconds)) + return self.response + + +def test_qianfan_reranker_sends_top_n_and_preserves_ranked_indexes() -> None: + client = FakeJsonHttpClient( + { + "results": [ + {"document": "退款周期为七天", "relevance_score": 0.93, "index": 1}, + {"document": "住宿标准说明", "relevance_score": 0.21, "index": 0}, + ] + } + ) + provider = QianfanRerankerProvider( + api_key="test-key", + endpoint="https://qianfan.example/v2/rerank", + timeout_seconds=12, + http_client=client, + ) + + results = provider.rerank(" 产品A退款周期 ", ("住宿标准说明", "退款周期为七天"), 2) + + assert provider.model_name == "qwen3-reranker-0.6b" + assert provider.max_documents == 64 + assert [result.candidate_index for result in results] == [1, 0] + url, headers, payload, timeout = client.calls[0] + assert url.endswith("/v2/rerank") + assert headers["Authorization"] == "Bearer test-key" + assert payload == { + "model": "qwen3-reranker-0.6b", + "query": "产品A退款周期", + "documents": ["住宿标准说明", "退款周期为七天"], + "top_n": 2, + } + assert timeout == 12 + + +@pytest.mark.parametrize( + "response", + [ + {"results": []}, + { + "results": [ + {"relevance_score": 0.9, "index": 0}, + {"relevance_score": 0.8, "index": 0}, + ] + }, + { + "results": [ + {"relevance_score": 0.3, "index": 0}, + {"relevance_score": 0.8, "index": 1}, + ] + }, + { + "results": [ + {"document": "错误内容", "relevance_score": 0.9, "index": 0}, + {"relevance_score": 0.8, "index": 1}, + ] + }, + { + "results": [ + {"relevance_score": float("nan"), "index": 0}, + {"relevance_score": 0.8, "index": 1}, + ] + }, + ], +) +def test_qianfan_reranker_rejects_invalid_responses(response: dict[str, object]) -> None: + provider = QianfanRerankerProvider( + api_key="test-key", + http_client=FakeJsonHttpClient(response), + ) + + with pytest.raises(RerankerProviderError): + provider.rerank("查询", ("文档一", "文档二"), 2) + + +def test_qianfan_reranker_enforces_document_and_character_limits() -> None: + provider = QianfanRerankerProvider( + api_key="test-key", + http_client=FakeJsonHttpClient({"results": []}), + ) + + with pytest.raises(RerankerProviderError, match="1600"): + provider.rerank("问" * 1601, ("内容",), 1) + with pytest.raises(RerankerProviderError, match="64"): + provider.rerank("查询", ("内容",) * 65, 5) + with pytest.raises(RerankerProviderError, match="4096"): + provider.rerank("查询", ("内" * 4097,), 1) diff --git a/tests/unit/test_reranker.py b/tests/unit/test_reranker.py new file mode 100644 index 0000000..efa898c --- /dev/null +++ b/tests/unit/test_reranker.py @@ -0,0 +1,41 @@ +"""Unit tests for provider-neutral reranking behavior.""" + +import pytest + +from ekp_reranker import FakeRerankerProvider, RerankerProviderError, RerankResult + + +def test_fake_reranker_orders_by_query_coverage_and_keeps_source_index() -> None: + provider = FakeRerankerProvider() + + results = provider.rerank( + "员工出差住宿标准", + ("提交报销材料", "员工出差住宿标准说明", "其他制度"), + top_n=2, + ) + + assert [result.candidate_index for result in results] == [1, 0] + assert results[0].score > results[1].score + assert provider.calls == [ + ( + "员工出差住宿标准", + ("提交报销材料", "员工出差住宿标准说明", "其他制度"), + 2, + ) + ] + + +def test_fake_reranker_rejects_invalid_batch_and_top_n() -> None: + provider = FakeRerankerProvider() + + with pytest.raises(RerankerProviderError, match="documents"): + provider.rerank("查询", (), 1) + with pytest.raises(RerankerProviderError, match="top_n"): + provider.rerank("查询", ("内容",), 2) + + +def test_rerank_result_rejects_invalid_index_and_score() -> None: + with pytest.raises(ValueError, match="candidate_index"): + RerankResult(candidate_index=True, score=0.5) + with pytest.raises(ValueError, match="finite"): + RerankResult(candidate_index=0, score=float("nan")) diff --git a/tests/unit/test_search.py b/tests/unit/test_search.py new file mode 100644 index 0000000..34f7da8 --- /dev/null +++ b/tests/unit/test_search.py @@ -0,0 +1,167 @@ +"""Unit tests for reranked search and canonical source enrichment.""" + +from collections.abc import Sequence +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker + +from ekp_core.database import create_session_factory +from ekp_core.dense_retrieval import DenseRetrievalResult +from ekp_core.search import CanonicalSourceError, RerankerContractError, SearchService +from ekp_domain import Base, Chunk, Document, DocumentStatus +from ekp_reranker import FakeRerankerProvider, RerankResult +from ekp_vector_store import DenseSearchCandidate + + +class StubDenseRetriever: + """Return prebuilt candidates while recording the query.""" + + def __init__(self, candidates: tuple[DenseSearchCandidate, ...]) -> None: + self.candidates = candidates + self.queries: list[str] = [] + + def search(self, query: str, top_n: int | None = None) -> DenseRetrievalResult: + self.queries.append(query) + return DenseRetrievalResult(query.strip(), "profile-v1", self.candidates) + + +def _candidate( + chunk_id: UUID, + document_id: UUID, + content: str, + *, + dense_score: float, +) -> DenseSearchCandidate: + return DenseSearchCandidate( + index_record_id=uuid4(), + chunk_id=chunk_id, + document_id=document_id, + filename="stale-index-name.md", + page=None, + page_end=None, + section="stale-index-section", + chunk_index=0, + content=content, + score=dense_score, + index_profile_id="profile-v1", + ) + + +def _canonical_source_session() -> tuple[sessionmaker[Session], UUID, UUID, UUID]: + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + document_id = uuid4() + first_chunk_id = uuid4() + second_chunk_id = uuid4() + with session_factory() as session: + session.add( + Document( + id=document_id, + filename="policy.md", + media_type="text/markdown", + size_bytes=100, + storage_key=f"documents/{document_id}/policy.md", + status=DocumentStatus.READY, + ) + ) + session.add_all( + [ + Chunk( + chunk_id=first_chunk_id, + document_id=document_id, + filename="policy.md", + page=1, + page_end=1, + section="报销材料", + section_path=["报销材料"], + chunk_index=0, + content="提交报销材料", + token_count=6, + strategy_version="structure_v1", + ), + Chunk( + chunk_id=second_chunk_id, + document_id=document_id, + filename="policy.md", + page=2, + page_end=2, + section="住宿标准", + section_path=["住宿标准"], + chunk_index=1, + content="员工出差住宿标准说明", + token_count=9, + strategy_version="structure_v1", + ), + ] + ) + session.commit() + return session_factory, document_id, first_chunk_id, second_chunk_id + + +def test_search_reranks_and_rehydrates_source_from_canonical_chunk() -> None: + session_factory, document_id, first_chunk_id, second_chunk_id = _canonical_source_session() + dense = StubDenseRetriever( + ( + _candidate(first_chunk_id, document_id, "提交报销材料", dense_score=0.91), + _candidate(second_chunk_id, document_id, "员工出差住宿标准说明", dense_score=0.72), + ) + ) + reranker = FakeRerankerProvider() + service = SearchService(session_factory, dense, reranker, default_top_k=1) + + result = service.search("员工出差住宿标准") + + assert len(result.results) == 1 + source = result.results[0] + assert source.chunk_id == second_chunk_id + assert source.filename == "policy.md" + assert source.page == 2 + assert source.section == "住宿标准" + assert source.content == "员工出差住宿标准说明" + assert source.score == 0.72 + assert source.rerank_score == 1.0 + + +def test_search_rejects_stale_vector_content() -> None: + session_factory, document_id, first_chunk_id, _ = _canonical_source_session() + dense = StubDenseRetriever( + (_candidate(first_chunk_id, document_id, "索引中的旧内容", dense_score=0.8),) + ) + service = SearchService( + session_factory, + dense, + FakeRerankerProvider(), + ) + + with pytest.raises(CanonicalSourceError, match="stale"): + service.search("旧内容", top_k=1) + + +class DuplicateReranker: + model_name = "bad" + max_documents = 64 + + def rerank( + self, + query: str, + documents: Sequence[str], + top_n: int, + ) -> tuple[RerankResult, ...]: + return (RerankResult(0, 0.9), RerankResult(0, 0.8)) + + +def test_search_rejects_duplicate_reranker_indexes() -> None: + session_factory, document_id, first_chunk_id, second_chunk_id = _canonical_source_session() + dense = StubDenseRetriever( + ( + _candidate(first_chunk_id, document_id, "提交报销材料", dense_score=0.9), + _candidate(second_chunk_id, document_id, "员工出差住宿标准说明", dense_score=0.8), + ) + ) + service = SearchService(session_factory, dense, DuplicateReranker(), default_top_k=2) + + with pytest.raises(RerankerContractError, match="duplicate"): + service.search("住宿标准")