merge: ARCH-003 OceanBase vector adapter
This commit is contained in:
commit
92d098cc25
@ -31,6 +31,14 @@ MOCHOW_TABLE=knowledge_chunks
|
||||
MOCHOW_BATCH_SIZE=100
|
||||
MOCHOW_SEARCH_EF=200
|
||||
MOCHOW_TIMEOUT_SECONDS=30
|
||||
OCEANBASE_VECTOR_URI=
|
||||
OCEANBASE_VECTOR_USER=
|
||||
OCEANBASE_VECTOR_PASSWORD=
|
||||
OCEANBASE_VECTOR_DATABASE=
|
||||
OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1
|
||||
OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding
|
||||
OCEANBASE_VECTOR_BATCH_SIZE=100
|
||||
OCEANBASE_VECTOR_TIMEOUT_SECONDS=30
|
||||
|
||||
EMBEDDING_PROVIDER=qianfan
|
||||
EMBEDDING_MODEL=bge-large-zh
|
||||
|
||||
@ -97,6 +97,9 @@ demo providers without changing PostgreSQL canonical ownership. Until the runtim
|
||||
cutover task is merged, the paragraphs above describe the current implementation.
|
||||
ARCH-002 adds the selectable BOS adapter and safe runtime configuration documented
|
||||
in `docs/development/BOS_SETUP.md`; local storage remains the default until ARCH-005.
|
||||
ARCH-003 adds the selectable OceanBase Vector adapter and pre-provisioned cosine
|
||||
HNSW schema documented in `docs/development/OCEANBASE_VECTOR_SETUP.md`; Mochow
|
||||
remains the runtime default until ARCH-005.
|
||||
|
||||
DEMO-007 adds query embedding and profile-scoped Dense Top-N retrieval. The
|
||||
default is Top 20 with a configurable Mochow HNSW `ef`; the in-memory fake uses
|
||||
|
||||
@ -23,7 +23,11 @@ from ekp_object_storage import ObjectStorage, create_bos_object_storage
|
||||
from ekp_object_storage.local import LocalObjectStorage
|
||||
from ekp_parser import ParserRouter
|
||||
from ekp_reranker import QianfanRerankerProvider, RerankerProvider
|
||||
from ekp_vector_store import MochowVectorStore, VectorStore, create_mochow_vector_store
|
||||
from ekp_vector_store import (
|
||||
VectorStore,
|
||||
create_mochow_vector_store,
|
||||
create_oceanbase_vector_store,
|
||||
)
|
||||
|
||||
BGE_LARGE_ZH_DIMENSIONS = 1024
|
||||
DEMO_CHUNK_STRATEGY_VERSION = "structure_v1"
|
||||
@ -85,8 +89,29 @@ def create_object_storage(settings: Settings) -> ObjectStorage:
|
||||
)
|
||||
|
||||
|
||||
def create_vector_store(settings: Settings) -> MochowVectorStore:
|
||||
"""Create the Mochow adapter for an explicitly configured database/table."""
|
||||
def create_vector_store(settings: Settings) -> VectorStore:
|
||||
"""Create the selected vector adapter from explicit, secret-safe settings."""
|
||||
if settings.vector_store_provider == "oceanbase":
|
||||
if settings.oceanbase_vector_uri is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_URI is required")
|
||||
if settings.oceanbase_vector_user is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_USER is required")
|
||||
if settings.oceanbase_vector_password is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_PASSWORD is required")
|
||||
if settings.oceanbase_vector_database is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_DATABASE is required")
|
||||
return create_oceanbase_vector_store(
|
||||
uri=settings.oceanbase_vector_uri,
|
||||
user=settings.oceanbase_vector_user,
|
||||
password=settings.oceanbase_vector_password.get_secret_value(),
|
||||
database_name=settings.oceanbase_vector_database,
|
||||
table_name=settings.oceanbase_vector_table,
|
||||
index_name=settings.oceanbase_vector_index,
|
||||
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
|
||||
batch_size=settings.oceanbase_vector_batch_size,
|
||||
timeout_seconds=settings.oceanbase_vector_timeout_seconds,
|
||||
)
|
||||
|
||||
if settings.baidu_mochow_api_key is None:
|
||||
raise ProviderConfigurationError("BAIDU_MOCHOW_API_KEY is required")
|
||||
if not settings.baidu_mochow_endpoint:
|
||||
|
||||
30
deploy/oceanbase/001_knowledge_chunks_v1.sql
Normal file
30
deploy/oceanbase/001_knowledge_chunks_v1.sql
Normal file
@ -0,0 +1,30 @@
|
||||
-- ARCH-003 rebuildable OceanBase Vector index for bge-large-zh (1024 dimensions).
|
||||
-- PostgreSQL remains canonical; this table may be dropped and rebuilt from canonical chunks.
|
||||
-- Requires OceanBase 4.6.0 or newer for explicit ANN scalar pre-filtering.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS knowledge_chunks_v1 (
|
||||
index_record_id VARCHAR(36) NOT NULL,
|
||||
chunk_id VARCHAR(36) NOT NULL,
|
||||
document_id VARCHAR(36) NOT NULL,
|
||||
filename VARCHAR(255) NOT NULL,
|
||||
page INT NULL,
|
||||
page_end INT NULL,
|
||||
section VARCHAR(1024) NULL,
|
||||
chunk_index INT NOT NULL,
|
||||
content MEDIUMTEXT NOT NULL,
|
||||
embedding VECTOR(1024) NOT NULL,
|
||||
embedding_model VARCHAR(128) NOT NULL,
|
||||
index_profile_id VARCHAR(128) NOT NULL,
|
||||
PRIMARY KEY (index_record_id),
|
||||
KEY idx_knowledge_chunks_document_profile (document_id, index_profile_id),
|
||||
KEY idx_knowledge_chunks_profile (index_profile_id),
|
||||
VECTOR KEY vidx_knowledge_chunks_embedding (embedding)
|
||||
WITH (
|
||||
DISTANCE = COSINE,
|
||||
TYPE = HNSW,
|
||||
LIB = VSAG,
|
||||
M = 16,
|
||||
EF_CONSTRUCTION = 256,
|
||||
EF_SEARCH = 200
|
||||
) BLOCK_SIZE 16384
|
||||
) ORGANIZATION = HEAP;
|
||||
@ -364,7 +364,8 @@ DEMO-002 Document 数据模型 + 本地文件存储
|
||||
DEMO-003 PDF/DOCX/TXT/MD 上传
|
||||
DEMO-004 Parser + Cleaner
|
||||
DEMO-005 Chunk Pipeline
|
||||
DEMO-006 bge-large-zh Embedding + 初始 Mochow 入库(将由 ARCH-003 迁移)
|
||||
DEMO-006 bge-large-zh Embedding + 初始 Mochow 入库(ARCH-003 已提供 OceanBase Adapter,
|
||||
ARCH-005 切换运行时)
|
||||
DEMO-007 Dense Retrieval
|
||||
DEMO-008 Reranker + Source Result
|
||||
DEMO-009 Document/Search REST API
|
||||
|
||||
72
docs/development/OCEANBASE_VECTOR_SETUP.md
Normal file
72
docs/development/OCEANBASE_VECTOR_SETUP.md
Normal file
@ -0,0 +1,72 @@
|
||||
# ARCH-003 OceanBase Vector 配置与索引契约
|
||||
|
||||
ARCH-003 在既有 `VectorStore` 契约后新增 OceanBase Vector Adapter。PostgreSQL 仍保存
|
||||
canonical Document、Chunk、Index Profile 和 Chunk Index State;OceanBase 表只保存可由这些
|
||||
数据重建的检索行。运行时默认切换和 Mochow 配置清理留在 ARCH-005。
|
||||
|
||||
## 数据库前提
|
||||
|
||||
- OceanBase MySQL 租户版本至少为 4.6.0:1024 维 Cosine HNSW 本身可由更早版本支持,
|
||||
但本项目要求使用 `VECTOR_INDEX(... pre_filter)` 显式保证 Profile 在 ANN 前过滤;
|
||||
- 目标数据库和最小权限账号由受控部署流程预创建;
|
||||
- Vector 内存、资源规格和 HNSW 参数由 DBA 在受控环境确认;
|
||||
- 应用启动和请求路径不会创建、修改或删除表结构。
|
||||
|
||||
使用 [001_knowledge_chunks_v1.sql](../../deploy/oceanbase/001_knowledge_chunks_v1.sql)
|
||||
在专用数据库中创建索引表。脚本固定:
|
||||
|
||||
```text
|
||||
VECTOR(1024)
|
||||
DISTANCE = COSINE
|
||||
TYPE = HNSW
|
||||
EF_CONSTRUCTION = 256
|
||||
EF_SEARCH = 200
|
||||
```
|
||||
|
||||
表的 `index_record_id` 是确定性的 Chunk/Profile 主键;`document_id + index_profile_id` 和
|
||||
`index_profile_id` 是清理与 Profile 过滤的标量索引。表名或向量维度变化必须建立新表或
|
||||
Shadow Profile,验证后再激活,禁止在生产索引上原地修改。
|
||||
|
||||
## 运行配置
|
||||
|
||||
配置只能由部署环境或 Secret Manager 注入,禁止把真实地址、用户名和密码写入 `.env`、
|
||||
镜像、日志、测试快照或提交记录。
|
||||
|
||||
```text
|
||||
VECTOR_STORE_PROVIDER=oceanbase
|
||||
OCEANBASE_VECTOR_URI=<host:port>
|
||||
OCEANBASE_VECTOR_USER=<运行时注入>
|
||||
OCEANBASE_VECTOR_PASSWORD=<运行时注入>
|
||||
OCEANBASE_VECTOR_DATABASE=<预创建数据库>
|
||||
OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1
|
||||
OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding
|
||||
OCEANBASE_VECTOR_BATCH_SIZE=100
|
||||
OCEANBASE_VECTOR_TIMEOUT_SECONDS=30
|
||||
```
|
||||
|
||||
- URI 只接受 `host:port`,不得包含 Scheme、用户信息、Path、Query 或 Fragment;
|
||||
- Database/Table/Index 只接受安全标识符;
|
||||
- Password 使用 `SecretStr`,Provider 异常只返回收敛后的错误,不回显 SQL、连接信息、
|
||||
Chunk 内容或 SDK Response;
|
||||
- `pyobvector` 只存在于 Adapter/Factory 内,业务服务不依赖其 Row、Engine 或距离函数类型。
|
||||
|
||||
## 写入、删除和检索语义
|
||||
|
||||
- Upsert 使用 `index_record_id` 主键,按可配置批次写入;
|
||||
- 删除使用参数化的 `document_id + index_profile_id` 条件,不拼接用户输入;
|
||||
- Dense Search 在同一 ANN 查询中使用 `VECTOR_INDEX(... pre_filter)`,保证
|
||||
`index_profile_id` 在候选近邻搜索前过滤,并返回 Dense Top20;禁止依赖普通 `WHERE`
|
||||
的自适应过滤语义,也禁止全局 TopK 后再过滤;
|
||||
- OceanBase `cosine_distance` 是低值优先距离,Adapter 统一转换为
|
||||
`score = 1 - cosine_distance`,保持 `VectorStore` 的高分优先契约;
|
||||
- 返回 UUID、Profile、字段数、距离范围和排序不满足契约时失败关闭,不生成 Source Result。
|
||||
|
||||
默认测试使用 Fake Client,不连接 OceanBase。真实版本、DDL、权限、Cosine HNSW 召回、过滤
|
||||
行为与 30~50 条脱敏 Golden Query 统一在 ARCH-006 的受控资源中验证,并且只允许使用轮换
|
||||
后的凭据。
|
||||
|
||||
官方参考:
|
||||
|
||||
- [pyobvector 官方仓库](https://github.com/oceanbase/pyobvector)
|
||||
- [OceanBase 创建向量索引](https://en.oceanbase.com/docs/common-oceanbase-database-10000000003075357)
|
||||
- [OceanBase 向量索引查询 Hint](https://en.oceanbase.com/docs/common-oceanbase-database-10000000004311163)
|
||||
@ -2,11 +2,11 @@
|
||||
|
||||
当前阶段:Provider 架构迁移
|
||||
|
||||
已完成:DEMO-001~DEMO-009、ARCH-001、ARCH-002
|
||||
已完成:DEMO-001~DEMO-009、ARCH-001、ARCH-002、ARCH-003
|
||||
进行中:无
|
||||
阻塞:无
|
||||
|
||||
下一任务:ARCH-003
|
||||
下一任务:ARCH-004
|
||||
|
||||
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
|
||||
执行;每个 Task 验证后独立推送并合并 `main`。全部迁移完成后再继续 DEMO-010。
|
||||
@ -154,3 +154,17 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
|
||||
Prefix 和超时均由环境注入。Local 仍是测试/开发及 ARCH-005 前的默认实现。
|
||||
- 新增完全离线的 BOS Adapter、配置、Provider wiring 测试和中文配置说明;未使用内部配置
|
||||
清单中的真实地址或凭据,真实 BOS 联调留在 ARCH-006。
|
||||
|
||||
### ARCH-003
|
||||
|
||||
- 新增官方 `pyobvector` 驱动的 `OceanBaseVectorStore`,覆盖批量 Upsert、按 Document/Profile
|
||||
参数化删除和 Profile 条件下推的 Cosine ANN TopN;SDK 类型不进入业务服务。检索固定使用
|
||||
OceanBase 4.6 的 `VECTOR_INDEX(... pre_filter)`,不依赖普通 `WHERE` 的自适应过滤,也不做
|
||||
全局 TopK 后过滤。
|
||||
- 将低值优先的 `cosine_distance` 转为 `score = 1 - distance`,并对向量维度、有限值、
|
||||
非零范数、返回字段、UUID、Profile、距离范围和排序执行失败关闭校验。
|
||||
- 新增 `VECTOR(1024)`、Cosine HNSW、Profile/Document 标量索引的预置 DDL;应用不自动执行
|
||||
外部索引 DDL,PostgreSQL 继续保存 canonical 数据,OceanBase 表可重建。
|
||||
- 新增 `mochow|oceanbase` 类型化配置、SecretStr 密码和运行时工厂;Mochow 保持 ARCH-005
|
||||
前的默认实现。全部 Adapter、配置、Schema 和 wiring 测试均离线运行,真实数据库验收留在
|
||||
ARCH-006。
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
|
||||
- [x] ARCH-001 Architecture decision, PRD, Scope and execution baseline
|
||||
- [x] ARCH-002 BOS ObjectStorage adapter and runtime configuration
|
||||
- [ ] ARCH-003 OceanBase VectorStore adapter and index schema
|
||||
- [x] ARCH-003 OceanBase VectorStore adapter and index schema
|
||||
- [ ] ARCH-004 Qianfan `bce-reranker-base` alignment
|
||||
- [ ] ARCH-005 Runtime cutover, shadow rebuild workflow and legacy configuration cleanup
|
||||
- [ ] ARCH-006 Controlled live integration and RAG acceptance
|
||||
|
||||
@ -42,7 +42,7 @@ class Settings(BaseSettings):
|
||||
embedding_endpoint: str = "https://qianfan.baidubce.com/v2/embeddings"
|
||||
baidu_qianfan_api_key: SecretStr | None = None
|
||||
embedding_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
vector_store_provider: Literal["mochow"] = "mochow"
|
||||
vector_store_provider: Literal["mochow", "oceanbase"] = "mochow"
|
||||
baidu_mochow_account: str = "root"
|
||||
baidu_mochow_api_key: SecretStr | None = None
|
||||
baidu_mochow_endpoint: str | None = None
|
||||
@ -51,6 +51,14 @@ class Settings(BaseSettings):
|
||||
mochow_batch_size: int = Field(default=100, ge=1, le=1000)
|
||||
mochow_search_ef: int = Field(default=200, ge=1, le=10000)
|
||||
mochow_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
oceanbase_vector_uri: str | None = None
|
||||
oceanbase_vector_user: str | None = None
|
||||
oceanbase_vector_password: SecretStr | None = None
|
||||
oceanbase_vector_database: str | None = None
|
||||
oceanbase_vector_table: str = "knowledge_chunks_v1"
|
||||
oceanbase_vector_index: str = "vidx_knowledge_chunks_embedding"
|
||||
oceanbase_vector_batch_size: int = Field(default=100, ge=1, le=1000)
|
||||
oceanbase_vector_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
index_profile_id: str = "demo-bge-large-zh-structure-v1"
|
||||
dense_top_n: int = Field(default=20, ge=1, le=64)
|
||||
reranker_provider: Literal["qianfan"] = "qianfan"
|
||||
|
||||
@ -68,7 +68,7 @@ class DocumentIndexingService:
|
||||
self._index_profile = index_profile
|
||||
|
||||
def process(self, document_id: UUID) -> DocumentIndexingResult:
|
||||
"""Index all chunks for one profile, compensating any partial Mochow write."""
|
||||
"""Index all chunks for one profile, compensating any partial provider write."""
|
||||
chunks = self._start_indexing(document_id)
|
||||
upsert_attempted = False
|
||||
try:
|
||||
|
||||
@ -12,6 +12,12 @@ from ekp_vector_store.mochow import (
|
||||
MochowVectorStoreError,
|
||||
create_mochow_vector_store,
|
||||
)
|
||||
from ekp_vector_store.oceanbase import (
|
||||
OceanBaseConfigurationError,
|
||||
OceanBaseVectorStore,
|
||||
OceanBaseVectorStoreError,
|
||||
create_oceanbase_vector_store,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"FakeVectorStore",
|
||||
@ -19,7 +25,11 @@ __all__ = [
|
||||
"DenseSearchRequest",
|
||||
"MochowVectorStore",
|
||||
"MochowVectorStoreError",
|
||||
"OceanBaseConfigurationError",
|
||||
"OceanBaseVectorStore",
|
||||
"OceanBaseVectorStoreError",
|
||||
"VectorRecord",
|
||||
"VectorStore",
|
||||
"create_mochow_vector_store",
|
||||
"create_oceanbase_vector_store",
|
||||
]
|
||||
|
||||
@ -0,0 +1,403 @@
|
||||
"""OceanBase Vector adapter isolated behind the provider-neutral VectorStore contract."""
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from math import isfinite
|
||||
from numbers import Real
|
||||
from typing import Any, Protocol
|
||||
from urllib.parse import urlsplit
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import and_, column, text
|
||||
|
||||
from ekp_vector_store.base import DenseSearchCandidate, DenseSearchRequest, VectorRecord
|
||||
|
||||
OCEANBASE_IDENTIFIER = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,63}$")
|
||||
PROFILE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$")
|
||||
SEARCH_OUTPUT_FIELDS = [
|
||||
"index_record_id",
|
||||
"chunk_id",
|
||||
"document_id",
|
||||
"filename",
|
||||
"page",
|
||||
"page_end",
|
||||
"section",
|
||||
"chunk_index",
|
||||
"content",
|
||||
"index_profile_id",
|
||||
]
|
||||
|
||||
|
||||
class OceanBaseConfigurationError(ValueError):
|
||||
"""Raised when OceanBase connection or index settings are invalid."""
|
||||
|
||||
|
||||
class OceanBaseVectorStoreError(RuntimeError):
|
||||
"""Sanitized OceanBase failure without SQL, endpoints, content, or credentials."""
|
||||
|
||||
|
||||
class _OceanBaseVectorClient(Protocol):
|
||||
"""Narrow pyobvector client surface used by the EKP adapter."""
|
||||
|
||||
def upsert(self, table_name: str, data: list[dict[str, object]]) -> object:
|
||||
"""Insert or replace a row batch by primary key."""
|
||||
|
||||
def delete(
|
||||
self,
|
||||
table_name: str,
|
||||
*,
|
||||
where_clause: list[object],
|
||||
) -> object:
|
||||
"""Delete rows matching bound SQLAlchemy predicates."""
|
||||
|
||||
def dense_search(
|
||||
self,
|
||||
table_name: str,
|
||||
*,
|
||||
index_name: str,
|
||||
query_vector: tuple[float, ...],
|
||||
index_profile_id: str,
|
||||
top_n: int,
|
||||
output_column_names: list[str],
|
||||
) -> Iterable[object]:
|
||||
"""Return explicitly pre-filtered ANN rows followed by their distance."""
|
||||
|
||||
|
||||
class _PyObVectorClient:
|
||||
"""Keep SDK operations and the required pre-filter SQL inside the adapter."""
|
||||
|
||||
def __init__(self, sdk_client: Any) -> None:
|
||||
self._sdk_client = sdk_client
|
||||
|
||||
def upsert(self, table_name: str, data: list[dict[str, object]]) -> object:
|
||||
"""Delegate bounded row writes to the official client."""
|
||||
result: object = self._sdk_client.upsert(table_name, data)
|
||||
return result
|
||||
|
||||
def delete(
|
||||
self,
|
||||
table_name: str,
|
||||
*,
|
||||
where_clause: list[object],
|
||||
) -> object:
|
||||
"""Delegate parameterized cleanup to the official client."""
|
||||
result: object = self._sdk_client.delete(table_name, where_clause=where_clause)
|
||||
return result
|
||||
|
||||
def dense_search(
|
||||
self,
|
||||
table_name: str,
|
||||
*,
|
||||
index_name: str,
|
||||
query_vector: tuple[float, ...],
|
||||
index_profile_id: str,
|
||||
top_n: int,
|
||||
output_column_names: list[str],
|
||||
) -> Iterable[object]:
|
||||
"""Execute cosine ANN with an explicit OceanBase pre-filter hint."""
|
||||
statement = text(
|
||||
_build_dense_search_sql(
|
||||
table_name=table_name,
|
||||
index_name=index_name,
|
||||
output_column_names=output_column_names,
|
||||
)
|
||||
)
|
||||
parameters = {
|
||||
"query_vector": _serialize_vector(query_vector),
|
||||
"index_profile_id": index_profile_id,
|
||||
"top_n": top_n,
|
||||
}
|
||||
# Materialize rows before closing the connection; no SDK Result leaks upward.
|
||||
with self._sdk_client.engine.connect() as connection:
|
||||
rows: list[object] = list(connection.execute(statement, parameters))
|
||||
return rows
|
||||
|
||||
|
||||
class OceanBaseVectorStore:
|
||||
"""Translate neutral vector records into one rebuildable OceanBase index table."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
client: _OceanBaseVectorClient,
|
||||
table_name: str,
|
||||
*,
|
||||
index_name: str = "vidx_knowledge_chunks_embedding",
|
||||
vector_dimension: int = 1024,
|
||||
batch_size: int = 100,
|
||||
) -> None:
|
||||
if not OCEANBASE_IDENTIFIER.fullmatch(table_name):
|
||||
raise OceanBaseConfigurationError("invalid OceanBase vector table name")
|
||||
if not OCEANBASE_IDENTIFIER.fullmatch(index_name):
|
||||
raise OceanBaseConfigurationError("invalid OceanBase vector index name")
|
||||
if vector_dimension <= 0:
|
||||
raise OceanBaseConfigurationError("OceanBase vector dimension must be positive")
|
||||
if not 1 <= batch_size <= 1000:
|
||||
raise OceanBaseConfigurationError(
|
||||
"OceanBase vector batch_size must be between 1 and 1000"
|
||||
)
|
||||
self._client = client
|
||||
self._table_name = table_name
|
||||
self._index_name = index_name
|
||||
self._vector_dimension = vector_dimension
|
||||
self._batch_size = batch_size
|
||||
|
||||
def upsert(self, records: tuple[VectorRecord, ...]) -> None:
|
||||
"""Validate and upsert records in bounded pyobvector batches."""
|
||||
rows = [self._to_row(record) for record in records]
|
||||
for start in range(0, len(rows), self._batch_size):
|
||||
try:
|
||||
self._client.upsert(
|
||||
self._table_name,
|
||||
rows[start : start + self._batch_size],
|
||||
)
|
||||
except Exception as exc:
|
||||
raise OceanBaseVectorStoreError("OceanBase vector upsert failed") from exc
|
||||
|
||||
def delete_document(self, document_id: UUID, index_profile_id: str) -> None:
|
||||
"""Delete only one document/profile using bound scalar predicates."""
|
||||
_validate_profile_id(index_profile_id)
|
||||
predicate = and_(
|
||||
column("document_id") == str(document_id),
|
||||
column("index_profile_id") == index_profile_id,
|
||||
)
|
||||
try:
|
||||
self._client.delete(
|
||||
self._table_name,
|
||||
where_clause=[predicate],
|
||||
)
|
||||
except Exception as exc:
|
||||
raise OceanBaseVectorStoreError("OceanBase vector cleanup failed") from exc
|
||||
|
||||
def dense_search(self, request: DenseSearchRequest) -> tuple[DenseSearchCandidate, ...]:
|
||||
"""Run profile-scoped cosine ANN and convert distance to high-is-better score."""
|
||||
_validate_profile_id(request.index_profile_id)
|
||||
self._validate_vector(request.query_vector)
|
||||
|
||||
try:
|
||||
rows = self._client.dense_search(
|
||||
self._table_name,
|
||||
index_name=self._index_name,
|
||||
query_vector=request.query_vector,
|
||||
index_profile_id=request.index_profile_id,
|
||||
top_n=request.top_n,
|
||||
output_column_names=list(SEARCH_OUTPUT_FIELDS),
|
||||
)
|
||||
candidates = tuple(
|
||||
_to_candidate(row, expected_profile=request.index_profile_id) for row in rows
|
||||
)
|
||||
if len(candidates) > request.top_n:
|
||||
raise ValueError("OceanBase returned more rows than requested")
|
||||
if any(
|
||||
earlier.score < later.score
|
||||
for earlier, later in zip(candidates, candidates[1:], strict=False)
|
||||
):
|
||||
raise ValueError("OceanBase ANN rows are not ordered by ascending distance")
|
||||
return candidates
|
||||
except Exception as exc:
|
||||
raise OceanBaseVectorStoreError("OceanBase dense search failed") from exc
|
||||
|
||||
def _to_row(self, record: VectorRecord) -> dict[str, object]:
|
||||
"""Map one rebuildable row to the pre-provisioned OceanBase schema."""
|
||||
_validate_profile_id(record.index_profile_id)
|
||||
self._validate_vector(record.embedding)
|
||||
return {
|
||||
"index_record_id": str(record.index_record_id),
|
||||
"chunk_id": str(record.chunk_id),
|
||||
"document_id": str(record.document_id),
|
||||
"filename": record.filename,
|
||||
"page": record.page,
|
||||
"page_end": record.page_end,
|
||||
"section": record.section,
|
||||
"chunk_index": record.chunk_index,
|
||||
"content": record.content,
|
||||
"embedding": list(record.embedding),
|
||||
"embedding_model": record.embedding_model,
|
||||
"index_profile_id": record.index_profile_id,
|
||||
}
|
||||
|
||||
def _validate_vector(self, vector: tuple[float, ...]) -> None:
|
||||
"""Reject dimensions and values that cannot satisfy the cosine index contract."""
|
||||
if len(vector) != self._vector_dimension:
|
||||
raise OceanBaseConfigurationError("OceanBase vector dimension mismatch")
|
||||
if any(not isfinite(value) for value in vector):
|
||||
raise OceanBaseConfigurationError("OceanBase vector values must be finite")
|
||||
if not any(value != 0 for value in vector):
|
||||
raise OceanBaseConfigurationError("OceanBase cosine vector norm must be positive")
|
||||
|
||||
|
||||
def create_oceanbase_vector_store(
|
||||
*,
|
||||
uri: str,
|
||||
user: str,
|
||||
password: str,
|
||||
database_name: str,
|
||||
table_name: str,
|
||||
index_name: str = "vidx_knowledge_chunks_embedding",
|
||||
vector_dimension: int = 1024,
|
||||
batch_size: int = 100,
|
||||
timeout_seconds: float = 30.0,
|
||||
) -> OceanBaseVectorStore:
|
||||
"""Create the official pyobvector client from runtime-injected settings."""
|
||||
_validate_uri(uri)
|
||||
if not user or not password:
|
||||
raise OceanBaseConfigurationError("OceanBase vector user and password are required")
|
||||
if not OCEANBASE_IDENTIFIER.fullmatch(database_name):
|
||||
raise OceanBaseConfigurationError("invalid OceanBase vector database name")
|
||||
if timeout_seconds <= 0:
|
||||
raise OceanBaseConfigurationError("OceanBase timeout_seconds must be positive")
|
||||
|
||||
try:
|
||||
from pyobvector import ObVecClient # type: ignore[import-untyped]
|
||||
|
||||
timeout = max(1, int(timeout_seconds))
|
||||
sdk_client = ObVecClient(
|
||||
uri=uri,
|
||||
user=user,
|
||||
password=password,
|
||||
db_name=database_name,
|
||||
pool_pre_ping=True,
|
||||
connect_args={
|
||||
"connect_timeout": timeout,
|
||||
"read_timeout": timeout,
|
||||
"write_timeout": timeout,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
raise OceanBaseVectorStoreError("OceanBase vector client initialization failed") from exc
|
||||
_validate_oceanbase_version(sdk_client.ob_version.version_nums)
|
||||
return OceanBaseVectorStore(
|
||||
_PyObVectorClient(sdk_client),
|
||||
table_name,
|
||||
index_name=index_name,
|
||||
vector_dimension=vector_dimension,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
|
||||
def _to_candidate(row: object, *, expected_profile: str) -> DenseSearchCandidate:
|
||||
"""Validate one pyobvector result tuple and normalize cosine distance to score."""
|
||||
if isinstance(row, (str, bytes, dict)) or not isinstance(row, Iterable):
|
||||
raise TypeError("OceanBase ANN row must be tuple-like")
|
||||
values = tuple(row)
|
||||
if len(values) != len(SEARCH_OUTPUT_FIELDS) + 1:
|
||||
raise ValueError("OceanBase ANN row has an unexpected field count")
|
||||
|
||||
profile = _required_str(values[9], "index_profile_id")
|
||||
if profile != expected_profile:
|
||||
raise ValueError("OceanBase returned a row from another index profile")
|
||||
distance = values[10]
|
||||
if isinstance(distance, bool) or not isinstance(distance, Real):
|
||||
raise ValueError("OceanBase cosine distance must be numeric")
|
||||
numeric_distance = float(distance)
|
||||
if not isfinite(numeric_distance) or not -1e-6 <= numeric_distance <= 2.000001:
|
||||
raise ValueError("OceanBase cosine distance is outside the supported range")
|
||||
score = max(-1.0, min(1.0, 1.0 - numeric_distance))
|
||||
|
||||
return DenseSearchCandidate(
|
||||
index_record_id=UUID(_required_str(values[0], "index_record_id")),
|
||||
chunk_id=UUID(_required_str(values[1], "chunk_id")),
|
||||
document_id=UUID(_required_str(values[2], "document_id")),
|
||||
filename=_required_str(values[3], "filename"),
|
||||
page=_optional_int(values[4]),
|
||||
page_end=_optional_int(values[5]),
|
||||
section=_optional_str(values[6]),
|
||||
chunk_index=_required_int(values[7], "chunk_index"),
|
||||
content=_required_str(values[8], "content"),
|
||||
score=score,
|
||||
index_profile_id=profile,
|
||||
)
|
||||
|
||||
|
||||
def _validate_profile_id(profile_id: str) -> None:
|
||||
"""Apply the canonical IndexProfile identifier subset at the provider edge."""
|
||||
if not PROFILE_ID.fullmatch(profile_id):
|
||||
raise OceanBaseConfigurationError("invalid OceanBase index profile ID")
|
||||
|
||||
|
||||
def _validate_uri(uri: str) -> None:
|
||||
"""Accept only a host:port target without credentials, paths, or query data."""
|
||||
if "://" in uri:
|
||||
raise OceanBaseConfigurationError("invalid OceanBase vector URI")
|
||||
try:
|
||||
parsed = urlsplit(f"//{uri}")
|
||||
port = parsed.port
|
||||
except ValueError as exc:
|
||||
raise OceanBaseConfigurationError("invalid OceanBase vector URI") from exc
|
||||
if (
|
||||
not parsed.hostname
|
||||
or port is None
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or parsed.path
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
raise OceanBaseConfigurationError("invalid OceanBase vector URI")
|
||||
|
||||
|
||||
def _validate_oceanbase_version(version_nums: object) -> None:
|
||||
"""Require the first release with an explicit vector pre-filter hint."""
|
||||
if (
|
||||
not isinstance(version_nums, (list, tuple))
|
||||
or len(version_nums) != 4
|
||||
or any(isinstance(part, bool) or not isinstance(part, int) for part in version_nums)
|
||||
):
|
||||
raise OceanBaseVectorStoreError("OceanBase version validation failed")
|
||||
if tuple(version_nums) < (4, 6, 0, 0):
|
||||
raise OceanBaseConfigurationError(
|
||||
"OceanBase 4.6.0 or newer is required for vector pre-filtering"
|
||||
)
|
||||
|
||||
|
||||
def _build_dense_search_sql(
|
||||
*,
|
||||
table_name: str,
|
||||
index_name: str,
|
||||
output_column_names: list[str],
|
||||
) -> str:
|
||||
"""Build a fixed-identifier query whose values remain bind parameters."""
|
||||
identifiers = [table_name, index_name, *output_column_names]
|
||||
if any(not OCEANBASE_IDENTIFIER.fullmatch(value) for value in identifiers):
|
||||
raise OceanBaseConfigurationError("invalid OceanBase dense-search identifier")
|
||||
fields = ", ".join(output_column_names)
|
||||
return (
|
||||
f"SELECT /*+ VECTOR_INDEX({table_name} {index_name} pre_filter) */ "
|
||||
f"{fields}, cosine_distance(embedding, :query_vector) AS distance "
|
||||
f"FROM {table_name} "
|
||||
"WHERE index_profile_id = :index_profile_id "
|
||||
"ORDER BY cosine_distance(embedding, :query_vector) APPROXIMATE "
|
||||
"LIMIT :top_n"
|
||||
)
|
||||
|
||||
|
||||
def _serialize_vector(vector: tuple[float, ...]) -> str:
|
||||
"""Serialize already-validated finite floats for OceanBase's VECTOR parameter."""
|
||||
return "[" + ",".join(format(value, ".17g") for value in vector) + "]"
|
||||
|
||||
|
||||
def _required_str(value: object, name: str) -> str:
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ValueError(f"OceanBase field {name} must be non-empty text")
|
||||
return value
|
||||
|
||||
|
||||
def _required_int(value: object, name: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise ValueError(f"OceanBase field {name} must be an integer")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_int(value: object) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise ValueError("OceanBase nullable integer metadata must be an integer")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_str(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("OceanBase nullable text metadata must be text")
|
||||
return value
|
||||
@ -20,6 +20,7 @@ dependencies = [
|
||||
"python-docx>=1.2,<2",
|
||||
"sqlalchemy>=2.0,<3",
|
||||
"uvicorn[standard]>=0.34,<1",
|
||||
"pyobvector>=0.2.28,<0.3",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
|
||||
@ -36,6 +36,14 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
|
||||
assert settings.mochow_table == "knowledge_chunks"
|
||||
assert settings.mochow_search_ef == 200
|
||||
assert settings.mochow_timeout_seconds == 30
|
||||
assert settings.oceanbase_vector_uri is None
|
||||
assert settings.oceanbase_vector_user is None
|
||||
assert settings.oceanbase_vector_password is None
|
||||
assert settings.oceanbase_vector_database is None
|
||||
assert settings.oceanbase_vector_table == "knowledge_chunks_v1"
|
||||
assert settings.oceanbase_vector_index == "vidx_knowledge_chunks_embedding"
|
||||
assert settings.oceanbase_vector_batch_size == 100
|
||||
assert settings.oceanbase_vector_timeout_seconds == 30
|
||||
assert settings.index_profile_id == "demo-bge-large-zh-structure-v1"
|
||||
assert settings.dense_top_n == 20
|
||||
assert settings.reranker_provider == "qianfan"
|
||||
@ -70,6 +78,7 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) -
|
||||
baidu_mochow_api_key=SecretStr("mochow-secret"),
|
||||
bos_access_key_id=SecretStr("bos-access-key"),
|
||||
bos_secret_access_key=SecretStr("bos-secret-key"),
|
||||
oceanbase_vector_password=SecretStr("oceanbase-secret"),
|
||||
reranker_api_key=SecretStr("reranker-secret"),
|
||||
)
|
||||
|
||||
@ -78,4 +87,5 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) -
|
||||
assert "mochow-secret" not in rendered
|
||||
assert "bos-access-key" not in rendered
|
||||
assert "bos-secret-key" not in rendered
|
||||
assert "oceanbase-secret" not in rendered
|
||||
assert "reranker-secret" not in rendered
|
||||
|
||||
310
tests/unit/test_oceanbase_vector_store.py
Normal file
310
tests/unit/test_oceanbase_vector_store.py
Normal file
@ -0,0 +1,310 @@
|
||||
"""Offline tests for neutral-record translation into pyobvector operations."""
|
||||
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from ekp_vector_store import DenseSearchRequest, OceanBaseVectorStore, VectorRecord
|
||||
from ekp_vector_store.oceanbase import (
|
||||
OceanBaseConfigurationError,
|
||||
OceanBaseVectorStoreError,
|
||||
_build_dense_search_sql,
|
||||
_PyObVectorClient,
|
||||
_serialize_vector,
|
||||
_validate_oceanbase_version,
|
||||
)
|
||||
|
||||
|
||||
class FakeOceanBaseClient:
|
||||
"""Capture ObVecClient-shaped calls without a database connection."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.upsert_calls: list[tuple[str, list[dict[str, object]]]] = []
|
||||
self.delete_calls: list[tuple[str, list[object]]] = []
|
||||
self.search_calls: list[dict[str, object]] = []
|
||||
self.search_rows: list[tuple[object, ...]] = []
|
||||
self.failure: Exception | None = None
|
||||
|
||||
def upsert(self, table_name: str, data: list[dict[str, object]]) -> object:
|
||||
self._raise_failure()
|
||||
self.upsert_calls.append((table_name, data))
|
||||
return object()
|
||||
|
||||
def delete(
|
||||
self,
|
||||
table_name: str,
|
||||
*,
|
||||
where_clause: list[object],
|
||||
) -> object:
|
||||
self._raise_failure()
|
||||
self.delete_calls.append((table_name, where_clause))
|
||||
return object()
|
||||
|
||||
def dense_search(
|
||||
self,
|
||||
table_name: str,
|
||||
*,
|
||||
index_name: str,
|
||||
query_vector: tuple[float, ...],
|
||||
index_profile_id: str,
|
||||
top_n: int,
|
||||
output_column_names: list[str],
|
||||
) -> list[tuple[object, ...]]:
|
||||
self._raise_failure()
|
||||
self.search_calls.append(
|
||||
{
|
||||
"table_name": table_name,
|
||||
"index_name": index_name,
|
||||
"query_vector": query_vector,
|
||||
"index_profile_id": index_profile_id,
|
||||
"top_n": top_n,
|
||||
"output_column_names": output_column_names,
|
||||
}
|
||||
)
|
||||
return self.search_rows
|
||||
|
||||
def _raise_failure(self) -> None:
|
||||
if self.failure is not None:
|
||||
raise self.failure
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
"""Capture SQLAlchemy text and bound values without executing vendor SQL."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.executions: list[tuple[str, dict[str, object]]] = []
|
||||
|
||||
def __enter__(self) -> "FakeConnection":
|
||||
return self
|
||||
|
||||
def __exit__(self, *args: object) -> None:
|
||||
return None
|
||||
|
||||
def execute(self, statement: object, parameters: dict[str, object]) -> list[object]:
|
||||
self.executions.append((str(statement), parameters))
|
||||
return []
|
||||
|
||||
|
||||
class FakeEngine:
|
||||
"""Return one inspectable fake connection."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.connection = FakeConnection()
|
||||
|
||||
def connect(self) -> FakeConnection:
|
||||
return self.connection
|
||||
|
||||
|
||||
class FakeSdkClient:
|
||||
"""Provide the official client's engine shape for the SQL boundary test."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.engine = FakeEngine()
|
||||
|
||||
|
||||
def _record(document_id: UUID, chunk_index: int) -> VectorRecord:
|
||||
return VectorRecord(
|
||||
index_record_id=uuid4(),
|
||||
chunk_id=uuid4(),
|
||||
document_id=document_id,
|
||||
filename="policy.md",
|
||||
page=1,
|
||||
page_end=2,
|
||||
section="产品 / 退款",
|
||||
chunk_index=chunk_index,
|
||||
content="退款周期为七天。",
|
||||
embedding=(0.1, 0.2, 0.3),
|
||||
embedding_model="fake",
|
||||
index_profile_id="profile-v1",
|
||||
)
|
||||
|
||||
|
||||
def _search_row(record: VectorRecord, distance: float) -> tuple[object, ...]:
|
||||
return (
|
||||
str(record.index_record_id),
|
||||
str(record.chunk_id),
|
||||
str(record.document_id),
|
||||
record.filename,
|
||||
record.page,
|
||||
record.page_end,
|
||||
record.section,
|
||||
record.chunk_index,
|
||||
record.content,
|
||||
record.index_profile_id,
|
||||
distance,
|
||||
)
|
||||
|
||||
|
||||
def test_oceanbase_adapter_maps_and_batches_upserts() -> None:
|
||||
client = FakeOceanBaseClient()
|
||||
store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3, batch_size=2)
|
||||
document_id = uuid4()
|
||||
records = tuple(_record(document_id, index) for index in range(3))
|
||||
|
||||
store.upsert(records)
|
||||
|
||||
assert [len(data) for _, data in client.upsert_calls] == [2, 1]
|
||||
first = client.upsert_calls[0][1][0]
|
||||
assert first["index_record_id"] == str(records[0].index_record_id)
|
||||
assert first["document_id"] == str(document_id)
|
||||
assert first["embedding"] == [0.1, 0.2, 0.3]
|
||||
assert first["index_profile_id"] == "profile-v1"
|
||||
|
||||
|
||||
def test_oceanbase_delete_uses_bound_document_and_profile_predicate() -> None:
|
||||
client = FakeOceanBaseClient()
|
||||
store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3)
|
||||
document_id = uuid4()
|
||||
|
||||
store.delete_document(document_id, "profile-v1")
|
||||
|
||||
table_name, clauses = client.delete_calls[0]
|
||||
compiled = cast(Any, clauses[0]).compile()
|
||||
assert table_name == "knowledge_chunks_v1"
|
||||
assert (
|
||||
str(compiled) == "document_id = :document_id_1 AND index_profile_id = :index_profile_id_1"
|
||||
)
|
||||
assert set(compiled.params.values()) == {str(document_id), "profile-v1"}
|
||||
|
||||
|
||||
def test_oceanbase_dense_search_filters_profile_and_normalizes_cosine_distance() -> None:
|
||||
client = FakeOceanBaseClient()
|
||||
first = _record(uuid4(), 0)
|
||||
second = _record(uuid4(), 1)
|
||||
client.search_rows = [_search_row(first, 0.1), _search_row(second, 0.4)]
|
||||
store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3)
|
||||
|
||||
candidates = store.dense_search(
|
||||
DenseSearchRequest((0.3, 0.2, 0.1), index_profile_id="profile-v1", top_n=20)
|
||||
)
|
||||
|
||||
call = client.search_calls[0]
|
||||
assert call["table_name"] == "knowledge_chunks_v1"
|
||||
assert call["index_name"] == "vidx_knowledge_chunks_embedding"
|
||||
assert call["query_vector"] == (0.3, 0.2, 0.1)
|
||||
assert call["index_profile_id"] == "profile-v1"
|
||||
assert call["top_n"] == 20
|
||||
assert [candidate.score for candidate in candidates] == pytest.approx([0.9, 0.6])
|
||||
assert candidates[0].chunk_id == first.chunk_id
|
||||
|
||||
|
||||
def test_oceanbase_rejects_wrong_dimension_and_zero_cosine_vector() -> None:
|
||||
store = OceanBaseVectorStore(
|
||||
FakeOceanBaseClient(),
|
||||
"knowledge_chunks_v1",
|
||||
vector_dimension=3,
|
||||
)
|
||||
|
||||
with pytest.raises(OceanBaseConfigurationError, match="dimension mismatch"):
|
||||
store.dense_search(DenseSearchRequest((0.1, 0.2), "profile-v1"))
|
||||
with pytest.raises(OceanBaseConfigurationError, match="norm must be positive"):
|
||||
store.upsert((replace(_record(uuid4(), 0), embedding=(0.0, 0.0, 0.0)),))
|
||||
|
||||
|
||||
def test_oceanbase_sanitizes_malformed_or_failed_search() -> None:
|
||||
client = FakeOceanBaseClient()
|
||||
client.search_rows = [("malformed",)]
|
||||
store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3)
|
||||
|
||||
with pytest.raises(OceanBaseVectorStoreError, match="dense search failed"):
|
||||
store.dense_search(DenseSearchRequest((0.1, 0.2, 0.3), "profile-v1"))
|
||||
|
||||
client.failure = RuntimeError("private-host password=secret-value")
|
||||
with pytest.raises(OceanBaseVectorStoreError, match="cleanup failed") as caught:
|
||||
store.delete_document(uuid4(), "profile-v1")
|
||||
assert "private-host" not in str(caught.value)
|
||||
assert "secret-value" not in str(caught.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"uri",
|
||||
["", "https://db.example:2881", "user:password@db.example:2881", "db.example"],
|
||||
)
|
||||
def test_oceanbase_factory_rejects_unsafe_uri_without_connecting(uri: str) -> None:
|
||||
from ekp_vector_store.oceanbase import create_oceanbase_vector_store
|
||||
|
||||
with pytest.raises(OceanBaseConfigurationError, match="URI"):
|
||||
create_oceanbase_vector_store(
|
||||
uri=uri,
|
||||
user="test-user",
|
||||
password="test-password",
|
||||
database_name="test_database",
|
||||
table_name="knowledge_chunks_v1",
|
||||
)
|
||||
|
||||
|
||||
def test_oceanbase_dense_sql_requires_profile_pre_filter_and_bound_values() -> None:
|
||||
sql = _build_dense_search_sql(
|
||||
table_name="knowledge_chunks_v1",
|
||||
index_name="vidx_knowledge_chunks_embedding",
|
||||
output_column_names=["chunk_id", "index_profile_id"],
|
||||
)
|
||||
|
||||
assert "VECTOR_INDEX(knowledge_chunks_v1 vidx_knowledge_chunks_embedding pre_filter)" in sql
|
||||
assert "WHERE index_profile_id = :index_profile_id" in sql
|
||||
assert "cosine_distance(embedding, :query_vector)" in sql
|
||||
assert "APPROXIMATE LIMIT :top_n" in sql
|
||||
assert "profile-v1" not in sql
|
||||
assert (
|
||||
_serialize_vector((0.1, -0.2, 0.3))
|
||||
== "[0.10000000000000001,-0.20000000000000001,0.29999999999999999]"
|
||||
)
|
||||
|
||||
|
||||
def test_oceanbase_rejects_unsafe_index_identifier() -> None:
|
||||
with pytest.raises(OceanBaseConfigurationError, match="index name"):
|
||||
OceanBaseVectorStore(
|
||||
FakeOceanBaseClient(),
|
||||
"knowledge_chunks_v1",
|
||||
index_name="vector-index;drop",
|
||||
vector_dimension=3,
|
||||
)
|
||||
|
||||
|
||||
def test_oceanbase_requires_version_with_explicit_pre_filter() -> None:
|
||||
with pytest.raises(OceanBaseConfigurationError, match="4.6.0"):
|
||||
_validate_oceanbase_version([4, 3, 5, 1])
|
||||
|
||||
_validate_oceanbase_version([4, 6, 0, 0])
|
||||
|
||||
with pytest.raises(OceanBaseVectorStoreError, match="version validation"):
|
||||
_validate_oceanbase_version("4.6.0.0")
|
||||
|
||||
|
||||
def test_pyobvector_boundary_binds_profile_vector_and_limit() -> None:
|
||||
sdk_client = FakeSdkClient()
|
||||
client = _PyObVectorClient(sdk_client)
|
||||
|
||||
rows = client.dense_search(
|
||||
"knowledge_chunks_v1",
|
||||
index_name="vidx_knowledge_chunks_embedding",
|
||||
query_vector=(0.1, 0.2, 0.3),
|
||||
index_profile_id="profile-v1",
|
||||
top_n=20,
|
||||
output_column_names=["chunk_id", "index_profile_id"],
|
||||
)
|
||||
|
||||
assert list(rows) == []
|
||||
sql, parameters = sdk_client.engine.connection.executions[0]
|
||||
assert "pre_filter" in sql
|
||||
assert parameters == {
|
||||
"query_vector": "[0.10000000000000001,0.20000000000000001,0.29999999999999999]",
|
||||
"index_profile_id": "profile-v1",
|
||||
"top_n": 20,
|
||||
}
|
||||
assert "profile-v1" not in sql
|
||||
|
||||
|
||||
def test_oceanbase_schema_declares_rebuildable_cosine_index() -> None:
|
||||
schema_path = Path(__file__).parents[2] / "deploy" / "oceanbase" / "001_knowledge_chunks_v1.sql"
|
||||
|
||||
schema = schema_path.read_text(encoding="utf-8")
|
||||
|
||||
assert "embedding VECTOR(1024) NOT NULL" in schema
|
||||
assert "index_profile_id VARCHAR(128) NOT NULL" in schema
|
||||
assert "DISTANCE = COSINE" in schema
|
||||
assert "TYPE = HNSW" in schema
|
||||
assert "document_id, index_profile_id" in schema
|
||||
@ -42,6 +42,44 @@ def test_provider_wiring_requires_explicit_mochow_target(
|
||||
create_vector_store(settings)
|
||||
|
||||
|
||||
def test_provider_wiring_requires_complete_oceanbase_target(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
settings = Settings(vector_store_provider="oceanbase")
|
||||
|
||||
with pytest.raises(ProviderConfigurationError, match="OCEANBASE_VECTOR_URI"):
|
||||
create_vector_store(settings)
|
||||
|
||||
|
||||
def test_provider_wiring_builds_oceanbase_from_secret_safe_settings(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
expected = FakeVectorStore()
|
||||
|
||||
def fake_factory(**kwargs: object) -> FakeVectorStore:
|
||||
captured.update(kwargs)
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr("ekp_api.providers.create_oceanbase_vector_store", fake_factory)
|
||||
settings = Settings(
|
||||
vector_store_provider="oceanbase",
|
||||
oceanbase_vector_uri="db.example:2881",
|
||||
oceanbase_vector_user="test-user@test-tenant",
|
||||
oceanbase_vector_password=SecretStr("test-password"),
|
||||
oceanbase_vector_database="test_database",
|
||||
)
|
||||
|
||||
vector_store = create_vector_store(settings)
|
||||
|
||||
assert vector_store is expected
|
||||
assert captured["uri"] == "db.example:2881"
|
||||
assert captured["password"] == "test-password"
|
||||
assert captured["index_name"] == "vidx_knowledge_chunks_embedding"
|
||||
assert captured["vector_dimension"] == 1024
|
||||
|
||||
|
||||
def test_provider_wiring_selects_local_storage_by_default(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
|
||||
79
uv.lock
generated
79
uv.lock
generated
@ -2,6 +2,18 @@ version = 1
|
||||
revision = 3
|
||||
requires-python = ">=3.12, <3.14"
|
||||
|
||||
[[package]]
|
||||
name = "aiomysql"
|
||||
version = "0.3.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pymysql" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/29/e0/302aeffe8d90853556f47f3106b89c16cc2ec2a4d269bdfd82e3f4ae12cc/aiomysql-0.3.2.tar.gz", hash = "sha256:72d15ef5cfc34c03468eb41e1b90adb9fd9347b0b589114bd23ead569a02ac1a", size = 108311, upload-time = "2025-10-22T00:15:21.278Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/4c/af/aae0153c3e28712adaf462328f6c7a3c196a1c1c27b491de4377dd3e6b52/aiomysql-0.3.2-py3-none-any.whl", hash = "sha256:c82c5ba04137d7afd5c693a258bea8ead2aad77101668044143a991e04632eb2", size = 71834, upload-time = "2025-10-22T00:15:15.905Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alembic"
|
||||
version = "1.19.1"
|
||||
@ -230,6 +242,7 @@ dependencies = [
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "pymochow" },
|
||||
{ name = "pymupdf" },
|
||||
{ name = "pyobvector" },
|
||||
{ name = "python-docx" },
|
||||
{ name = "python-multipart" },
|
||||
{ name = "sqlalchemy" },
|
||||
@ -254,6 +267,7 @@ requires-dist = [
|
||||
{ name = "pydantic-settings", specifier = ">=2.7,<3" },
|
||||
{ name = "pymochow", specifier = ">=2.4,<3" },
|
||||
{ name = "pymupdf", specifier = ">=1.26,<2" },
|
||||
{ name = "pyobvector", specifier = ">=0.2.28,<0.3" },
|
||||
{ name = "python-docx", specifier = ">=1.2,<2" },
|
||||
{ name = "python-multipart", specifier = ">=0.0.20,<1" },
|
||||
{ name = "sqlalchemy", specifier = ">=2.0,<3" },
|
||||
@ -580,6 +594,36 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "numpy"
|
||||
version = "2.5.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9a/80/db0b4559e57ec36362bedbb05530a87fafbcb6067708c946967a41d449e7/numpy-2.5.2.tar.gz", hash = "sha256:d482d171c406ae88c5b19cad3b6a1c4c5209f886ab74bc44c2c865c23f52d860", size = 20773161, upload-time = "2026-08-09T13:48:27.962Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/69/72/dccb0aaf40972777283303919f613964227266d0c13adebb79ac124f1c3e/numpy-2.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:14e373cfc6387177e8409dac3c7159be8eb05cd77096cd7c950268b86f62831c", size = 16891693, upload-time = "2026-08-09T13:44:51.702Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/60/2e/b5aee50a1f74ac815cf8331812cb8251e29024025de462e0c047641c614c/numpy-2.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bbd96c833ecc8cc069ce518078fc8c60cb9cbfb0fea5b7a803ad65035596d03", size = 11903109, upload-time = "2026-08-09T13:44:55.501Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/f4/29e78102a80601cf034d4e9767022cffeca2c3b4c926e1754572ca95593d/numpy-2.5.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:6e8172ddfcf5cf74b811d372b570b83c60bd2de87a6fbfbebdadb4a9bd9c6cbb", size = 5350202, upload-time = "2026-08-09T13:44:58.401Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/11/4b/dcd3b7eadaf4035d2c7a4289d232523a6964f602598ef7674e4bd7291f93/numpy-2.5.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:65f188481f1669e26f62b701e8205d19e460fa4a9b52a1414ba382330e4a3414", size = 6687736, upload-time = "2026-08-09T13:45:00.813Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/21/4947e0e9d6c9fc2e2ff15b8949049ee44f63adb9cacc729ab8793f97e712/numpy-2.5.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ee9c4eeb8454b3660a8b53493563c3e121c2fc94fbd72b848ef814ed7b676a9", size = 15612696, upload-time = "2026-08-09T13:45:04.151Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3a/5f/62d28cf019460c7f1394105b4d49d9911a9c444cb77ab0bd95a204c5a6de/numpy-2.5.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3cdec01fa790a186d430433fdd4d4ffb70eed6f0eeb4bf05c8dbe2dce0a9bcb8", size = 16722264, upload-time = "2026-08-09T13:45:07.714Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/25/3f0be4c1b9fdf5dd5e708a6806978564d7c46a055c000496309ff2a2f8af/numpy-2.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7999d4ddb0c4025018373fd787510d46e04c769467af22869707b3c1cfd459ab", size = 16974396, upload-time = "2026-08-09T13:45:11.316Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/72/6262cbdeeb45da9d971e40715f579d791603ba8ec0b5e2db1ac55454421d/numpy-2.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c1f017dc0875c9209d219f97feceb7d54c2661bb243deb4114478e1295808af7", size = 18476044, upload-time = "2026-08-09T13:45:14.869Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/36/33/29208b8b075bde62d26a81d14b358c42b0f69b6cabd98d4ff97f37f22b05/numpy-2.5.2-cp312-cp312-win32.whl", hash = "sha256:d6a48072864e3324e194a8fbb3c657bcc5b5c869dbc64c9537b1d5c862572c0a", size = 6072817, upload-time = "2026-08-09T13:45:17.867Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/b9/87fea2769fe1c47c1b5b01d8310772c9d1a85d485de7cf386ef7a3332b02/numpy-2.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:28ac63476ec7651484215ee7fa15a1f78b57c14621f01e392afe17b9a1390ce4", size = 12464674, upload-time = "2026-08-09T13:45:20.734Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/52/032b97e00461ab0809bbe4c588b035620e5a14b8cdee47ecddefc7b17d33/numpy-2.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:27650bb0e7140fa3d37b9923b4803645e0b125d190f326eecfd3f4dad8e8ade1", size = 10397131, upload-time = "2026-08-09T13:45:23.73Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f5/d2/6b24738a0ef4557d189b150046cd07823c50e4273e8aebd651222e24306f/numpy-2.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8e4cb9a754c8a0c62eaa88273a5fba3391f4a610d1dee893c0755da31c083f15", size = 16886595, upload-time = "2026-08-09T13:45:27.323Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/65/60/f2d208d366f263f39c6e69ed309290717aab41078b6d04c9be2a84fa2a07/numpy-2.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:52c808f96484f5571a5cc863775ce50247c17dfb3b0361f8ed6b4b0456f80080", size = 11896845, upload-time = "2026-08-09T13:45:31.638Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/79/81e0bf24f4d020a2b1d5cd297a9f60c3f24eeb116f9bba5870443f7b6a4a/numpy-2.5.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:29d81e97f668489cba8ebfd796b9bdd453525d35dd9e162e2daec94bf3fc7740", size = 5343880, upload-time = "2026-08-09T13:45:34.373Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/cc/e3141cf06d1a8a2c7e107543fe1269c1d1af760d4d683c0794a4ee1127c2/numpy-2.5.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:afb3f0632d6b2e3ba04dbce8d1e48d321b369138b73830b5ca371a0e8d479d56", size = 6682264, upload-time = "2026-08-09T13:45:36.7Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/29/f1/2a64a307d92c5d98f5255a4014eb43bb6103ee477087b61ecae44a3aa9b9/numpy-2.5.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0aadf13b60048d501e05fa699efaf7734e2494f3498a4c2a5521d822640324f3", size = 15609566, upload-time = "2026-08-09T13:45:39.518Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/44/59a1eb68e773c4098d107ef34a0dbdeca501d72ffcfbff9a7707343921ce/numpy-2.5.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29b86ff8a6cc556b47ec6b64b194815cc80e6bf5eedcc6cddfd65318cb0b4eee", size = 16709995, upload-time = "2026-08-09T13:45:43.661Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8a/4c/3e54d4ddbc359a1295f8b633e8106bcd4d7d4a206e82df051bdfb3058755/numpy-2.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6950c4b7dd562453090548ba7f5da7e59f57f85663f15d5dcc60e249192f7e59", size = 16972511, upload-time = "2026-08-09T13:45:47.094Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f2/9f/02e371638ebf19b66d46231e4be52999e87f32d1961b113bc45656608b22/numpy-2.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b9727f472d2f3888053b8a75ab0cb94745a9de224bb5846dbadc0092101bc71d", size = 18465609, upload-time = "2026-08-09T13:45:50.808Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/ae/ad6645abc7a3510fe48e8ea1ab4598166f500057ef4ebf38bfad4f1577de/numpy-2.5.2-cp313-cp313-win32.whl", hash = "sha256:4f9744f9fbdcea0bc552e8f19e1f141f811a3f9bc2be2cc6e86d982cab23e3f4", size = 6070204, upload-time = "2026-08-09T13:45:54.111Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/20/f3489f86d81ea460b2bcdceaed094142ca6579f6be0ec527b781d39afe68/numpy-2.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:85aaccb24182c25df891ad0ec333585967e115269d5f1b17f2c9ae005bc96657", size = 12460532, upload-time = "2026-08-09T13:45:57.167Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/21/35b31dde1b283b79de828b80f876afd8c94e28fe1e9c375f89e261cc4c0d/numpy-2.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:bd68ece1553d2023c09a4226d9e41c586ad2d20594d1a456186c33513d2cb3f2", size = 10396725, upload-time = "2026-08-09T13:46:00.478Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "orjson"
|
||||
version = "3.12.0"
|
||||
@ -828,6 +872,32 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/58/8c/d897dcd32a25b58186c968b15ce4324ca029e9d96460de12325314e390be/pymupdf-1.28.2-cp313-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2e1b574c0fd2cb238021033fd3c0f9c4388816638df064e4bfb56d9d81736dc8", size = 18399403, upload-time = "2026-08-06T21:39:25.008Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pymysql"
|
||||
version = "1.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c9/bc/1c6a92f385940f727daeecf3bacaf186e03875dff57197801046c583bcf0/pymysql-1.2.0.tar.gz", hash = "sha256:6c7b17ca686988104d7426c27895b455cdeea3e9d3ceb1270f0c3704fead8c33", size = 49021, upload-time = "2026-05-19T08:26:22.302Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/bd/2534e130295c8cfd4f0a2e31623baab7502278f1e97bcfe61db75656a77f/pymysql-1.2.0-py3-none-any.whl", hash = "sha256:62169ce6d5510f08e140c5e7990ee884a9764024e4a9a27b2cc11f1099322ae0", size = 45716, upload-time = "2026-05-19T08:26:20.974Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyobvector"
|
||||
version = "0.2.29"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "aiomysql" },
|
||||
{ name = "numpy" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pymysql" },
|
||||
{ name = "sqlalchemy" },
|
||||
{ name = "sqlglot" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/32/47/80f5f62572b00348d6864eb14d495cf26e52d7b9dba0ad7365cdbac86851/pyobvector-0.2.29.tar.gz", hash = "sha256:3f0feac0ac964bb56c2e3ed661b5de84ef5bc97253ae8e9935a65fea367e3736", size = 85589, upload-time = "2026-07-29T02:53:22.676Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/82/66/1aa4ca29ee2d1780aa4761d58ccac82771747a75f9146db0d7285a245bdb/pyobvector-0.2.29-py3-none-any.whl", hash = "sha256:f7cfdf1ca0c5c6ccbc10fd096aef944cd3970970f1382441a77dff1be78744cf", size = 70210, upload-time = "2026-07-29T02:53:21.307Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "9.1.1"
|
||||
@ -993,6 +1063,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/3f/3582293d1e185e71d19d7c731c3e2ee20ba21981c4a1115c0806c1f62120/sqlalchemy-2.0.52-py3-none-any.whl", hash = "sha256:3b81b8363a919ce53453591cdb93702e6bd54ade6c4fa2f468fc053baee5ed89", size = 1950700, upload-time = "2026-08-11T20:47:21.603Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlglot"
|
||||
version = "30.17.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/56/d4/da49abcc81beebbb25f29ddf87f2980c63c949569d7f2da40c06d95fa415/sqlglot-30.17.0.tar.gz", hash = "sha256:2d6b8def93304fa300f4d20f48e3909e7f436fda56ca1fafd8975f6c561ef62c", size = 5999019, upload-time = "2026-08-12T19:36:50.587Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/bc/2a07cef49046e6cf5d1a8b1de553aaef8491b4170dbfe254c8687c764408/sqlglot-30.17.0-py3-none-any.whl", hash = "sha256:84435ac283a60173da31b5fd7d11a725037a1c3fd6ed1e21fb065de74ddb579f", size = 741795, upload-time = "2026-08-12T19:36:48.699Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "starlette"
|
||||
version = "1.6.0"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user