diff --git a/.env.example b/.env.example index a641b2d..c5808ae 100644 --- a/.env.example +++ b/.env.example @@ -31,6 +31,14 @@ MOCHOW_TABLE=knowledge_chunks MOCHOW_BATCH_SIZE=100 MOCHOW_SEARCH_EF=200 MOCHOW_TIMEOUT_SECONDS=30 +OCEANBASE_VECTOR_URI= +OCEANBASE_VECTOR_USER= +OCEANBASE_VECTOR_PASSWORD= +OCEANBASE_VECTOR_DATABASE= +OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1 +OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding +OCEANBASE_VECTOR_BATCH_SIZE=100 +OCEANBASE_VECTOR_TIMEOUT_SECONDS=30 EMBEDDING_PROVIDER=qianfan EMBEDDING_MODEL=bge-large-zh diff --git a/README.md b/README.md index d440f2e..64c974c 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,9 @@ demo providers without changing PostgreSQL canonical ownership. Until the runtim cutover task is merged, the paragraphs above describe the current implementation. ARCH-002 adds the selectable BOS adapter and safe runtime configuration documented in `docs/development/BOS_SETUP.md`; local storage remains the default until ARCH-005. +ARCH-003 adds the selectable OceanBase Vector adapter and pre-provisioned cosine +HNSW schema documented in `docs/development/OCEANBASE_VECTOR_SETUP.md`; Mochow +remains the runtime default until ARCH-005. DEMO-007 adds query embedding and profile-scoped Dense Top-N retrieval. The default is Top 20 with a configurable Mochow HNSW `ef`; the in-memory fake uses diff --git a/apps/api/src/ekp_api/providers.py b/apps/api/src/ekp_api/providers.py index 5478c03..c869b28 100644 --- a/apps/api/src/ekp_api/providers.py +++ b/apps/api/src/ekp_api/providers.py @@ -23,7 +23,11 @@ from ekp_object_storage import ObjectStorage, create_bos_object_storage from ekp_object_storage.local import LocalObjectStorage from ekp_parser import ParserRouter from ekp_reranker import QianfanRerankerProvider, RerankerProvider -from ekp_vector_store import MochowVectorStore, VectorStore, create_mochow_vector_store +from ekp_vector_store import ( + VectorStore, + create_mochow_vector_store, + create_oceanbase_vector_store, +) BGE_LARGE_ZH_DIMENSIONS = 1024 DEMO_CHUNK_STRATEGY_VERSION = "structure_v1" @@ -85,8 +89,29 @@ def create_object_storage(settings: Settings) -> ObjectStorage: ) -def create_vector_store(settings: Settings) -> MochowVectorStore: - """Create the Mochow adapter for an explicitly configured database/table.""" +def create_vector_store(settings: Settings) -> VectorStore: + """Create the selected vector adapter from explicit, secret-safe settings.""" + if settings.vector_store_provider == "oceanbase": + if settings.oceanbase_vector_uri is None: + raise ProviderConfigurationError("OCEANBASE_VECTOR_URI is required") + if settings.oceanbase_vector_user is None: + raise ProviderConfigurationError("OCEANBASE_VECTOR_USER is required") + if settings.oceanbase_vector_password is None: + raise ProviderConfigurationError("OCEANBASE_VECTOR_PASSWORD is required") + if settings.oceanbase_vector_database is None: + raise ProviderConfigurationError("OCEANBASE_VECTOR_DATABASE is required") + return create_oceanbase_vector_store( + uri=settings.oceanbase_vector_uri, + user=settings.oceanbase_vector_user, + password=settings.oceanbase_vector_password.get_secret_value(), + database_name=settings.oceanbase_vector_database, + table_name=settings.oceanbase_vector_table, + index_name=settings.oceanbase_vector_index, + vector_dimension=BGE_LARGE_ZH_DIMENSIONS, + batch_size=settings.oceanbase_vector_batch_size, + timeout_seconds=settings.oceanbase_vector_timeout_seconds, + ) + if settings.baidu_mochow_api_key is None: raise ProviderConfigurationError("BAIDU_MOCHOW_API_KEY is required") if not settings.baidu_mochow_endpoint: diff --git a/deploy/oceanbase/001_knowledge_chunks_v1.sql b/deploy/oceanbase/001_knowledge_chunks_v1.sql new file mode 100644 index 0000000..72c682a --- /dev/null +++ b/deploy/oceanbase/001_knowledge_chunks_v1.sql @@ -0,0 +1,30 @@ +-- ARCH-003 rebuildable OceanBase Vector index for bge-large-zh (1024 dimensions). +-- PostgreSQL remains canonical; this table may be dropped and rebuilt from canonical chunks. +-- Requires OceanBase 4.6.0 or newer for explicit ANN scalar pre-filtering. + +CREATE TABLE IF NOT EXISTS knowledge_chunks_v1 ( + index_record_id VARCHAR(36) NOT NULL, + chunk_id VARCHAR(36) NOT NULL, + document_id VARCHAR(36) NOT NULL, + filename VARCHAR(255) NOT NULL, + page INT NULL, + page_end INT NULL, + section VARCHAR(1024) NULL, + chunk_index INT NOT NULL, + content MEDIUMTEXT NOT NULL, + embedding VECTOR(1024) NOT NULL, + embedding_model VARCHAR(128) NOT NULL, + index_profile_id VARCHAR(128) NOT NULL, + PRIMARY KEY (index_record_id), + KEY idx_knowledge_chunks_document_profile (document_id, index_profile_id), + KEY idx_knowledge_chunks_profile (index_profile_id), + VECTOR KEY vidx_knowledge_chunks_embedding (embedding) + WITH ( + DISTANCE = COSINE, + TYPE = HNSW, + LIB = VSAG, + M = 16, + EF_CONSTRUCTION = 256, + EF_SEARCH = 200 + ) BLOCK_SIZE 16384 +) ORGANIZATION = HEAP; diff --git a/docs/demo/DEMO_SCOPE.md b/docs/demo/DEMO_SCOPE.md index 030eb15..e315329 100644 --- a/docs/demo/DEMO_SCOPE.md +++ b/docs/demo/DEMO_SCOPE.md @@ -364,7 +364,8 @@ DEMO-002 Document 数据模型 + 本地文件存储 DEMO-003 PDF/DOCX/TXT/MD 上传 DEMO-004 Parser + Cleaner DEMO-005 Chunk Pipeline -DEMO-006 bge-large-zh Embedding + 初始 Mochow 入库(将由 ARCH-003 迁移) +DEMO-006 bge-large-zh Embedding + 初始 Mochow 入库(ARCH-003 已提供 OceanBase Adapter, +ARCH-005 切换运行时) DEMO-007 Dense Retrieval DEMO-008 Reranker + Source Result DEMO-009 Document/Search REST API diff --git a/docs/development/OCEANBASE_VECTOR_SETUP.md b/docs/development/OCEANBASE_VECTOR_SETUP.md new file mode 100644 index 0000000..8d52672 --- /dev/null +++ b/docs/development/OCEANBASE_VECTOR_SETUP.md @@ -0,0 +1,72 @@ +# ARCH-003 OceanBase Vector 配置与索引契约 + +ARCH-003 在既有 `VectorStore` 契约后新增 OceanBase Vector Adapter。PostgreSQL 仍保存 +canonical Document、Chunk、Index Profile 和 Chunk Index State;OceanBase 表只保存可由这些 +数据重建的检索行。运行时默认切换和 Mochow 配置清理留在 ARCH-005。 + +## 数据库前提 + +- OceanBase MySQL 租户版本至少为 4.6.0:1024 维 Cosine HNSW 本身可由更早版本支持, + 但本项目要求使用 `VECTOR_INDEX(... pre_filter)` 显式保证 Profile 在 ANN 前过滤; +- 目标数据库和最小权限账号由受控部署流程预创建; +- Vector 内存、资源规格和 HNSW 参数由 DBA 在受控环境确认; +- 应用启动和请求路径不会创建、修改或删除表结构。 + +使用 [001_knowledge_chunks_v1.sql](../../deploy/oceanbase/001_knowledge_chunks_v1.sql) +在专用数据库中创建索引表。脚本固定: + +```text +VECTOR(1024) +DISTANCE = COSINE +TYPE = HNSW +EF_CONSTRUCTION = 256 +EF_SEARCH = 200 +``` + +表的 `index_record_id` 是确定性的 Chunk/Profile 主键;`document_id + index_profile_id` 和 +`index_profile_id` 是清理与 Profile 过滤的标量索引。表名或向量维度变化必须建立新表或 +Shadow Profile,验证后再激活,禁止在生产索引上原地修改。 + +## 运行配置 + +配置只能由部署环境或 Secret Manager 注入,禁止把真实地址、用户名和密码写入 `.env`、 +镜像、日志、测试快照或提交记录。 + +```text +VECTOR_STORE_PROVIDER=oceanbase +OCEANBASE_VECTOR_URI= +OCEANBASE_VECTOR_USER=<运行时注入> +OCEANBASE_VECTOR_PASSWORD=<运行时注入> +OCEANBASE_VECTOR_DATABASE=<预创建数据库> +OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1 +OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding +OCEANBASE_VECTOR_BATCH_SIZE=100 +OCEANBASE_VECTOR_TIMEOUT_SECONDS=30 +``` + +- URI 只接受 `host:port`,不得包含 Scheme、用户信息、Path、Query 或 Fragment; +- Database/Table/Index 只接受安全标识符; +- Password 使用 `SecretStr`,Provider 异常只返回收敛后的错误,不回显 SQL、连接信息、 + Chunk 内容或 SDK Response; +- `pyobvector` 只存在于 Adapter/Factory 内,业务服务不依赖其 Row、Engine 或距离函数类型。 + +## 写入、删除和检索语义 + +- Upsert 使用 `index_record_id` 主键,按可配置批次写入; +- 删除使用参数化的 `document_id + index_profile_id` 条件,不拼接用户输入; +- Dense Search 在同一 ANN 查询中使用 `VECTOR_INDEX(... pre_filter)`,保证 + `index_profile_id` 在候选近邻搜索前过滤,并返回 Dense Top20;禁止依赖普通 `WHERE` + 的自适应过滤语义,也禁止全局 TopK 后再过滤; +- OceanBase `cosine_distance` 是低值优先距离,Adapter 统一转换为 + `score = 1 - cosine_distance`,保持 `VectorStore` 的高分优先契约; +- 返回 UUID、Profile、字段数、距离范围和排序不满足契约时失败关闭,不生成 Source Result。 + +默认测试使用 Fake Client,不连接 OceanBase。真实版本、DDL、权限、Cosine HNSW 召回、过滤 +行为与 30~50 条脱敏 Golden Query 统一在 ARCH-006 的受控资源中验证,并且只允许使用轮换 +后的凭据。 + +官方参考: + +- [pyobvector 官方仓库](https://github.com/oceanbase/pyobvector) +- [OceanBase 创建向量索引](https://en.oceanbase.com/docs/common-oceanbase-database-10000000003075357) +- [OceanBase 向量索引查询 Hint](https://en.oceanbase.com/docs/common-oceanbase-database-10000000004311163) diff --git a/docs/plans/PROGRESS.md b/docs/plans/PROGRESS.md index d543a78..e04b63a 100644 --- a/docs/plans/PROGRESS.md +++ b/docs/plans/PROGRESS.md @@ -2,11 +2,11 @@ 当前阶段:Provider 架构迁移 -已完成:DEMO-001~DEMO-009、ARCH-001、ARCH-002 +已完成:DEMO-001~DEMO-009、ARCH-001、ARCH-002、ARCH-003 进行中:无 阻塞:无 -下一任务:ARCH-003 +下一任务:ARCH-004 Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006` 执行;每个 Task 验证后独立推送并合并 `main`。全部迁移完成后再继续 DEMO-010。 @@ -154,3 +154,17 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR Prefix 和超时均由环境注入。Local 仍是测试/开发及 ARCH-005 前的默认实现。 - 新增完全离线的 BOS Adapter、配置、Provider wiring 测试和中文配置说明;未使用内部配置 清单中的真实地址或凭据,真实 BOS 联调留在 ARCH-006。 + +### ARCH-003 + +- 新增官方 `pyobvector` 驱动的 `OceanBaseVectorStore`,覆盖批量 Upsert、按 Document/Profile + 参数化删除和 Profile 条件下推的 Cosine ANN TopN;SDK 类型不进入业务服务。检索固定使用 + OceanBase 4.6 的 `VECTOR_INDEX(... pre_filter)`,不依赖普通 `WHERE` 的自适应过滤,也不做 + 全局 TopK 后过滤。 +- 将低值优先的 `cosine_distance` 转为 `score = 1 - distance`,并对向量维度、有限值、 + 非零范数、返回字段、UUID、Profile、距离范围和排序执行失败关闭校验。 +- 新增 `VECTOR(1024)`、Cosine HNSW、Profile/Document 标量索引的预置 DDL;应用不自动执行 + 外部索引 DDL,PostgreSQL 继续保存 canonical 数据,OceanBase 表可重建。 +- 新增 `mochow|oceanbase` 类型化配置、SecretStr 密码和运行时工厂;Mochow 保持 ARCH-005 + 前的默认实现。全部 Adapter、配置、Schema 和 wiring 测试均离线运行,真实数据库验收留在 + ARCH-006。 diff --git a/docs/plans/TASKS.md b/docs/plans/TASKS.md index bb1d81b..b12fffa 100644 --- a/docs/plans/TASKS.md +++ b/docs/plans/TASKS.md @@ -4,7 +4,7 @@ - [x] ARCH-001 Architecture decision, PRD, Scope and execution baseline - [x] ARCH-002 BOS ObjectStorage adapter and runtime configuration -- [ ] ARCH-003 OceanBase VectorStore adapter and index schema +- [x] ARCH-003 OceanBase VectorStore adapter and index schema - [ ] ARCH-004 Qianfan `bce-reranker-base` alignment - [ ] ARCH-005 Runtime cutover, shadow rebuild workflow and legacy configuration cleanup - [ ] ARCH-006 Controlled live integration and RAG acceptance diff --git a/packages/core/src/ekp_core/config.py b/packages/core/src/ekp_core/config.py index 7d23c72..2120a1d 100644 --- a/packages/core/src/ekp_core/config.py +++ b/packages/core/src/ekp_core/config.py @@ -42,7 +42,7 @@ class Settings(BaseSettings): embedding_endpoint: str = "https://qianfan.baidubce.com/v2/embeddings" baidu_qianfan_api_key: SecretStr | None = None embedding_timeout_seconds: float = Field(default=30.0, gt=0) - vector_store_provider: Literal["mochow"] = "mochow" + vector_store_provider: Literal["mochow", "oceanbase"] = "mochow" baidu_mochow_account: str = "root" baidu_mochow_api_key: SecretStr | None = None baidu_mochow_endpoint: str | None = None @@ -51,6 +51,14 @@ class Settings(BaseSettings): mochow_batch_size: int = Field(default=100, ge=1, le=1000) mochow_search_ef: int = Field(default=200, ge=1, le=10000) mochow_timeout_seconds: float = Field(default=30.0, gt=0) + oceanbase_vector_uri: str | None = None + oceanbase_vector_user: str | None = None + oceanbase_vector_password: SecretStr | None = None + oceanbase_vector_database: str | None = None + oceanbase_vector_table: str = "knowledge_chunks_v1" + oceanbase_vector_index: str = "vidx_knowledge_chunks_embedding" + oceanbase_vector_batch_size: int = Field(default=100, ge=1, le=1000) + oceanbase_vector_timeout_seconds: float = Field(default=30.0, gt=0) index_profile_id: str = "demo-bge-large-zh-structure-v1" dense_top_n: int = Field(default=20, ge=1, le=64) reranker_provider: Literal["qianfan"] = "qianfan" diff --git a/packages/core/src/ekp_core/document_indexing.py b/packages/core/src/ekp_core/document_indexing.py index fc22d03..2d8100b 100644 --- a/packages/core/src/ekp_core/document_indexing.py +++ b/packages/core/src/ekp_core/document_indexing.py @@ -68,7 +68,7 @@ class DocumentIndexingService: self._index_profile = index_profile def process(self, document_id: UUID) -> DocumentIndexingResult: - """Index all chunks for one profile, compensating any partial Mochow write.""" + """Index all chunks for one profile, compensating any partial provider write.""" chunks = self._start_indexing(document_id) upsert_attempted = False try: diff --git a/packages/providers/vector_store/src/ekp_vector_store/__init__.py b/packages/providers/vector_store/src/ekp_vector_store/__init__.py index ba3b95f..ab3df9f 100644 --- a/packages/providers/vector_store/src/ekp_vector_store/__init__.py +++ b/packages/providers/vector_store/src/ekp_vector_store/__init__.py @@ -12,6 +12,12 @@ from ekp_vector_store.mochow import ( MochowVectorStoreError, create_mochow_vector_store, ) +from ekp_vector_store.oceanbase import ( + OceanBaseConfigurationError, + OceanBaseVectorStore, + OceanBaseVectorStoreError, + create_oceanbase_vector_store, +) __all__ = [ "FakeVectorStore", @@ -19,7 +25,11 @@ __all__ = [ "DenseSearchRequest", "MochowVectorStore", "MochowVectorStoreError", + "OceanBaseConfigurationError", + "OceanBaseVectorStore", + "OceanBaseVectorStoreError", "VectorRecord", "VectorStore", "create_mochow_vector_store", + "create_oceanbase_vector_store", ] diff --git a/packages/providers/vector_store/src/ekp_vector_store/oceanbase.py b/packages/providers/vector_store/src/ekp_vector_store/oceanbase.py new file mode 100644 index 0000000..c140da9 --- /dev/null +++ b/packages/providers/vector_store/src/ekp_vector_store/oceanbase.py @@ -0,0 +1,403 @@ +"""OceanBase Vector adapter isolated behind the provider-neutral VectorStore contract.""" + +import re +from collections.abc import Iterable +from math import isfinite +from numbers import Real +from typing import Any, Protocol +from urllib.parse import urlsplit +from uuid import UUID + +from sqlalchemy import and_, column, text + +from ekp_vector_store.base import DenseSearchCandidate, DenseSearchRequest, VectorRecord + +OCEANBASE_IDENTIFIER = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,63}$") +PROFILE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$") +SEARCH_OUTPUT_FIELDS = [ + "index_record_id", + "chunk_id", + "document_id", + "filename", + "page", + "page_end", + "section", + "chunk_index", + "content", + "index_profile_id", +] + + +class OceanBaseConfigurationError(ValueError): + """Raised when OceanBase connection or index settings are invalid.""" + + +class OceanBaseVectorStoreError(RuntimeError): + """Sanitized OceanBase failure without SQL, endpoints, content, or credentials.""" + + +class _OceanBaseVectorClient(Protocol): + """Narrow pyobvector client surface used by the EKP adapter.""" + + def upsert(self, table_name: str, data: list[dict[str, object]]) -> object: + """Insert or replace a row batch by primary key.""" + + def delete( + self, + table_name: str, + *, + where_clause: list[object], + ) -> object: + """Delete rows matching bound SQLAlchemy predicates.""" + + def dense_search( + self, + table_name: str, + *, + index_name: str, + query_vector: tuple[float, ...], + index_profile_id: str, + top_n: int, + output_column_names: list[str], + ) -> Iterable[object]: + """Return explicitly pre-filtered ANN rows followed by their distance.""" + + +class _PyObVectorClient: + """Keep SDK operations and the required pre-filter SQL inside the adapter.""" + + def __init__(self, sdk_client: Any) -> None: + self._sdk_client = sdk_client + + def upsert(self, table_name: str, data: list[dict[str, object]]) -> object: + """Delegate bounded row writes to the official client.""" + result: object = self._sdk_client.upsert(table_name, data) + return result + + def delete( + self, + table_name: str, + *, + where_clause: list[object], + ) -> object: + """Delegate parameterized cleanup to the official client.""" + result: object = self._sdk_client.delete(table_name, where_clause=where_clause) + return result + + def dense_search( + self, + table_name: str, + *, + index_name: str, + query_vector: tuple[float, ...], + index_profile_id: str, + top_n: int, + output_column_names: list[str], + ) -> Iterable[object]: + """Execute cosine ANN with an explicit OceanBase pre-filter hint.""" + statement = text( + _build_dense_search_sql( + table_name=table_name, + index_name=index_name, + output_column_names=output_column_names, + ) + ) + parameters = { + "query_vector": _serialize_vector(query_vector), + "index_profile_id": index_profile_id, + "top_n": top_n, + } + # Materialize rows before closing the connection; no SDK Result leaks upward. + with self._sdk_client.engine.connect() as connection: + rows: list[object] = list(connection.execute(statement, parameters)) + return rows + + +class OceanBaseVectorStore: + """Translate neutral vector records into one rebuildable OceanBase index table.""" + + def __init__( + self, + client: _OceanBaseVectorClient, + table_name: str, + *, + index_name: str = "vidx_knowledge_chunks_embedding", + vector_dimension: int = 1024, + batch_size: int = 100, + ) -> None: + if not OCEANBASE_IDENTIFIER.fullmatch(table_name): + raise OceanBaseConfigurationError("invalid OceanBase vector table name") + if not OCEANBASE_IDENTIFIER.fullmatch(index_name): + raise OceanBaseConfigurationError("invalid OceanBase vector index name") + if vector_dimension <= 0: + raise OceanBaseConfigurationError("OceanBase vector dimension must be positive") + if not 1 <= batch_size <= 1000: + raise OceanBaseConfigurationError( + "OceanBase vector batch_size must be between 1 and 1000" + ) + self._client = client + self._table_name = table_name + self._index_name = index_name + self._vector_dimension = vector_dimension + self._batch_size = batch_size + + def upsert(self, records: tuple[VectorRecord, ...]) -> None: + """Validate and upsert records in bounded pyobvector batches.""" + rows = [self._to_row(record) for record in records] + for start in range(0, len(rows), self._batch_size): + try: + self._client.upsert( + self._table_name, + rows[start : start + self._batch_size], + ) + except Exception as exc: + raise OceanBaseVectorStoreError("OceanBase vector upsert failed") from exc + + def delete_document(self, document_id: UUID, index_profile_id: str) -> None: + """Delete only one document/profile using bound scalar predicates.""" + _validate_profile_id(index_profile_id) + predicate = and_( + column("document_id") == str(document_id), + column("index_profile_id") == index_profile_id, + ) + try: + self._client.delete( + self._table_name, + where_clause=[predicate], + ) + except Exception as exc: + raise OceanBaseVectorStoreError("OceanBase vector cleanup failed") from exc + + def dense_search(self, request: DenseSearchRequest) -> tuple[DenseSearchCandidate, ...]: + """Run profile-scoped cosine ANN and convert distance to high-is-better score.""" + _validate_profile_id(request.index_profile_id) + self._validate_vector(request.query_vector) + + try: + rows = self._client.dense_search( + self._table_name, + index_name=self._index_name, + query_vector=request.query_vector, + index_profile_id=request.index_profile_id, + top_n=request.top_n, + output_column_names=list(SEARCH_OUTPUT_FIELDS), + ) + candidates = tuple( + _to_candidate(row, expected_profile=request.index_profile_id) for row in rows + ) + if len(candidates) > request.top_n: + raise ValueError("OceanBase returned more rows than requested") + if any( + earlier.score < later.score + for earlier, later in zip(candidates, candidates[1:], strict=False) + ): + raise ValueError("OceanBase ANN rows are not ordered by ascending distance") + return candidates + except Exception as exc: + raise OceanBaseVectorStoreError("OceanBase dense search failed") from exc + + def _to_row(self, record: VectorRecord) -> dict[str, object]: + """Map one rebuildable row to the pre-provisioned OceanBase schema.""" + _validate_profile_id(record.index_profile_id) + self._validate_vector(record.embedding) + return { + "index_record_id": str(record.index_record_id), + "chunk_id": str(record.chunk_id), + "document_id": str(record.document_id), + "filename": record.filename, + "page": record.page, + "page_end": record.page_end, + "section": record.section, + "chunk_index": record.chunk_index, + "content": record.content, + "embedding": list(record.embedding), + "embedding_model": record.embedding_model, + "index_profile_id": record.index_profile_id, + } + + def _validate_vector(self, vector: tuple[float, ...]) -> None: + """Reject dimensions and values that cannot satisfy the cosine index contract.""" + if len(vector) != self._vector_dimension: + raise OceanBaseConfigurationError("OceanBase vector dimension mismatch") + if any(not isfinite(value) for value in vector): + raise OceanBaseConfigurationError("OceanBase vector values must be finite") + if not any(value != 0 for value in vector): + raise OceanBaseConfigurationError("OceanBase cosine vector norm must be positive") + + +def create_oceanbase_vector_store( + *, + uri: str, + user: str, + password: str, + database_name: str, + table_name: str, + index_name: str = "vidx_knowledge_chunks_embedding", + vector_dimension: int = 1024, + batch_size: int = 100, + timeout_seconds: float = 30.0, +) -> OceanBaseVectorStore: + """Create the official pyobvector client from runtime-injected settings.""" + _validate_uri(uri) + if not user or not password: + raise OceanBaseConfigurationError("OceanBase vector user and password are required") + if not OCEANBASE_IDENTIFIER.fullmatch(database_name): + raise OceanBaseConfigurationError("invalid OceanBase vector database name") + if timeout_seconds <= 0: + raise OceanBaseConfigurationError("OceanBase timeout_seconds must be positive") + + try: + from pyobvector import ObVecClient # type: ignore[import-untyped] + + timeout = max(1, int(timeout_seconds)) + sdk_client = ObVecClient( + uri=uri, + user=user, + password=password, + db_name=database_name, + pool_pre_ping=True, + connect_args={ + "connect_timeout": timeout, + "read_timeout": timeout, + "write_timeout": timeout, + }, + ) + except Exception as exc: + raise OceanBaseVectorStoreError("OceanBase vector client initialization failed") from exc + _validate_oceanbase_version(sdk_client.ob_version.version_nums) + return OceanBaseVectorStore( + _PyObVectorClient(sdk_client), + table_name, + index_name=index_name, + vector_dimension=vector_dimension, + batch_size=batch_size, + ) + + +def _to_candidate(row: object, *, expected_profile: str) -> DenseSearchCandidate: + """Validate one pyobvector result tuple and normalize cosine distance to score.""" + if isinstance(row, (str, bytes, dict)) or not isinstance(row, Iterable): + raise TypeError("OceanBase ANN row must be tuple-like") + values = tuple(row) + if len(values) != len(SEARCH_OUTPUT_FIELDS) + 1: + raise ValueError("OceanBase ANN row has an unexpected field count") + + profile = _required_str(values[9], "index_profile_id") + if profile != expected_profile: + raise ValueError("OceanBase returned a row from another index profile") + distance = values[10] + if isinstance(distance, bool) or not isinstance(distance, Real): + raise ValueError("OceanBase cosine distance must be numeric") + numeric_distance = float(distance) + if not isfinite(numeric_distance) or not -1e-6 <= numeric_distance <= 2.000001: + raise ValueError("OceanBase cosine distance is outside the supported range") + score = max(-1.0, min(1.0, 1.0 - numeric_distance)) + + return DenseSearchCandidate( + index_record_id=UUID(_required_str(values[0], "index_record_id")), + chunk_id=UUID(_required_str(values[1], "chunk_id")), + document_id=UUID(_required_str(values[2], "document_id")), + filename=_required_str(values[3], "filename"), + page=_optional_int(values[4]), + page_end=_optional_int(values[5]), + section=_optional_str(values[6]), + chunk_index=_required_int(values[7], "chunk_index"), + content=_required_str(values[8], "content"), + score=score, + index_profile_id=profile, + ) + + +def _validate_profile_id(profile_id: str) -> None: + """Apply the canonical IndexProfile identifier subset at the provider edge.""" + if not PROFILE_ID.fullmatch(profile_id): + raise OceanBaseConfigurationError("invalid OceanBase index profile ID") + + +def _validate_uri(uri: str) -> None: + """Accept only a host:port target without credentials, paths, or query data.""" + if "://" in uri: + raise OceanBaseConfigurationError("invalid OceanBase vector URI") + try: + parsed = urlsplit(f"//{uri}") + port = parsed.port + except ValueError as exc: + raise OceanBaseConfigurationError("invalid OceanBase vector URI") from exc + if ( + not parsed.hostname + or port is None + or parsed.username is not None + or parsed.password is not None + or parsed.path + or parsed.query + or parsed.fragment + ): + raise OceanBaseConfigurationError("invalid OceanBase vector URI") + + +def _validate_oceanbase_version(version_nums: object) -> None: + """Require the first release with an explicit vector pre-filter hint.""" + if ( + not isinstance(version_nums, (list, tuple)) + or len(version_nums) != 4 + or any(isinstance(part, bool) or not isinstance(part, int) for part in version_nums) + ): + raise OceanBaseVectorStoreError("OceanBase version validation failed") + if tuple(version_nums) < (4, 6, 0, 0): + raise OceanBaseConfigurationError( + "OceanBase 4.6.0 or newer is required for vector pre-filtering" + ) + + +def _build_dense_search_sql( + *, + table_name: str, + index_name: str, + output_column_names: list[str], +) -> str: + """Build a fixed-identifier query whose values remain bind parameters.""" + identifiers = [table_name, index_name, *output_column_names] + if any(not OCEANBASE_IDENTIFIER.fullmatch(value) for value in identifiers): + raise OceanBaseConfigurationError("invalid OceanBase dense-search identifier") + fields = ", ".join(output_column_names) + return ( + f"SELECT /*+ VECTOR_INDEX({table_name} {index_name} pre_filter) */ " + f"{fields}, cosine_distance(embedding, :query_vector) AS distance " + f"FROM {table_name} " + "WHERE index_profile_id = :index_profile_id " + "ORDER BY cosine_distance(embedding, :query_vector) APPROXIMATE " + "LIMIT :top_n" + ) + + +def _serialize_vector(vector: tuple[float, ...]) -> str: + """Serialize already-validated finite floats for OceanBase's VECTOR parameter.""" + return "[" + ",".join(format(value, ".17g") for value in vector) + "]" + + +def _required_str(value: object, name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"OceanBase field {name} must be non-empty text") + return value + + +def _required_int(value: object, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"OceanBase field {name} must be an integer") + return value + + +def _optional_int(value: object) -> int | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError("OceanBase nullable integer metadata must be an integer") + return value + + +def _optional_str(value: object) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ValueError("OceanBase nullable text metadata must be text") + return value diff --git a/pyproject.toml b/pyproject.toml index 7bfb059..a0883af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "python-docx>=1.2,<2", "sqlalchemy>=2.0,<3", "uvicorn[standard]>=0.34,<1", + "pyobvector>=0.2.28,<0.3", ] [dependency-groups] diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index a7d9698..9405caf 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -36,6 +36,14 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non assert settings.mochow_table == "knowledge_chunks" assert settings.mochow_search_ef == 200 assert settings.mochow_timeout_seconds == 30 + assert settings.oceanbase_vector_uri is None + assert settings.oceanbase_vector_user is None + assert settings.oceanbase_vector_password is None + assert settings.oceanbase_vector_database is None + assert settings.oceanbase_vector_table == "knowledge_chunks_v1" + assert settings.oceanbase_vector_index == "vidx_knowledge_chunks_embedding" + assert settings.oceanbase_vector_batch_size == 100 + assert settings.oceanbase_vector_timeout_seconds == 30 assert settings.index_profile_id == "demo-bge-large-zh-structure-v1" assert settings.dense_top_n == 20 assert settings.reranker_provider == "qianfan" @@ -70,6 +78,7 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) - baidu_mochow_api_key=SecretStr("mochow-secret"), bos_access_key_id=SecretStr("bos-access-key"), bos_secret_access_key=SecretStr("bos-secret-key"), + oceanbase_vector_password=SecretStr("oceanbase-secret"), reranker_api_key=SecretStr("reranker-secret"), ) @@ -78,4 +87,5 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) - assert "mochow-secret" not in rendered assert "bos-access-key" not in rendered assert "bos-secret-key" not in rendered + assert "oceanbase-secret" not in rendered assert "reranker-secret" not in rendered diff --git a/tests/unit/test_oceanbase_vector_store.py b/tests/unit/test_oceanbase_vector_store.py new file mode 100644 index 0000000..53d55eb --- /dev/null +++ b/tests/unit/test_oceanbase_vector_store.py @@ -0,0 +1,310 @@ +"""Offline tests for neutral-record translation into pyobvector operations.""" + +from dataclasses import replace +from pathlib import Path +from typing import Any, cast +from uuid import UUID, uuid4 + +import pytest + +from ekp_vector_store import DenseSearchRequest, OceanBaseVectorStore, VectorRecord +from ekp_vector_store.oceanbase import ( + OceanBaseConfigurationError, + OceanBaseVectorStoreError, + _build_dense_search_sql, + _PyObVectorClient, + _serialize_vector, + _validate_oceanbase_version, +) + + +class FakeOceanBaseClient: + """Capture ObVecClient-shaped calls without a database connection.""" + + def __init__(self) -> None: + self.upsert_calls: list[tuple[str, list[dict[str, object]]]] = [] + self.delete_calls: list[tuple[str, list[object]]] = [] + self.search_calls: list[dict[str, object]] = [] + self.search_rows: list[tuple[object, ...]] = [] + self.failure: Exception | None = None + + def upsert(self, table_name: str, data: list[dict[str, object]]) -> object: + self._raise_failure() + self.upsert_calls.append((table_name, data)) + return object() + + def delete( + self, + table_name: str, + *, + where_clause: list[object], + ) -> object: + self._raise_failure() + self.delete_calls.append((table_name, where_clause)) + return object() + + def dense_search( + self, + table_name: str, + *, + index_name: str, + query_vector: tuple[float, ...], + index_profile_id: str, + top_n: int, + output_column_names: list[str], + ) -> list[tuple[object, ...]]: + self._raise_failure() + self.search_calls.append( + { + "table_name": table_name, + "index_name": index_name, + "query_vector": query_vector, + "index_profile_id": index_profile_id, + "top_n": top_n, + "output_column_names": output_column_names, + } + ) + return self.search_rows + + def _raise_failure(self) -> None: + if self.failure is not None: + raise self.failure + + +class FakeConnection: + """Capture SQLAlchemy text and bound values without executing vendor SQL.""" + + def __init__(self) -> None: + self.executions: list[tuple[str, dict[str, object]]] = [] + + def __enter__(self) -> "FakeConnection": + return self + + def __exit__(self, *args: object) -> None: + return None + + def execute(self, statement: object, parameters: dict[str, object]) -> list[object]: + self.executions.append((str(statement), parameters)) + return [] + + +class FakeEngine: + """Return one inspectable fake connection.""" + + def __init__(self) -> None: + self.connection = FakeConnection() + + def connect(self) -> FakeConnection: + return self.connection + + +class FakeSdkClient: + """Provide the official client's engine shape for the SQL boundary test.""" + + def __init__(self) -> None: + self.engine = FakeEngine() + + +def _record(document_id: UUID, chunk_index: int) -> VectorRecord: + return VectorRecord( + index_record_id=uuid4(), + chunk_id=uuid4(), + document_id=document_id, + filename="policy.md", + page=1, + page_end=2, + section="产品 / 退款", + chunk_index=chunk_index, + content="退款周期为七天。", + embedding=(0.1, 0.2, 0.3), + embedding_model="fake", + index_profile_id="profile-v1", + ) + + +def _search_row(record: VectorRecord, distance: float) -> tuple[object, ...]: + return ( + str(record.index_record_id), + str(record.chunk_id), + str(record.document_id), + record.filename, + record.page, + record.page_end, + record.section, + record.chunk_index, + record.content, + record.index_profile_id, + distance, + ) + + +def test_oceanbase_adapter_maps_and_batches_upserts() -> None: + client = FakeOceanBaseClient() + store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3, batch_size=2) + document_id = uuid4() + records = tuple(_record(document_id, index) for index in range(3)) + + store.upsert(records) + + assert [len(data) for _, data in client.upsert_calls] == [2, 1] + first = client.upsert_calls[0][1][0] + assert first["index_record_id"] == str(records[0].index_record_id) + assert first["document_id"] == str(document_id) + assert first["embedding"] == [0.1, 0.2, 0.3] + assert first["index_profile_id"] == "profile-v1" + + +def test_oceanbase_delete_uses_bound_document_and_profile_predicate() -> None: + client = FakeOceanBaseClient() + store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3) + document_id = uuid4() + + store.delete_document(document_id, "profile-v1") + + table_name, clauses = client.delete_calls[0] + compiled = cast(Any, clauses[0]).compile() + assert table_name == "knowledge_chunks_v1" + assert ( + str(compiled) == "document_id = :document_id_1 AND index_profile_id = :index_profile_id_1" + ) + assert set(compiled.params.values()) == {str(document_id), "profile-v1"} + + +def test_oceanbase_dense_search_filters_profile_and_normalizes_cosine_distance() -> None: + client = FakeOceanBaseClient() + first = _record(uuid4(), 0) + second = _record(uuid4(), 1) + client.search_rows = [_search_row(first, 0.1), _search_row(second, 0.4)] + store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3) + + candidates = store.dense_search( + DenseSearchRequest((0.3, 0.2, 0.1), index_profile_id="profile-v1", top_n=20) + ) + + call = client.search_calls[0] + assert call["table_name"] == "knowledge_chunks_v1" + assert call["index_name"] == "vidx_knowledge_chunks_embedding" + assert call["query_vector"] == (0.3, 0.2, 0.1) + assert call["index_profile_id"] == "profile-v1" + assert call["top_n"] == 20 + assert [candidate.score for candidate in candidates] == pytest.approx([0.9, 0.6]) + assert candidates[0].chunk_id == first.chunk_id + + +def test_oceanbase_rejects_wrong_dimension_and_zero_cosine_vector() -> None: + store = OceanBaseVectorStore( + FakeOceanBaseClient(), + "knowledge_chunks_v1", + vector_dimension=3, + ) + + with pytest.raises(OceanBaseConfigurationError, match="dimension mismatch"): + store.dense_search(DenseSearchRequest((0.1, 0.2), "profile-v1")) + with pytest.raises(OceanBaseConfigurationError, match="norm must be positive"): + store.upsert((replace(_record(uuid4(), 0), embedding=(0.0, 0.0, 0.0)),)) + + +def test_oceanbase_sanitizes_malformed_or_failed_search() -> None: + client = FakeOceanBaseClient() + client.search_rows = [("malformed",)] + store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3) + + with pytest.raises(OceanBaseVectorStoreError, match="dense search failed"): + store.dense_search(DenseSearchRequest((0.1, 0.2, 0.3), "profile-v1")) + + client.failure = RuntimeError("private-host password=secret-value") + with pytest.raises(OceanBaseVectorStoreError, match="cleanup failed") as caught: + store.delete_document(uuid4(), "profile-v1") + assert "private-host" not in str(caught.value) + assert "secret-value" not in str(caught.value) + + +@pytest.mark.parametrize( + "uri", + ["", "https://db.example:2881", "user:password@db.example:2881", "db.example"], +) +def test_oceanbase_factory_rejects_unsafe_uri_without_connecting(uri: str) -> None: + from ekp_vector_store.oceanbase import create_oceanbase_vector_store + + with pytest.raises(OceanBaseConfigurationError, match="URI"): + create_oceanbase_vector_store( + uri=uri, + user="test-user", + password="test-password", + database_name="test_database", + table_name="knowledge_chunks_v1", + ) + + +def test_oceanbase_dense_sql_requires_profile_pre_filter_and_bound_values() -> None: + sql = _build_dense_search_sql( + table_name="knowledge_chunks_v1", + index_name="vidx_knowledge_chunks_embedding", + output_column_names=["chunk_id", "index_profile_id"], + ) + + assert "VECTOR_INDEX(knowledge_chunks_v1 vidx_knowledge_chunks_embedding pre_filter)" in sql + assert "WHERE index_profile_id = :index_profile_id" in sql + assert "cosine_distance(embedding, :query_vector)" in sql + assert "APPROXIMATE LIMIT :top_n" in sql + assert "profile-v1" not in sql + assert ( + _serialize_vector((0.1, -0.2, 0.3)) + == "[0.10000000000000001,-0.20000000000000001,0.29999999999999999]" + ) + + +def test_oceanbase_rejects_unsafe_index_identifier() -> None: + with pytest.raises(OceanBaseConfigurationError, match="index name"): + OceanBaseVectorStore( + FakeOceanBaseClient(), + "knowledge_chunks_v1", + index_name="vector-index;drop", + vector_dimension=3, + ) + + +def test_oceanbase_requires_version_with_explicit_pre_filter() -> None: + with pytest.raises(OceanBaseConfigurationError, match="4.6.0"): + _validate_oceanbase_version([4, 3, 5, 1]) + + _validate_oceanbase_version([4, 6, 0, 0]) + + with pytest.raises(OceanBaseVectorStoreError, match="version validation"): + _validate_oceanbase_version("4.6.0.0") + + +def test_pyobvector_boundary_binds_profile_vector_and_limit() -> None: + sdk_client = FakeSdkClient() + client = _PyObVectorClient(sdk_client) + + rows = client.dense_search( + "knowledge_chunks_v1", + index_name="vidx_knowledge_chunks_embedding", + query_vector=(0.1, 0.2, 0.3), + index_profile_id="profile-v1", + top_n=20, + output_column_names=["chunk_id", "index_profile_id"], + ) + + assert list(rows) == [] + sql, parameters = sdk_client.engine.connection.executions[0] + assert "pre_filter" in sql + assert parameters == { + "query_vector": "[0.10000000000000001,0.20000000000000001,0.29999999999999999]", + "index_profile_id": "profile-v1", + "top_n": 20, + } + assert "profile-v1" not in sql + + +def test_oceanbase_schema_declares_rebuildable_cosine_index() -> None: + schema_path = Path(__file__).parents[2] / "deploy" / "oceanbase" / "001_knowledge_chunks_v1.sql" + + schema = schema_path.read_text(encoding="utf-8") + + assert "embedding VECTOR(1024) NOT NULL" in schema + assert "index_profile_id VARCHAR(128) NOT NULL" in schema + assert "DISTANCE = COSINE" in schema + assert "TYPE = HNSW" in schema + assert "document_id, index_profile_id" in schema diff --git a/tests/unit/test_provider_wiring.py b/tests/unit/test_provider_wiring.py index 1ee607c..84bb53d 100644 --- a/tests/unit/test_provider_wiring.py +++ b/tests/unit/test_provider_wiring.py @@ -42,6 +42,44 @@ def test_provider_wiring_requires_explicit_mochow_target( create_vector_store(settings) +def test_provider_wiring_requires_complete_oceanbase_target( + isolated_settings_environment: None, +) -> None: + settings = Settings(vector_store_provider="oceanbase") + + with pytest.raises(ProviderConfigurationError, match="OCEANBASE_VECTOR_URI"): + create_vector_store(settings) + + +def test_provider_wiring_builds_oceanbase_from_secret_safe_settings( + monkeypatch: pytest.MonkeyPatch, + isolated_settings_environment: None, +) -> None: + captured: dict[str, object] = {} + expected = FakeVectorStore() + + def fake_factory(**kwargs: object) -> FakeVectorStore: + captured.update(kwargs) + return expected + + monkeypatch.setattr("ekp_api.providers.create_oceanbase_vector_store", fake_factory) + settings = Settings( + vector_store_provider="oceanbase", + oceanbase_vector_uri="db.example:2881", + oceanbase_vector_user="test-user@test-tenant", + oceanbase_vector_password=SecretStr("test-password"), + oceanbase_vector_database="test_database", + ) + + vector_store = create_vector_store(settings) + + assert vector_store is expected + assert captured["uri"] == "db.example:2881" + assert captured["password"] == "test-password" + assert captured["index_name"] == "vidx_knowledge_chunks_embedding" + assert captured["vector_dimension"] == 1024 + + def test_provider_wiring_selects_local_storage_by_default( isolated_settings_environment: None, ) -> None: diff --git a/uv.lock b/uv.lock index 887625d..78bf5b4 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,18 @@ version = 1 revision = 3 requires-python = ">=3.12, <3.14" +[[package]] +name = "aiomysql" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pymysql" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/e0/302aeffe8d90853556f47f3106b89c16cc2ec2a4d269bdfd82e3f4ae12cc/aiomysql-0.3.2.tar.gz", hash = "sha256:72d15ef5cfc34c03468eb41e1b90adb9fd9347b0b589114bd23ead569a02ac1a", size = 108311, upload-time = "2025-10-22T00:15:21.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/af/aae0153c3e28712adaf462328f6c7a3c196a1c1c27b491de4377dd3e6b52/aiomysql-0.3.2-py3-none-any.whl", hash = "sha256:c82c5ba04137d7afd5c693a258bea8ead2aad77101668044143a991e04632eb2", size = 71834, upload-time = "2025-10-22T00:15:15.905Z" }, +] + [[package]] name = "alembic" version = "1.19.1" @@ -230,6 +242,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "pymochow" }, { name = "pymupdf" }, + { name = "pyobvector" }, { name = "python-docx" }, { name = "python-multipart" }, { name = "sqlalchemy" }, @@ -254,6 +267,7 @@ requires-dist = [ { name = "pydantic-settings", specifier = ">=2.7,<3" }, { name = "pymochow", specifier = ">=2.4,<3" }, { name = "pymupdf", specifier = ">=1.26,<2" }, + { name = "pyobvector", specifier = ">=0.2.28,<0.3" }, { name = "python-docx", specifier = ">=1.2,<2" }, { name = "python-multipart", specifier = ">=0.0.20,<1" }, { name = "sqlalchemy", specifier = ">=2.0,<3" }, @@ -580,6 +594,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "numpy" +version = "2.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/80/db0b4559e57ec36362bedbb05530a87fafbcb6067708c946967a41d449e7/numpy-2.5.2.tar.gz", hash = "sha256:d482d171c406ae88c5b19cad3b6a1c4c5209f886ab74bc44c2c865c23f52d860", size = 20773161, upload-time = "2026-08-09T13:48:27.962Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/72/dccb0aaf40972777283303919f613964227266d0c13adebb79ac124f1c3e/numpy-2.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:14e373cfc6387177e8409dac3c7159be8eb05cd77096cd7c950268b86f62831c", size = 16891693, upload-time = "2026-08-09T13:44:51.702Z" }, + { url = "https://files.pythonhosted.org/packages/60/2e/b5aee50a1f74ac815cf8331812cb8251e29024025de462e0c047641c614c/numpy-2.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bbd96c833ecc8cc069ce518078fc8c60cb9cbfb0fea5b7a803ad65035596d03", size = 11903109, upload-time = "2026-08-09T13:44:55.501Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f4/29e78102a80601cf034d4e9767022cffeca2c3b4c926e1754572ca95593d/numpy-2.5.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:6e8172ddfcf5cf74b811d372b570b83c60bd2de87a6fbfbebdadb4a9bd9c6cbb", size = 5350202, upload-time = "2026-08-09T13:44:58.401Z" }, + { url = "https://files.pythonhosted.org/packages/11/4b/dcd3b7eadaf4035d2c7a4289d232523a6964f602598ef7674e4bd7291f93/numpy-2.5.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:65f188481f1669e26f62b701e8205d19e460fa4a9b52a1414ba382330e4a3414", size = 6687736, upload-time = "2026-08-09T13:45:00.813Z" }, + { url = "https://files.pythonhosted.org/packages/e5/21/4947e0e9d6c9fc2e2ff15b8949049ee44f63adb9cacc729ab8793f97e712/numpy-2.5.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ee9c4eeb8454b3660a8b53493563c3e121c2fc94fbd72b848ef814ed7b676a9", size = 15612696, upload-time = "2026-08-09T13:45:04.151Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5f/62d28cf019460c7f1394105b4d49d9911a9c444cb77ab0bd95a204c5a6de/numpy-2.5.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3cdec01fa790a186d430433fdd4d4ffb70eed6f0eeb4bf05c8dbe2dce0a9bcb8", size = 16722264, upload-time = "2026-08-09T13:45:07.714Z" }, + { url = "https://files.pythonhosted.org/packages/14/25/3f0be4c1b9fdf5dd5e708a6806978564d7c46a055c000496309ff2a2f8af/numpy-2.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7999d4ddb0c4025018373fd787510d46e04c769467af22869707b3c1cfd459ab", size = 16974396, upload-time = "2026-08-09T13:45:11.316Z" }, + { url = "https://files.pythonhosted.org/packages/22/72/6262cbdeeb45da9d971e40715f579d791603ba8ec0b5e2db1ac55454421d/numpy-2.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c1f017dc0875c9209d219f97feceb7d54c2661bb243deb4114478e1295808af7", size = 18476044, upload-time = "2026-08-09T13:45:14.869Z" }, + { url = "https://files.pythonhosted.org/packages/36/33/29208b8b075bde62d26a81d14b358c42b0f69b6cabd98d4ff97f37f22b05/numpy-2.5.2-cp312-cp312-win32.whl", hash = "sha256:d6a48072864e3324e194a8fbb3c657bcc5b5c869dbc64c9537b1d5c862572c0a", size = 6072817, upload-time = "2026-08-09T13:45:17.867Z" }, + { url = "https://files.pythonhosted.org/packages/7f/b9/87fea2769fe1c47c1b5b01d8310772c9d1a85d485de7cf386ef7a3332b02/numpy-2.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:28ac63476ec7651484215ee7fa15a1f78b57c14621f01e392afe17b9a1390ce4", size = 12464674, upload-time = "2026-08-09T13:45:20.734Z" }, + { url = "https://files.pythonhosted.org/packages/14/52/032b97e00461ab0809bbe4c588b035620e5a14b8cdee47ecddefc7b17d33/numpy-2.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:27650bb0e7140fa3d37b9923b4803645e0b125d190f326eecfd3f4dad8e8ade1", size = 10397131, upload-time = "2026-08-09T13:45:23.73Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d2/6b24738a0ef4557d189b150046cd07823c50e4273e8aebd651222e24306f/numpy-2.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8e4cb9a754c8a0c62eaa88273a5fba3391f4a610d1dee893c0755da31c083f15", size = 16886595, upload-time = "2026-08-09T13:45:27.323Z" }, + { url = "https://files.pythonhosted.org/packages/65/60/f2d208d366f263f39c6e69ed309290717aab41078b6d04c9be2a84fa2a07/numpy-2.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:52c808f96484f5571a5cc863775ce50247c17dfb3b0361f8ed6b4b0456f80080", size = 11896845, upload-time = "2026-08-09T13:45:31.638Z" }, + { url = "https://files.pythonhosted.org/packages/3c/79/81e0bf24f4d020a2b1d5cd297a9f60c3f24eeb116f9bba5870443f7b6a4a/numpy-2.5.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:29d81e97f668489cba8ebfd796b9bdd453525d35dd9e162e2daec94bf3fc7740", size = 5343880, upload-time = "2026-08-09T13:45:34.373Z" }, + { url = "https://files.pythonhosted.org/packages/ba/cc/e3141cf06d1a8a2c7e107543fe1269c1d1af760d4d683c0794a4ee1127c2/numpy-2.5.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:afb3f0632d6b2e3ba04dbce8d1e48d321b369138b73830b5ca371a0e8d479d56", size = 6682264, upload-time = "2026-08-09T13:45:36.7Z" }, + { url = "https://files.pythonhosted.org/packages/29/f1/2a64a307d92c5d98f5255a4014eb43bb6103ee477087b61ecae44a3aa9b9/numpy-2.5.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0aadf13b60048d501e05fa699efaf7734e2494f3498a4c2a5521d822640324f3", size = 15609566, upload-time = "2026-08-09T13:45:39.518Z" }, + { url = "https://files.pythonhosted.org/packages/7b/44/59a1eb68e773c4098d107ef34a0dbdeca501d72ffcfbff9a7707343921ce/numpy-2.5.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29b86ff8a6cc556b47ec6b64b194815cc80e6bf5eedcc6cddfd65318cb0b4eee", size = 16709995, upload-time = "2026-08-09T13:45:43.661Z" }, + { url = "https://files.pythonhosted.org/packages/8a/4c/3e54d4ddbc359a1295f8b633e8106bcd4d7d4a206e82df051bdfb3058755/numpy-2.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6950c4b7dd562453090548ba7f5da7e59f57f85663f15d5dcc60e249192f7e59", size = 16972511, upload-time = "2026-08-09T13:45:47.094Z" }, + { url = "https://files.pythonhosted.org/packages/f2/9f/02e371638ebf19b66d46231e4be52999e87f32d1961b113bc45656608b22/numpy-2.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b9727f472d2f3888053b8a75ab0cb94745a9de224bb5846dbadc0092101bc71d", size = 18465609, upload-time = "2026-08-09T13:45:50.808Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ae/ad6645abc7a3510fe48e8ea1ab4598166f500057ef4ebf38bfad4f1577de/numpy-2.5.2-cp313-cp313-win32.whl", hash = "sha256:4f9744f9fbdcea0bc552e8f19e1f141f811a3f9bc2be2cc6e86d982cab23e3f4", size = 6070204, upload-time = "2026-08-09T13:45:54.111Z" }, + { url = "https://files.pythonhosted.org/packages/15/20/f3489f86d81ea460b2bcdceaed094142ca6579f6be0ec527b781d39afe68/numpy-2.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:85aaccb24182c25df891ad0ec333585967e115269d5f1b17f2c9ae005bc96657", size = 12460532, upload-time = "2026-08-09T13:45:57.167Z" }, + { url = "https://files.pythonhosted.org/packages/d5/21/35b31dde1b283b79de828b80f876afd8c94e28fe1e9c375f89e261cc4c0d/numpy-2.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:bd68ece1553d2023c09a4226d9e41c586ad2d20594d1a456186c33513d2cb3f2", size = 10396725, upload-time = "2026-08-09T13:46:00.478Z" }, +] + [[package]] name = "orjson" version = "3.12.0" @@ -828,6 +872,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/8c/d897dcd32a25b58186c968b15ce4324ca029e9d96460de12325314e390be/pymupdf-1.28.2-cp313-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2e1b574c0fd2cb238021033fd3c0f9c4388816638df064e4bfb56d9d81736dc8", size = 18399403, upload-time = "2026-08-06T21:39:25.008Z" }, ] +[[package]] +name = "pymysql" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/bc/1c6a92f385940f727daeecf3bacaf186e03875dff57197801046c583bcf0/pymysql-1.2.0.tar.gz", hash = "sha256:6c7b17ca686988104d7426c27895b455cdeea3e9d3ceb1270f0c3704fead8c33", size = 49021, upload-time = "2026-05-19T08:26:22.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/bd/2534e130295c8cfd4f0a2e31623baab7502278f1e97bcfe61db75656a77f/pymysql-1.2.0-py3-none-any.whl", hash = "sha256:62169ce6d5510f08e140c5e7990ee884a9764024e4a9a27b2cc11f1099322ae0", size = 45716, upload-time = "2026-05-19T08:26:20.974Z" }, +] + +[[package]] +name = "pyobvector" +version = "0.2.29" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiomysql" }, + { name = "numpy" }, + { name = "pydantic" }, + { name = "pymysql" }, + { name = "sqlalchemy" }, + { name = "sqlglot" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/47/80f5f62572b00348d6864eb14d495cf26e52d7b9dba0ad7365cdbac86851/pyobvector-0.2.29.tar.gz", hash = "sha256:3f0feac0ac964bb56c2e3ed661b5de84ef5bc97253ae8e9935a65fea367e3736", size = 85589, upload-time = "2026-07-29T02:53:22.676Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/66/1aa4ca29ee2d1780aa4761d58ccac82771747a75f9146db0d7285a245bdb/pyobvector-0.2.29-py3-none-any.whl", hash = "sha256:f7cfdf1ca0c5c6ccbc10fd096aef944cd3970970f1382441a77dff1be78744cf", size = 70210, upload-time = "2026-07-29T02:53:21.307Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -993,6 +1063,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/3f/3582293d1e185e71d19d7c731c3e2ee20ba21981c4a1115c0806c1f62120/sqlalchemy-2.0.52-py3-none-any.whl", hash = "sha256:3b81b8363a919ce53453591cdb93702e6bd54ade6c4fa2f468fc053baee5ed89", size = 1950700, upload-time = "2026-08-11T20:47:21.603Z" }, ] +[[package]] +name = "sqlglot" +version = "30.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/d4/da49abcc81beebbb25f29ddf87f2980c63c949569d7f2da40c06d95fa415/sqlglot-30.17.0.tar.gz", hash = "sha256:2d6b8def93304fa300f4d20f48e3909e7f436fda56ca1fafd8975f6c561ef62c", size = 5999019, upload-time = "2026-08-12T19:36:50.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/bc/2a07cef49046e6cf5d1a8b1de553aaef8491b4170dbfe254c8687c764408/sqlglot-30.17.0-py3-none-any.whl", hash = "sha256:84435ac283a60173da31b5fd7d11a725037a1c3fd6ed1e21fb065de74ddb579f", size = 741795, upload-time = "2026-08-12T19:36:48.699Z" }, +] + [[package]] name = "starlette" version = "1.6.0"