feat: complete ARCH-005 runtime cutover
This commit is contained in:
parent
4ce74604ba
commit
df4b0ae91e
14
.env.example
14
.env.example
@ -9,7 +9,7 @@ DATABASE_URL=postgresql+psycopg://ekp:ekp@localhost:5432/ekp
|
||||
REDIS_URL=redis://localhost:6379/0
|
||||
RABBITMQ_URL=amqp://guest:guest@localhost:5672//
|
||||
|
||||
OBJECT_STORAGE_PROVIDER=local
|
||||
OBJECT_STORAGE_PROVIDER=bos
|
||||
OBJECT_STORAGE_PATH=storage
|
||||
BOS_ENDPOINT=
|
||||
BOS_ACCESS_KEY_ID=
|
||||
@ -22,15 +22,7 @@ CHUNK_MAX_TOKENS=450
|
||||
CHUNK_OVERLAP_TOKENS=75
|
||||
CHUNK_MAX_CHARACTERS=2000
|
||||
|
||||
VECTOR_STORE_PROVIDER=mochow
|
||||
BAIDU_MOCHOW_ACCOUNT=root
|
||||
BAIDU_MOCHOW_API_KEY=
|
||||
BAIDU_MOCHOW_ENDPOINT=
|
||||
KNOWLEDGE_VDB_DB_NAME=
|
||||
MOCHOW_TABLE=knowledge_chunks
|
||||
MOCHOW_BATCH_SIZE=100
|
||||
MOCHOW_SEARCH_EF=200
|
||||
MOCHOW_TIMEOUT_SECONDS=30
|
||||
VECTOR_STORE_PROVIDER=oceanbase
|
||||
OCEANBASE_VECTOR_URI=
|
||||
OCEANBASE_VECTOR_USER=
|
||||
OCEANBASE_VECTOR_PASSWORD=
|
||||
@ -45,7 +37,7 @@ EMBEDDING_MODEL=bge-large-zh
|
||||
EMBEDDING_ENDPOINT=https://qianfan.baidubce.com/v2/embeddings
|
||||
BAIDU_QIANFAN_API_KEY=
|
||||
EMBEDDING_TIMEOUT_SECONDS=30
|
||||
INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1
|
||||
INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1-oceanbase-v1
|
||||
DENSE_TOP_N=20
|
||||
|
||||
RERANKER_PROVIDER=qianfan
|
||||
|
||||
23
README.md
23
README.md
@ -83,26 +83,23 @@ Documents remain `processing` until embedding/indexing completes; pipeline
|
||||
failures become `failed` without deleting the immutable original. DEMO-009
|
||||
exposes this pipeline through the versioned upload endpoint.
|
||||
|
||||
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, a
|
||||
validated Qianfan `bge-large-zh` adapter, and the initial isolated `pymochow` adapter.
|
||||
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a
|
||||
validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase.
|
||||
Chunk vectors are upserted with source metadata under a versioned index profile;
|
||||
PostgreSQL records canonical index state while Mochow remains rebuildable. A
|
||||
PostgreSQL records canonical index state while OceanBase remains rebuildable. A
|
||||
document becomes `ready` only after every chunk is indexed. See
|
||||
`docs/development/MOCHOW_SETUP.md` for the required environment variables and
|
||||
pre-provisioned table schema.
|
||||
`docs/development/OCEANBASE_VECTOR_SETUP.md` for the required environment variables
|
||||
and pre-provisioned table schema.
|
||||
|
||||
The accepted provider migration is tracked by `ARCH-001` through `ARCH-006`:
|
||||
Baidu BOS, OceanBase Vector, and Qianfan `bce-reranker-base` replace the initial
|
||||
demo providers without changing PostgreSQL canonical ownership. Until the runtime
|
||||
cutover task is merged, the paragraphs above describe the current implementation.
|
||||
ARCH-002 adds the selectable BOS adapter and safe runtime configuration documented
|
||||
in `docs/development/BOS_SETUP.md`; local storage remains the default until ARCH-005.
|
||||
ARCH-003 adds the selectable OceanBase Vector adapter and pre-provisioned cosine
|
||||
HNSW schema documented in `docs/development/OCEANBASE_VECTOR_SETUP.md`; Mochow
|
||||
remains the runtime default until ARCH-005.
|
||||
demo providers without changing PostgreSQL canonical ownership. BOS and OceanBase
|
||||
are now the runtime defaults; LocalObjectStorage and FakeVectorStore remain available
|
||||
for explicit local/offline tests. ARCH-005 also adds the controlled shadow rebuild
|
||||
workflow documented in `docs/development/INDEX_REBUILD.md`.
|
||||
|
||||
DEMO-007 adds query embedding and profile-scoped Dense Top-N retrieval. The
|
||||
default is Top 20 with a configurable Mochow HNSW `ef`; the in-memory fake uses
|
||||
default is Top 20; the in-memory fake uses
|
||||
exact cosine search for deterministic offline regression. Returned objects are
|
||||
raw dense candidates only.
|
||||
|
||||
|
||||
67
apps/api/src/ekp_api/index_rebuild.py
Normal file
67
apps/api/src/ekp_api/index_rebuild.py
Normal file
@ -0,0 +1,67 @@
|
||||
"""Controlled CLI for the ARCH-005 shadow index cutover workflow."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import asdict
|
||||
|
||||
from ekp_api.providers import (
|
||||
create_embedding_provider,
|
||||
create_index_profile,
|
||||
create_vector_store,
|
||||
)
|
||||
from ekp_core.config import Settings
|
||||
from ekp_core.database import create_database_engine, create_session_factory
|
||||
from ekp_core.index_rebuild import IndexBuildSummary, IndexRebuildService
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build, verify, and activate a versioned shadow vector index."
|
||||
)
|
||||
parser.add_argument(
|
||||
"command",
|
||||
choices=("build", "verify", "activate"),
|
||||
help="Lifecycle step to execute for INDEX_PROFILE_ID.",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _create_service(settings: Settings) -> IndexRebuildService:
|
||||
"""Compose the command from the same provider settings as the API runtime."""
|
||||
engine = create_database_engine(settings)
|
||||
session_factory = create_session_factory(engine)
|
||||
return IndexRebuildService(
|
||||
session_factory=session_factory,
|
||||
embedding_provider=create_embedding_provider(settings),
|
||||
vector_store=create_vector_store(settings),
|
||||
target_profile=create_index_profile(settings),
|
||||
)
|
||||
|
||||
|
||||
def _execute(service: IndexRebuildService, command: str) -> IndexBuildSummary:
|
||||
"""Run exactly one explicit lifecycle step."""
|
||||
if command == "build":
|
||||
return service.rebuild()
|
||||
if command == "verify":
|
||||
return service.verify()
|
||||
if command == "activate":
|
||||
return service.activate()
|
||||
raise ValueError("unsupported index rebuild command")
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
"""Return a shell-safe status without printing endpoints, SQL, or credentials."""
|
||||
args = _parser().parse_args(argv)
|
||||
try:
|
||||
summary = _execute(_create_service(Settings()), args.command)
|
||||
except Exception:
|
||||
print("index profile command failed; inspect sanitized service logs", file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps(asdict(summary), sort_keys=True, separators=(",", ":")))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -16,6 +16,7 @@ from ekp_core.document_indexing import (
|
||||
from ekp_core.document_management import DocumentManagementService
|
||||
from ekp_core.document_parsing import DocumentParsingService
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.index_rebuild import resolve_active_index_profile
|
||||
from ekp_core.search import SearchService
|
||||
from ekp_domain import IndexProfile
|
||||
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
|
||||
@ -25,7 +26,6 @@ from ekp_parser import ParserRouter
|
||||
from ekp_reranker import QianfanRerankerProvider, RerankerProvider
|
||||
from ekp_vector_store import (
|
||||
VectorStore,
|
||||
create_mochow_vector_store,
|
||||
create_oceanbase_vector_store,
|
||||
)
|
||||
|
||||
@ -46,14 +46,20 @@ class ApiServices:
|
||||
search: SearchService
|
||||
|
||||
|
||||
def create_index_profile(settings: Settings) -> IndexProfile:
|
||||
"""Bind the Demo's chunk and embedding versions without vendor SDK types."""
|
||||
return IndexProfile(
|
||||
def create_index_profile(
|
||||
settings: Settings,
|
||||
session_factory: sessionmaker[Session] | None = None,
|
||||
) -> IndexProfile:
|
||||
"""Resolve the canonical active profile or the fresh-deployment baseline."""
|
||||
configured = IndexProfile(
|
||||
profile_id=settings.index_profile_id,
|
||||
chunk_strategy_version=DEMO_CHUNK_STRATEGY_VERSION,
|
||||
embedding_model=settings.embedding_model,
|
||||
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
|
||||
)
|
||||
if session_factory is None:
|
||||
return configured
|
||||
return resolve_active_index_profile(session_factory, configured)
|
||||
|
||||
|
||||
def create_embedding_provider(settings: Settings) -> QianfanEmbeddingProvider:
|
||||
@ -90,43 +96,25 @@ def create_object_storage(settings: Settings) -> ObjectStorage:
|
||||
|
||||
|
||||
def create_vector_store(settings: Settings) -> VectorStore:
|
||||
"""Create the selected vector adapter from explicit, secret-safe settings."""
|
||||
if settings.vector_store_provider == "oceanbase":
|
||||
if settings.oceanbase_vector_uri is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_URI is required")
|
||||
if settings.oceanbase_vector_user is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_USER is required")
|
||||
if settings.oceanbase_vector_password is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_PASSWORD is required")
|
||||
if settings.oceanbase_vector_database is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_DATABASE is required")
|
||||
return create_oceanbase_vector_store(
|
||||
uri=settings.oceanbase_vector_uri,
|
||||
user=settings.oceanbase_vector_user,
|
||||
password=settings.oceanbase_vector_password.get_secret_value(),
|
||||
database_name=settings.oceanbase_vector_database,
|
||||
table_name=settings.oceanbase_vector_table,
|
||||
index_name=settings.oceanbase_vector_index,
|
||||
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
|
||||
batch_size=settings.oceanbase_vector_batch_size,
|
||||
timeout_seconds=settings.oceanbase_vector_timeout_seconds,
|
||||
)
|
||||
|
||||
if settings.baidu_mochow_api_key is None:
|
||||
raise ProviderConfigurationError("BAIDU_MOCHOW_API_KEY is required")
|
||||
if not settings.baidu_mochow_endpoint:
|
||||
raise ProviderConfigurationError("BAIDU_MOCHOW_ENDPOINT is required")
|
||||
if not settings.knowledge_vdb_db_name:
|
||||
raise ProviderConfigurationError("KNOWLEDGE_VDB_DB_NAME is required")
|
||||
return create_mochow_vector_store(
|
||||
account=settings.baidu_mochow_account,
|
||||
api_key=settings.baidu_mochow_api_key.get_secret_value(),
|
||||
endpoint=settings.baidu_mochow_endpoint,
|
||||
database_name=settings.knowledge_vdb_db_name,
|
||||
table_name=settings.mochow_table,
|
||||
batch_size=settings.mochow_batch_size,
|
||||
search_ef=settings.mochow_search_ef,
|
||||
timeout_seconds=settings.mochow_timeout_seconds,
|
||||
"""Create the OceanBase adapter from explicit, secret-safe settings."""
|
||||
if settings.oceanbase_vector_uri is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_URI is required")
|
||||
if settings.oceanbase_vector_user is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_USER is required")
|
||||
if settings.oceanbase_vector_password is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_PASSWORD is required")
|
||||
if settings.oceanbase_vector_database is None:
|
||||
raise ProviderConfigurationError("OCEANBASE_VECTOR_DATABASE is required")
|
||||
return create_oceanbase_vector_store(
|
||||
uri=settings.oceanbase_vector_uri,
|
||||
user=settings.oceanbase_vector_user,
|
||||
password=settings.oceanbase_vector_password.get_secret_value(),
|
||||
database_name=settings.oceanbase_vector_database,
|
||||
table_name=settings.oceanbase_vector_table,
|
||||
index_name=settings.oceanbase_vector_index,
|
||||
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
|
||||
batch_size=settings.oceanbase_vector_batch_size,
|
||||
timeout_seconds=settings.oceanbase_vector_timeout_seconds,
|
||||
)
|
||||
|
||||
|
||||
@ -178,7 +166,7 @@ def create_api_services(settings: Settings) -> ApiServices:
|
||||
engine = create_database_engine(settings)
|
||||
session_factory = create_session_factory(engine)
|
||||
storage = create_object_storage(settings)
|
||||
index_profile = create_index_profile(settings)
|
||||
index_profile = create_index_profile(settings, session_factory)
|
||||
embedding_provider = create_embedding_provider(settings)
|
||||
vector_store = create_vector_store(settings)
|
||||
|
||||
|
||||
@ -326,6 +326,11 @@ profile_v2
|
||||
|
||||
避免直接原地覆盖现有生产索引。
|
||||
|
||||
ARCH-005 将 Profile 生命周期以 `shadow / active / retired` 记录在 PostgreSQL;任一时刻至多
|
||||
一个 Active Profile。重建从 canonical Chunk 生成确定性 `chunk_id + profile_id` 索引行,
|
||||
重复执行跳过已完整文档,激活前重新核对全部 ChunkIndexState,并在同一事务内锁定 Chunk
|
||||
写入、退役旧 Profile、激活新 Profile。OceanBase 表本身仍不是 canonical source。
|
||||
|
||||
---
|
||||
|
||||
## 8. Vector Store 设计
|
||||
|
||||
@ -364,8 +364,7 @@ DEMO-002 Document 数据模型 + 本地文件存储
|
||||
DEMO-003 PDF/DOCX/TXT/MD 上传
|
||||
DEMO-004 Parser + Cleaner
|
||||
DEMO-005 Chunk Pipeline
|
||||
DEMO-006 bge-large-zh Embedding + 初始 Mochow 入库(ARCH-003 已提供 OceanBase Adapter,
|
||||
ARCH-005 切换运行时)
|
||||
DEMO-006 bge-large-zh Embedding + 初始 Mochow 入库(ARCH-003~005 已迁移为 OceanBase)
|
||||
DEMO-007 Dense Retrieval
|
||||
DEMO-008 Reranker + Source Result
|
||||
DEMO-009 Document/Search REST API
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
# ARCH-002 BOS 配置与对象契约
|
||||
|
||||
ARCH-002 在既有 `ObjectStorage` 契约后新增百度 BOS Adapter。业务服务仍只使用逻辑 key,
|
||||
不知道 Bucket、Prefix、SDK 类型或凭据。Local Adapter 继续用于单元测试和本地开发;正式
|
||||
默认切换留在 ARCH-005,避免在 Vector/Reranker 迁移完成前形成半套运行配置。
|
||||
不知道 Bucket、Prefix、SDK 类型或凭据。ARCH-005 已将 BOS 设为正式运行默认值;Local
|
||||
Adapter 继续用于单元测试和显式配置的本地开发。
|
||||
|
||||
## 运行配置
|
||||
|
||||
|
||||
53
docs/development/INDEX_REBUILD.md
Normal file
53
docs/development/INDEX_REBUILD.md
Normal file
@ -0,0 +1,53 @@
|
||||
# ARCH-005 Shadow Index 重建与切换
|
||||
|
||||
ARCH-005 将正式运行默认值切换为 BOS、OceanBase Vector 和千帆 BCE Reranker。PostgreSQL
|
||||
继续保存 canonical Document、Chunk、ChunkIndexState 和 Index Profile 生命周期;OceanBase
|
||||
只保存可以重建的检索行。旧 Mochow 运行配置、依赖和代码已移除,但本流程不会自动删除
|
||||
旧外部索引,物理退役必须由受控运维流程另行确认。
|
||||
|
||||
## Profile 与迁移前提
|
||||
|
||||
默认新 Profile 为:
|
||||
|
||||
```text
|
||||
demo-bge-large-zh-structure-v1-oceanbase-v1
|
||||
```
|
||||
|
||||
它与旧 Demo Profile 使用不同 ID,避免已有 Mochow `ChunkIndexState` 被误认为 OceanBase
|
||||
写入证据。自定义 ID 仍由 `INDEX_PROFILE_ID` 注入,并且必须是全新的、稳定的版本标识。
|
||||
|
||||
执行前必须:
|
||||
|
||||
1. 运行 `uv run alembic upgrade head`,创建 canonical `index_profiles` 表;
|
||||
2. 使用 `deploy/oceanbase/001_knowledge_chunks_v1.sql` 预创建目标表和 HNSW 索引;
|
||||
3. 注入 OceanBase 与千帆 Embedding 的轮换凭据,不使用配置清单中的历史 Secret;
|
||||
4. 确认目标 OceanBase 为 4.6.0+,且应用账号只有目标数据库/表所需权限;
|
||||
5. 在重建期间避免修改 Chunk 策略;激活事务会阻止 PostgreSQL Chunk 并发写入穿过最终核对。
|
||||
|
||||
## 顺序执行
|
||||
|
||||
每一步只输出 Profile ID 和计数,不输出 Endpoint、SQL、Chunk 内容或凭据。
|
||||
|
||||
```bash
|
||||
uv run ekp-index-rebuild build
|
||||
uv run ekp-index-rebuild verify
|
||||
uv run ekp-index-rebuild activate
|
||||
```
|
||||
|
||||
- `build` 注册 Shadow Profile,从 PostgreSQL canonical Chunk 分文档生成 Embedding 并写入
|
||||
OceanBase。命令中断后可以原样重试;已完整文档会跳过,部分目标 Profile 数据会先定向
|
||||
清理再重建,不触碰 Active Profile;
|
||||
- `verify` 要求目标策略下至少存在一个 canonical Chunk,并核对全部 Chunk 与匹配模型、
|
||||
维度的 ChunkIndexState。输出中的 `complete` 必须为 `true`;
|
||||
- `activate` 在 PostgreSQL 事务内再次验证,锁定 Chunk 写入,将原 Active Profile 标为
|
||||
`retired`,再将目标 Shadow Profile 标为 `active`。重复执行已成功的激活是幂等的。
|
||||
|
||||
激活后 API 从 PostgreSQL 解析 Active Profile,不再仅依赖进程环境中的 ID。新上传与查询
|
||||
统一使用 OceanBase Active Profile。部署仍应保留前一版本制品和旧索引回滚窗口;本 Task
|
||||
不自动销毁任何旧外部数据。
|
||||
|
||||
## 验证与已知边界
|
||||
|
||||
离线测试覆盖多文档构建、重试跳过、完整性失败关闭、单 Active 约束、原子激活、迁移和
|
||||
CLI 输出脱敏。真实 BOS/OceanBase/Qianfan 连通性、DDL 权限、30~50 条脱敏 Golden Query、
|
||||
召回质量和延迟属于 ARCH-006,且只能使用轮换后的凭据与受控测试资源。
|
||||
@ -1,90 +0,0 @@
|
||||
# DEMO-006/007 千帆与 Mochow 配置
|
||||
|
||||
本文记录 Knowledge Platform Demo 的安全连接方式和 Mochow 表契约。配置参考现有
|
||||
`Embedding与数据库配置说明.md`,但不复制其中的环境地址或任何凭证。
|
||||
|
||||
## Canonical 数据库
|
||||
|
||||
本项目继续使用 `DATABASE_URL` 连接 PostgreSQL。参考项目中的 MySQL 配置不适用于
|
||||
本仓库,也不会替换 Accepted ADR 中的 PostgreSQL canonical storage。
|
||||
|
||||
## 千帆 Embedding
|
||||
|
||||
```dotenv
|
||||
EMBEDDING_PROVIDER=qianfan
|
||||
EMBEDDING_MODEL=bge-large-zh
|
||||
EMBEDDING_ENDPOINT=https://qianfan.baidubce.com/v2/embeddings
|
||||
BAIDU_QIANFAN_API_KEY=<运行环境注入>
|
||||
EMBEDDING_TIMEOUT_SECONDS=30
|
||||
```
|
||||
|
||||
`bge-large-zh` 固定输出 1024 维向量。Adapter 将每次请求限制为最多 16 条,并校验
|
||||
输入字符数、响应数量、顺序、维度和有限数值。API Key 使用 `SecretStr` 保存,异常中
|
||||
不记录请求头、请求正文或响应正文。
|
||||
|
||||
## Mochow 连接
|
||||
|
||||
```dotenv
|
||||
VECTOR_STORE_PROVIDER=mochow
|
||||
BAIDU_MOCHOW_ACCOUNT=root
|
||||
BAIDU_MOCHOW_API_KEY=<运行环境注入>
|
||||
BAIDU_MOCHOW_ENDPOINT=<已授权实例地址>
|
||||
KNOWLEDGE_VDB_DB_NAME=<专用数据库名>
|
||||
MOCHOW_TABLE=knowledge_chunks
|
||||
MOCHOW_BATCH_SIZE=100
|
||||
MOCHOW_SEARCH_EF=200
|
||||
MOCHOW_TIMEOUT_SECONDS=30
|
||||
INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1
|
||||
DENSE_TOP_N=20
|
||||
```
|
||||
|
||||
配置说明中的 endpoint 可作为运行环境的值,但不得提交到 `.env.example` 或代码。
|
||||
建议为本 Demo 使用独立表,禁止直接写入字段契约未知的既有业务表。
|
||||
|
||||
## Mochow 表契约
|
||||
|
||||
`MOCHOW_TABLE` 必须由受控的部署/运维步骤预先创建。应用不会在启动或入库时隐式创建、
|
||||
修改或删除数据库结构。
|
||||
|
||||
| 字段 | Mochow 类型 | 要求 |
|
||||
| --- | --- | --- |
|
||||
| `index_record_id` | STRING | 主键、分区键、非空;由 Chunk/profile 确定性生成 |
|
||||
| `chunk_id` | STRING | 非空 |
|
||||
| `document_id` | STRING | 非空,建议建立过滤索引 |
|
||||
| `filename` | STRING | 非空 |
|
||||
| `page` | UINT32 | 可空,一基页码 |
|
||||
| `page_end` | UINT32 | 可空,一基页码 |
|
||||
| `section` | TEXT | 可空 |
|
||||
| `chunk_index` | UINT32 | 非空 |
|
||||
| `content` | TEXT | 非空 |
|
||||
| `embedding` | FLOAT_VECTOR | 非空,dimension=1024 |
|
||||
| `embedding_model` | STRING | 非空 |
|
||||
| `index_profile_id` | STRING | 非空,建议建立过滤索引 |
|
||||
|
||||
向量字段建议建立 HNSW/COSINE 索引。`document_id` 与 `index_profile_id` 必须支持标量
|
||||
过滤,以便失败补偿和后续按文档删除。切换模型或 Chunk 策略时应使用新的
|
||||
`INDEX_PROFILE_ID` 和 shadow index,不得原地覆盖已记录的 profile。
|
||||
|
||||
## Dense Retrieval 契约
|
||||
|
||||
DEMO-007 使用同一 `EmbeddingProvider` 将 Query 转换为 1024 维向量,然后调用 Mochow
|
||||
`VectorTopkSearchRequest`:
|
||||
|
||||
- 向量字段固定为 `embedding`;
|
||||
- 默认 `DENSE_TOP_N=20`,调用方可在 1~1000 范围内覆盖;
|
||||
- HNSW 搜索候选参数默认 `MOCHOW_SEARCH_EF=200`,运行时至少提升到本次 Top-N;
|
||||
- `index_profile_id` 过滤随 Dense Search 请求下推,禁止混入其他模型/Chunk 版本;
|
||||
- 返回值是尚未 Rerank 的候选集,`score` 原样保留 Mochow 返回值,不在此阶段设置阈值。
|
||||
|
||||
Demo 明确不实现 RBAC/Document ACL;正式 V1 引入权限后,允许范围仍必须在召回阶段下推,
|
||||
不得先做全局 Top-K 再过滤。Reranker 与最终 Source Result 的配置参见
|
||||
`docs/development/RERANKER_SETUP.md`;REST API 参见
|
||||
`docs/development/REST_API_SETUP.md`。
|
||||
|
||||
## 验证边界
|
||||
|
||||
默认测试使用 `FakeEmbeddingProvider` 和 `FakeVectorStore`,不访问互联网或生产凭证。
|
||||
真实联调前需确认:专用表已按上述 Schema 创建、1024 维向量索引正常、Endpoint 网络
|
||||
可达、运行身份拥有 upsert/delete/vector_search 权限。默认回归已离线验证 Query
|
||||
Embedding → profile 过滤 → Dense Top20 及来源元数据保持;真实 Mochow 的网络联调需要由
|
||||
受控运行环境注入凭证后单独执行。
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
ARCH-003 在既有 `VectorStore` 契约后新增 OceanBase Vector Adapter。PostgreSQL 仍保存
|
||||
canonical Document、Chunk、Index Profile 和 Chunk Index State;OceanBase 表只保存可由这些
|
||||
数据重建的检索行。运行时默认切换和 Mochow 配置清理留在 ARCH-005。
|
||||
数据重建的检索行。ARCH-005 已完成运行时默认切换,并移除 Mochow 运行配置与依赖。
|
||||
|
||||
## 数据库前提
|
||||
|
||||
|
||||
@ -3,16 +3,18 @@
|
||||
## 范围
|
||||
|
||||
DEMO-009 将已经完成的同步入库和检索服务公开为版本化 REST API。API 层只负责输入校验、
|
||||
错误映射和响应序列化,不重复实现 Parser、Embedding、Mochow 或 Reranker 业务逻辑。
|
||||
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
|
||||
|
||||
本任务不包含 Admin Console、ACL/RBAC、异步任务、软删除或 Hybrid Search。
|
||||
|
||||
## 启动前提
|
||||
|
||||
1. 按 `docs/development/MOCHOW_SETUP.md` 配置 PostgreSQL、Mochow 和千帆 Embedding;
|
||||
2. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
|
||||
3. 执行 `uv run alembic upgrade head`;
|
||||
4. 执行 `make run`。
|
||||
1. 按 `docs/development/BOS_SETUP.md` 配置 BOS;
|
||||
2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding;
|
||||
3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
|
||||
4. 执行 `uv run alembic upgrade head`;
|
||||
5. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
|
||||
6. 执行 `make run`。
|
||||
|
||||
`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时
|
||||
延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。
|
||||
@ -38,14 +40,14 @@ curl -H 'Content-Type: application/json' \
|
||||
http://127.0.0.1:8000/api/v1/search
|
||||
```
|
||||
|
||||
Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 Mochow Upsert 完成。大文件或
|
||||
云端调用较慢时,请求耗时会相应增加;异步任务属于 Demo 后置能力。
|
||||
Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase Upsert 完成。大文
|
||||
件或云端调用较慢时,请求耗时会相应增加;异步任务属于 Demo 后置能力。
|
||||
|
||||
## 删除一致性
|
||||
|
||||
删除按以下顺序执行:
|
||||
|
||||
1. 删除所有已记录 Index Profile 以及当前 Profile 的 Mochow 行;
|
||||
1. 删除所有已记录 Index Profile 以及当前 Profile 的 OceanBase 行;
|
||||
2. 删除 PostgreSQL 中的 ChunkIndexState、Chunk 和 Document;
|
||||
3. 删除解析 JSON 与不可变原始文件。
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# Test Strategy
|
||||
|
||||
Unit tests must run without Mochow, Qianfan, internet, GPU or production credentials.
|
||||
Unit tests must run without OceanBase, BOS, Qianfan, internet, GPU or production credentials.
|
||||
|
||||
Provide deterministic fakes:
|
||||
- FakeEmbeddingProvider
|
||||
|
||||
@ -2,11 +2,11 @@
|
||||
|
||||
当前阶段:Provider 架构迁移
|
||||
|
||||
已完成:DEMO-001~DEMO-009、ARCH-001、ARCH-002、ARCH-003、ARCH-004
|
||||
已完成:DEMO-001~DEMO-009、ARCH-001、ARCH-002、ARCH-003、ARCH-004、ARCH-005
|
||||
进行中:无
|
||||
阻塞:无
|
||||
|
||||
下一任务:ARCH-005
|
||||
下一任务:ARCH-006
|
||||
|
||||
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
|
||||
执行;每个 Task 验证后独立推送并合并 `main`。全部迁移完成后再继续 DEMO-010。
|
||||
@ -180,3 +180,18 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
|
||||
- 更新 Adapter、环境示例、Provider wiring、配置与请求契约测试;Golden Dataset 继续使用
|
||||
确定性 Fake 离线评估,但显式绑定 `bce-reranker-base` 当前模型标识。真实 BCE 质量与延迟
|
||||
统一留在 ARCH-006 使用轮换凭据验收。
|
||||
|
||||
### ARCH-005
|
||||
|
||||
- 将正式运行默认值切换为 BOS、OceanBase Vector 和 `bce-reranker-base`,新 Profile ID 为
|
||||
`demo-bge-large-zh-structure-v1-oceanbase-v1`;Local/Fake 仍用于显式本地和离线测试。
|
||||
- 新增 PostgreSQL canonical `index_profiles` 迁移与 `shadow / active / retired` 生命周期,
|
||||
通过局部唯一索引保证至多一个 Active Profile;API 启动时优先解析数据库 Active Profile,
|
||||
无记录的新部署才使用配置基线。
|
||||
- 新增 `ekp-index-rebuild build|verify|activate`:从 canonical Chunk 分文档重建,确定性 Upsert,
|
||||
重试时跳过完整文档并清理目标 Profile 的部分写入;激活前核对全部 ChunkIndexState,并在
|
||||
PostgreSQL 事务内锁定 Chunk 写入、退役旧 Profile、激活新 Profile。
|
||||
- 移除 Mochow 运行配置、公开 Adapter、测试、配置文档及 `pymochow` 依赖;不自动删除旧
|
||||
外部索引,保留受控回滚和运维退役边界。
|
||||
- 新增重建/重试/完整性失败/单 Active/激活/迁移/CLI 脱敏测试及中文操作文档。真实资源
|
||||
构建、BOS/OceanBase/Qianfan 连通性和 30~50 条脱敏查询验收仍属于 ARCH-006。
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
- [x] ARCH-002 BOS ObjectStorage adapter and runtime configuration
|
||||
- [x] ARCH-003 OceanBase VectorStore adapter and index schema
|
||||
- [x] ARCH-004 Qianfan `bce-reranker-base` alignment
|
||||
- [ ] ARCH-005 Runtime cutover, shadow rebuild workflow and legacy configuration cleanup
|
||||
- [x] ARCH-005 Runtime cutover, shadow rebuild workflow and legacy configuration cleanup
|
||||
- [ ] ARCH-006 Controlled live integration and RAG acceptance
|
||||
|
||||
Dependencies:
|
||||
|
||||
@ -2374,6 +2374,10 @@ Backend:
|
||||
API First
|
||||
```
|
||||
|
||||
运行时切换约束:索引 Provider 或 Profile 变更必须以新 Profile 构建 Shadow Index;Profile
|
||||
生命周期和激活状态保存在 PostgreSQL。只有 canonical Chunk 与 ChunkIndexState 完整核对
|
||||
通过后才能原子激活,禁止复用旧 Profile ID 或原地覆盖当前 Active Index。
|
||||
|
||||
---
|
||||
|
||||
# 74. 已确认产品决策
|
||||
|
||||
77
migrations/versions/20260902_0005_create_index_profiles.py
Normal file
77
migrations/versions/20260902_0005_create_index_profiles.py
Normal file
@ -0,0 +1,77 @@
|
||||
"""Create canonical index-profile lifecycle records.
|
||||
|
||||
Revision ID: 20260902_0005
|
||||
Revises: 20260901_0004
|
||||
Create Date: 2026-09-02
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260902_0005"
|
||||
down_revision: str | None = "20260901_0004"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Persist shadow/active/retired profiles without storing vector data."""
|
||||
op.create_table(
|
||||
"index_profiles",
|
||||
sa.Column("profile_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("chunk_strategy_version", sa.String(length=64), nullable=False),
|
||||
sa.Column("embedding_model", sa.String(length=128), nullable=False),
|
||||
sa.Column("vector_dimension", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.Enum(
|
||||
"shadow",
|
||||
"active",
|
||||
"retired",
|
||||
name="index_profile_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("activated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.CheckConstraint("length(profile_id) > 0", name="profile_id_not_empty"),
|
||||
sa.CheckConstraint(
|
||||
"length(chunk_strategy_version) > 0",
|
||||
name="profile_chunk_strategy_not_empty",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"length(embedding_model) > 0",
|
||||
name="profile_embedding_model_not_empty",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"vector_dimension > 0",
|
||||
name="profile_dimension_positive",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('shadow', 'active', 'retired')",
|
||||
name="profile_status_valid",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("profile_id", name="pk_index_profiles"),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_index_profiles_single_active",
|
||||
"index_profiles",
|
||||
["status"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("status = 'active'"),
|
||||
sqlite_where=sa.text("status = 'active'"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -25,7 +25,7 @@ class Settings(BaseSettings):
|
||||
api_port: int = Field(default=8000, ge=1, le=65535)
|
||||
api_prefix: str = "/api/v1"
|
||||
database_url: str = "postgresql+psycopg://localhost:5432/ekp"
|
||||
object_storage_provider: Literal["local", "bos"] = "local"
|
||||
object_storage_provider: Literal["local", "bos"] = "bos"
|
||||
object_storage_path: Path = Path("storage")
|
||||
bos_endpoint: str | None = None
|
||||
bos_access_key_id: SecretStr | None = None
|
||||
@ -42,15 +42,7 @@ class Settings(BaseSettings):
|
||||
embedding_endpoint: str = "https://qianfan.baidubce.com/v2/embeddings"
|
||||
baidu_qianfan_api_key: SecretStr | None = None
|
||||
embedding_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
vector_store_provider: Literal["mochow", "oceanbase"] = "mochow"
|
||||
baidu_mochow_account: str = "root"
|
||||
baidu_mochow_api_key: SecretStr | None = None
|
||||
baidu_mochow_endpoint: str | None = None
|
||||
knowledge_vdb_db_name: str | None = None
|
||||
mochow_table: str = "knowledge_chunks"
|
||||
mochow_batch_size: int = Field(default=100, ge=1, le=1000)
|
||||
mochow_search_ef: int = Field(default=200, ge=1, le=10000)
|
||||
mochow_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
vector_store_provider: Literal["oceanbase"] = "oceanbase"
|
||||
oceanbase_vector_uri: str | None = None
|
||||
oceanbase_vector_user: str | None = None
|
||||
oceanbase_vector_password: SecretStr | None = None
|
||||
@ -59,7 +51,7 @@ class Settings(BaseSettings):
|
||||
oceanbase_vector_index: str = "vidx_knowledge_chunks_embedding"
|
||||
oceanbase_vector_batch_size: int = Field(default=100, ge=1, le=1000)
|
||||
oceanbase_vector_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
index_profile_id: str = "demo-bge-large-zh-structure-v1"
|
||||
index_profile_id: str = "demo-bge-large-zh-structure-v1-oceanbase-v1"
|
||||
dense_top_n: int = Field(default=20, ge=1, le=64)
|
||||
reranker_provider: Literal["qianfan"] = "qianfan"
|
||||
reranker_model: Literal["bce-reranker-base"] = "bce-reranker-base"
|
||||
|
||||
279
packages/core/src/ekp_core/index_rebuild.py
Normal file
279
packages/core/src/ekp_core/index_rebuild.py
Normal file
@ -0,0 +1,279 @@
|
||||
"""Retry-safe shadow index rebuild, verification, and activation services."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import delete, select, text
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.document_indexing import DocumentIndexingService
|
||||
from ekp_domain import (
|
||||
Chunk,
|
||||
ChunkIndexState,
|
||||
IndexProfile,
|
||||
IndexProfileStatus,
|
||||
StoredIndexProfile,
|
||||
)
|
||||
from ekp_embedding import EmbeddingProvider
|
||||
from ekp_vector_store import VectorStore
|
||||
|
||||
|
||||
class IndexProfileLifecycleError(RuntimeError):
|
||||
"""Raised when a shadow profile cannot safely move through its lifecycle."""
|
||||
|
||||
|
||||
class IndexProfileMetadataConflictError(IndexProfileLifecycleError):
|
||||
"""Raised when a profile ID is reused with different immutable metadata."""
|
||||
|
||||
|
||||
class IndexProfileNotFoundError(IndexProfileLifecycleError):
|
||||
"""Raised when verification or activation targets an unknown profile."""
|
||||
|
||||
|
||||
class IndexProfileIncompleteError(IndexProfileLifecycleError):
|
||||
"""Raised when activation is attempted before every canonical chunk is indexed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class IndexBuildSummary:
|
||||
"""Safe progress counters for rebuild commands and deployment checks."""
|
||||
|
||||
profile_id: str
|
||||
expected_documents: int
|
||||
built_documents: int
|
||||
skipped_documents: int
|
||||
expected_chunks: int
|
||||
indexed_chunks: int
|
||||
complete: bool
|
||||
|
||||
|
||||
class IndexRebuildService:
|
||||
"""Build a new profile from PostgreSQL chunks without mutating an active index."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker[Session],
|
||||
embedding_provider: EmbeddingProvider,
|
||||
vector_store: VectorStore,
|
||||
target_profile: IndexProfile,
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._vector_store = vector_store
|
||||
self._target_profile = target_profile
|
||||
self._indexing = DocumentIndexingService(
|
||||
session_factory,
|
||||
embedding_provider,
|
||||
vector_store,
|
||||
target_profile,
|
||||
)
|
||||
|
||||
def rebuild(self) -> IndexBuildSummary:
|
||||
"""Register and populate a shadow profile, skipping verified documents on retry."""
|
||||
self._register_shadow()
|
||||
document_ids = self._target_document_ids()
|
||||
built_documents = 0
|
||||
skipped_documents = 0
|
||||
for document_id in document_ids:
|
||||
if self._document_is_complete(document_id):
|
||||
skipped_documents += 1
|
||||
continue
|
||||
self._reset_partial_document(document_id)
|
||||
self._indexing.process(document_id)
|
||||
built_documents += 1
|
||||
|
||||
verified = self.verify()
|
||||
return IndexBuildSummary(
|
||||
profile_id=verified.profile_id,
|
||||
expected_documents=verified.expected_documents,
|
||||
built_documents=built_documents,
|
||||
skipped_documents=skipped_documents,
|
||||
expected_chunks=verified.expected_chunks,
|
||||
indexed_chunks=verified.indexed_chunks,
|
||||
complete=verified.complete,
|
||||
)
|
||||
|
||||
def verify(self) -> IndexBuildSummary:
|
||||
"""Compare canonical chunks and successful index states for the target profile."""
|
||||
with self._session_factory() as session:
|
||||
self._require_registered_profile(session)
|
||||
return self._verification_summary(session)
|
||||
|
||||
def activate(self) -> IndexBuildSummary:
|
||||
"""Atomically retire the previous profile after target completeness is proven."""
|
||||
with self._session_factory() as session:
|
||||
target = self._require_registered_profile(session, for_update=True)
|
||||
self._lock_canonical_chunks(session)
|
||||
summary = self._verification_summary(session)
|
||||
if not summary.complete:
|
||||
raise IndexProfileIncompleteError(
|
||||
"shadow index is incomplete and cannot be activated"
|
||||
)
|
||||
if target.status is IndexProfileStatus.ACTIVE:
|
||||
return summary
|
||||
if target.status is not IndexProfileStatus.SHADOW:
|
||||
raise IndexProfileLifecycleError("only a shadow profile can be activated")
|
||||
|
||||
active_profiles = session.scalars(
|
||||
select(StoredIndexProfile)
|
||||
.where(StoredIndexProfile.status == IndexProfileStatus.ACTIVE)
|
||||
.with_for_update()
|
||||
).all()
|
||||
for active in active_profiles:
|
||||
active.status = IndexProfileStatus.RETIRED
|
||||
target.status = IndexProfileStatus.ACTIVE
|
||||
target.activated_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
return summary
|
||||
|
||||
def _register_shadow(self) -> None:
|
||||
"""Create an idempotent shadow registration without reusing active IDs."""
|
||||
with self._session_factory() as session:
|
||||
stored = session.get(StoredIndexProfile, self._target_profile.profile_id)
|
||||
if stored is None:
|
||||
session.add(
|
||||
StoredIndexProfile(
|
||||
profile_id=self._target_profile.profile_id,
|
||||
chunk_strategy_version=self._target_profile.chunk_strategy_version,
|
||||
embedding_model=self._target_profile.embedding_model,
|
||||
vector_dimension=self._target_profile.vector_dimension,
|
||||
status=IndexProfileStatus.SHADOW,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return
|
||||
self._validate_stored_metadata(stored)
|
||||
if stored.status is not IndexProfileStatus.SHADOW:
|
||||
raise IndexProfileLifecycleError(
|
||||
"rebuild target must be a shadow profile with a new profile ID"
|
||||
)
|
||||
|
||||
def _target_document_ids(self) -> tuple[UUID, ...]:
|
||||
"""List documents that have canonical chunks for the target strategy."""
|
||||
with self._session_factory() as session:
|
||||
return tuple(
|
||||
session.scalars(
|
||||
select(Chunk.document_id)
|
||||
.where(Chunk.strategy_version == self._target_profile.chunk_strategy_version)
|
||||
.distinct()
|
||||
.order_by(Chunk.document_id)
|
||||
).all()
|
||||
)
|
||||
|
||||
def _document_is_complete(self, document_id: UUID) -> bool:
|
||||
"""Return true only when every selected chunk has matching profile evidence."""
|
||||
with self._session_factory() as session:
|
||||
chunks = session.scalars(
|
||||
select(Chunk.chunk_id).where(
|
||||
Chunk.document_id == document_id,
|
||||
Chunk.strategy_version == self._target_profile.chunk_strategy_version,
|
||||
)
|
||||
).all()
|
||||
states = session.scalars(
|
||||
select(ChunkIndexState).where(
|
||||
ChunkIndexState.chunk_id.in_(chunks),
|
||||
ChunkIndexState.index_profile_id == self._target_profile.profile_id,
|
||||
)
|
||||
).all()
|
||||
return len(states) == len(chunks) and all(
|
||||
self._state_matches_profile(state) for state in states
|
||||
)
|
||||
|
||||
def _reset_partial_document(self, document_id: UUID) -> None:
|
||||
"""Remove only target-profile remnants before retrying one document."""
|
||||
self._vector_store.delete_document(document_id, self._target_profile.profile_id)
|
||||
with self._session_factory() as session:
|
||||
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
|
||||
session.execute(
|
||||
delete(ChunkIndexState).where(
|
||||
ChunkIndexState.chunk_id.in_(chunk_ids),
|
||||
ChunkIndexState.index_profile_id == self._target_profile.profile_id,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
def _require_registered_profile(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
for_update: bool = False,
|
||||
) -> StoredIndexProfile:
|
||||
statement = select(StoredIndexProfile).where(
|
||||
StoredIndexProfile.profile_id == self._target_profile.profile_id
|
||||
)
|
||||
if for_update:
|
||||
statement = statement.with_for_update()
|
||||
stored = session.scalar(statement)
|
||||
if stored is None:
|
||||
raise IndexProfileNotFoundError(self._target_profile.profile_id)
|
||||
self._validate_stored_metadata(stored)
|
||||
return stored
|
||||
|
||||
def _validate_stored_metadata(self, stored: StoredIndexProfile) -> None:
|
||||
expected = self._target_profile
|
||||
if (
|
||||
stored.chunk_strategy_version != expected.chunk_strategy_version
|
||||
or stored.embedding_model != expected.embedding_model
|
||||
or stored.vector_dimension != expected.vector_dimension
|
||||
):
|
||||
raise IndexProfileMetadataConflictError(
|
||||
"index profile ID is already registered with different metadata"
|
||||
)
|
||||
|
||||
def _verification_summary(self, session: Session) -> IndexBuildSummary:
|
||||
expected_rows = session.execute(
|
||||
select(Chunk.chunk_id, Chunk.document_id).where(
|
||||
Chunk.strategy_version == self._target_profile.chunk_strategy_version
|
||||
)
|
||||
).all()
|
||||
expected_chunk_ids = {row.chunk_id for row in expected_rows}
|
||||
expected_document_ids = {row.document_id for row in expected_rows}
|
||||
states = session.scalars(
|
||||
select(ChunkIndexState).where(
|
||||
ChunkIndexState.index_profile_id == self._target_profile.profile_id
|
||||
)
|
||||
).all()
|
||||
indexed_chunk_ids = {
|
||||
state.chunk_id
|
||||
for state in states
|
||||
if state.chunk_id in expected_chunk_ids and self._state_matches_profile(state)
|
||||
}
|
||||
complete = (
|
||||
bool(expected_chunk_ids)
|
||||
and len(states) == len(expected_chunk_ids)
|
||||
and indexed_chunk_ids == expected_chunk_ids
|
||||
)
|
||||
return IndexBuildSummary(
|
||||
profile_id=self._target_profile.profile_id,
|
||||
expected_documents=len(expected_document_ids),
|
||||
built_documents=0,
|
||||
skipped_documents=0,
|
||||
expected_chunks=len(expected_chunk_ids),
|
||||
indexed_chunks=len(indexed_chunk_ids),
|
||||
complete=complete,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _lock_canonical_chunks(session: Session) -> None:
|
||||
"""Prevent PostgreSQL chunk writes between final verification and activation."""
|
||||
bind = session.get_bind()
|
||||
if bind.dialect.name == "postgresql":
|
||||
session.execute(text("LOCK TABLE chunks IN SHARE MODE"))
|
||||
|
||||
def _state_matches_profile(self, state: ChunkIndexState) -> bool:
|
||||
return (
|
||||
state.embedding_model == self._target_profile.embedding_model
|
||||
and state.vector_dimension == self._target_profile.vector_dimension
|
||||
)
|
||||
|
||||
|
||||
def resolve_active_index_profile(
|
||||
session_factory: sessionmaker[Session],
|
||||
fallback: IndexProfile,
|
||||
) -> IndexProfile:
|
||||
"""Resolve the canonical active profile, with a fresh-deployment fallback."""
|
||||
with session_factory() as session:
|
||||
stored = session.scalar(
|
||||
select(StoredIndexProfile).where(StoredIndexProfile.status == IndexProfileStatus.ACTIVE)
|
||||
)
|
||||
return fallback if stored is None else stored.to_profile()
|
||||
@ -2,7 +2,12 @@
|
||||
|
||||
from ekp_domain.chunk import Chunk
|
||||
from ekp_domain.document import Base, Document, DocumentStatus
|
||||
from ekp_domain.index_profile import ChunkIndexState, IndexProfile
|
||||
from ekp_domain.index_profile import (
|
||||
ChunkIndexState,
|
||||
IndexProfile,
|
||||
IndexProfileStatus,
|
||||
StoredIndexProfile,
|
||||
)
|
||||
from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocument
|
||||
|
||||
__all__ = [
|
||||
@ -12,7 +17,9 @@ __all__ = [
|
||||
"Document",
|
||||
"DocumentStatus",
|
||||
"IndexProfile",
|
||||
"IndexProfileStatus",
|
||||
"ParsedBlock",
|
||||
"ParsedBlockType",
|
||||
"ParsedDocument",
|
||||
"StoredIndexProfile",
|
||||
]
|
||||
|
||||
@ -3,9 +3,11 @@
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Integer, String, func
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, Integer, String, func, text
|
||||
from sqlalchemy import Enum as SqlEnum
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ekp_domain.document import Base
|
||||
@ -13,6 +15,14 @@ from ekp_domain.document import Base
|
||||
PROFILE_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$")
|
||||
|
||||
|
||||
class IndexProfileStatus(StrEnum):
|
||||
"""Lifecycle state for a canonical index-profile registration."""
|
||||
|
||||
SHADOW = "shadow"
|
||||
ACTIVE = "active"
|
||||
RETIRED = "retired"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class IndexProfile:
|
||||
"""Small Demo profile binding chunk and embedding versions together."""
|
||||
@ -31,6 +41,71 @@ class IndexProfile:
|
||||
raise ValueError("vector_dimension must be positive")
|
||||
|
||||
|
||||
class StoredIndexProfile(Base):
|
||||
"""Canonical registration used to build, verify, and activate shadow indexes."""
|
||||
|
||||
__tablename__ = "index_profiles"
|
||||
__table_args__ = (
|
||||
CheckConstraint("length(profile_id) > 0", name="profile_id_not_empty"),
|
||||
CheckConstraint(
|
||||
"length(chunk_strategy_version) > 0",
|
||||
name="profile_chunk_strategy_not_empty",
|
||||
),
|
||||
CheckConstraint(
|
||||
"length(embedding_model) > 0",
|
||||
name="profile_embedding_model_not_empty",
|
||||
),
|
||||
CheckConstraint("vector_dimension > 0", name="profile_dimension_positive"),
|
||||
CheckConstraint(
|
||||
"status IN ('shadow', 'active', 'retired')",
|
||||
name="profile_status_valid",
|
||||
),
|
||||
# Both supported test/runtime dialects enforce at most one active profile.
|
||||
Index(
|
||||
"uq_index_profiles_single_active",
|
||||
"status",
|
||||
unique=True,
|
||||
postgresql_where=text("status = 'active'"),
|
||||
sqlite_where=text("status = 'active'"),
|
||||
),
|
||||
)
|
||||
|
||||
profile_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
chunk_strategy_version: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
embedding_model: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
vector_dimension: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
status: Mapped[IndexProfileStatus] = mapped_column(
|
||||
SqlEnum(
|
||||
IndexProfileStatus,
|
||||
name="index_profile_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
validate_strings=True,
|
||||
values_callable=lambda enum_type: [item.value for item in enum_type],
|
||||
),
|
||||
nullable=False,
|
||||
default=IndexProfileStatus.SHADOW,
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
)
|
||||
activated_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
def to_profile(self) -> IndexProfile:
|
||||
"""Return the immutable business value used by indexing and retrieval."""
|
||||
return IndexProfile(
|
||||
profile_id=self.profile_id,
|
||||
chunk_strategy_version=self.chunk_strategy_version,
|
||||
embedding_model=self.embedding_model,
|
||||
vector_dimension=self.vector_dimension,
|
||||
)
|
||||
|
||||
|
||||
class ChunkIndexState(Base):
|
||||
"""Canonical evidence that one chunk was written to a rebuildable index."""
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
"""Vector store abstraction, fake, and Mochow adapter."""
|
||||
"""Vector store abstraction with fake and OceanBase implementations."""
|
||||
|
||||
from ekp_vector_store.base import (
|
||||
DenseSearchCandidate,
|
||||
@ -7,11 +7,6 @@ from ekp_vector_store.base import (
|
||||
VectorStore,
|
||||
)
|
||||
from ekp_vector_store.fake import FakeVectorStore
|
||||
from ekp_vector_store.mochow import (
|
||||
MochowVectorStore,
|
||||
MochowVectorStoreError,
|
||||
create_mochow_vector_store,
|
||||
)
|
||||
from ekp_vector_store.oceanbase import (
|
||||
OceanBaseConfigurationError,
|
||||
OceanBaseVectorStore,
|
||||
@ -23,13 +18,10 @@ __all__ = [
|
||||
"FakeVectorStore",
|
||||
"DenseSearchCandidate",
|
||||
"DenseSearchRequest",
|
||||
"MochowVectorStore",
|
||||
"MochowVectorStoreError",
|
||||
"OceanBaseConfigurationError",
|
||||
"OceanBaseVectorStore",
|
||||
"OceanBaseVectorStoreError",
|
||||
"VectorRecord",
|
||||
"VectorStore",
|
||||
"create_mochow_vector_store",
|
||||
"create_oceanbase_vector_store",
|
||||
]
|
||||
|
||||
@ -1,268 +0,0 @@
|
||||
"""Mochow SDK adapter isolated behind the provider-neutral VectorStore contract."""
|
||||
|
||||
import re
|
||||
from collections.abc import Callable, Mapping
|
||||
from math import isfinite
|
||||
from numbers import Real
|
||||
from typing import Any, Protocol
|
||||
from uuid import UUID
|
||||
|
||||
from ekp_vector_store.base import DenseSearchCandidate, DenseSearchRequest, VectorRecord
|
||||
|
||||
MOCHOW_IDENTIFIER = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,63}$")
|
||||
PROFILE_FILTER_VALUE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$")
|
||||
SEARCH_PROJECTIONS = [
|
||||
"index_record_id",
|
||||
"chunk_id",
|
||||
"document_id",
|
||||
"filename",
|
||||
"page",
|
||||
"page_end",
|
||||
"section",
|
||||
"chunk_index",
|
||||
"content",
|
||||
"index_profile_id",
|
||||
]
|
||||
|
||||
|
||||
class MochowConfigurationError(ValueError):
|
||||
"""Raised when connection or table identifiers are unsafe or incomplete."""
|
||||
|
||||
|
||||
class MochowVectorStoreError(RuntimeError):
|
||||
"""Sanitized failure raised without SDK request details or credentials."""
|
||||
|
||||
|
||||
class _MochowTable(Protocol):
|
||||
def upsert(self, rows: list[object]) -> object:
|
||||
"""Upsert one SDK row batch."""
|
||||
|
||||
def delete(self, *, filter: str) -> object:
|
||||
"""Delete rows matching a scalar filter."""
|
||||
|
||||
def vector_search(self, *, request: object, projections: list[str]) -> object:
|
||||
"""Run one vector Top-K request with scalar projections."""
|
||||
|
||||
|
||||
class _ClosableClient(Protocol):
|
||||
def close(self) -> object:
|
||||
"""Release SDK network resources."""
|
||||
|
||||
|
||||
class MochowVectorStore:
|
||||
"""Translate neutral records into configured Mochow table rows."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
table: _MochowTable,
|
||||
row_factory: Callable[..., object],
|
||||
batch_size: int = 100,
|
||||
search_ef: int = 200,
|
||||
client: _ClosableClient | None = None,
|
||||
) -> None:
|
||||
if not 1 <= batch_size <= 1000:
|
||||
raise ValueError("Mochow batch_size must be between 1 and 1000")
|
||||
if not 1 <= search_ef <= 10000:
|
||||
raise ValueError("Mochow search_ef must be between 1 and 10000")
|
||||
self._table = table
|
||||
self._row_factory = row_factory
|
||||
self._batch_size = batch_size
|
||||
self._search_ef = search_ef
|
||||
self._client = client
|
||||
|
||||
def upsert(self, records: tuple[VectorRecord, ...]) -> None:
|
||||
"""Map records to Mochow rows and upsert in bounded SDK batches."""
|
||||
rows = [self._to_row(record) for record in records]
|
||||
for start in range(0, len(rows), self._batch_size):
|
||||
try:
|
||||
self._table.upsert(rows[start : start + self._batch_size])
|
||||
except Exception as exc:
|
||||
raise MochowVectorStoreError("Mochow vector upsert failed") from exc
|
||||
|
||||
def delete_document(self, document_id: UUID, index_profile_id: str) -> None:
|
||||
"""Delete only one profile so shadow or active indexes are not mixed."""
|
||||
if not PROFILE_FILTER_VALUE.fullmatch(index_profile_id):
|
||||
raise MochowConfigurationError("unsafe index profile filter value")
|
||||
try:
|
||||
self._table.delete(
|
||||
filter=(
|
||||
f"document_id == '{document_id}' AND index_profile_id == '{index_profile_id}'"
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
raise MochowVectorStoreError("Mochow vector cleanup failed") from exc
|
||||
|
||||
def dense_search(self, request: DenseSearchRequest) -> tuple[DenseSearchCandidate, ...]:
|
||||
"""Run profile-filtered Mochow Top-K and translate its scalar rows."""
|
||||
if not PROFILE_FILTER_VALUE.fullmatch(request.index_profile_id):
|
||||
raise MochowConfigurationError("unsafe index profile filter value")
|
||||
|
||||
# Keep SDK types isolated in this adapter so core retrieval remains portable.
|
||||
from pymochow.model.table import ( # type: ignore[import-untyped]
|
||||
FloatVector,
|
||||
VectorSearchConfig,
|
||||
VectorTopkSearchRequest,
|
||||
)
|
||||
|
||||
sdk_request = VectorTopkSearchRequest(
|
||||
vector_field="embedding",
|
||||
vector=FloatVector(list(request.query_vector)),
|
||||
limit=request.top_n,
|
||||
filter=f"index_profile_id = '{request.index_profile_id}'",
|
||||
# HNSW must inspect at least Top-N candidates even if operators tune ef lower.
|
||||
config=VectorSearchConfig(ef=max(self._search_ef, request.top_n)),
|
||||
)
|
||||
try:
|
||||
response = self._table.vector_search(
|
||||
request=sdk_request,
|
||||
projections=list(SEARCH_PROJECTIONS),
|
||||
)
|
||||
rows = getattr(response, "rows", None)
|
||||
if not isinstance(rows, list):
|
||||
raise TypeError("Mochow response rows must be a list")
|
||||
candidates = tuple(self._to_candidate(row) for row in rows)
|
||||
if len(candidates) > request.top_n:
|
||||
raise ValueError("Mochow returned more rows than requested")
|
||||
if any(
|
||||
candidate.index_profile_id != request.index_profile_id for candidate in candidates
|
||||
):
|
||||
raise ValueError("Mochow returned a row from another index profile")
|
||||
return candidates
|
||||
except Exception as exc:
|
||||
raise MochowVectorStoreError("Mochow dense search failed") from exc
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the owning SDK client when this adapter created it."""
|
||||
if self._client is not None:
|
||||
self._client.close()
|
||||
|
||||
def _to_row(self, record: VectorRecord) -> object:
|
||||
"""Build only fields in the documented EKP Demo Mochow schema."""
|
||||
fields: dict[str, object] = {
|
||||
"index_record_id": str(record.index_record_id),
|
||||
"chunk_id": str(record.chunk_id),
|
||||
"document_id": str(record.document_id),
|
||||
"filename": record.filename,
|
||||
"chunk_index": record.chunk_index,
|
||||
"content": record.content,
|
||||
"embedding": list(record.embedding),
|
||||
"embedding_model": record.embedding_model,
|
||||
"index_profile_id": record.index_profile_id,
|
||||
}
|
||||
if record.page is not None:
|
||||
fields["page"] = record.page
|
||||
if record.page_end is not None:
|
||||
fields["page_end"] = record.page_end
|
||||
if record.section is not None:
|
||||
fields["section"] = record.section
|
||||
return self._row_factory(**fields)
|
||||
|
||||
@staticmethod
|
||||
def _to_candidate(row: object) -> DenseSearchCandidate:
|
||||
"""Validate the documented Mochow scalar row and its raw dense score."""
|
||||
if not isinstance(row, dict):
|
||||
raise TypeError("Mochow search row must be an object")
|
||||
fields = row.get("row")
|
||||
if not isinstance(fields, dict):
|
||||
raise TypeError("Mochow search result must contain a scalar row object")
|
||||
scalar_fields: Mapping[str, object] = fields
|
||||
score = row.get("score")
|
||||
if isinstance(score, bool) or not isinstance(score, Real) or not isfinite(float(score)):
|
||||
raise ValueError("Mochow search row has an invalid score")
|
||||
return DenseSearchCandidate(
|
||||
index_record_id=UUID(_required_str(scalar_fields, "index_record_id")),
|
||||
chunk_id=UUID(_required_str(scalar_fields, "chunk_id")),
|
||||
document_id=UUID(_required_str(scalar_fields, "document_id")),
|
||||
filename=_required_str(scalar_fields, "filename"),
|
||||
page=_optional_int(scalar_fields.get("page")),
|
||||
page_end=_optional_int(scalar_fields.get("page_end")),
|
||||
section=_optional_str(scalar_fields.get("section")),
|
||||
chunk_index=_required_int(scalar_fields, "chunk_index"),
|
||||
content=_required_str(scalar_fields, "content"),
|
||||
score=float(score),
|
||||
index_profile_id=_required_str(scalar_fields, "index_profile_id"),
|
||||
)
|
||||
|
||||
|
||||
def _required_str(fields: Mapping[str, object], name: str) -> str:
|
||||
"""Read a required, non-empty text field from a Mochow scalar row."""
|
||||
value = fields.get(name)
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ValueError(f"Mochow scalar field {name} must be non-empty text")
|
||||
return value
|
||||
|
||||
|
||||
def _required_int(fields: Mapping[str, object], name: str) -> int:
|
||||
"""Read required integer metadata without accepting Python booleans."""
|
||||
value = fields.get(name)
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise ValueError(f"Mochow scalar field {name} must be an integer")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_int(value: object) -> int | None:
|
||||
"""Convert a nullable scalar while rejecting booleans as integer metadata."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise ValueError("Mochow integer metadata must be an integer")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_str(value: object) -> str | None:
|
||||
"""Preserve nullable text metadata without producing the string 'None'."""
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("Mochow text metadata must be text")
|
||||
return value
|
||||
|
||||
|
||||
def create_mochow_vector_store(
|
||||
*,
|
||||
account: str,
|
||||
api_key: str,
|
||||
endpoint: str,
|
||||
database_name: str,
|
||||
table_name: str,
|
||||
batch_size: int = 100,
|
||||
search_ef: int = 200,
|
||||
timeout_seconds: float = 30.0,
|
||||
) -> MochowVectorStore:
|
||||
"""Create the official pymochow client without leaking it into core code."""
|
||||
if not account or not api_key or not endpoint:
|
||||
raise MochowConfigurationError("Mochow account, API key, and endpoint are required")
|
||||
if not MOCHOW_IDENTIFIER.fullmatch(database_name):
|
||||
raise MochowConfigurationError("invalid Mochow database name")
|
||||
if not MOCHOW_IDENTIFIER.fullmatch(table_name):
|
||||
raise MochowConfigurationError("invalid Mochow table name")
|
||||
if timeout_seconds <= 0:
|
||||
raise MochowConfigurationError("Mochow timeout_seconds must be positive")
|
||||
if not 1 <= batch_size <= 1000:
|
||||
raise MochowConfigurationError("Mochow batch_size must be between 1 and 1000")
|
||||
if not 1 <= search_ef <= 10000:
|
||||
raise MochowConfigurationError("Mochow search_ef must be between 1 and 10000")
|
||||
|
||||
import pymochow # type: ignore[import-untyped]
|
||||
from pymochow.auth.bce_credentials import BceCredentials # type: ignore[import-untyped]
|
||||
from pymochow.configuration import Configuration # type: ignore[import-untyped]
|
||||
from pymochow.model.table import Row
|
||||
|
||||
config = Configuration(
|
||||
credentials=BceCredentials(account, api_key),
|
||||
endpoint=endpoint,
|
||||
connection_timeout_in_mills=int(timeout_seconds * 1000),
|
||||
request_timeout_ms=int(timeout_seconds * 1000),
|
||||
)
|
||||
try:
|
||||
client: Any = pymochow.MochowClient(config)
|
||||
table: Any = client.database(database_name).table(table_name)
|
||||
except Exception as exc:
|
||||
raise MochowVectorStoreError("Mochow client initialization failed") from exc
|
||||
return MochowVectorStore(
|
||||
table=table,
|
||||
row_factory=Row,
|
||||
batch_size=batch_size,
|
||||
search_ef=search_ef,
|
||||
client=client,
|
||||
)
|
||||
@ -13,7 +13,6 @@ dependencies = [
|
||||
"bce-python-sdk>=0.9.72,<0.10",
|
||||
"fastapi>=0.115,<1",
|
||||
"psycopg[binary]>=3.2,<4",
|
||||
"pymochow>=2.4,<3",
|
||||
"pymupdf>=1.26,<2",
|
||||
"pydantic-settings>=2.7,<3",
|
||||
"python-multipart>=0.0.20,<1",
|
||||
@ -23,6 +22,9 @@ dependencies = [
|
||||
"pyobvector>=0.2.28,<0.3",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
ekp-index-rebuild = "ekp_api.index_rebuild:main"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"httpx2>=2,<3",
|
||||
|
||||
@ -38,7 +38,7 @@ def test_business_providers_are_lazy_for_health_checks(
|
||||
tmp_path: Path,
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
"""Starting or probing the API must not require Qianfan/Mochow credentials."""
|
||||
"""Starting or probing the API must not require cloud provider credentials."""
|
||||
settings = Settings(app_env="test", object_storage_path=tmp_path / "storage")
|
||||
|
||||
with TestClient(create_app(settings)) as client:
|
||||
|
||||
@ -25,11 +25,11 @@ from ekp_vector_store import FakeVectorStore, VectorRecord
|
||||
|
||||
|
||||
class PartiallyFailingVectorStore(FakeVectorStore):
|
||||
"""Expose one partial upsert before simulating a Mochow batch failure."""
|
||||
"""Expose one partial upsert before simulating a provider batch failure."""
|
||||
|
||||
def upsert(self, records: tuple[VectorRecord, ...]) -> None:
|
||||
super().upsert(records[:1])
|
||||
raise RuntimeError("simulated Mochow failure")
|
||||
raise RuntimeError("simulated vector provider failure")
|
||||
|
||||
|
||||
def _build_services(
|
||||
@ -132,7 +132,7 @@ def test_partial_vector_write_is_compensated_and_document_fails(tmp_path: Path)
|
||||
vector_store=failing_store,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="simulated Mochow failure"):
|
||||
with pytest.raises(RuntimeError, match="simulated vector provider failure"):
|
||||
pipeline.ingest("policy.txt", "text/plain", BytesIO("退款周期为七天。".encode()))
|
||||
|
||||
with session_factory() as session:
|
||||
@ -140,7 +140,7 @@ def test_partial_vector_write_is_compensated_and_document_fails(tmp_path: Path)
|
||||
states = session.scalars(select(ChunkIndexState)).all()
|
||||
assert stored_document is not None
|
||||
assert stored_document.status is DocumentStatus.FAILED
|
||||
assert "simulated Mochow failure" in (stored_document.error_message or "")
|
||||
assert "simulated vector provider failure" in (stored_document.error_message or "")
|
||||
assert states == []
|
||||
assert failing_store.records == {}
|
||||
assert failing_store.delete_calls == 1
|
||||
|
||||
@ -18,7 +18,9 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
|
||||
command.check(config)
|
||||
|
||||
inspector = inspect(create_engine(database_url))
|
||||
assert {"documents", "chunks", "chunk_index_states"}.issubset(inspector.get_table_names())
|
||||
assert {"documents", "chunks", "chunk_index_states", "index_profiles"}.issubset(
|
||||
inspector.get_table_names()
|
||||
)
|
||||
assert {column["name"] for column in inspector.get_columns("documents")} == {
|
||||
"id",
|
||||
"filename",
|
||||
@ -52,3 +54,12 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
|
||||
"vector_dimension",
|
||||
"indexed_at",
|
||||
}
|
||||
assert {column["name"] for column in inspector.get_columns("index_profiles")} == {
|
||||
"profile_id",
|
||||
"chunk_strategy_version",
|
||||
"embedding_model",
|
||||
"vector_dimension",
|
||||
"status",
|
||||
"created_at",
|
||||
"activated_at",
|
||||
}
|
||||
|
||||
177
tests/integration/test_index_rebuild.py
Normal file
177
tests/integration/test_index_rebuild.py
Normal file
@ -0,0 +1,177 @@
|
||||
"""Integration tests for retry-safe shadow index build and activation."""
|
||||
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from ekp_core.database import create_session_factory
|
||||
from ekp_core.index_rebuild import (
|
||||
IndexProfileIncompleteError,
|
||||
IndexRebuildService,
|
||||
resolve_active_index_profile,
|
||||
)
|
||||
from ekp_domain import (
|
||||
Base,
|
||||
Chunk,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
IndexProfile,
|
||||
IndexProfileStatus,
|
||||
StoredIndexProfile,
|
||||
)
|
||||
from ekp_embedding import FakeEmbeddingProvider
|
||||
from ekp_vector_store import FakeVectorStore
|
||||
|
||||
|
||||
def _add_document(session: object, *, filename: str, chunks: int) -> UUID:
|
||||
"""Persist one canonical document and deterministic strategy-v1 chunks."""
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
assert isinstance(session, Session)
|
||||
document = Document(
|
||||
filename=filename,
|
||||
media_type="text/plain",
|
||||
size_bytes=10,
|
||||
storage_key=f"documents/{uuid4()}/source.txt",
|
||||
status=DocumentStatus.READY,
|
||||
)
|
||||
session.add(document)
|
||||
session.flush()
|
||||
for index in range(chunks):
|
||||
session.add(
|
||||
Chunk(
|
||||
chunk_id=uuid4(),
|
||||
document_id=document.id,
|
||||
filename=filename,
|
||||
page=None,
|
||||
page_end=None,
|
||||
section="测试",
|
||||
section_path=["测试"],
|
||||
chunk_index=index,
|
||||
content=f"{filename} canonical chunk {index}",
|
||||
token_count=5,
|
||||
strategy_version="structure_v1",
|
||||
)
|
||||
)
|
||||
return document.id
|
||||
|
||||
|
||||
def _profiles() -> tuple[IndexProfile, IndexProfile]:
|
||||
return (
|
||||
IndexProfile("legacy-profile-v1", "structure_v1", "fake-bge", 4),
|
||||
IndexProfile("oceanbase-shadow-v1", "structure_v1", "fake-bge", 4),
|
||||
)
|
||||
|
||||
|
||||
def test_shadow_rebuild_is_retry_safe_and_activation_is_atomic() -> None:
|
||||
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
session_factory = create_session_factory(engine)
|
||||
legacy, target = _profiles()
|
||||
with session_factory() as session:
|
||||
_add_document(session, filename="first.txt", chunks=2)
|
||||
_add_document(session, filename="second.txt", chunks=1)
|
||||
session.add(
|
||||
StoredIndexProfile(
|
||||
profile_id=legacy.profile_id,
|
||||
chunk_strategy_version=legacy.chunk_strategy_version,
|
||||
embedding_model=legacy.embedding_model,
|
||||
vector_dimension=legacy.vector_dimension,
|
||||
status=IndexProfileStatus.ACTIVE,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
vector_store = FakeVectorStore()
|
||||
service = IndexRebuildService(
|
||||
session_factory,
|
||||
FakeEmbeddingProvider("fake-bge", dimensions=4),
|
||||
vector_store,
|
||||
target,
|
||||
)
|
||||
|
||||
first = service.rebuild()
|
||||
retry = service.rebuild()
|
||||
|
||||
assert first.complete is True
|
||||
assert first.expected_documents == 2
|
||||
assert first.expected_chunks == first.indexed_chunks == 3
|
||||
assert first.built_documents == 2
|
||||
assert retry.built_documents == 0
|
||||
assert retry.skipped_documents == 2
|
||||
assert vector_store.upsert_calls == 2
|
||||
|
||||
activated = service.activate()
|
||||
assert activated.complete is True
|
||||
assert resolve_active_index_profile(session_factory, legacy) == target
|
||||
with session_factory() as session:
|
||||
rows = session.scalars(select(StoredIndexProfile)).all()
|
||||
statuses = {row.profile_id: row.status for row in rows}
|
||||
assert statuses == {
|
||||
legacy.profile_id: IndexProfileStatus.RETIRED,
|
||||
target.profile_id: IndexProfileStatus.ACTIVE,
|
||||
}
|
||||
|
||||
|
||||
def test_incomplete_shadow_cannot_be_activated() -> None:
|
||||
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
session_factory = create_session_factory(engine)
|
||||
_, target = _profiles()
|
||||
with session_factory() as session:
|
||||
_add_document(session, filename="pending.txt", chunks=1)
|
||||
session.add(
|
||||
StoredIndexProfile(
|
||||
profile_id=target.profile_id,
|
||||
chunk_strategy_version=target.chunk_strategy_version,
|
||||
embedding_model=target.embedding_model,
|
||||
vector_dimension=target.vector_dimension,
|
||||
status=IndexProfileStatus.SHADOW,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
service = IndexRebuildService(
|
||||
session_factory,
|
||||
FakeEmbeddingProvider("fake-bge", dimensions=4),
|
||||
FakeVectorStore(),
|
||||
target,
|
||||
)
|
||||
|
||||
with pytest.raises(IndexProfileIncompleteError, match="incomplete"):
|
||||
service.activate()
|
||||
|
||||
with session_factory() as session:
|
||||
stored = session.get(StoredIndexProfile, target.profile_id)
|
||||
assert stored is not None
|
||||
assert stored.status is IndexProfileStatus.SHADOW
|
||||
|
||||
|
||||
def test_profile_registry_allows_multiple_shadows_but_only_one_active() -> None:
|
||||
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
session_factory = create_session_factory(engine)
|
||||
|
||||
def stored(profile_id: str, status: IndexProfileStatus) -> StoredIndexProfile:
|
||||
return StoredIndexProfile(
|
||||
profile_id=profile_id,
|
||||
chunk_strategy_version="structure_v1",
|
||||
embedding_model="fake-bge",
|
||||
vector_dimension=4,
|
||||
status=status,
|
||||
)
|
||||
|
||||
with session_factory() as session:
|
||||
session.add_all(
|
||||
[
|
||||
stored("shadow-v1", IndexProfileStatus.SHADOW),
|
||||
stored("shadow-v2", IndexProfileStatus.SHADOW),
|
||||
stored("active-v1", IndexProfileStatus.ACTIVE),
|
||||
]
|
||||
)
|
||||
session.commit()
|
||||
session.add(stored("active-v2", IndexProfileStatus.ACTIVE))
|
||||
with pytest.raises(IntegrityError):
|
||||
session.commit()
|
||||
@ -15,7 +15,7 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
|
||||
assert settings.api_prefix == "/api/v1"
|
||||
assert settings.api_port == 8000
|
||||
assert settings.database_url.startswith("postgresql+psycopg://")
|
||||
assert settings.object_storage_provider == "local"
|
||||
assert settings.object_storage_provider == "bos"
|
||||
assert settings.object_storage_path == Path("storage")
|
||||
assert settings.bos_endpoint is None
|
||||
assert settings.bos_access_key_id is None
|
||||
@ -31,11 +31,7 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
|
||||
assert settings.embedding_model == "bge-large-zh"
|
||||
assert settings.embedding_endpoint.endswith("/v2/embeddings")
|
||||
assert settings.baidu_qianfan_api_key is None
|
||||
assert settings.vector_store_provider == "mochow"
|
||||
assert settings.baidu_mochow_endpoint is None
|
||||
assert settings.mochow_table == "knowledge_chunks"
|
||||
assert settings.mochow_search_ef == 200
|
||||
assert settings.mochow_timeout_seconds == 30
|
||||
assert settings.vector_store_provider == "oceanbase"
|
||||
assert settings.oceanbase_vector_uri is None
|
||||
assert settings.oceanbase_vector_user is None
|
||||
assert settings.oceanbase_vector_password is None
|
||||
@ -44,7 +40,7 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
|
||||
assert settings.oceanbase_vector_index == "vidx_knowledge_chunks_embedding"
|
||||
assert settings.oceanbase_vector_batch_size == 100
|
||||
assert settings.oceanbase_vector_timeout_seconds == 30
|
||||
assert settings.index_profile_id == "demo-bge-large-zh-structure-v1"
|
||||
assert settings.index_profile_id == "demo-bge-large-zh-structure-v1-oceanbase-v1"
|
||||
assert settings.dense_top_n == 20
|
||||
assert settings.reranker_provider == "qianfan"
|
||||
assert settings.reranker_model == "bce-reranker-base"
|
||||
@ -75,7 +71,6 @@ def test_settings_reject_final_top_k_larger_than_dense_candidates(
|
||||
def test_settings_redact_provider_secrets(isolated_settings_environment: None) -> None:
|
||||
settings = Settings(
|
||||
baidu_qianfan_api_key=SecretStr("qianfan-secret"),
|
||||
baidu_mochow_api_key=SecretStr("mochow-secret"),
|
||||
bos_access_key_id=SecretStr("bos-access-key"),
|
||||
bos_secret_access_key=SecretStr("bos-secret-key"),
|
||||
oceanbase_vector_password=SecretStr("oceanbase-secret"),
|
||||
@ -84,7 +79,6 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) -
|
||||
|
||||
rendered = repr(settings)
|
||||
assert "qianfan-secret" not in rendered
|
||||
assert "mochow-secret" not in rendered
|
||||
assert "bos-access-key" not in rendered
|
||||
assert "bos-secret-key" not in rendered
|
||||
assert "oceanbase-secret" not in rendered
|
||||
|
||||
@ -14,7 +14,7 @@ from ekp_vector_store import FakeVectorStore
|
||||
|
||||
|
||||
class FailingDeleteVectorStore(FakeVectorStore):
|
||||
"""Simulate Mochow refusing cleanup before canonical deletion begins."""
|
||||
"""Simulate the active vector provider refusing cleanup before deletion."""
|
||||
|
||||
def delete_document(self, document_id: object, index_profile_id: str) -> None:
|
||||
raise RuntimeError("simulated vector deletion failure")
|
||||
|
||||
82
tests/unit/test_index_rebuild_cli.py
Normal file
82
tests/unit/test_index_rebuild_cli.py
Normal file
@ -0,0 +1,82 @@
|
||||
"""Tests for the secret-safe shadow index command boundary."""
|
||||
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
from ekp_api import index_rebuild
|
||||
from ekp_core.index_rebuild import IndexBuildSummary, IndexRebuildService
|
||||
|
||||
|
||||
class FakeRebuildService:
|
||||
"""Return one stable summary for every explicit lifecycle operation."""
|
||||
|
||||
def __init__(self, *, failure: Exception | None = None) -> None:
|
||||
self.failure = failure
|
||||
self.calls: list[str] = []
|
||||
|
||||
def _result(self, command: str) -> IndexBuildSummary:
|
||||
self.calls.append(command)
|
||||
if self.failure is not None:
|
||||
raise self.failure
|
||||
return IndexBuildSummary(
|
||||
profile_id="oceanbase-shadow-v1",
|
||||
expected_documents=2,
|
||||
built_documents=2 if command == "build" else 0,
|
||||
skipped_documents=0,
|
||||
expected_chunks=3,
|
||||
indexed_chunks=3,
|
||||
complete=True,
|
||||
)
|
||||
|
||||
def rebuild(self) -> IndexBuildSummary:
|
||||
return self._result("build")
|
||||
|
||||
def verify(self) -> IndexBuildSummary:
|
||||
return self._result("verify")
|
||||
|
||||
def activate(self) -> IndexBuildSummary:
|
||||
return self._result("activate")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("command", ["build", "verify", "activate"])
|
||||
def test_index_rebuild_cli_executes_one_step_and_prints_safe_json(
|
||||
command: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
service = FakeRebuildService()
|
||||
monkeypatch.setattr(
|
||||
index_rebuild,
|
||||
"_create_service",
|
||||
lambda settings: cast(IndexRebuildService, service),
|
||||
)
|
||||
|
||||
exit_code = index_rebuild.main([command])
|
||||
|
||||
output = capsys.readouterr()
|
||||
assert exit_code == 0
|
||||
assert service.calls == [command]
|
||||
assert '"complete":true' in output.out
|
||||
assert '"profile_id":"oceanbase-shadow-v1"' in output.out
|
||||
assert output.err == ""
|
||||
|
||||
|
||||
def test_index_rebuild_cli_does_not_print_provider_failure_details(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
service = FakeRebuildService(failure=RuntimeError("private-db.example password=secret-value"))
|
||||
monkeypatch.setattr(
|
||||
index_rebuild,
|
||||
"_create_service",
|
||||
lambda settings: cast(IndexRebuildService, service),
|
||||
)
|
||||
|
||||
exit_code = index_rebuild.main(["build"])
|
||||
|
||||
output = capsys.readouterr()
|
||||
assert exit_code == 1
|
||||
assert "private-db.example" not in output.err
|
||||
assert "secret-value" not in output.err
|
||||
assert "command failed" in output.err
|
||||
@ -1,130 +0,0 @@
|
||||
"""Unit tests for neutral-record translation into Mochow SDK operations."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from ekp_vector_store import DenseSearchRequest, MochowVectorStore, VectorRecord
|
||||
from ekp_vector_store.mochow import MochowConfigurationError, MochowVectorStoreError
|
||||
|
||||
|
||||
class FakeMochowTable:
|
||||
def __init__(self) -> None:
|
||||
self.upsert_batches: list[list[object]] = []
|
||||
self.delete_filters: list[str] = []
|
||||
self.search_calls: list[tuple[object, list[str]]] = []
|
||||
self.search_rows: list[dict[str, object]] = []
|
||||
|
||||
def upsert(self, rows: list[object]) -> object:
|
||||
self.upsert_batches.append(rows)
|
||||
return None
|
||||
|
||||
def delete(self, *, filter: str) -> object:
|
||||
self.delete_filters.append(filter)
|
||||
return None
|
||||
|
||||
def vector_search(self, *, request: object, projections: list[str]) -> object:
|
||||
self.search_calls.append((request, projections))
|
||||
return SimpleNamespace(rows=self.search_rows)
|
||||
|
||||
|
||||
def _record(document_id: UUID, chunk_number: int) -> VectorRecord:
|
||||
return VectorRecord(
|
||||
index_record_id=uuid4(),
|
||||
chunk_id=uuid4(),
|
||||
document_id=document_id,
|
||||
filename="policy.md",
|
||||
page=1,
|
||||
page_end=2,
|
||||
section="产品 / 退款",
|
||||
chunk_index=chunk_number,
|
||||
content="退款周期为七天。",
|
||||
embedding=(0.1, 0.2),
|
||||
embedding_model="fake",
|
||||
index_profile_id="profile-v1",
|
||||
)
|
||||
|
||||
|
||||
def test_mochow_adapter_maps_rows_and_batches_upserts() -> None:
|
||||
table = FakeMochowTable()
|
||||
store = MochowVectorStore(table, lambda **fields: fields, batch_size=2)
|
||||
document_id = uuid4()
|
||||
records = tuple(_record(document_id, index) for index in range(3))
|
||||
|
||||
store.upsert(records)
|
||||
|
||||
assert [len(batch) for batch in table.upsert_batches] == [2, 1]
|
||||
first_row = table.upsert_batches[0][0]
|
||||
assert isinstance(first_row, dict)
|
||||
assert first_row["index_record_id"] == str(records[0].index_record_id)
|
||||
assert first_row["document_id"] == str(document_id)
|
||||
assert first_row["embedding"] == [0.1, 0.2]
|
||||
assert first_row["section"] == "产品 / 退款"
|
||||
|
||||
|
||||
def test_mochow_adapter_deletes_only_document_profile_rows() -> None:
|
||||
table = FakeMochowTable()
|
||||
store = MochowVectorStore(table, lambda **fields: fields)
|
||||
document_id = uuid4()
|
||||
|
||||
store.delete_document(document_id, "profile-v1")
|
||||
|
||||
assert table.delete_filters == [
|
||||
f"document_id == '{document_id}' AND index_profile_id == 'profile-v1'"
|
||||
]
|
||||
|
||||
|
||||
def test_mochow_adapter_rejects_unsafe_profile_filter() -> None:
|
||||
store = MochowVectorStore(FakeMochowTable(), lambda **fields: fields)
|
||||
|
||||
with pytest.raises(MochowConfigurationError):
|
||||
store.delete_document(uuid4(), "bad' OR true")
|
||||
|
||||
|
||||
def test_mochow_adapter_builds_profile_scoped_dense_topk_request() -> None:
|
||||
table = FakeMochowTable()
|
||||
source = _record(uuid4(), 3)
|
||||
table.search_rows = [
|
||||
{
|
||||
"row": {
|
||||
"index_record_id": str(source.index_record_id),
|
||||
"chunk_id": str(source.chunk_id),
|
||||
"document_id": str(source.document_id),
|
||||
"filename": source.filename,
|
||||
"page": source.page,
|
||||
"page_end": source.page_end,
|
||||
"section": source.section,
|
||||
"chunk_index": source.chunk_index,
|
||||
"content": source.content,
|
||||
"index_profile_id": source.index_profile_id,
|
||||
},
|
||||
"score": 0.87,
|
||||
"distance": 0.13,
|
||||
}
|
||||
]
|
||||
store = MochowVectorStore(table, lambda **fields: fields, search_ef=10)
|
||||
|
||||
candidates = store.dense_search(
|
||||
DenseSearchRequest((0.1, 0.2), index_profile_id="profile-v1", top_n=20)
|
||||
)
|
||||
|
||||
sdk_request, projections = table.search_calls[0]
|
||||
request_body = sdk_request.to_dict() # type: ignore[attr-defined]
|
||||
assert request_body["anns"]["vectorField"] == "embedding"
|
||||
assert request_body["anns"]["vectorFloats"] == [0.1, 0.2]
|
||||
assert request_body["anns"]["filter"] == "index_profile_id = 'profile-v1'"
|
||||
assert request_body["anns"]["params"]["limit"] == 20
|
||||
assert request_body["anns"]["params"]["ef"] == 20
|
||||
assert "content" in projections
|
||||
assert candidates[0].chunk_id == source.chunk_id
|
||||
assert candidates[0].score == pytest.approx(0.87)
|
||||
|
||||
|
||||
def test_mochow_adapter_sanitizes_malformed_search_response() -> None:
|
||||
table = FakeMochowTable()
|
||||
table.search_rows = [{"score": "not-a-number"}]
|
||||
store = MochowVectorStore(table, lambda **fields: fields)
|
||||
|
||||
with pytest.raises(MochowVectorStoreError, match="dense search failed"):
|
||||
store.dense_search(DenseSearchRequest((0.1, 0.2), "profile-v1"))
|
||||
@ -27,25 +27,44 @@ def test_provider_wiring_builds_profile_and_qianfan_without_network(
|
||||
profile = create_index_profile(settings)
|
||||
embedding = create_embedding_provider(settings)
|
||||
|
||||
assert profile.profile_id == "demo-bge-large-zh-structure-v1"
|
||||
assert profile.profile_id == "demo-bge-large-zh-structure-v1-oceanbase-v1"
|
||||
assert profile.chunk_strategy_version == "structure_v1"
|
||||
assert profile.vector_dimension == 1024
|
||||
assert embedding.model_name == profile.embedding_model
|
||||
|
||||
|
||||
def test_provider_wiring_requires_explicit_mochow_target(
|
||||
def test_provider_wiring_resolves_canonical_active_profile(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
settings = Settings(baidu_mochow_api_key=SecretStr("test-key"))
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
with pytest.raises(ProviderConfigurationError, match="ENDPOINT"):
|
||||
create_vector_store(settings)
|
||||
from ekp_core.database import create_session_factory
|
||||
from ekp_domain import Base, IndexProfileStatus, StoredIndexProfile
|
||||
|
||||
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
session_factory = create_session_factory(engine)
|
||||
with session_factory() as session:
|
||||
session.add(
|
||||
StoredIndexProfile(
|
||||
profile_id="activated-oceanbase-v2",
|
||||
chunk_strategy_version="structure_v1",
|
||||
embedding_model="bge-large-zh",
|
||||
vector_dimension=1024,
|
||||
status=IndexProfileStatus.ACTIVE,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
profile = create_index_profile(Settings(), session_factory)
|
||||
|
||||
assert profile.profile_id == "activated-oceanbase-v2"
|
||||
|
||||
|
||||
def test_provider_wiring_requires_complete_oceanbase_target(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
settings = Settings(vector_store_provider="oceanbase")
|
||||
settings = Settings()
|
||||
|
||||
with pytest.raises(ProviderConfigurationError, match="OCEANBASE_VECTOR_URI"):
|
||||
create_vector_store(settings)
|
||||
@ -64,7 +83,6 @@ def test_provider_wiring_builds_oceanbase_from_secret_safe_settings(
|
||||
|
||||
monkeypatch.setattr("ekp_api.providers.create_oceanbase_vector_store", fake_factory)
|
||||
settings = Settings(
|
||||
vector_store_provider="oceanbase",
|
||||
oceanbase_vector_uri="db.example:2881",
|
||||
oceanbase_vector_user="test-user@test-tenant",
|
||||
oceanbase_vector_password=SecretStr("test-password"),
|
||||
@ -80,10 +98,10 @@ def test_provider_wiring_builds_oceanbase_from_secret_safe_settings(
|
||||
assert captured["vector_dimension"] == 1024
|
||||
|
||||
|
||||
def test_provider_wiring_selects_local_storage_by_default(
|
||||
def test_provider_wiring_keeps_local_storage_for_explicit_development_use(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
storage = create_object_storage(Settings())
|
||||
storage = create_object_storage(Settings(object_storage_provider="local"))
|
||||
|
||||
assert isinstance(storage, LocalObjectStorage)
|
||||
|
||||
|
||||
141
uv.lock
generated
141
uv.lock
generated
@ -74,77 +74,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/de/52b763b183e23f4dd73811a2dd2664824ae7fc6ceb1aa8dcd3f1102b1094/bce_python_sdk-0.9.79-py3-none-any.whl", hash = "sha256:71799ac8740505e0759d30873f6f1a478fa8f83aedf425d511f1419a5f30082e", size = 440268, upload-time = "2026-09-01T05:48:50.459Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.7.22"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "charset-normalizer"
|
||||
version = "3.5.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e5/3f/143b048436775b0f76ac3eec145c019e8173ccc2885c8f20319b996d5e83/charset_normalizer-3.5.1.tar.gz", hash = "sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3", size = 171764, upload-time = "2026-08-15T08:20:44.807Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/30/27/78873dc8b6a56357517b74b6bb9568b80450e7bb4f6ef7e3fa9d22aa0bd7/charset_normalizer-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f", size = 344456, upload-time = "2026-08-15T08:17:10.072Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/4c/be49ada26b1f0232d57aa89bbebf997a5cc2332a5616b6eca26ff680044d/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa", size = 238530, upload-time = "2026-08-15T08:17:11.563Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/84/6f1290fa07ae6978d3960caa3eb1b8019bf9284ab7c2297b00c099ef4250/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369", size = 230200, upload-time = "2026-08-15T08:17:12.919Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/a0/47b18adeed31c8f16ba9700f32c1b18594cfa09f47eb672a488c273c22bf/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893", size = 262222, upload-time = "2026-08-15T08:17:14.571Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/38/fe/341861ac118dae06f3ec0eb487488af52128f2ef2faf0b11003944d22259/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0", size = 258951, upload-time = "2026-08-15T08:17:16.158Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6f/89/bb5108dc6c3651dca963f2b0a3ba19bbcb370c94e1b6d3e0e844a58e6dca/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08", size = 248801, upload-time = "2026-08-15T08:17:17.683Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/ba/ef83ae3aca816393decfa3530976f38a79812d707b80b580ac33b83f9877/charset_normalizer-3.5.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada", size = 244070, upload-time = "2026-08-15T08:17:19.191Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/0b/c5292a2462d69b7378ea89793bbb5b2b6fcf6f7dd6d1667f9619094ad553/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9", size = 240110, upload-time = "2026-08-15T08:17:20.547Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/22/111e5be3b740d5c2a5bfcedb3d237b6591e5c2e82ae9d6ffcb121fe0909c/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e", size = 232836, upload-time = "2026-08-15T08:17:21.895Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/d2/d2aad6fe0dbb44b194bf3becb60f5a0ac48446ade999a47fe7bb41eb09a7/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6", size = 262712, upload-time = "2026-08-15T08:17:23.727Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/5a/337e4663a5eae6de99db940ee8066d4145caafb61327db62deda15313cce/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf", size = 242977, upload-time = "2026-08-15T08:17:25.157Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/85/f82f8a92e31c7519410e2e1afdc630f28ec47490ce2c09a11c1a43cbb459/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71", size = 260207, upload-time = "2026-08-15T08:17:26.602Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/52/643d11ffd60e9ac2fd1fb87e167a19285b9eefeff4a40e63c87cbfbeab36/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573", size = 250562, upload-time = "2026-08-15T08:17:27.971Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/16/46556278c2168d12df9da7fede5dc6fc70e60301b26a82bbeec238c9cfe3/charset_normalizer-3.5.1-cp312-cp312-win32.whl", hash = "sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2", size = 178507, upload-time = "2026-08-15T08:17:29.277Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/7a/4c6c298171e6b3e745633180ff59350fc0ca0db1ffd28df1e369e0579f71/charset_normalizer-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2", size = 200551, upload-time = "2026-08-15T08:17:30.668Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/d7/eb95a042f0dd22e304b0b6472b154f3546a1a039a9ee89ccb2a7f61591fc/charset_normalizer-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a", size = 180700, upload-time = "2026-08-15T08:17:32.028Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/61/2cb6ad133dbbb449fa2d37ccae973232f4827e799af258d15e589a3d1e9e/charset_normalizer-3.5.1-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9", size = 211584, upload-time = "2026-08-15T08:17:33.597Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/57/a305c968be1ca13f3dd1b32f445877e97addf55d80b65c7cb35fac82b777/charset_normalizer-3.5.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491", size = 223359, upload-time = "2026-08-15T08:17:35.022Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/09/0a/d3646670292ce8d8f8cc11ac067d44885e697a5591f57a9221128da5e7b3/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7", size = 194464, upload-time = "2026-08-15T08:17:36.452Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/93/d51ec556e01042fed6f993ea859311bc7917b466684182fbbceb6ca24762/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e", size = 197676, upload-time = "2026-08-15T08:17:37.819Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a4/a0/562247944386f7d4ef94467e84876600cc1e0f1b93239aaa9213d2bc3cbd/charset_normalizer-3.5.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d", size = 340473, upload-time = "2026-08-15T08:17:39.303Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/e7/1d994be1b93d41e9502b8b0460eaa88a1dd8df335df415db87d6c3e91ab2/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a", size = 240156, upload-time = "2026-08-15T08:17:40.66Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/09/53/27923ce5cc6cbccb832037b27dca98882d9c53e9b69e866bbbef4aae7fc8/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe", size = 228246, upload-time = "2026-08-15T08:17:42.003Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/48/5a97e84d63af1d55c07439cb80e56d99a8efb4295700eb4e18c0d1615d2c/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac", size = 263660, upload-time = "2026-08-15T08:17:43.627Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7a/c2/071575791dcc88316c0a9a65ce38897a82e4cfe4a325f0f7fe1b1ac47bcf/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e", size = 260354, upload-time = "2026-08-15T08:17:45.094Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/af/63240b0c0248c075c2535a1f1bd992821d8251b9f173abc13329661d09e4/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3", size = 250638, upload-time = "2026-08-15T08:17:46.496Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4d/66/70dfad64f15be09c15ccfee81330a7e515895dbe296dd23114e9a231268a/charset_normalizer-3.5.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876", size = 244583, upload-time = "2026-08-15T08:17:47.963Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c0/24/ef36367d38b9ddd4bccbf72888c342e8de1f5ae506fa0b2dcf970e2732a1/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6", size = 242038, upload-time = "2026-08-15T08:17:49.481Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/ab/55e683ba0fff2e43adafc10daa3001eac90fdaa419a97227d5a7067eedde/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2", size = 233677, upload-time = "2026-08-15T08:17:50.845Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/67/0f40eaf8d1b6e7cf15e82382a2965efaca787fc1c2794b7021d37aaf5036/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591", size = 264491, upload-time = "2026-08-15T08:17:52.61Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/64/12b4c2a11ee8df4fcc518c78b0d93e3a92bd3d5253d1617ce74ff0e8c7ef/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c", size = 245196, upload-time = "2026-08-15T08:17:54.023Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/37/2e/651d910af6d0fba325eee1cda37ec5443462ed25360e666c144166eb6091/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c", size = 261660, upload-time = "2026-08-15T08:17:55.491Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/90/c6/b09e05e6db7f64338e0dc067c79577b1138da86c1e38369096851d96be88/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f", size = 252618, upload-time = "2026-08-15T08:17:57.025Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/4e/362d4f9fdcdf5556fb2aa3ce7d4a58ebce03ed1ff03aa1d9aca8d02f13f3/charset_normalizer-3.5.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4", size = 140362, upload-time = "2026-08-15T08:17:58.425Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b4/d4/703be739b26acce318bd29eb3b25b7209e1b1f527f9eae3d1f1f01fdde2b/charset_normalizer-3.5.1-cp313-cp313-win32.whl", hash = "sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3", size = 177755, upload-time = "2026-08-15T08:18:00.037Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8a/33/56d97ade41c8db611e727168c52ae46c9224c362ec28d4b65d7e9869e8da/charset_normalizer-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6", size = 199295, upload-time = "2026-08-15T08:18:01.506Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/75/5b20dd1e6573a01a08158fe104104fa2c8abf941745596954185726cd46c/charset_normalizer-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0", size = 179856, upload-time = "2026-08-15T08:18:02.929Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/97/fb4e82231aba271ffd775a1b4993b0defc4e3059f286ae41d9433409fe85/charset_normalizer-3.5.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2", size = 331467, upload-time = "2026-08-15T08:19:50.959Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9f/2f/fe3f187327aac18e2d54e9d2b08e15d27bf9b642d9e51c219f130fc34d1a/charset_normalizer-3.5.1-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99", size = 253057, upload-time = "2026-08-15T08:19:52.654Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/c7/9e48cee5c161fe24da823b61bf381921d77cb994a0a4de148e95018c1984/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2", size = 240930, upload-time = "2026-08-15T08:19:54.163Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/e0/716601f3cc69be7b198951150c75ead1ece33c3c8036ff6ffa46029659a0/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235", size = 230822, upload-time = "2026-08-15T08:19:55.807Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/05/71bfc5caa0abcc45aea1f6a4d50ac68e59605ddc7666fe8494f4cd229665/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598", size = 260037, upload-time = "2026-08-15T08:19:57.312Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/92/de7e32ed05341e7a9c4c877c318418197b7f2d66a3b68d561bf2ac57ca3e/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96", size = 255097, upload-time = "2026-08-15T08:19:59.056Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f5/7b/ade0a122600319dfa0b1000ab0f9731c94a817904cf3c5de408c73a4ede7/charset_normalizer-3.5.1-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962", size = 250166, upload-time = "2026-08-15T08:20:00.612Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/75/9c/019fbb9f4834491a160951349b1a3714439376f66e5f7cf18b4f18f0c7aa/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3", size = 241821, upload-time = "2026-08-15T08:20:02.321Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2b/b8/11d4840bfc99330cc7fbcc2681ee5a044553a6e77655508d8f9b2bff7b34/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950", size = 232529, upload-time = "2026-08-15T08:20:04.008Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/96/2b3a21492d9f65171ac75d872f5018260013d00bfa0ff70ec9f179148cbd/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8", size = 260348, upload-time = "2026-08-15T08:20:05.877Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/aa/a69a2028e8bd052476c245460ab19d7de595de084dd968f2d75cd50c3e25/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031", size = 247234, upload-time = "2026-08-15T08:20:07.487Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/8a/3d130aeabcaf3d2466af76b7b141c08d9e89c9016ab4b7cdd0f7dc2d1c62/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072", size = 256917, upload-time = "2026-08-15T08:20:09.142Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/c2/a7379b840292d0c1ab9fbd17d1f3967aa81794dc95bc74be8999d7fedcf7/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d", size = 254846, upload-time = "2026-08-15T08:20:10.727Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/01/65/d43b714731bb2f40d4053dfa00ecfc1c5a301f8e3316c5db3a09af59fe94/charset_normalizer-3.5.1-cp37-abi3-win32.whl", hash = "sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc", size = 174216, upload-time = "2026-08-15T08:20:12.334Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/4f/b911ed898b26a09789eba9c9200c999aff6c61b4bafaf4838e56d1a1e1a3/charset_normalizer-3.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959", size = 199764, upload-time = "2026-08-15T08:20:13.908Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/a7/920baf467bfd9bf689f3b318340f37aee4572a71f162bd8db51da55ba4fa/charset_normalizer-3.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e", size = 287318, upload-time = "2026-08-15T08:20:15.551Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/61/d01fc49b8dea277640b55a9e15960dbca9fdc8c9fde18e572d39c59f4019/charset_normalizer-3.5.1-py3-none-any.whl", hash = "sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6", size = 68658, upload-time = "2026-08-15T08:20:43.306Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "click"
|
||||
version = "8.5.0"
|
||||
@ -240,7 +169,6 @@ dependencies = [
|
||||
{ name = "fastapi" },
|
||||
{ name = "psycopg", extra = ["binary"] },
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "pymochow" },
|
||||
{ name = "pymupdf" },
|
||||
{ name = "pyobvector" },
|
||||
{ name = "python-docx" },
|
||||
@ -265,7 +193,6 @@ requires-dist = [
|
||||
{ name = "fastapi", specifier = ">=0.115,<1" },
|
||||
{ name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.7,<3" },
|
||||
{ name = "pymochow", specifier = ">=2.4,<3" },
|
||||
{ name = "pymupdf", specifier = ">=1.26,<2" },
|
||||
{ name = "pyobvector", specifier = ">=0.2.28,<0.3" },
|
||||
{ name = "python-docx", specifier = ">=1.2,<2" },
|
||||
@ -624,36 +551,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/21/35b31dde1b283b79de828b80f876afd8c94e28fe1e9c375f89e261cc4c0d/numpy-2.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:bd68ece1553d2023c09a4226d9e41c586ad2d20594d1a456186c33513d2cb3f2", size = 10396725, upload-time = "2026-08-09T13:46:00.478Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "orjson"
|
||||
version = "3.12.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0f/f3/742fb1f62b825f2c010697eaf4e828004bc2a81e7e806666989c132c7c42/orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5", size = 4142915, upload-time = "2026-08-14T16:13:30.607Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/be/4a/295da39c651c2faac8bd351a2a346f0fdedd9d50b847ee9dfc27d2207ef6/orjson-3.12.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:aa3e43a6846e91d7bde3d5a9c66090fcd8744f569a9b6cffc5e1ca38f6a461c0", size = 223427, upload-time = "2026-08-14T16:12:28.525Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/29/98/758cf90fbeaaafb7f8141bfac75a432099959f3a2f5db93a412e876415d8/orjson-3.12.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:11edb4660a6680abee9788a3a9072208a2c96538cc1322bd79542065229d8e54", size = 123725, upload-time = "2026-08-14T16:12:30.013Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/32/b5/5b934d251f8651f7e41df180ad0c57a6e1cabe15c7bd331638413a50ebc9/orjson-3.12.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2d3a9da945a4d96ae758fdaaca56742e6b73b6fd554c5d8876f252a6dad70b83", size = 113375, upload-time = "2026-08-14T16:12:31.209Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/d2/37efb5b12a176ce3ced29f4144f20da57d02757f78ce549637dc1b4e1fc8/orjson-3.12.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:92ffc09e07233a6ab6d4e067f7841edcbcc134cb4812155cf171ea5255a421d7", size = 129983, upload-time = "2026-08-14T16:12:32.721Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/22/0644b87c73f13e0092df8f35a1fe280d991e5e90072087411e0dd7e44e0c/orjson-3.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf44e374aadde77b1f6109f1030be51433eb61984379852766b6f4e187db7b1e", size = 130629, upload-time = "2026-08-14T16:12:34.084Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/57/80b986ebfecd9c6a177ddf1c2319717f0cd8feffb2b78946595a18a2fc88/orjson-3.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1192a7021b6d071aaf909864f6e924d6a2675ca360485b972b8401749311750b", size = 131245, upload-time = "2026-08-14T16:12:35.713Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/3d/75c5ac5a69161f44492a68fbdde66f4cc4ce48cd5e1fb05918e46f0c8848/orjson-3.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:53c0c474a9d9aff9aebfc0c88de1f28f843d940e6e3a80729abdf6a20274356f", size = 135397, upload-time = "2026-08-14T16:12:37.128Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/93/4d71f2df314a97ff0d27a4559bf5888fc8406e3c6dec90e92291e3511215/orjson-3.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:532ff8cd4bd59a327a953a7dcde922c7fc25b85e29721bb8633265430d3a3873", size = 127693, upload-time = "2026-08-14T16:12:38.627Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/1d/0dbc6be5adfd1730491072fb60beb6bcdf5d7b2596ee41b7fc2e298bfc09/orjson-3.12.0-cp312-cp312-win32.whl", hash = "sha256:a6cf4b18e7de173f209f2084ffbd736dd72389a396326ee80a7022168be232e5", size = 128000, upload-time = "2026-08-14T16:12:39.954Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2d/c9/97b1ce0112ebf5e949c775ed5b1755e562233179f3584579673cc24d6378/orjson-3.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:010811c1b69773450a01cef97727a67b223242f350b77d4ca000e59a9ef2155a", size = 122106, upload-time = "2026-08-14T16:12:41.324Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/6a/facd8b312e4a0d3a7fa978c7e15821f74a336adf1d65529faec33b48e18b/orjson-3.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:ad29eece0c601737f2a60edc2752a84e7a0785df3efb62e3012834700a5afe0d", size = 126869, upload-time = "2026-08-14T16:12:42.651Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/cb/d7b78218a987eb8a8ce4eeae0286b1bb679333eb631ea0eeaf6371680bfc/orjson-3.12.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9a36ec60f1796f9a3f13e3b98390295e17a1c7c10155b448d264098bf9ee5900", size = 223397, upload-time = "2026-08-14T16:12:44.003Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/4a/bc87c45e7ec639d35ebefd62618e01939531ac8e171426606a01bda05914/orjson-3.12.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ad0422b92d5195443a39f80c3bcf731cc2e00f153bd32063a47b73b057bd0f03", size = 123662, upload-time = "2026-08-14T16:12:45.433Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/ee/c9a4ff3f2dbedbbe9e635d0fa72c8866adede09b6335ef9644f53752f0d8/orjson-3.12.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5a0fdbc216388f653d3752ff310e710f59253bd4ed6a2bfb3f4f06b84714bbd8", size = 113374, upload-time = "2026-08-14T16:12:46.755Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/75/09/3f330a026a796c8b4c97a6f429652a5e912e7065039bf96ed25e42aa7b25/orjson-3.12.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2eb5c56e534127b2b8fa38d2363c8b1b8190367ee0d1d16c041517d880843b94", size = 130029, upload-time = "2026-08-14T16:12:48.06Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7d/40/094cc53126a3d22f76cdf83b6ea67338bed01d774037621a785aa8e6e5ea/orjson-3.12.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:784106539f4b9d4b930e0b4eb8d45168507dae001945e71b4675a367f1e5e806", size = 130528, upload-time = "2026-08-14T16:12:49.362Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/74/89bb236deb9565f99434b13052bb40ddfcce4adf3afbfa3132ee7e421468/orjson-3.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c680706fc8396d95e7c4c1f9482563f552137aef91b57237a3ad5aaf64629df", size = 131075, upload-time = "2026-08-14T16:12:50.692Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/ac/1176360d762c01b5bd34acd56fc098e936c491363d8b6b397ad4aa475547/orjson-3.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:83445adc40cba26d6d621185a45128ce455b766af368cad2ab64b970603a7978", size = 135321, upload-time = "2026-08-14T16:12:52.114Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7a/02/bbd881c8b9276d50b998de38b4e97de8ace1aac940b0ee545aedbf65ed00/orjson-3.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:644d005bc82f917337a95ce270c9f6f92f9834c2bed7b1477572f8db00784222", size = 127472, upload-time = "2026-08-14T16:12:53.517Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8e/02/a0934d7503e6dcbedd6afac3e7f3f8597fd09389949ad94d0f7540e9dbca/orjson-3.12.0-cp313-cp313-win32.whl", hash = "sha256:d8e78d3d93705e3d27cc17cdb209e44d7a8ea203010cac6ce9c7ffc1ae1996f1", size = 128000, upload-time = "2026-08-14T16:12:55.14Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/52/87/69f98f8d40faff103a965a5fbb83f08241b01beaf92badb5413fbc9358cc/orjson-3.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b85931be5b6763c31283805c9bdaae1ca03ad9f6f12a15f1cbf6745b907932c2", size = 121841, upload-time = "2026-08-14T16:12:56.507Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/07/b83046a4e3cadcc0987d0f160696107c4af706a619b56e4ad01940cadadf/orjson-3.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:6a31348d7dfa64cd9c78bd1f510ff44c48fe64d71094e6b90e364dba3b55949e", size = 126765, upload-time = "2026-08-14T16:12:57.806Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "26.3"
|
||||
@ -841,20 +738,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pymochow"
|
||||
version = "2.4.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "future" },
|
||||
{ name = "orjson" },
|
||||
{ name = "requests" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1c/17/16e8db9d6299cbb85a73bee162601ad2d4bd619567f0b71fef8f710b51af/pymochow-2.4.1.tar.gz", hash = "sha256:e53a700337c9e538bdd62597e13c05532136451a7722459b90474ce38c7c62b0", size = 51571, upload-time = "2026-05-09T10:24:59.328Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e8/ec/c8ec3d989c7391e58403ad670d996611b630e4969b74298cc977442425ac/pymochow-2.4.1-py3-none-any.whl", hash = "sha256:a8c7b269e9b5997347b15806c953b62b1305c6a770f4f83af3f96199498252c3", size = 79391, upload-time = "2026-05-09T10:24:56.259Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pymupdf"
|
||||
version = "1.28.2"
|
||||
@ -987,21 +870,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "requests"
|
||||
version = "2.34.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
{ name = "charset-normalizer" },
|
||||
{ name = "idna" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.16.5"
|
||||
@ -1124,15 +992,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "urllib3"
|
||||
version = "2.7.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "uvicorn"
|
||||
version = "0.52.4"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user