311 lines
10 KiB
Python
311 lines
10 KiB
Python
"""Offline tests for neutral-record translation into pyobvector operations."""
|
|
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from typing import Any, cast
|
|
from uuid import UUID, uuid4
|
|
|
|
import pytest
|
|
|
|
from ekp_vector_store import DenseSearchRequest, OceanBaseVectorStore, VectorRecord
|
|
from ekp_vector_store.oceanbase import (
|
|
OceanBaseConfigurationError,
|
|
OceanBaseVectorStoreError,
|
|
_build_dense_search_sql,
|
|
_PyObVectorClient,
|
|
_serialize_vector,
|
|
_validate_oceanbase_version,
|
|
)
|
|
|
|
|
|
class FakeOceanBaseClient:
|
|
"""Capture ObVecClient-shaped calls without a database connection."""
|
|
|
|
def __init__(self) -> None:
|
|
self.upsert_calls: list[tuple[str, list[dict[str, object]]]] = []
|
|
self.delete_calls: list[tuple[str, list[object]]] = []
|
|
self.search_calls: list[dict[str, object]] = []
|
|
self.search_rows: list[tuple[object, ...]] = []
|
|
self.failure: Exception | None = None
|
|
|
|
def upsert(self, table_name: str, data: list[dict[str, object]]) -> object:
|
|
self._raise_failure()
|
|
self.upsert_calls.append((table_name, data))
|
|
return object()
|
|
|
|
def delete(
|
|
self,
|
|
table_name: str,
|
|
*,
|
|
where_clause: list[object],
|
|
) -> object:
|
|
self._raise_failure()
|
|
self.delete_calls.append((table_name, where_clause))
|
|
return object()
|
|
|
|
def dense_search(
|
|
self,
|
|
table_name: str,
|
|
*,
|
|
index_name: str,
|
|
query_vector: tuple[float, ...],
|
|
index_profile_id: str,
|
|
top_n: int,
|
|
output_column_names: list[str],
|
|
) -> list[tuple[object, ...]]:
|
|
self._raise_failure()
|
|
self.search_calls.append(
|
|
{
|
|
"table_name": table_name,
|
|
"index_name": index_name,
|
|
"query_vector": query_vector,
|
|
"index_profile_id": index_profile_id,
|
|
"top_n": top_n,
|
|
"output_column_names": output_column_names,
|
|
}
|
|
)
|
|
return self.search_rows
|
|
|
|
def _raise_failure(self) -> None:
|
|
if self.failure is not None:
|
|
raise self.failure
|
|
|
|
|
|
class FakeConnection:
|
|
"""Capture SQLAlchemy text and bound values without executing vendor SQL."""
|
|
|
|
def __init__(self) -> None:
|
|
self.executions: list[tuple[str, dict[str, object]]] = []
|
|
|
|
def __enter__(self) -> "FakeConnection":
|
|
return self
|
|
|
|
def __exit__(self, *args: object) -> None:
|
|
return None
|
|
|
|
def execute(self, statement: object, parameters: dict[str, object]) -> list[object]:
|
|
self.executions.append((str(statement), parameters))
|
|
return []
|
|
|
|
|
|
class FakeEngine:
|
|
"""Return one inspectable fake connection."""
|
|
|
|
def __init__(self) -> None:
|
|
self.connection = FakeConnection()
|
|
|
|
def connect(self) -> FakeConnection:
|
|
return self.connection
|
|
|
|
|
|
class FakeSdkClient:
|
|
"""Provide the official client's engine shape for the SQL boundary test."""
|
|
|
|
def __init__(self) -> None:
|
|
self.engine = FakeEngine()
|
|
|
|
|
|
def _record(document_id: UUID, chunk_index: int) -> VectorRecord:
|
|
return VectorRecord(
|
|
index_record_id=uuid4(),
|
|
chunk_id=uuid4(),
|
|
document_id=document_id,
|
|
filename="policy.md",
|
|
page=1,
|
|
page_end=2,
|
|
section="产品 / 退款",
|
|
chunk_index=chunk_index,
|
|
content="退款周期为七天。",
|
|
embedding=(0.1, 0.2, 0.3),
|
|
embedding_model="fake",
|
|
index_profile_id="profile-v1",
|
|
)
|
|
|
|
|
|
def _search_row(record: VectorRecord, distance: float) -> tuple[object, ...]:
|
|
return (
|
|
str(record.index_record_id),
|
|
str(record.chunk_id),
|
|
str(record.document_id),
|
|
record.filename,
|
|
record.page,
|
|
record.page_end,
|
|
record.section,
|
|
record.chunk_index,
|
|
record.content,
|
|
record.index_profile_id,
|
|
distance,
|
|
)
|
|
|
|
|
|
def test_oceanbase_adapter_maps_and_batches_upserts() -> None:
|
|
client = FakeOceanBaseClient()
|
|
store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3, batch_size=2)
|
|
document_id = uuid4()
|
|
records = tuple(_record(document_id, index) for index in range(3))
|
|
|
|
store.upsert(records)
|
|
|
|
assert [len(data) for _, data in client.upsert_calls] == [2, 1]
|
|
first = client.upsert_calls[0][1][0]
|
|
assert first["index_record_id"] == str(records[0].index_record_id)
|
|
assert first["document_id"] == str(document_id)
|
|
assert first["embedding"] == [0.1, 0.2, 0.3]
|
|
assert first["index_profile_id"] == "profile-v1"
|
|
|
|
|
|
def test_oceanbase_delete_uses_bound_document_and_profile_predicate() -> None:
|
|
client = FakeOceanBaseClient()
|
|
store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3)
|
|
document_id = uuid4()
|
|
|
|
store.delete_document(document_id, "profile-v1")
|
|
|
|
table_name, clauses = client.delete_calls[0]
|
|
compiled = cast(Any, clauses[0]).compile()
|
|
assert table_name == "knowledge_chunks_v1"
|
|
assert (
|
|
str(compiled) == "document_id = :document_id_1 AND index_profile_id = :index_profile_id_1"
|
|
)
|
|
assert set(compiled.params.values()) == {str(document_id), "profile-v1"}
|
|
|
|
|
|
def test_oceanbase_dense_search_filters_profile_and_normalizes_cosine_distance() -> None:
|
|
client = FakeOceanBaseClient()
|
|
first = _record(uuid4(), 0)
|
|
second = _record(uuid4(), 1)
|
|
client.search_rows = [_search_row(first, 0.1), _search_row(second, 0.4)]
|
|
store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3)
|
|
|
|
candidates = store.dense_search(
|
|
DenseSearchRequest((0.3, 0.2, 0.1), index_profile_id="profile-v1", top_n=20)
|
|
)
|
|
|
|
call = client.search_calls[0]
|
|
assert call["table_name"] == "knowledge_chunks_v1"
|
|
assert call["index_name"] == "vidx_knowledge_chunks_embedding"
|
|
assert call["query_vector"] == (0.3, 0.2, 0.1)
|
|
assert call["index_profile_id"] == "profile-v1"
|
|
assert call["top_n"] == 20
|
|
assert [candidate.score for candidate in candidates] == pytest.approx([0.9, 0.6])
|
|
assert candidates[0].chunk_id == first.chunk_id
|
|
|
|
|
|
def test_oceanbase_rejects_wrong_dimension_and_zero_cosine_vector() -> None:
|
|
store = OceanBaseVectorStore(
|
|
FakeOceanBaseClient(),
|
|
"knowledge_chunks_v1",
|
|
vector_dimension=3,
|
|
)
|
|
|
|
with pytest.raises(OceanBaseConfigurationError, match="dimension mismatch"):
|
|
store.dense_search(DenseSearchRequest((0.1, 0.2), "profile-v1"))
|
|
with pytest.raises(OceanBaseConfigurationError, match="norm must be positive"):
|
|
store.upsert((replace(_record(uuid4(), 0), embedding=(0.0, 0.0, 0.0)),))
|
|
|
|
|
|
def test_oceanbase_sanitizes_malformed_or_failed_search() -> None:
|
|
client = FakeOceanBaseClient()
|
|
client.search_rows = [("malformed",)]
|
|
store = OceanBaseVectorStore(client, "knowledge_chunks_v1", vector_dimension=3)
|
|
|
|
with pytest.raises(OceanBaseVectorStoreError, match="dense search failed"):
|
|
store.dense_search(DenseSearchRequest((0.1, 0.2, 0.3), "profile-v1"))
|
|
|
|
client.failure = RuntimeError("private-host password=secret-value")
|
|
with pytest.raises(OceanBaseVectorStoreError, match="cleanup failed") as caught:
|
|
store.delete_document(uuid4(), "profile-v1")
|
|
assert "private-host" not in str(caught.value)
|
|
assert "secret-value" not in str(caught.value)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"uri",
|
|
["", "https://db.example:2881", "user:password@db.example:2881", "db.example"],
|
|
)
|
|
def test_oceanbase_factory_rejects_unsafe_uri_without_connecting(uri: str) -> None:
|
|
from ekp_vector_store.oceanbase import create_oceanbase_vector_store
|
|
|
|
with pytest.raises(OceanBaseConfigurationError, match="URI"):
|
|
create_oceanbase_vector_store(
|
|
uri=uri,
|
|
user="test-user",
|
|
password="test-password",
|
|
database_name="test_database",
|
|
table_name="knowledge_chunks_v1",
|
|
)
|
|
|
|
|
|
def test_oceanbase_dense_sql_requires_profile_pre_filter_and_bound_values() -> None:
|
|
sql = _build_dense_search_sql(
|
|
table_name="knowledge_chunks_v1",
|
|
index_name="vidx_knowledge_chunks_embedding",
|
|
output_column_names=["chunk_id", "index_profile_id"],
|
|
)
|
|
|
|
assert "VECTOR_INDEX(knowledge_chunks_v1 vidx_knowledge_chunks_embedding pre_filter)" in sql
|
|
assert "WHERE index_profile_id = :index_profile_id" in sql
|
|
assert "cosine_distance(embedding, :query_vector)" in sql
|
|
assert "APPROXIMATE LIMIT :top_n" in sql
|
|
assert "profile-v1" not in sql
|
|
assert (
|
|
_serialize_vector((0.1, -0.2, 0.3))
|
|
== "[0.10000000000000001,-0.20000000000000001,0.29999999999999999]"
|
|
)
|
|
|
|
|
|
def test_oceanbase_rejects_unsafe_index_identifier() -> None:
|
|
with pytest.raises(OceanBaseConfigurationError, match="index name"):
|
|
OceanBaseVectorStore(
|
|
FakeOceanBaseClient(),
|
|
"knowledge_chunks_v1",
|
|
index_name="vector-index;drop",
|
|
vector_dimension=3,
|
|
)
|
|
|
|
|
|
def test_oceanbase_requires_version_with_explicit_pre_filter() -> None:
|
|
with pytest.raises(OceanBaseConfigurationError, match="4.3.5.4"):
|
|
_validate_oceanbase_version([4, 3, 5, 3])
|
|
|
|
_validate_oceanbase_version([4, 3, 5, 4])
|
|
|
|
with pytest.raises(OceanBaseVectorStoreError, match="version validation"):
|
|
_validate_oceanbase_version("4.3.5.4")
|
|
|
|
|
|
def test_pyobvector_boundary_binds_profile_vector_and_limit() -> None:
|
|
sdk_client = FakeSdkClient()
|
|
client = _PyObVectorClient(sdk_client)
|
|
|
|
rows = client.dense_search(
|
|
"knowledge_chunks_v1",
|
|
index_name="vidx_knowledge_chunks_embedding",
|
|
query_vector=(0.1, 0.2, 0.3),
|
|
index_profile_id="profile-v1",
|
|
top_n=20,
|
|
output_column_names=["chunk_id", "index_profile_id"],
|
|
)
|
|
|
|
assert list(rows) == []
|
|
sql, parameters = sdk_client.engine.connection.executions[0]
|
|
assert "pre_filter" in sql
|
|
assert parameters == {
|
|
"query_vector": "[0.10000000000000001,0.20000000000000001,0.29999999999999999]",
|
|
"index_profile_id": "profile-v1",
|
|
"top_n": 20,
|
|
}
|
|
assert "profile-v1" not in sql
|
|
|
|
|
|
def test_oceanbase_schema_declares_rebuildable_cosine_index() -> None:
|
|
schema_path = Path(__file__).parents[2] / "deploy" / "oceanbase" / "001_knowledge_chunks_v1.sql"
|
|
|
|
schema = schema_path.read_text(encoding="utf-8")
|
|
|
|
assert "embedding VECTOR(1024) NOT NULL" in schema
|
|
assert "index_profile_id VARCHAR(128) NOT NULL" in schema
|
|
assert "DISTANCE = COSINE" in schema
|
|
assert "TYPE = HNSW" in schema
|
|
assert "document_id, index_profile_id" in schema
|