knowledge-platform/tests/integration/test_document_search_api.py
2026-09-01 16:10:14 +08:00

207 lines
7.5 KiB
Python

"""End-to-end REST contract tests using only deterministic local providers."""
from pathlib import Path
from uuid import UUID, uuid4
from fastapi.testclient import TestClient
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool
from ekp_api.main import create_app
from ekp_api.providers import ApiServices
from ekp_core.chunking import StructureAwareChunker
from ekp_core.config import Settings
from ekp_core.database import create_session_factory
from ekp_core.dense_retrieval import DenseRetrievalService
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.search import SearchService
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile
from ekp_embedding import FakeEmbeddingProvider
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_reranker import FakeRerankerProvider
from ekp_vector_store import FakeVectorStore
REQUIRED_SOURCE_FIELDS = {
"document_id",
"filename",
"page",
"section",
"chunk_id",
"content",
"score",
"rerank_score",
}
def _build_api_services(
tmp_path: Path,
) -> tuple[
ApiServices,
sessionmaker[Session],
LocalObjectStorage,
FakeVectorStore,
]:
"""Build the production service graph with offline providers and shared SQLite."""
engine = create_engine(
"sqlite+pysqlite:///:memory:",
connect_args={"check_same_thread": False},
poolclass=StaticPool,
)
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
vector_store = FakeVectorStore()
embedding = FakeEmbeddingProvider(
model_name="fake-bge-large-zh",
dimensions=8,
)
profile = IndexProfile(
profile_id="test-profile-v1",
chunk_strategy_version="structure_v1",
embedding_model=embedding.model_name,
vector_dimension=embedding.dimensions,
)
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
)
pipeline = DocumentPipelineService(upload, parsing, chunking)
indexing = DocumentIndexingService(
session_factory,
embedding,
vector_store,
profile,
)
ingestion = DocumentIndexingPipelineService(pipeline, indexing)
dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20)
search = SearchService(
session_factory,
dense,
FakeRerankerProvider(),
default_top_k=5,
)
documents = DocumentManagementService(
session_factory,
storage,
vector_store,
active_index_profile_id=profile.profile_id,
)
return (
ApiServices(ingestion=ingestion, documents=documents, search=search),
session_factory,
storage,
vector_store,
)
def test_document_and_search_rest_flow(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
services, session_factory, storage, vector_store = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
upload_response = client.post(
"/api/v1/documents",
files={
"file": (
"退款政策.md",
"# 产品A\n\n## 退款政策\n产品A退款周期为七天。".encode(),
"text/markdown",
)
},
)
assert upload_response.status_code == 201
uploaded = upload_response.json()
document_id = UUID(uploaded["id"])
assert uploaded["status"] == "ready"
assert uploaded["filename"] == "退款政策.md"
assert "storage_key" not in uploaded
list_response = client.get("/api/v1/documents")
detail_response = client.get(f"/api/v1/documents/{document_id}")
assert list_response.status_code == 200
assert [item["id"] for item in list_response.json()] == [str(document_id)]
assert detail_response.status_code == 200
assert detail_response.json() == uploaded
search_response = client.post(
"/api/v1/search",
json={"query": "产品A退款周期", "top_k": 1},
)
assert search_response.status_code == 200
results = search_response.json()["results"]
assert len(results) == 1
assert set(results[0]) == REQUIRED_SOURCE_FIELDS
assert results[0]["document_id"] == str(document_id)
assert results[0]["filename"] == "退款政策.md"
assert results[0]["section"] == "产品A / 退款政策"
assert "七天" in results[0]["content"]
with session_factory() as session:
document = session.get(Document, document_id)
assert document is not None
source_key = document.storage_key
parsed_key = document.parsed_storage_key
assert storage.exists(source_key)
assert parsed_key is not None and storage.exists(parsed_key)
delete_response = client.delete(f"/api/v1/documents/{document_id}")
assert delete_response.status_code == 204
assert delete_response.content == b""
with session_factory() as session:
assert session.get(Document, document_id) is None
assert session.scalars(select(Chunk)).all() == []
assert session.scalars(select(ChunkIndexState)).all() == []
assert vector_store.records == {}
assert not storage.exists(source_key)
assert not storage.exists(parsed_key)
def test_rest_validation_not_found_and_openapi_contract(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
services, _, _, _ = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
missing_id = uuid4()
with TestClient(app) as client:
openapi = client.get("/api/v1/openapi.json").json()
assert set(openapi["paths"]) >= {
"/api/v1/documents",
"/api/v1/documents/{document_id}",
"/api/v1/search",
}
assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"}
assert set(openapi["paths"]["/api/v1/documents/{document_id}"]) >= {
"get",
"delete",
}
unsupported = client.post(
"/api/v1/documents",
files={"file": ("notes.csv", b"value", "text/csv")},
)
blank_query = client.post("/api/v1/search", json={"query": " ", "top_k": 5})
boolean_top_k = client.post("/api/v1/search", json={"query": "退款", "top_k": True})
missing_get = client.get(f"/api/v1/documents/{missing_id}")
missing_delete = client.delete(f"/api/v1/documents/{missing_id}")
assert unsupported.status_code == 415
assert blank_query.status_code == 422
assert boolean_top_k.status_code == 422
assert missing_get.status_code == 404
assert missing_delete.status_code == 404