knowledge-platform/tests/integration/test_document_search_api.py
2026-09-02 20:18:58 +08:00

264 lines
9.2 KiB
Python

"""End-to-end REST contract tests using only deterministic local providers."""
from pathlib import Path
from uuid import UUID, uuid4
from fastapi.testclient import TestClient
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool
from ekp_api.main import create_app
from ekp_api.providers import ApiServices
from ekp_core.chunking import StructureAwareChunker
from ekp_core.config import Settings
from ekp_core.database import create_session_factory
from ekp_core.dense_retrieval import DenseRetrievalService
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
)
from ekp_core.search import SearchService
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile
from ekp_embedding import FakeEmbeddingProvider
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_reranker import FakeRerankerProvider
from ekp_vector_store import FakeVectorStore
REQUIRED_SOURCE_FIELDS = {
"document_id",
"filename",
"page",
"section",
"chunk_id",
"content",
"score",
"rerank_score",
}
class RecordingDispatcher:
"""Capture durable Job deliveries without starting a real broker."""
def __init__(self) -> None:
self.job_ids: list[UUID] = []
def enqueue(self, job_id: UUID) -> None:
self.job_ids.append(job_id)
def _build_api_services(
tmp_path: Path,
) -> tuple[
ApiServices,
sessionmaker[Session],
LocalObjectStorage,
FakeVectorStore,
IngestionJobRunner,
]:
"""Build the production service graph with offline providers and shared SQLite."""
engine = create_engine(
"sqlite+pysqlite:///:memory:",
connect_args={"check_same_thread": False},
poolclass=StaticPool,
)
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
vector_store = FakeVectorStore()
embedding = FakeEmbeddingProvider(
model_name="fake-bge-large-zh",
dimensions=8,
)
profile = IndexProfile(
profile_id="test-profile-v1",
chunk_strategy_version="structure_v1",
embedding_model=embedding.model_name,
vector_dimension=embedding.dimensions,
)
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
)
pipeline = DocumentPipelineService(upload, parsing, chunking)
indexing = DocumentIndexingService(
session_factory,
embedding,
vector_store,
profile,
)
ingestion = DocumentIndexingPipelineService(pipeline, indexing)
jobs = IngestionJobService(session_factory, max_attempts=3)
dispatcher = RecordingDispatcher()
submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher)
runner = IngestionJobRunner(
session_factory,
parsing,
chunking,
indexing,
DerivedDocumentCleanupService(
session_factory,
storage,
vector_store,
profile.profile_id,
),
retry_backoff_seconds=1,
retry_backoff_max_seconds=4,
lease_seconds=30,
)
dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20)
search = SearchService(
session_factory,
dense,
FakeRerankerProvider(),
default_top_k=5,
)
documents = DocumentManagementService(
session_factory,
storage,
vector_store,
active_index_profile_id=profile.profile_id,
)
return (
ApiServices(
ingestion=ingestion,
submissions=submissions,
jobs=jobs,
documents=documents,
search=search,
),
session_factory,
storage,
vector_store,
runner,
)
def test_document_and_search_rest_flow(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
services, session_factory, storage, vector_store, runner = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
upload_response = client.post(
"/api/v1/documents",
files={
"file": (
"退款政策.md",
"# 产品A\n\n## 退款政策\n产品A退款周期为七天。".encode(),
"text/markdown",
)
},
)
assert upload_response.status_code == 202
uploaded = upload_response.json()
document_id = UUID(uploaded["id"])
job_id = UUID(uploaded["job_id"])
assert uploaded["status"] == "uploaded"
assert uploaded["job_status"] == "queued"
assert uploaded["filename"] == "退款政策.md"
assert "storage_key" not in uploaded
queued = client.get(f"/api/v1/jobs/{job_id}")
assert queued.status_code == 200
assert queued.json()["attempt"] == 0
assert queued.json()["status"] == "queued"
assert client.delete(f"/api/v1/documents/{document_id}").status_code == 409
runner.run(job_id)
list_response = client.get("/api/v1/documents")
detail_response = client.get(f"/api/v1/documents/{document_id}")
assert list_response.status_code == 200
assert [item["id"] for item in list_response.json()] == [str(document_id)]
assert detail_response.status_code == 200
assert detail_response.json()["status"] == "ready"
completed = client.get(f"/api/v1/jobs/{job_id}").json()
assert completed["status"] == "succeeded"
assert completed["stage"] == "indexing"
assert completed["attempt"] == 1
search_response = client.post(
"/api/v1/search",
json={"query": "产品A退款周期", "top_k": 1},
)
assert search_response.status_code == 200
results = search_response.json()["results"]
assert len(results) == 1
assert set(results[0]) == REQUIRED_SOURCE_FIELDS
assert results[0]["document_id"] == str(document_id)
assert results[0]["filename"] == "退款政策.md"
assert results[0]["section"] == "产品A / 退款政策"
assert "七天" in results[0]["content"]
with session_factory() as session:
document = session.get(Document, document_id)
assert document is not None
source_key = document.storage_key
parsed_key = document.parsed_storage_key
assert storage.exists(source_key)
assert parsed_key is not None and storage.exists(parsed_key)
delete_response = client.delete(f"/api/v1/documents/{document_id}")
assert delete_response.status_code == 204
assert delete_response.content == b""
with session_factory() as session:
assert session.get(Document, document_id) is None
assert session.scalars(select(Chunk)).all() == []
assert session.scalars(select(ChunkIndexState)).all() == []
assert vector_store.records == {}
assert not storage.exists(source_key)
assert not storage.exists(parsed_key)
def test_rest_validation_not_found_and_openapi_contract(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
services, _, _, _, _ = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
missing_id = uuid4()
with TestClient(app) as client:
openapi = client.get("/api/v1/openapi.json").json()
assert set(openapi["paths"]) >= {
"/api/v1/documents",
"/api/v1/documents/{document_id}",
"/api/v1/jobs/{job_id}",
"/api/v1/search",
}
assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"}
assert set(openapi["paths"]["/api/v1/documents/{document_id}"]) >= {
"get",
"delete",
}
unsupported = client.post(
"/api/v1/documents",
files={"file": ("notes.csv", b"value", "text/csv")},
)
blank_query = client.post("/api/v1/search", json={"query": " ", "top_k": 5})
boolean_top_k = client.post("/api/v1/search", json={"query": "退款", "top_k": True})
missing_get = client.get(f"/api/v1/documents/{missing_id}")
missing_delete = client.delete(f"/api/v1/documents/{missing_id}")
assert unsupported.status_code == 415
assert blank_query.status_code == 422
assert boolean_top_k.status_code == 422
assert missing_get.status_code == 404
assert missing_delete.status_code == 404