knowledge-platform/tests/integration/test_document_search_api.py
2026-09-02 20:42:52 +08:00

420 lines
15 KiB
Python

"""End-to-end REST contract tests using only deterministic local providers."""
from pathlib import Path
from uuid import UUID, uuid4
from fastapi.testclient import TestClient
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool
from ekp_api.main import create_app
from ekp_api.providers import ApiServices
from ekp_core.chunking import StructureAwareChunker
from ekp_core.config import Settings
from ekp_core.database import create_session_factory
from ekp_core.dense_retrieval import DenseRetrievalService
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.ingestion_batches import IngestionBatchService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
)
from ekp_core.search import SearchService
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile, IngestionBatch
from ekp_embedding import FakeEmbeddingProvider
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_reranker import FakeRerankerProvider
from ekp_vector_store import FakeVectorStore
REQUIRED_SOURCE_FIELDS = {
"document_id",
"filename",
"page",
"section",
"chunk_id",
"content",
"score",
"rerank_score",
}
class RecordingDispatcher:
"""Capture durable Job deliveries without starting a real broker."""
def __init__(self) -> None:
self.job_ids: list[UUID] = []
def enqueue(self, job_id: UUID) -> None:
self.job_ids.append(job_id)
def _build_api_services(
tmp_path: Path,
*,
max_upload_bytes: int = 1024 * 1024,
batch_max_files: int = 20,
batch_max_total_bytes: int = 2 * 1024 * 1024,
) -> tuple[
ApiServices,
sessionmaker[Session],
LocalObjectStorage,
FakeVectorStore,
IngestionJobRunner,
]:
"""Build the production service graph with offline providers and shared SQLite."""
engine = create_engine(
"sqlite+pysqlite:///:memory:",
connect_args={"check_same_thread": False},
poolclass=StaticPool,
)
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
vector_store = FakeVectorStore()
embedding = FakeEmbeddingProvider(
model_name="fake-bge-large-zh",
dimensions=8,
)
profile = IndexProfile(
profile_id="test-profile-v1",
chunk_strategy_version="structure_v1",
embedding_model=embedding.model_name,
vector_dimension=embedding.dimensions,
)
upload = DocumentUploadService(
session_factory,
storage,
max_upload_bytes=max_upload_bytes,
)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
)
pipeline = DocumentPipelineService(upload, parsing, chunking)
indexing = DocumentIndexingService(
session_factory,
embedding,
vector_store,
profile,
)
ingestion = DocumentIndexingPipelineService(pipeline, indexing)
jobs = IngestionJobService(session_factory, max_attempts=3)
dispatcher = RecordingDispatcher()
submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher)
batches = IngestionBatchService(
session_factory,
submissions,
max_files=batch_max_files,
max_total_bytes=batch_max_total_bytes,
)
runner = IngestionJobRunner(
session_factory,
parsing,
chunking,
indexing,
DerivedDocumentCleanupService(
session_factory,
storage,
vector_store,
profile.profile_id,
),
retry_backoff_seconds=1,
retry_backoff_max_seconds=4,
lease_seconds=30,
)
dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20)
search = SearchService(
session_factory,
dense,
FakeRerankerProvider(),
default_top_k=5,
)
documents = DocumentManagementService(
session_factory,
storage,
vector_store,
active_index_profile_id=profile.profile_id,
)
return (
ApiServices(
ingestion=ingestion,
submissions=submissions,
batches=batches,
jobs=jobs,
documents=documents,
search=search,
),
session_factory,
storage,
vector_store,
runner,
)
def test_document_and_search_rest_flow(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
services, session_factory, storage, vector_store, runner = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
upload_response = client.post(
"/api/v1/documents",
files={
"file": (
"退款政策.md",
"# 产品A\n\n## 退款政策\n产品A退款周期为七天。".encode(),
"text/markdown",
)
},
)
assert upload_response.status_code == 202
uploaded = upload_response.json()
document_id = UUID(uploaded["id"])
job_id = UUID(uploaded["job_id"])
assert uploaded["status"] == "uploaded"
assert uploaded["job_status"] == "queued"
assert uploaded["filename"] == "退款政策.md"
assert "storage_key" not in uploaded
queued = client.get(f"/api/v1/jobs/{job_id}")
assert queued.status_code == 200
assert queued.json()["attempt"] == 0
assert queued.json()["status"] == "queued"
assert client.delete(f"/api/v1/documents/{document_id}").status_code == 409
runner.run(job_id)
list_response = client.get("/api/v1/documents")
detail_response = client.get(f"/api/v1/documents/{document_id}")
assert list_response.status_code == 200
assert [item["id"] for item in list_response.json()] == [str(document_id)]
assert detail_response.status_code == 200
assert detail_response.json()["status"] == "ready"
completed = client.get(f"/api/v1/jobs/{job_id}").json()
assert completed["status"] == "succeeded"
assert completed["stage"] == "indexing"
assert completed["attempt"] == 1
search_response = client.post(
"/api/v1/search",
json={"query": "产品A退款周期", "top_k": 1},
)
assert search_response.status_code == 200
results = search_response.json()["results"]
assert len(results) == 1
assert set(results[0]) == REQUIRED_SOURCE_FIELDS
assert results[0]["document_id"] == str(document_id)
assert results[0]["filename"] == "退款政策.md"
assert results[0]["section"] == "产品A / 退款政策"
assert "七天" in results[0]["content"]
with session_factory() as session:
document = session.get(Document, document_id)
assert document is not None
source_key = document.storage_key
parsed_key = document.parsed_storage_key
assert storage.exists(source_key)
assert parsed_key is not None and storage.exists(parsed_key)
delete_response = client.delete(f"/api/v1/documents/{document_id}")
assert delete_response.status_code == 204
assert delete_response.content == b""
with session_factory() as session:
assert session.get(Document, document_id) is None
assert session.scalars(select(Chunk)).all() == []
assert session.scalars(select(ChunkIndexState)).all() == []
assert vector_store.records == {}
assert not storage.exists(source_key)
assert not storage.exists(parsed_key)
def test_rest_validation_not_found_and_openapi_contract(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
services, _, _, _, _ = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
missing_id = uuid4()
with TestClient(app) as client:
openapi = client.get("/api/v1/openapi.json").json()
assert set(openapi["paths"]) >= {
"/api/v1/documents",
"/api/v1/documents/{document_id}",
"/api/v1/ingestion-batches",
"/api/v1/ingestion-batches/{batch_id}",
"/api/v1/jobs/{job_id}",
"/api/v1/search",
}
assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"}
assert set(openapi["paths"]["/api/v1/documents/{document_id}"]) >= {
"get",
"delete",
}
unsupported = client.post(
"/api/v1/documents",
files={"file": ("notes.csv", b"value", "text/csv")},
)
blank_query = client.post("/api/v1/search", json={"query": " ", "top_k": 5})
boolean_top_k = client.post("/api/v1/search", json={"query": "退款", "top_k": True})
missing_get = client.get(f"/api/v1/documents/{missing_id}")
missing_delete = client.delete(f"/api/v1/documents/{missing_id}")
assert unsupported.status_code == 415
assert blank_query.status_code == 422
assert boolean_top_k.status_code == 422
assert missing_get.status_code == 404
assert missing_delete.status_code == 404
def test_batch_upload_isolates_files_and_reports_terminal_status(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
"""A rejected file must not roll back valid siblings in the same batch."""
services, _, _, _, runner = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
accepted = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("alpha.md", b"# Alpha\n\nFirst policy.", "text/markdown")),
("files", ("blocked.csv", b"secret=must-not-persist", "text/csv")),
("files", ("beta.txt", b"Second policy.", "text/plain")),
],
)
assert accepted.status_code == 202
body = accepted.json()
assert body["status"] == "processing"
assert body["total_files"] == 3
assert body["accepted_files"] == 2
assert body["rejected_files"] == 1
assert body["completed_files"] == 1
assert [item["filename"] for item in body["items"]] == [
"alpha.md",
"blocked.csv",
"beta.txt",
]
assert [item["acceptance_status"] for item in body["items"]] == [
"accepted",
"rejected",
"accepted",
]
assert body["items"][1]["error_message"] == ("unsupported document type or media type")
assert "must-not-persist" not in str(body)
for item in body["items"]:
if item["job_id"] is not None:
runner.run(UUID(item["job_id"]))
terminal = client.get(f"/api/v1/ingestion-batches/{body['batch_id']}")
assert terminal.status_code == 200
terminal_body = terminal.json()
assert terminal_body["status"] == "partial_failure"
assert terminal_body["completed_files"] == 3
assert [item["job_status"] for item in terminal_body["items"]] == [
"succeeded",
None,
"succeeded",
]
def test_batch_upload_enforces_count_and_shared_byte_limits(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
"""Request limits are bounded while already accepted siblings remain durable."""
services, session_factory, _, _, _ = _build_api_services(
tmp_path,
max_upload_bytes=10,
batch_max_files=2,
batch_max_total_bytes=15,
)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
too_many = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("one.txt", b"1", "text/plain")),
("files", ("two.txt", b"2", "text/plain")),
("files", ("three.txt", b"3", "text/plain")),
],
)
bounded = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("first.txt", b"1234567890", "text/plain")),
("files", ("second.txt", b"abcdefghij", "text/plain")),
],
)
oversized = client.post(
"/api/v1/ingestion-batches",
files=[("files", ("large.txt", b"12345678901", "text/plain"))],
)
assert too_many.status_code == 413
assert too_many.json()["detail"] == "batch file count exceeds the configured limit"
assert bounded.status_code == 202
body = bounded.json()
assert body["accepted_files"] == 1
assert body["rejected_files"] == 1
assert body["items"][1]["error_message"] == ("batch exceeds the configured total upload limit")
assert oversized.status_code == 202
assert oversized.json()["status"] == "failed"
assert oversized.json()["accepted_files"] == 0
assert oversized.json()["items"][0]["error_message"] == (
"document exceeds the configured upload limit"
)
# A count-rejected request creates no batch; the two accepted requests do.
with session_factory() as session:
assert len(session.scalars(select(IngestionBatch)).all()) == 2
def test_batch_upload_completed_and_missing_contract(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
"""All-success batches become completed and unknown IDs stay explicit."""
services, _, _, _, runner = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
missing_id = uuid4()
with TestClient(app) as client:
accepted = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("one.md", b"# One\n\nContent.", "text/markdown")),
("files", ("two.txt", b"More content.", "text/plain")),
],
).json()
for item in accepted["items"]:
runner.run(UUID(item["job_id"]))
completed = client.get(f"/api/v1/ingestion-batches/{accepted['batch_id']}")
missing = client.get(f"/api/v1/ingestion-batches/{missing_id}")
assert completed.status_code == 200
assert completed.json()["status"] == "completed"
assert completed.json()["completed_files"] == 2
assert missing.status_code == 404
assert missing.json()["detail"] == "ingestion batch not found"