264 lines
9.2 KiB
Python
264 lines
9.2 KiB
Python
"""End-to-end REST contract tests using only deterministic local providers."""
|
|
|
|
from pathlib import Path
|
|
from uuid import UUID, uuid4
|
|
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import create_engine, select
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from ekp_api.main import create_app
|
|
from ekp_api.providers import ApiServices
|
|
from ekp_core.chunking import StructureAwareChunker
|
|
from ekp_core.config import Settings
|
|
from ekp_core.database import create_session_factory
|
|
from ekp_core.dense_retrieval import DenseRetrievalService
|
|
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
|
|
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
|
|
from ekp_core.document_management import DocumentManagementService
|
|
from ekp_core.document_parsing import DocumentParsingService
|
|
from ekp_core.document_upload import DocumentUploadService
|
|
from ekp_core.ingestion_jobs import (
|
|
AsyncIngestionSubmissionService,
|
|
DerivedDocumentCleanupService,
|
|
IngestionJobRunner,
|
|
IngestionJobService,
|
|
)
|
|
from ekp_core.search import SearchService
|
|
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile
|
|
from ekp_embedding import FakeEmbeddingProvider
|
|
from ekp_object_storage import LocalObjectStorage
|
|
from ekp_parser import ParserRouter
|
|
from ekp_reranker import FakeRerankerProvider
|
|
from ekp_vector_store import FakeVectorStore
|
|
|
|
REQUIRED_SOURCE_FIELDS = {
|
|
"document_id",
|
|
"filename",
|
|
"page",
|
|
"section",
|
|
"chunk_id",
|
|
"content",
|
|
"score",
|
|
"rerank_score",
|
|
}
|
|
|
|
|
|
class RecordingDispatcher:
|
|
"""Capture durable Job deliveries without starting a real broker."""
|
|
|
|
def __init__(self) -> None:
|
|
self.job_ids: list[UUID] = []
|
|
|
|
def enqueue(self, job_id: UUID) -> None:
|
|
self.job_ids.append(job_id)
|
|
|
|
|
|
def _build_api_services(
|
|
tmp_path: Path,
|
|
) -> tuple[
|
|
ApiServices,
|
|
sessionmaker[Session],
|
|
LocalObjectStorage,
|
|
FakeVectorStore,
|
|
IngestionJobRunner,
|
|
]:
|
|
"""Build the production service graph with offline providers and shared SQLite."""
|
|
engine = create_engine(
|
|
"sqlite+pysqlite:///:memory:",
|
|
connect_args={"check_same_thread": False},
|
|
poolclass=StaticPool,
|
|
)
|
|
Base.metadata.create_all(engine)
|
|
session_factory = create_session_factory(engine)
|
|
storage = LocalObjectStorage(tmp_path / "storage")
|
|
vector_store = FakeVectorStore()
|
|
embedding = FakeEmbeddingProvider(
|
|
model_name="fake-bge-large-zh",
|
|
dimensions=8,
|
|
)
|
|
profile = IndexProfile(
|
|
profile_id="test-profile-v1",
|
|
chunk_strategy_version="structure_v1",
|
|
embedding_model=embedding.model_name,
|
|
vector_dimension=embedding.dimensions,
|
|
)
|
|
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
|
|
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
|
|
chunking = DocumentChunkingService(
|
|
session_factory,
|
|
storage,
|
|
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
|
|
)
|
|
pipeline = DocumentPipelineService(upload, parsing, chunking)
|
|
indexing = DocumentIndexingService(
|
|
session_factory,
|
|
embedding,
|
|
vector_store,
|
|
profile,
|
|
)
|
|
ingestion = DocumentIndexingPipelineService(pipeline, indexing)
|
|
jobs = IngestionJobService(session_factory, max_attempts=3)
|
|
dispatcher = RecordingDispatcher()
|
|
submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher)
|
|
runner = IngestionJobRunner(
|
|
session_factory,
|
|
parsing,
|
|
chunking,
|
|
indexing,
|
|
DerivedDocumentCleanupService(
|
|
session_factory,
|
|
storage,
|
|
vector_store,
|
|
profile.profile_id,
|
|
),
|
|
retry_backoff_seconds=1,
|
|
retry_backoff_max_seconds=4,
|
|
lease_seconds=30,
|
|
)
|
|
dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20)
|
|
search = SearchService(
|
|
session_factory,
|
|
dense,
|
|
FakeRerankerProvider(),
|
|
default_top_k=5,
|
|
)
|
|
documents = DocumentManagementService(
|
|
session_factory,
|
|
storage,
|
|
vector_store,
|
|
active_index_profile_id=profile.profile_id,
|
|
)
|
|
return (
|
|
ApiServices(
|
|
ingestion=ingestion,
|
|
submissions=submissions,
|
|
jobs=jobs,
|
|
documents=documents,
|
|
search=search,
|
|
),
|
|
session_factory,
|
|
storage,
|
|
vector_store,
|
|
runner,
|
|
)
|
|
|
|
|
|
def test_document_and_search_rest_flow(
|
|
tmp_path: Path,
|
|
isolated_settings_environment: None,
|
|
) -> None:
|
|
services, session_factory, storage, vector_store, runner = _build_api_services(tmp_path)
|
|
app = create_app(Settings(app_env="test"), api_services=services)
|
|
|
|
with TestClient(app) as client:
|
|
upload_response = client.post(
|
|
"/api/v1/documents",
|
|
files={
|
|
"file": (
|
|
"退款政策.md",
|
|
"# 产品A\n\n## 退款政策\n产品A退款周期为七天。".encode(),
|
|
"text/markdown",
|
|
)
|
|
},
|
|
)
|
|
assert upload_response.status_code == 202
|
|
uploaded = upload_response.json()
|
|
document_id = UUID(uploaded["id"])
|
|
job_id = UUID(uploaded["job_id"])
|
|
assert uploaded["status"] == "uploaded"
|
|
assert uploaded["job_status"] == "queued"
|
|
assert uploaded["filename"] == "退款政策.md"
|
|
assert "storage_key" not in uploaded
|
|
|
|
queued = client.get(f"/api/v1/jobs/{job_id}")
|
|
assert queued.status_code == 200
|
|
assert queued.json()["attempt"] == 0
|
|
assert queued.json()["status"] == "queued"
|
|
assert client.delete(f"/api/v1/documents/{document_id}").status_code == 409
|
|
|
|
runner.run(job_id)
|
|
|
|
list_response = client.get("/api/v1/documents")
|
|
detail_response = client.get(f"/api/v1/documents/{document_id}")
|
|
assert list_response.status_code == 200
|
|
assert [item["id"] for item in list_response.json()] == [str(document_id)]
|
|
assert detail_response.status_code == 200
|
|
assert detail_response.json()["status"] == "ready"
|
|
completed = client.get(f"/api/v1/jobs/{job_id}").json()
|
|
assert completed["status"] == "succeeded"
|
|
assert completed["stage"] == "indexing"
|
|
assert completed["attempt"] == 1
|
|
|
|
search_response = client.post(
|
|
"/api/v1/search",
|
|
json={"query": "产品A退款周期", "top_k": 1},
|
|
)
|
|
assert search_response.status_code == 200
|
|
results = search_response.json()["results"]
|
|
assert len(results) == 1
|
|
assert set(results[0]) == REQUIRED_SOURCE_FIELDS
|
|
assert results[0]["document_id"] == str(document_id)
|
|
assert results[0]["filename"] == "退款政策.md"
|
|
assert results[0]["section"] == "产品A / 退款政策"
|
|
assert "七天" in results[0]["content"]
|
|
|
|
with session_factory() as session:
|
|
document = session.get(Document, document_id)
|
|
assert document is not None
|
|
source_key = document.storage_key
|
|
parsed_key = document.parsed_storage_key
|
|
assert storage.exists(source_key)
|
|
assert parsed_key is not None and storage.exists(parsed_key)
|
|
|
|
delete_response = client.delete(f"/api/v1/documents/{document_id}")
|
|
assert delete_response.status_code == 204
|
|
assert delete_response.content == b""
|
|
|
|
with session_factory() as session:
|
|
assert session.get(Document, document_id) is None
|
|
assert session.scalars(select(Chunk)).all() == []
|
|
assert session.scalars(select(ChunkIndexState)).all() == []
|
|
assert vector_store.records == {}
|
|
assert not storage.exists(source_key)
|
|
assert not storage.exists(parsed_key)
|
|
|
|
|
|
def test_rest_validation_not_found_and_openapi_contract(
|
|
tmp_path: Path,
|
|
isolated_settings_environment: None,
|
|
) -> None:
|
|
services, _, _, _, _ = _build_api_services(tmp_path)
|
|
app = create_app(Settings(app_env="test"), api_services=services)
|
|
missing_id = uuid4()
|
|
|
|
with TestClient(app) as client:
|
|
openapi = client.get("/api/v1/openapi.json").json()
|
|
assert set(openapi["paths"]) >= {
|
|
"/api/v1/documents",
|
|
"/api/v1/documents/{document_id}",
|
|
"/api/v1/jobs/{job_id}",
|
|
"/api/v1/search",
|
|
}
|
|
assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"}
|
|
assert set(openapi["paths"]["/api/v1/documents/{document_id}"]) >= {
|
|
"get",
|
|
"delete",
|
|
}
|
|
|
|
unsupported = client.post(
|
|
"/api/v1/documents",
|
|
files={"file": ("notes.csv", b"value", "text/csv")},
|
|
)
|
|
blank_query = client.post("/api/v1/search", json={"query": " ", "top_k": 5})
|
|
boolean_top_k = client.post("/api/v1/search", json={"query": "退款", "top_k": True})
|
|
missing_get = client.get(f"/api/v1/documents/{missing_id}")
|
|
missing_delete = client.delete(f"/api/v1/documents/{missing_id}")
|
|
|
|
assert unsupported.status_code == 415
|
|
assert blank_query.status_code == 422
|
|
assert boolean_top_k.status_code == 422
|
|
assert missing_get.status_code == 404
|
|
assert missing_delete.status_code == 404
|