420 lines
15 KiB
Python
420 lines
15 KiB
Python
"""End-to-end REST contract tests using only deterministic local providers."""
|
|
|
|
from pathlib import Path
|
|
from uuid import UUID, uuid4
|
|
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import create_engine, select
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from ekp_api.main import create_app
|
|
from ekp_api.providers import ApiServices
|
|
from ekp_core.chunking import StructureAwareChunker
|
|
from ekp_core.config import Settings
|
|
from ekp_core.database import create_session_factory
|
|
from ekp_core.dense_retrieval import DenseRetrievalService
|
|
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
|
|
from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService
|
|
from ekp_core.document_management import DocumentManagementService
|
|
from ekp_core.document_parsing import DocumentParsingService
|
|
from ekp_core.document_upload import DocumentUploadService
|
|
from ekp_core.ingestion_batches import IngestionBatchService
|
|
from ekp_core.ingestion_jobs import (
|
|
AsyncIngestionSubmissionService,
|
|
DerivedDocumentCleanupService,
|
|
IngestionJobRunner,
|
|
IngestionJobService,
|
|
)
|
|
from ekp_core.search import SearchService
|
|
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile, IngestionBatch
|
|
from ekp_embedding import FakeEmbeddingProvider
|
|
from ekp_object_storage import LocalObjectStorage
|
|
from ekp_parser import ParserRouter
|
|
from ekp_reranker import FakeRerankerProvider
|
|
from ekp_vector_store import FakeVectorStore
|
|
|
|
REQUIRED_SOURCE_FIELDS = {
|
|
"document_id",
|
|
"filename",
|
|
"page",
|
|
"section",
|
|
"chunk_id",
|
|
"content",
|
|
"score",
|
|
"rerank_score",
|
|
}
|
|
|
|
|
|
class RecordingDispatcher:
|
|
"""Capture durable Job deliveries without starting a real broker."""
|
|
|
|
def __init__(self) -> None:
|
|
self.job_ids: list[UUID] = []
|
|
|
|
def enqueue(self, job_id: UUID) -> None:
|
|
self.job_ids.append(job_id)
|
|
|
|
|
|
def _build_api_services(
|
|
tmp_path: Path,
|
|
*,
|
|
max_upload_bytes: int = 1024 * 1024,
|
|
batch_max_files: int = 20,
|
|
batch_max_total_bytes: int = 2 * 1024 * 1024,
|
|
) -> tuple[
|
|
ApiServices,
|
|
sessionmaker[Session],
|
|
LocalObjectStorage,
|
|
FakeVectorStore,
|
|
IngestionJobRunner,
|
|
]:
|
|
"""Build the production service graph with offline providers and shared SQLite."""
|
|
engine = create_engine(
|
|
"sqlite+pysqlite:///:memory:",
|
|
connect_args={"check_same_thread": False},
|
|
poolclass=StaticPool,
|
|
)
|
|
Base.metadata.create_all(engine)
|
|
session_factory = create_session_factory(engine)
|
|
storage = LocalObjectStorage(tmp_path / "storage")
|
|
vector_store = FakeVectorStore()
|
|
embedding = FakeEmbeddingProvider(
|
|
model_name="fake-bge-large-zh",
|
|
dimensions=8,
|
|
)
|
|
profile = IndexProfile(
|
|
profile_id="test-profile-v1",
|
|
chunk_strategy_version="structure_v1",
|
|
embedding_model=embedding.model_name,
|
|
vector_dimension=embedding.dimensions,
|
|
)
|
|
upload = DocumentUploadService(
|
|
session_factory,
|
|
storage,
|
|
max_upload_bytes=max_upload_bytes,
|
|
)
|
|
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
|
|
chunking = DocumentChunkingService(
|
|
session_factory,
|
|
storage,
|
|
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
|
|
)
|
|
pipeline = DocumentPipelineService(upload, parsing, chunking)
|
|
indexing = DocumentIndexingService(
|
|
session_factory,
|
|
embedding,
|
|
vector_store,
|
|
profile,
|
|
)
|
|
ingestion = DocumentIndexingPipelineService(pipeline, indexing)
|
|
jobs = IngestionJobService(session_factory, max_attempts=3)
|
|
dispatcher = RecordingDispatcher()
|
|
submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher)
|
|
batches = IngestionBatchService(
|
|
session_factory,
|
|
submissions,
|
|
max_files=batch_max_files,
|
|
max_total_bytes=batch_max_total_bytes,
|
|
)
|
|
runner = IngestionJobRunner(
|
|
session_factory,
|
|
parsing,
|
|
chunking,
|
|
indexing,
|
|
DerivedDocumentCleanupService(
|
|
session_factory,
|
|
storage,
|
|
vector_store,
|
|
profile.profile_id,
|
|
),
|
|
retry_backoff_seconds=1,
|
|
retry_backoff_max_seconds=4,
|
|
lease_seconds=30,
|
|
)
|
|
dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20)
|
|
search = SearchService(
|
|
session_factory,
|
|
dense,
|
|
FakeRerankerProvider(),
|
|
default_top_k=5,
|
|
)
|
|
documents = DocumentManagementService(
|
|
session_factory,
|
|
storage,
|
|
vector_store,
|
|
active_index_profile_id=profile.profile_id,
|
|
)
|
|
return (
|
|
ApiServices(
|
|
ingestion=ingestion,
|
|
submissions=submissions,
|
|
batches=batches,
|
|
jobs=jobs,
|
|
documents=documents,
|
|
search=search,
|
|
),
|
|
session_factory,
|
|
storage,
|
|
vector_store,
|
|
runner,
|
|
)
|
|
|
|
|
|
def test_document_and_search_rest_flow(
|
|
tmp_path: Path,
|
|
isolated_settings_environment: None,
|
|
) -> None:
|
|
services, session_factory, storage, vector_store, runner = _build_api_services(tmp_path)
|
|
app = create_app(Settings(app_env="test"), api_services=services)
|
|
|
|
with TestClient(app) as client:
|
|
upload_response = client.post(
|
|
"/api/v1/documents",
|
|
files={
|
|
"file": (
|
|
"退款政策.md",
|
|
"# 产品A\n\n## 退款政策\n产品A退款周期为七天。".encode(),
|
|
"text/markdown",
|
|
)
|
|
},
|
|
)
|
|
assert upload_response.status_code == 202
|
|
uploaded = upload_response.json()
|
|
document_id = UUID(uploaded["id"])
|
|
job_id = UUID(uploaded["job_id"])
|
|
assert uploaded["status"] == "uploaded"
|
|
assert uploaded["job_status"] == "queued"
|
|
assert uploaded["filename"] == "退款政策.md"
|
|
assert "storage_key" not in uploaded
|
|
|
|
queued = client.get(f"/api/v1/jobs/{job_id}")
|
|
assert queued.status_code == 200
|
|
assert queued.json()["attempt"] == 0
|
|
assert queued.json()["status"] == "queued"
|
|
assert client.delete(f"/api/v1/documents/{document_id}").status_code == 409
|
|
|
|
runner.run(job_id)
|
|
|
|
list_response = client.get("/api/v1/documents")
|
|
detail_response = client.get(f"/api/v1/documents/{document_id}")
|
|
assert list_response.status_code == 200
|
|
assert [item["id"] for item in list_response.json()] == [str(document_id)]
|
|
assert detail_response.status_code == 200
|
|
assert detail_response.json()["status"] == "ready"
|
|
completed = client.get(f"/api/v1/jobs/{job_id}").json()
|
|
assert completed["status"] == "succeeded"
|
|
assert completed["stage"] == "indexing"
|
|
assert completed["attempt"] == 1
|
|
|
|
search_response = client.post(
|
|
"/api/v1/search",
|
|
json={"query": "产品A退款周期", "top_k": 1},
|
|
)
|
|
assert search_response.status_code == 200
|
|
results = search_response.json()["results"]
|
|
assert len(results) == 1
|
|
assert set(results[0]) == REQUIRED_SOURCE_FIELDS
|
|
assert results[0]["document_id"] == str(document_id)
|
|
assert results[0]["filename"] == "退款政策.md"
|
|
assert results[0]["section"] == "产品A / 退款政策"
|
|
assert "七天" in results[0]["content"]
|
|
|
|
with session_factory() as session:
|
|
document = session.get(Document, document_id)
|
|
assert document is not None
|
|
source_key = document.storage_key
|
|
parsed_key = document.parsed_storage_key
|
|
assert storage.exists(source_key)
|
|
assert parsed_key is not None and storage.exists(parsed_key)
|
|
|
|
delete_response = client.delete(f"/api/v1/documents/{document_id}")
|
|
assert delete_response.status_code == 204
|
|
assert delete_response.content == b""
|
|
|
|
with session_factory() as session:
|
|
assert session.get(Document, document_id) is None
|
|
assert session.scalars(select(Chunk)).all() == []
|
|
assert session.scalars(select(ChunkIndexState)).all() == []
|
|
assert vector_store.records == {}
|
|
assert not storage.exists(source_key)
|
|
assert not storage.exists(parsed_key)
|
|
|
|
|
|
def test_rest_validation_not_found_and_openapi_contract(
|
|
tmp_path: Path,
|
|
isolated_settings_environment: None,
|
|
) -> None:
|
|
services, _, _, _, _ = _build_api_services(tmp_path)
|
|
app = create_app(Settings(app_env="test"), api_services=services)
|
|
missing_id = uuid4()
|
|
|
|
with TestClient(app) as client:
|
|
openapi = client.get("/api/v1/openapi.json").json()
|
|
assert set(openapi["paths"]) >= {
|
|
"/api/v1/documents",
|
|
"/api/v1/documents/{document_id}",
|
|
"/api/v1/ingestion-batches",
|
|
"/api/v1/ingestion-batches/{batch_id}",
|
|
"/api/v1/jobs/{job_id}",
|
|
"/api/v1/search",
|
|
}
|
|
assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"}
|
|
assert set(openapi["paths"]["/api/v1/documents/{document_id}"]) >= {
|
|
"get",
|
|
"delete",
|
|
}
|
|
|
|
unsupported = client.post(
|
|
"/api/v1/documents",
|
|
files={"file": ("notes.csv", b"value", "text/csv")},
|
|
)
|
|
blank_query = client.post("/api/v1/search", json={"query": " ", "top_k": 5})
|
|
boolean_top_k = client.post("/api/v1/search", json={"query": "退款", "top_k": True})
|
|
missing_get = client.get(f"/api/v1/documents/{missing_id}")
|
|
missing_delete = client.delete(f"/api/v1/documents/{missing_id}")
|
|
|
|
assert unsupported.status_code == 415
|
|
assert blank_query.status_code == 422
|
|
assert boolean_top_k.status_code == 422
|
|
assert missing_get.status_code == 404
|
|
assert missing_delete.status_code == 404
|
|
|
|
|
|
def test_batch_upload_isolates_files_and_reports_terminal_status(
|
|
tmp_path: Path,
|
|
isolated_settings_environment: None,
|
|
) -> None:
|
|
"""A rejected file must not roll back valid siblings in the same batch."""
|
|
services, _, _, _, runner = _build_api_services(tmp_path)
|
|
app = create_app(Settings(app_env="test"), api_services=services)
|
|
|
|
with TestClient(app) as client:
|
|
accepted = client.post(
|
|
"/api/v1/ingestion-batches",
|
|
files=[
|
|
("files", ("alpha.md", b"# Alpha\n\nFirst policy.", "text/markdown")),
|
|
("files", ("blocked.csv", b"secret=must-not-persist", "text/csv")),
|
|
("files", ("beta.txt", b"Second policy.", "text/plain")),
|
|
],
|
|
)
|
|
|
|
assert accepted.status_code == 202
|
|
body = accepted.json()
|
|
assert body["status"] == "processing"
|
|
assert body["total_files"] == 3
|
|
assert body["accepted_files"] == 2
|
|
assert body["rejected_files"] == 1
|
|
assert body["completed_files"] == 1
|
|
assert [item["filename"] for item in body["items"]] == [
|
|
"alpha.md",
|
|
"blocked.csv",
|
|
"beta.txt",
|
|
]
|
|
assert [item["acceptance_status"] for item in body["items"]] == [
|
|
"accepted",
|
|
"rejected",
|
|
"accepted",
|
|
]
|
|
assert body["items"][1]["error_message"] == ("unsupported document type or media type")
|
|
assert "must-not-persist" not in str(body)
|
|
|
|
for item in body["items"]:
|
|
if item["job_id"] is not None:
|
|
runner.run(UUID(item["job_id"]))
|
|
|
|
terminal = client.get(f"/api/v1/ingestion-batches/{body['batch_id']}")
|
|
|
|
assert terminal.status_code == 200
|
|
terminal_body = terminal.json()
|
|
assert terminal_body["status"] == "partial_failure"
|
|
assert terminal_body["completed_files"] == 3
|
|
assert [item["job_status"] for item in terminal_body["items"]] == [
|
|
"succeeded",
|
|
None,
|
|
"succeeded",
|
|
]
|
|
|
|
|
|
def test_batch_upload_enforces_count_and_shared_byte_limits(
|
|
tmp_path: Path,
|
|
isolated_settings_environment: None,
|
|
) -> None:
|
|
"""Request limits are bounded while already accepted siblings remain durable."""
|
|
services, session_factory, _, _, _ = _build_api_services(
|
|
tmp_path,
|
|
max_upload_bytes=10,
|
|
batch_max_files=2,
|
|
batch_max_total_bytes=15,
|
|
)
|
|
app = create_app(Settings(app_env="test"), api_services=services)
|
|
|
|
with TestClient(app) as client:
|
|
too_many = client.post(
|
|
"/api/v1/ingestion-batches",
|
|
files=[
|
|
("files", ("one.txt", b"1", "text/plain")),
|
|
("files", ("two.txt", b"2", "text/plain")),
|
|
("files", ("three.txt", b"3", "text/plain")),
|
|
],
|
|
)
|
|
bounded = client.post(
|
|
"/api/v1/ingestion-batches",
|
|
files=[
|
|
("files", ("first.txt", b"1234567890", "text/plain")),
|
|
("files", ("second.txt", b"abcdefghij", "text/plain")),
|
|
],
|
|
)
|
|
oversized = client.post(
|
|
"/api/v1/ingestion-batches",
|
|
files=[("files", ("large.txt", b"12345678901", "text/plain"))],
|
|
)
|
|
|
|
assert too_many.status_code == 413
|
|
assert too_many.json()["detail"] == "batch file count exceeds the configured limit"
|
|
assert bounded.status_code == 202
|
|
body = bounded.json()
|
|
assert body["accepted_files"] == 1
|
|
assert body["rejected_files"] == 1
|
|
assert body["items"][1]["error_message"] == ("batch exceeds the configured total upload limit")
|
|
assert oversized.status_code == 202
|
|
assert oversized.json()["status"] == "failed"
|
|
assert oversized.json()["accepted_files"] == 0
|
|
assert oversized.json()["items"][0]["error_message"] == (
|
|
"document exceeds the configured upload limit"
|
|
)
|
|
|
|
# A count-rejected request creates no batch; the two accepted requests do.
|
|
with session_factory() as session:
|
|
assert len(session.scalars(select(IngestionBatch)).all()) == 2
|
|
|
|
|
|
def test_batch_upload_completed_and_missing_contract(
|
|
tmp_path: Path,
|
|
isolated_settings_environment: None,
|
|
) -> None:
|
|
"""All-success batches become completed and unknown IDs stay explicit."""
|
|
services, _, _, _, runner = _build_api_services(tmp_path)
|
|
app = create_app(Settings(app_env="test"), api_services=services)
|
|
missing_id = uuid4()
|
|
|
|
with TestClient(app) as client:
|
|
accepted = client.post(
|
|
"/api/v1/ingestion-batches",
|
|
files=[
|
|
("files", ("one.md", b"# One\n\nContent.", "text/markdown")),
|
|
("files", ("two.txt", b"More content.", "text/plain")),
|
|
],
|
|
).json()
|
|
for item in accepted["items"]:
|
|
runner.run(UUID(item["job_id"]))
|
|
|
|
completed = client.get(f"/api/v1/ingestion-batches/{accepted['batch_id']}")
|
|
missing = client.get(f"/api/v1/ingestion-batches/{missing_id}")
|
|
|
|
assert completed.status_code == 200
|
|
assert completed.json()["status"] == "completed"
|
|
assert completed.json()["completed_files"] == 2
|
|
assert missing.status_code == 404
|
|
assert missing.json()["detail"] == "ingestion batch not found"
|