"""End-to-end REST contract tests using only deterministic local providers.""" from pathlib import Path from uuid import UUID, uuid4 from fastapi.testclient import TestClient from sqlalchemy import create_engine, select from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import StaticPool from ekp_api.main import create_app from ekp_api.providers import ApiServices from ekp_core.chunking import StructureAwareChunker from ekp_core.config import Settings from ekp_core.database import create_session_factory from ekp_core.dense_retrieval import DenseRetrievalService from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService from ekp_core.document_management import DocumentManagementService from ekp_core.document_parsing import DocumentParsingService from ekp_core.document_upload import DocumentUploadService from ekp_core.ingestion_batches import IngestionBatchService from ekp_core.ingestion_jobs import ( AsyncIngestionSubmissionService, DerivedDocumentCleanupService, IngestionJobRunner, IngestionJobService, ) from ekp_core.search import SearchService from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile, IngestionBatch from ekp_embedding import FakeEmbeddingProvider from ekp_object_storage import LocalObjectStorage from ekp_parser import ParserRouter from ekp_reranker import FakeRerankerProvider from ekp_vector_store import FakeVectorStore REQUIRED_SOURCE_FIELDS = { "document_id", "filename", "page", "section", "chunk_id", "content", "score", "rerank_score", } class RecordingDispatcher: """Capture durable Job deliveries without starting a real broker.""" def __init__(self) -> None: self.job_ids: list[UUID] = [] def enqueue(self, job_id: UUID) -> None: self.job_ids.append(job_id) def _build_api_services( tmp_path: Path, *, max_upload_bytes: int = 1024 * 1024, batch_max_files: int = 20, batch_max_total_bytes: int = 2 * 1024 * 1024, ) -> tuple[ ApiServices, sessionmaker[Session], LocalObjectStorage, FakeVectorStore, IngestionJobRunner, ]: """Build the production service graph with offline providers and shared SQLite.""" engine = create_engine( "sqlite+pysqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool, ) Base.metadata.create_all(engine) session_factory = create_session_factory(engine) storage = LocalObjectStorage(tmp_path / "storage") vector_store = FakeVectorStore() embedding = FakeEmbeddingProvider( model_name="fake-bge-large-zh", dimensions=8, ) profile = IndexProfile( profile_id="test-profile-v1", chunk_strategy_version="structure_v1", embedding_model=embedding.model_name, vector_dimension=embedding.dimensions, ) upload = DocumentUploadService( session_factory, storage, max_upload_bytes=max_upload_bytes, ) parsing = DocumentParsingService(session_factory, storage, ParserRouter()) chunking = DocumentChunkingService( session_factory, storage, StructureAwareChunker(max_tokens=450, overlap_tokens=75), ) pipeline = DocumentPipelineService(upload, parsing, chunking) indexing = DocumentIndexingService( session_factory, embedding, vector_store, profile, ) ingestion = DocumentIndexingPipelineService(pipeline, indexing) jobs = IngestionJobService(session_factory, max_attempts=3) dispatcher = RecordingDispatcher() submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher) batches = IngestionBatchService( session_factory, submissions, max_files=batch_max_files, max_total_bytes=batch_max_total_bytes, ) runner = IngestionJobRunner( session_factory, parsing, chunking, indexing, DerivedDocumentCleanupService( session_factory, storage, vector_store, profile.profile_id, ), retry_backoff_seconds=1, retry_backoff_max_seconds=4, lease_seconds=30, ) dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20) search = SearchService( session_factory, dense, FakeRerankerProvider(), default_top_k=5, ) documents = DocumentManagementService( session_factory, storage, vector_store, active_index_profile_id=profile.profile_id, ) return ( ApiServices( ingestion=ingestion, submissions=submissions, batches=batches, jobs=jobs, documents=documents, search=search, ), session_factory, storage, vector_store, runner, ) def test_document_and_search_rest_flow( tmp_path: Path, isolated_settings_environment: None, ) -> None: services, session_factory, storage, vector_store, runner = _build_api_services(tmp_path) app = create_app(Settings(app_env="test"), api_services=services) with TestClient(app) as client: upload_response = client.post( "/api/v1/documents", files={ "file": ( "退款政策.md", "# 产品A\n\n## 退款政策\n产品A退款周期为七天。".encode(), "text/markdown", ) }, ) assert upload_response.status_code == 202 uploaded = upload_response.json() document_id = UUID(uploaded["id"]) job_id = UUID(uploaded["job_id"]) assert uploaded["status"] == "uploaded" assert uploaded["job_status"] == "queued" assert uploaded["filename"] == "退款政策.md" assert "storage_key" not in uploaded queued = client.get(f"/api/v1/jobs/{job_id}") assert queued.status_code == 200 assert queued.json()["attempt"] == 0 assert queued.json()["status"] == "queued" assert client.delete(f"/api/v1/documents/{document_id}").status_code == 409 runner.run(job_id) list_response = client.get("/api/v1/documents") detail_response = client.get(f"/api/v1/documents/{document_id}") assert list_response.status_code == 200 assert [item["id"] for item in list_response.json()] == [str(document_id)] assert detail_response.status_code == 200 assert detail_response.json()["status"] == "ready" completed = client.get(f"/api/v1/jobs/{job_id}").json() assert completed["status"] == "succeeded" assert completed["stage"] == "indexing" assert completed["attempt"] == 1 search_response = client.post( "/api/v1/search", json={"query": "产品A退款周期", "top_k": 1}, ) assert search_response.status_code == 200 results = search_response.json()["results"] assert len(results) == 1 assert set(results[0]) == REQUIRED_SOURCE_FIELDS assert results[0]["document_id"] == str(document_id) assert results[0]["filename"] == "退款政策.md" assert results[0]["section"] == "产品A / 退款政策" assert "七天" in results[0]["content"] with session_factory() as session: document = session.get(Document, document_id) assert document is not None source_key = document.storage_key parsed_key = document.parsed_storage_key assert storage.exists(source_key) assert parsed_key is not None and storage.exists(parsed_key) delete_response = client.delete(f"/api/v1/documents/{document_id}") assert delete_response.status_code == 204 assert delete_response.content == b"" with session_factory() as session: assert session.get(Document, document_id) is None assert session.scalars(select(Chunk)).all() == [] assert session.scalars(select(ChunkIndexState)).all() == [] assert vector_store.records == {} assert not storage.exists(source_key) assert not storage.exists(parsed_key) def test_rest_validation_not_found_and_openapi_contract( tmp_path: Path, isolated_settings_environment: None, ) -> None: services, _, _, _, _ = _build_api_services(tmp_path) app = create_app(Settings(app_env="test"), api_services=services) missing_id = uuid4() with TestClient(app) as client: openapi = client.get("/api/v1/openapi.json").json() assert set(openapi["paths"]) >= { "/api/v1/documents", "/api/v1/documents/{document_id}", "/api/v1/ingestion-batches", "/api/v1/ingestion-batches/{batch_id}", "/api/v1/jobs/{job_id}", "/api/v1/search", } assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"} assert set(openapi["paths"]["/api/v1/documents/{document_id}"]) >= { "get", "delete", } unsupported = client.post( "/api/v1/documents", files={"file": ("notes.csv", b"value", "text/csv")}, ) blank_query = client.post("/api/v1/search", json={"query": " ", "top_k": 5}) boolean_top_k = client.post("/api/v1/search", json={"query": "退款", "top_k": True}) missing_get = client.get(f"/api/v1/documents/{missing_id}") missing_delete = client.delete(f"/api/v1/documents/{missing_id}") assert unsupported.status_code == 415 assert blank_query.status_code == 422 assert boolean_top_k.status_code == 422 assert missing_get.status_code == 404 assert missing_delete.status_code == 404 def test_batch_upload_isolates_files_and_reports_terminal_status( tmp_path: Path, isolated_settings_environment: None, ) -> None: """A rejected file must not roll back valid siblings in the same batch.""" services, _, _, _, runner = _build_api_services(tmp_path) app = create_app(Settings(app_env="test"), api_services=services) with TestClient(app) as client: accepted = client.post( "/api/v1/ingestion-batches", files=[ ("files", ("alpha.md", b"# Alpha\n\nFirst policy.", "text/markdown")), ("files", ("blocked.csv", b"secret=must-not-persist", "text/csv")), ("files", ("beta.txt", b"Second policy.", "text/plain")), ], ) assert accepted.status_code == 202 body = accepted.json() assert body["status"] == "processing" assert body["total_files"] == 3 assert body["accepted_files"] == 2 assert body["rejected_files"] == 1 assert body["completed_files"] == 1 assert [item["filename"] for item in body["items"]] == [ "alpha.md", "blocked.csv", "beta.txt", ] assert [item["acceptance_status"] for item in body["items"]] == [ "accepted", "rejected", "accepted", ] assert body["items"][1]["error_message"] == ("unsupported document type or media type") assert "must-not-persist" not in str(body) for item in body["items"]: if item["job_id"] is not None: runner.run(UUID(item["job_id"])) terminal = client.get(f"/api/v1/ingestion-batches/{body['batch_id']}") assert terminal.status_code == 200 terminal_body = terminal.json() assert terminal_body["status"] == "partial_failure" assert terminal_body["completed_files"] == 3 assert [item["job_status"] for item in terminal_body["items"]] == [ "succeeded", None, "succeeded", ] def test_batch_upload_enforces_count_and_shared_byte_limits( tmp_path: Path, isolated_settings_environment: None, ) -> None: """Request limits are bounded while already accepted siblings remain durable.""" services, session_factory, _, _, _ = _build_api_services( tmp_path, max_upload_bytes=10, batch_max_files=2, batch_max_total_bytes=15, ) app = create_app(Settings(app_env="test"), api_services=services) with TestClient(app) as client: too_many = client.post( "/api/v1/ingestion-batches", files=[ ("files", ("one.txt", b"1", "text/plain")), ("files", ("two.txt", b"2", "text/plain")), ("files", ("three.txt", b"3", "text/plain")), ], ) bounded = client.post( "/api/v1/ingestion-batches", files=[ ("files", ("first.txt", b"1234567890", "text/plain")), ("files", ("second.txt", b"abcdefghij", "text/plain")), ], ) oversized = client.post( "/api/v1/ingestion-batches", files=[("files", ("large.txt", b"12345678901", "text/plain"))], ) assert too_many.status_code == 413 assert too_many.json()["detail"] == "batch file count exceeds the configured limit" assert bounded.status_code == 202 body = bounded.json() assert body["accepted_files"] == 1 assert body["rejected_files"] == 1 assert body["items"][1]["error_message"] == ("batch exceeds the configured total upload limit") assert oversized.status_code == 202 assert oversized.json()["status"] == "failed" assert oversized.json()["accepted_files"] == 0 assert oversized.json()["items"][0]["error_message"] == ( "document exceeds the configured upload limit" ) # A count-rejected request creates no batch; the two accepted requests do. with session_factory() as session: assert len(session.scalars(select(IngestionBatch)).all()) == 2 def test_batch_upload_completed_and_missing_contract( tmp_path: Path, isolated_settings_environment: None, ) -> None: """All-success batches become completed and unknown IDs stay explicit.""" services, _, _, _, runner = _build_api_services(tmp_path) app = create_app(Settings(app_env="test"), api_services=services) missing_id = uuid4() with TestClient(app) as client: accepted = client.post( "/api/v1/ingestion-batches", files=[ ("files", ("one.md", b"# One\n\nContent.", "text/markdown")), ("files", ("two.txt", b"More content.", "text/plain")), ], ).json() for item in accepted["items"]: runner.run(UUID(item["job_id"])) completed = client.get(f"/api/v1/ingestion-batches/{accepted['batch_id']}") missing = client.get(f"/api/v1/ingestion-batches/{missing_id}") assert completed.status_code == 200 assert completed.json()["status"] == "completed" assert completed.json()["completed_files"] == 2 assert missing.status_code == 404 assert missing.json()["detail"] == "ingestion batch not found"