diff --git a/README.md b/README.md index 968af24..445e61e 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,8 @@ Chunk metadata, including deterministic IDs, filename, page range, section, index, and strategy version, is stored atomically in PostgreSQL. The lightweight token counter is replaceable so DEMO-006 can inject model-specific behavior. Documents remain `processing` until embedding/indexing completes; pipeline -failures become `failed` without deleting the immutable original. Public upload -endpoints are introduced by DEMO-009. +failures become `failed` without deleting the immutable original. DEMO-009 +exposes this pipeline through the versioned upload endpoint. DEMO-006 adds a provider-neutral `EmbeddingProvider` and `VectorStore`, a validated Qianfan `bge-large-zh` adapter, and an isolated `pymochow` adapter. @@ -100,4 +100,8 @@ DEMO-008 adds a provider-neutral reranker, a validated Qianfan source results are rehydrated from canonical PostgreSQL chunks and retain both Dense and reranker scores; stale vector content fails closed. See `docs/development/RERANKER_SETUP.md` for configuration and validation limits. -Public search REST endpoints remain in DEMO-009. +DEMO-009 exposes the completed synchronous pipeline through versioned document +and search REST endpoints under `/api/v1`. Upload, list, detail, direct delete, +and reranked source search are documented in +`docs/development/REST_API_SETUP.md`; provider and storage internals remain +outside the public response contract. diff --git a/apps/api/src/ekp_api/main.py b/apps/api/src/ekp_api/main.py index e8215ad..c84ab72 100644 --- a/apps/api/src/ekp_api/main.py +++ b/apps/api/src/ekp_api/main.py @@ -3,6 +3,8 @@ from fastapi import FastAPI from pydantic import BaseModel +from ekp_api.providers import ApiServices +from ekp_api.routes import router from ekp_core.config import Settings, get_settings @@ -14,7 +16,10 @@ class HealthResponse(BaseModel): environment: str -def create_app(settings: Settings | None = None) -> FastAPI: +def create_app( + settings: Settings | None = None, + api_services: ApiServices | None = None, +) -> FastAPI: """Create the API application with explicit, testable settings.""" resolved_settings = settings or get_settings() application = FastAPI( @@ -23,6 +28,9 @@ def create_app(settings: Settings | None = None) -> FastAPI: docs_url=f"{resolved_settings.api_prefix}/docs", openapi_url=f"{resolved_settings.api_prefix}/openapi.json", ) + application.state.settings = resolved_settings + application.state.api_services = api_services + application.include_router(router, prefix=resolved_settings.api_prefix) @application.get("/healthz", response_model=HealthResponse, tags=["system"]) async def health() -> HealthResponse: diff --git a/apps/api/src/ekp_api/providers.py b/apps/api/src/ekp_api/providers.py index ec26e45..0c450ba 100644 --- a/apps/api/src/ekp_api/providers.py +++ b/apps/api/src/ekp_api/providers.py @@ -1,12 +1,26 @@ -"""Runtime provider composition from secret-safe application settings.""" +"""Runtime provider and application-service composition from safe settings.""" + +from dataclasses import dataclass from sqlalchemy.orm import Session, sessionmaker +from ekp_core.chunking import StructureAwareChunker from ekp_core.config import Settings +from ekp_core.database import create_database_engine, create_session_factory from ekp_core.dense_retrieval import DenseRetrievalService +from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService +from ekp_core.document_indexing import ( + DocumentIndexingPipelineService, + DocumentIndexingService, +) +from ekp_core.document_management import DocumentManagementService +from ekp_core.document_parsing import DocumentParsingService +from ekp_core.document_upload import DocumentUploadService from ekp_core.search import SearchService from ekp_domain import IndexProfile from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider +from ekp_object_storage import LocalObjectStorage +from ekp_parser import ParserRouter from ekp_reranker import QianfanRerankerProvider, RerankerProvider from ekp_vector_store import MochowVectorStore, VectorStore, create_mochow_vector_store @@ -18,6 +32,15 @@ class ProviderConfigurationError(ValueError): """Raised when a runtime provider lacks required injected configuration.""" +@dataclass(frozen=True, slots=True) +class ApiServices: + """Business services injected into REST routes without provider leakage.""" + + ingestion: DocumentIndexingPipelineService + documents: DocumentManagementService + search: SearchService + + def create_index_profile(settings: Settings) -> IndexProfile: """Bind the Demo's chunk and embedding versions without vendor SDK types.""" return IndexProfile( @@ -65,7 +88,7 @@ def create_dense_retrieval_service( vector_store: VectorStore, index_profile: IndexProfile, ) -> DenseRetrievalService: - """Compose DEMO-007 retrieval while leaving REST exposure to DEMO-009.""" + """Compose the profile-scoped dense stage reused by ingestion search APIs.""" return DenseRetrievalService( embedding_provider=embedding_provider, vector_store=vector_store, @@ -93,10 +116,68 @@ def create_search_service( dense_retriever: DenseRetrievalService, reranker_provider: RerankerProvider, ) -> SearchService: - """Compose DEMO-008 search while leaving REST serialization to DEMO-009.""" + """Compose reranked search independently from REST response schemas.""" return SearchService( session_factory=session_factory, dense_retriever=dense_retriever, reranker_provider=reranker_provider, default_top_k=settings.search_top_k, ) + + +def create_api_services(settings: Settings) -> ApiServices: + """Compose the synchronous Demo runtime once for all business endpoints.""" + engine = create_database_engine(settings) + session_factory = create_session_factory(engine) + storage = LocalObjectStorage(settings.object_storage_path) + index_profile = create_index_profile(settings) + embedding_provider = create_embedding_provider(settings) + vector_store = create_vector_store(settings) + + upload_service = DocumentUploadService( + session_factory, + storage, + max_upload_bytes=settings.max_upload_bytes, + ) + parsing_service = DocumentParsingService(session_factory, storage, ParserRouter()) + chunking_service = DocumentChunkingService( + session_factory, + storage, + StructureAwareChunker( + max_tokens=settings.chunk_max_tokens, + overlap_tokens=settings.chunk_overlap_tokens, + max_characters=settings.chunk_max_characters, + ), + ) + document_pipeline = DocumentPipelineService( + upload_service, + parsing_service, + chunking_service, + ) + indexing_service = DocumentIndexingService( + session_factory, + embedding_provider, + vector_store, + index_profile, + ) + ingestion = DocumentIndexingPipelineService(document_pipeline, indexing_service) + dense_retrieval = create_dense_retrieval_service( + settings, + embedding_provider, + vector_store, + index_profile, + ) + reranker_provider = create_reranker_provider(settings) + search = create_search_service( + settings, + session_factory, + dense_retrieval, + reranker_provider, + ) + documents = DocumentManagementService( + session_factory, + storage, + vector_store, + active_index_profile_id=index_profile.profile_id, + ) + return ApiServices(ingestion=ingestion, documents=documents, search=search) diff --git a/apps/api/src/ekp_api/routes.py b/apps/api/src/ekp_api/routes.py new file mode 100644 index 0000000..682fec8 --- /dev/null +++ b/apps/api/src/ekp_api/routes.py @@ -0,0 +1,146 @@ +"""Versioned document and search routes for the retrieval Demo.""" + +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status + +from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services +from ekp_api.schemas import DocumentResponse, SearchRequest, SearchResponse +from ekp_core.dense_retrieval import InvalidRetrievalQueryError +from ekp_core.document_management import ( + DocumentArtifactCleanupError, + DocumentNotFoundError, +) +from ekp_core.document_upload import ( + DocumentTooLargeError, + DocumentUploadError, + MediaTypeMismatchError, + UnsupportedDocumentTypeError, +) +from ekp_parser import DocumentParseError + +router = APIRouter() + + +def get_api_services(request: Request) -> ApiServices: + """Resolve provider composition lazily so health checks need no cloud credentials.""" + configured = getattr(request.app.state, "api_services", None) + if isinstance(configured, ApiServices): + return configured + try: + services = create_api_services(request.app.state.settings) + except ProviderConfigurationError as error: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="API providers are not fully configured", + ) from error + request.app.state.api_services = services + return services + + +Services = Annotated[ApiServices, Depends(get_api_services)] + + +@router.post( + "/documents", + response_model=DocumentResponse, + status_code=status.HTTP_201_CREATED, + tags=["documents"], +) +def upload_document( + services: Services, + file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")], +) -> DocumentResponse: + """Upload one file and synchronously run the complete Demo ingestion pipeline.""" + try: + result = services.ingestion.ingest(file.filename or "", file.content_type, file.file) + except DocumentTooLargeError as error: + raise HTTPException( + status_code=status.HTTP_413_CONTENT_TOO_LARGE, + detail="document exceeds the configured upload limit", + ) from error + except (UnsupportedDocumentTypeError, MediaTypeMismatchError) as error: + raise HTTPException( + status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, + detail="unsupported document type or media type", + ) from error + except (DocumentUploadError, DocumentParseError) as error: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail="document could not be accepted or parsed", + ) from error + except Exception as error: + # Provider and persistence exceptions are intentionally not reflected to clients. + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="document processing failed", + ) from error + # Reload after every pipeline stage so server-generated timestamps and the + # final persisted status, not a detached intermediate ORM object, are returned. + document = services.documents.get_document(result.ingestion.document.id) + return DocumentResponse.from_document(document) + + +@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"]) +def list_documents(services: Services) -> list[DocumentResponse]: + """List canonical document metadata for the Demo admin page.""" + return [ + DocumentResponse.from_document(document) for document in services.documents.list_documents() + ] + + +@router.get("/documents/{document_id}", response_model=DocumentResponse, tags=["documents"]) +def get_document(document_id: UUID, services: Services) -> DocumentResponse: + """Return one document's status and safe processing diagnostic.""" + try: + document = services.documents.get_document(document_id) + except DocumentNotFoundError as error: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="document not found" + ) from error + return DocumentResponse.from_document(document) + + +@router.delete( + "/documents/{document_id}", + status_code=status.HTTP_204_NO_CONTENT, + tags=["documents"], +) +def delete_document(document_id: UUID, services: Services) -> Response: + """Directly remove Demo metadata, source objects, and vector rows.""" + try: + services.documents.delete_document(document_id) + except DocumentNotFoundError as error: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="document not found" + ) from error + except DocumentArtifactCleanupError as error: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="document metadata was deleted but artifact cleanup failed", + ) from error + except Exception as error: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="document deletion failed", + ) from error + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@router.post("/search", response_model=SearchResponse, tags=["search"]) +def search_knowledge(payload: SearchRequest, services: Services) -> SearchResponse: + """Return reranked Top-K sources without leaking provider or index details.""" + try: + result = services.search.search(payload.query, payload.top_k) + except InvalidRetrievalQueryError as error: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail="search query must contain non-whitespace text", + ) from error + except Exception as error: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="search providers failed", + ) from error + return SearchResponse.from_result(result) diff --git a/apps/api/src/ekp_api/schemas.py b/apps/api/src/ekp_api/schemas.py new file mode 100644 index 0000000..3c13e68 --- /dev/null +++ b/apps/api/src/ekp_api/schemas.py @@ -0,0 +1,74 @@ +"""Provider-neutral REST request and response schemas for the Demo API.""" + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from ekp_core.search import SearchResult +from ekp_domain import Document, DocumentStatus + + +class DocumentResponse(BaseModel): + """Public document metadata without internal storage or provider fields.""" + + model_config = ConfigDict(from_attributes=True) + + id: UUID + filename: str + media_type: str + size_bytes: int + status: DocumentStatus + error_message: str | None + created_at: datetime + updated_at: datetime + + @classmethod + def from_document(cls, document: Document) -> "DocumentResponse": + """Serialize a detached canonical model through the explicit API shape.""" + return cls.model_validate(document) + + +class SearchRequest(BaseModel): + """Validated public search input; normalization remains in the core service.""" + + query: str = Field(min_length=1, max_length=1600) + top_k: int = Field(default=5, ge=1, le=64, strict=True) + + +class SourceResponse(BaseModel): + """Traceable source returned after dense retrieval and reranking.""" + + document_id: UUID + filename: str + page: int | None + section: str | None + chunk_id: UUID + content: str + score: float + rerank_score: float + + +class SearchResponse(BaseModel): + """Minimal response contract consumed by the DEMO-010 Search Lab.""" + + results: list[SourceResponse] + + @classmethod + def from_result(cls, result: SearchResult) -> "SearchResponse": + """Hide index-profile internals while preserving every required source field.""" + return cls( + results=[ + SourceResponse( + document_id=source.document_id, + filename=source.filename, + page=source.page, + section=source.section, + chunk_id=source.chunk_id, + content=source.content, + score=source.score, + rerank_score=source.rerank_score, + ) + for source in result.results + ] + ) diff --git a/docs/api/API_CONTRACT.md b/docs/api/API_CONTRACT.md index 809034b..2ceff7b 100644 --- a/docs/api/API_CONTRACT.md +++ b/docs/api/API_CONTRACT.md @@ -5,3 +5,52 @@ Version all public APIs under `/api/v1`. Initial domains: auth, users/groups, knowledge-bases, documents, compiled-knowledge, search, evaluations, jobs, audit-logs, providers and MCP administration. Use OpenAPI as executable source of truth once implementation starts. Provider-specific concepts must not leak into public contracts. + +## Demo v0.1 P0 + +DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一 +版本化为 `/api/v1/*`。 + +| Method | Path | 行为 | +|---|---|---| +| `POST` | `/api/v1/documents` | multipart 单文件上传并同步执行完整入库流水线 | +| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 | +| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 | +| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 | +| `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 | + +上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `201` 和 Document; +Demo 同步流水线完成后状态为 `ready`。Document 响应不暴露 `storage_key`、Provider 配置或 +索引实现。 + +Search 请求: + +```json +{ + "query": "产品A退款周期是多少?", + "top_k": 5 +} +``` + +Search 响应: + +```json +{ + "results": [ + { + "document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752", + "filename": "产品A说明书.pdf", + "page": 12, + "section": "退款政策", + "chunk_id": "76ba1c30-bff7-44e5-9989-fb8372f54f46", + "content": "……", + "score": 0.82, + "rerank_score": 0.94 + } + ] +} +``` + +公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`, +文档不存在返回 `404`,运行时 Provider 未配置返回 `503`。上游 Provider 或持久化异常只返回 +经过收敛的 `5xx` 信息,不回显凭证、请求正文或厂商响应。 diff --git a/docs/development/MOCHOW_SETUP.md b/docs/development/MOCHOW_SETUP.md index 0ff0e9c..3fd2213 100644 --- a/docs/development/MOCHOW_SETUP.md +++ b/docs/development/MOCHOW_SETUP.md @@ -78,7 +78,8 @@ DEMO-007 使用同一 `EmbeddingProvider` 将 Query 转换为 1024 维向量, Demo 明确不实现 RBAC/Document ACL;正式 V1 引入权限后,允许范围仍必须在召回阶段下推, 不得先做全局 Top-K 再过滤。Reranker 与最终 Source Result 的配置参见 -`docs/development/RERANKER_SETUP.md`;REST API 属于 DEMO-009。 +`docs/development/RERANKER_SETUP.md`;REST API 参见 +`docs/development/REST_API_SETUP.md`。 ## 验证边界 diff --git a/docs/development/RERANKER_SETUP.md b/docs/development/RERANKER_SETUP.md index 32e4e2f..c19d314 100644 --- a/docs/development/RERANKER_SETUP.md +++ b/docs/development/RERANKER_SETUP.md @@ -54,9 +54,9 @@ rerank_score 不一致或 Mochow content 已与 canonical content 不一致,搜索会失败关闭,避免返回来源与 评分文本不一致的结果。 -DEMO-008 不创建公开 REST Endpoint;`POST /api/search` 的序列化和错误映射属于 -DEMO-009。Demo 仍不实现 RBAC/Document ACL,正式 V1 权限范围必须在 Dense 召回阶段 -下推,不能在 Reranker 后过滤。 +DEMO-009 已通过 `POST /api/v1/search` 提供序列化和错误映射,参见 +`docs/development/REST_API_SETUP.md`。Demo 仍不实现 RBAC/Document ACL,正式 V1 权限 +范围必须在 Dense 召回阶段下推,不能在 Reranker 后过滤。 ## 验证边界 diff --git a/docs/development/REST_API_SETUP.md b/docs/development/REST_API_SETUP.md new file mode 100644 index 0000000..4f2840a --- /dev/null +++ b/docs/development/REST_API_SETUP.md @@ -0,0 +1,54 @@ +# Demo REST API 使用说明 + +## 范围 + +DEMO-009 将已经完成的同步入库和检索服务公开为版本化 REST API。API 层只负责输入校验、 +错误映射和响应序列化,不重复实现 Parser、Embedding、Mochow 或 Reranker 业务逻辑。 + +本任务不包含 Admin Console、ACL/RBAC、异步任务、软删除或 Hybrid Search。 + +## 启动前提 + +1. 按 `docs/development/MOCHOW_SETUP.md` 配置 PostgreSQL、Mochow 和千帆 Embedding; +2. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker; +3. 执行 `uv run alembic upgrade head`; +4. 执行 `make run`。 + +`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时 +延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。 + +OpenAPI 地址:`/api/v1/openapi.json` + +Swagger UI:`/api/v1/docs` + +## 示例 + +上传并同步入库: + +```bash +curl -F 'file=@./policy.md;type=text/markdown' \ + http://127.0.0.1:8000/api/v1/documents +``` + +检索: + +```bash +curl -H 'Content-Type: application/json' \ + -d '{"query":"产品A退款周期是多少?","top_k":5}' \ + http://127.0.0.1:8000/api/v1/search +``` + +Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 Mochow Upsert 完成。大文件或 +云端调用较慢时,请求耗时会相应增加;异步任务属于 Demo 后置能力。 + +## 删除一致性 + +删除按以下顺序执行: + +1. 删除所有已记录 Index Profile 以及当前 Profile 的 Mochow 行; +2. 删除 PostgreSQL 中的 ChunkIndexState、Chunk 和 Document; +3. 删除解析 JSON 与不可变原始文件。 + +向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口 +返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1 +应通过可重试清理任务和审计记录强化。 diff --git a/docs/plans/PROGRESS.md b/docs/plans/PROGRESS.md index 1dc7080..120bbb4 100644 --- a/docs/plans/PROGRESS.md +++ b/docs/plans/PROGRESS.md @@ -2,11 +2,11 @@ 当前阶段:Knowledge Retrieval Demo v0.1 -已完成:DEMO-001、DEMO-002、DEMO-003、DEMO-004、DEMO-005、DEMO-006、DEMO-007、DEMO-008 +已完成:DEMO-001、DEMO-002、DEMO-003、DEMO-004、DEMO-005、DEMO-006、DEMO-007、DEMO-008、DEMO-009 进行中:无 阻塞:无 -下一任务:DEMO-009 +下一任务:DEMO-010 完整 V1 待办将在 Demo Scope 完成后继续执行。当前任务顺序以 `docs/demo/DEMO_SCOPE.md` 为准。 @@ -117,3 +117,18 @@ - 新增 Provider、配置脱敏、排序映射、canonical 回查、一致性失败和 Golden Dataset 全链路回归。默认测试不调用真实千帆;真实联调边界记录在 `docs/development/RERANKER_SETUP.md`。公开 Search REST API 留在 DEMO-009。 + +### DEMO-009 + +- 新增 `/api/v1/documents` 上传/列表、`/api/v1/documents/{document_id}` 详情/直接删除和 + `/api/v1/search`,通过 OpenAPI 固化 multipart 上传、Document 状态和最小 Source Result + 契约;API 响应不暴露 storage key、Index Profile 或厂商实现。 +- 上传接口同步复用 DEMO-003~006 的完整入库流水线,并在完成后回读 PostgreSQL canonical + Document,确保返回最终 `ready` 状态和数据库生成时间戳。 +- 新增文档查询/删除应用服务。删除覆盖当前及历史已记录 Index Profile 的向量行、 + ChunkIndexState、Chunk、Document、解析 JSON 和原始文件;向量清理失败时保留 canonical + 数据,来源清理失败返回明确错误并记录 Demo 一致性边界。 +- 业务依赖延迟创建,因此健康检查和 OpenAPI 启动不要求云端凭证;Provider 配置不完整时 + 业务接口返回 `503`,上游异常使用收敛后的错误响应,避免泄露凭证或厂商正文。 +- 新增完全离线的 REST 全链路测试,覆盖上传、列表、详情、Dense/Reranker 检索、来源字段、 + 直接删除、OpenAPI 路径、输入校验、404、Provider 延迟加载及删除失败保护。 diff --git a/packages/core/src/ekp_core/document_management.py b/packages/core/src/ekp_core/document_management.py new file mode 100644 index 0000000..026cd84 --- /dev/null +++ b/packages/core/src/ekp_core/document_management.py @@ -0,0 +1,99 @@ +"""Document query and direct-delete services for the Demo REST API.""" + +from uuid import UUID + +from sqlalchemy import delete, select +from sqlalchemy.orm import Session, sessionmaker + +from ekp_domain import Chunk, ChunkIndexState, Document +from ekp_object_storage import ObjectStorage +from ekp_vector_store import VectorStore + + +class DocumentNotFoundError(LookupError): + """Raised when a requested document does not exist.""" + + +class DocumentArtifactCleanupError(RuntimeError): + """Raised when deleted metadata leaves an external storage artifact behind.""" + + +class DocumentManagementService: + """Read canonical documents and coordinate the Demo's direct deletion. + + Vector rows are removed before canonical metadata. Object bytes are removed + after the metadata commit, so a storage failure can only leave an unreachable + orphan rather than expose a searchable row without canonical provenance. + """ + + def __init__( + self, + session_factory: sessionmaker[Session], + storage: ObjectStorage, + vector_store: VectorStore, + active_index_profile_id: str, + ) -> None: + if not active_index_profile_id: + raise ValueError("active_index_profile_id must not be empty") + self._session_factory = session_factory + self._storage = storage + self._vector_store = vector_store + self._active_index_profile_id = active_index_profile_id + + def list_documents(self) -> tuple[Document, ...]: + """Return documents newest first with a deterministic UUID tie-break.""" + with self._session_factory() as session: + return tuple( + session.scalars( + select(Document).order_by(Document.created_at.desc(), Document.id.desc()) + ).all() + ) + + def get_document(self, document_id: UUID) -> Document: + """Return one canonical document or raise a stable not-found error.""" + with self._session_factory() as session: + document = session.get(Document, document_id) + if document is None: + raise DocumentNotFoundError(str(document_id)) + return document + + def delete_document(self, document_id: UUID) -> None: + """Delete vector rows, canonical metadata, and immutable source artifacts.""" + with self._session_factory() as session: + document = session.get(Document, document_id) + if document is None: + raise DocumentNotFoundError(str(document_id)) + + source_keys = tuple( + key + for key in (document.parsed_storage_key, document.storage_key) + if key is not None + ) + profile_ids = set( + session.scalars( + select(ChunkIndexState.index_profile_id) + .join(Chunk, Chunk.chunk_id == ChunkIndexState.chunk_id) + .where(Chunk.document_id == document_id) + .distinct() + ).all() + ) + # Also clean the active profile when an interrupted index write did + # not reach canonical ChunkIndexState persistence. + profile_ids.add(self._active_index_profile_id) + for profile_id in sorted(profile_ids): + self._vector_store.delete_document(document_id, profile_id) + + chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id) + session.execute(delete(ChunkIndexState).where(ChunkIndexState.chunk_id.in_(chunk_ids))) + session.execute(delete(Chunk).where(Chunk.document_id == document_id)) + session.execute(delete(Document).where(Document.id == document_id)) + session.commit() + + try: + for source_key in source_keys: + if self._storage.exists(source_key): + self._storage.delete(source_key) + except Exception as error: + raise DocumentArtifactCleanupError( + f"external artifact cleanup failed for document {document_id}" + ) from error diff --git a/packages/core/src/ekp_core/search.py b/packages/core/src/ekp_core/search.py index c6b465d..1a4d0f3 100644 --- a/packages/core/src/ekp_core/search.py +++ b/packages/core/src/ekp_core/search.py @@ -51,7 +51,7 @@ class SourceResult: @dataclass(frozen=True, slots=True) class SearchResult: - """Final Top-K result set ready for DEMO-009 REST serialization.""" + """Final Top-K result set consumed by provider-neutral API serialization.""" query: str index_profile_id: str diff --git a/pyproject.toml b/pyproject.toml index d4cfbae..11b5c53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "pymochow>=2.4,<3", "pymupdf>=1.26,<2", "pydantic-settings>=2.7,<3", + "python-multipart>=0.0.20,<1", "python-docx>=1.2,<2", "sqlalchemy>=2.0,<3", "uvicorn[standard]>=0.34,<1", diff --git a/tests/integration/test_api_startup.py b/tests/integration/test_api_startup.py index 266a0e9..0c9bfd5 100644 --- a/tests/integration/test_api_startup.py +++ b/tests/integration/test_api_startup.py @@ -1,5 +1,7 @@ """API startup and health checks.""" +from pathlib import Path + from fastapi.testclient import TestClient from ekp_api.main import create_app @@ -30,3 +32,18 @@ def test_openapi_is_versioned(isolated_settings_environment: None) -> None: response = client.get("/api/v1/openapi.json") assert response.status_code == 200 + + +def test_business_providers_are_lazy_for_health_checks( + tmp_path: Path, + isolated_settings_environment: None, +) -> None: + """Starting or probing the API must not require Qianfan/Mochow credentials.""" + settings = Settings(app_env="test", object_storage_path=tmp_path / "storage") + + with TestClient(create_app(settings)) as client: + health_response = client.get("/healthz") + documents_response = client.get("/api/v1/documents") + + assert health_response.status_code == 200 + assert documents_response.status_code == 503 diff --git a/tests/integration/test_document_search_api.py b/tests/integration/test_document_search_api.py new file mode 100644 index 0000000..b38a9e1 --- /dev/null +++ b/tests/integration/test_document_search_api.py @@ -0,0 +1,206 @@ +"""End-to-end REST contract tests using only deterministic local providers.""" + +from pathlib import Path +from uuid import UUID, uuid4 + +from fastapi.testclient import TestClient +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import StaticPool + +from ekp_api.main import create_app +from ekp_api.providers import ApiServices +from ekp_core.chunking import StructureAwareChunker +from ekp_core.config import Settings +from ekp_core.database import create_session_factory +from ekp_core.dense_retrieval import DenseRetrievalService +from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService +from ekp_core.document_indexing import DocumentIndexingPipelineService, DocumentIndexingService +from ekp_core.document_management import DocumentManagementService +from ekp_core.document_parsing import DocumentParsingService +from ekp_core.document_upload import DocumentUploadService +from ekp_core.search import SearchService +from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile +from ekp_embedding import FakeEmbeddingProvider +from ekp_object_storage import LocalObjectStorage +from ekp_parser import ParserRouter +from ekp_reranker import FakeRerankerProvider +from ekp_vector_store import FakeVectorStore + +REQUIRED_SOURCE_FIELDS = { + "document_id", + "filename", + "page", + "section", + "chunk_id", + "content", + "score", + "rerank_score", +} + + +def _build_api_services( + tmp_path: Path, +) -> tuple[ + ApiServices, + sessionmaker[Session], + LocalObjectStorage, + FakeVectorStore, +]: + """Build the production service graph with offline providers and shared SQLite.""" + engine = create_engine( + "sqlite+pysqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + storage = LocalObjectStorage(tmp_path / "storage") + vector_store = FakeVectorStore() + embedding = FakeEmbeddingProvider( + model_name="fake-bge-large-zh", + dimensions=8, + ) + profile = IndexProfile( + profile_id="test-profile-v1", + chunk_strategy_version="structure_v1", + embedding_model=embedding.model_name, + vector_dimension=embedding.dimensions, + ) + upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024) + parsing = DocumentParsingService(session_factory, storage, ParserRouter()) + chunking = DocumentChunkingService( + session_factory, + storage, + StructureAwareChunker(max_tokens=450, overlap_tokens=75), + ) + pipeline = DocumentPipelineService(upload, parsing, chunking) + indexing = DocumentIndexingService( + session_factory, + embedding, + vector_store, + profile, + ) + ingestion = DocumentIndexingPipelineService(pipeline, indexing) + dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20) + search = SearchService( + session_factory, + dense, + FakeRerankerProvider(), + default_top_k=5, + ) + documents = DocumentManagementService( + session_factory, + storage, + vector_store, + active_index_profile_id=profile.profile_id, + ) + return ( + ApiServices(ingestion=ingestion, documents=documents, search=search), + session_factory, + storage, + vector_store, + ) + + +def test_document_and_search_rest_flow( + tmp_path: Path, + isolated_settings_environment: None, +) -> None: + services, session_factory, storage, vector_store = _build_api_services(tmp_path) + app = create_app(Settings(app_env="test"), api_services=services) + + with TestClient(app) as client: + upload_response = client.post( + "/api/v1/documents", + files={ + "file": ( + "退款政策.md", + "# 产品A\n\n## 退款政策\n产品A退款周期为七天。".encode(), + "text/markdown", + ) + }, + ) + assert upload_response.status_code == 201 + uploaded = upload_response.json() + document_id = UUID(uploaded["id"]) + assert uploaded["status"] == "ready" + assert uploaded["filename"] == "退款政策.md" + assert "storage_key" not in uploaded + + list_response = client.get("/api/v1/documents") + detail_response = client.get(f"/api/v1/documents/{document_id}") + assert list_response.status_code == 200 + assert [item["id"] for item in list_response.json()] == [str(document_id)] + assert detail_response.status_code == 200 + assert detail_response.json() == uploaded + + search_response = client.post( + "/api/v1/search", + json={"query": "产品A退款周期", "top_k": 1}, + ) + assert search_response.status_code == 200 + results = search_response.json()["results"] + assert len(results) == 1 + assert set(results[0]) == REQUIRED_SOURCE_FIELDS + assert results[0]["document_id"] == str(document_id) + assert results[0]["filename"] == "退款政策.md" + assert results[0]["section"] == "产品A / 退款政策" + assert "七天" in results[0]["content"] + + with session_factory() as session: + document = session.get(Document, document_id) + assert document is not None + source_key = document.storage_key + parsed_key = document.parsed_storage_key + assert storage.exists(source_key) + assert parsed_key is not None and storage.exists(parsed_key) + + delete_response = client.delete(f"/api/v1/documents/{document_id}") + assert delete_response.status_code == 204 + assert delete_response.content == b"" + + with session_factory() as session: + assert session.get(Document, document_id) is None + assert session.scalars(select(Chunk)).all() == [] + assert session.scalars(select(ChunkIndexState)).all() == [] + assert vector_store.records == {} + assert not storage.exists(source_key) + assert not storage.exists(parsed_key) + + +def test_rest_validation_not_found_and_openapi_contract( + tmp_path: Path, + isolated_settings_environment: None, +) -> None: + services, _, _, _ = _build_api_services(tmp_path) + app = create_app(Settings(app_env="test"), api_services=services) + missing_id = uuid4() + + with TestClient(app) as client: + openapi = client.get("/api/v1/openapi.json").json() + assert set(openapi["paths"]) >= { + "/api/v1/documents", + "/api/v1/documents/{document_id}", + "/api/v1/search", + } + assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"} + assert set(openapi["paths"]["/api/v1/documents/{document_id}"]) >= { + "get", + "delete", + } + + unsupported = client.post( + "/api/v1/documents", + files={"file": ("notes.csv", b"value", "text/csv")}, + ) + blank_query = client.post("/api/v1/search", json={"query": " ", "top_k": 5}) + boolean_top_k = client.post("/api/v1/search", json={"query": "退款", "top_k": True}) + missing_get = client.get(f"/api/v1/documents/{missing_id}") + missing_delete = client.delete(f"/api/v1/documents/{missing_id}") + + assert unsupported.status_code == 415 + assert blank_query.status_code == 422 + assert boolean_top_k.status_code == 422 + assert missing_get.status_code == 404 + assert missing_delete.status_code == 404 diff --git a/tests/unit/test_document_management.py b/tests/unit/test_document_management.py new file mode 100644 index 0000000..79e4dfc --- /dev/null +++ b/tests/unit/test_document_management.py @@ -0,0 +1,53 @@ +"""Unit tests for direct document deletion across canonical and external stores.""" + +from io import BytesIO +from pathlib import Path + +import pytest +from sqlalchemy import create_engine + +from ekp_core.database import create_session_factory +from ekp_core.document_management import DocumentManagementService +from ekp_domain import Base, Document +from ekp_object_storage import LocalObjectStorage +from ekp_vector_store import FakeVectorStore + + +class FailingDeleteVectorStore(FakeVectorStore): + """Simulate Mochow refusing cleanup before canonical deletion begins.""" + + def delete_document(self, document_id: object, index_profile_id: str) -> None: + raise RuntimeError("simulated vector deletion failure") + + +def test_vector_failure_preserves_canonical_document_and_source(tmp_path: Path) -> None: + """A rebuildable-index failure must not delete canonical source data.""" + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + storage = LocalObjectStorage(tmp_path / "storage") + stored = storage.put("documents/test/source.txt", BytesIO(b"canonical source")) + with session_factory() as session: + document = Document( + filename="source.txt", + media_type="text/plain", + size_bytes=stored.size_bytes, + storage_key=stored.key, + ) + session.add(document) + session.commit() + document_id = document.id + + service = DocumentManagementService( + session_factory, + storage, + FailingDeleteVectorStore(), + active_index_profile_id="profile-v1", + ) + + with pytest.raises(RuntimeError, match="vector deletion failure"): + service.delete_document(document_id) + + with session_factory() as session: + assert session.get(Document, document_id) is not None + assert storage.exists(stored.key) diff --git a/tests/unit/test_provider_wiring.py b/tests/unit/test_provider_wiring.py index 81af822..12f056d 100644 --- a/tests/unit/test_provider_wiring.py +++ b/tests/unit/test_provider_wiring.py @@ -9,6 +9,7 @@ from ekp_api.providers import ( create_embedding_provider, create_index_profile, create_reranker_provider, + create_search_service, create_vector_store, ) from ekp_core.config import Settings @@ -68,3 +69,32 @@ def test_provider_wiring_reuses_qianfan_key_for_demo_reranker( assert reranker.model_name == "qwen3-reranker-0.6b" assert reranker.max_documents == 64 + + +def test_search_wiring_uses_configured_top_k( + isolated_settings_environment: None, +) -> None: + """Keep the REST-facing default in typed configuration, not route code.""" + from sqlalchemy import create_engine + + from ekp_core.database import create_session_factory + from ekp_core.dense_retrieval import DenseRetrievalService + from ekp_domain import Base, IndexProfile + from ekp_reranker import FakeRerankerProvider + + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + settings = Settings(search_top_k=3, dense_top_n=20) + embedding = FakeEmbeddingProvider("fake", dimensions=4) + profile = IndexProfile("profile-v1", "structure_v1", "fake", 4) + dense = DenseRetrievalService(embedding, FakeVectorStore(), profile) + + service = create_search_service( + settings, + session_factory, + dense, + FakeRerankerProvider(), + ) + + assert service.search("测试").results == () diff --git a/uv.lock b/uv.lock index 5cad782..8e62471 100644 --- a/uv.lock +++ b/uv.lock @@ -187,6 +187,7 @@ dependencies = [ { name = "pymochow" }, { name = "pymupdf" }, { name = "python-docx" }, + { name = "python-multipart" }, { name = "sqlalchemy" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -209,6 +210,7 @@ requires-dist = [ { name = "pymochow", specifier = ">=2.4,<3" }, { name = "pymupdf", specifier = ">=1.26,<2" }, { name = "python-docx", specifier = ">=1.2,<2" }, + { name = "python-multipart", specifier = ">=0.0.20,<1" }, { name = "sqlalchemy", specifier = ">=2.0,<3" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34,<1" }, ] @@ -803,6 +805,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" }, ] +[[package]] +name = "python-multipart" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3"