From 7938c0b9cf955b9b0f4e0a48ca25d1b77e4abd71 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Wed, 2 Sep 2026 20:18:58 +0800 Subject: [PATCH] feat: add durable async ingestion jobs --- .env.example | 9 +- Makefile | 5 +- README.md | 12 +- apps/admin-web/assets/app.js | 4 +- apps/admin-web/index.html | 2 +- apps/api/src/ekp_api/celery_queue.py | 55 ++ apps/api/src/ekp_api/providers.py | 153 ++++- apps/api/src/ekp_api/routes.py | 49 +- apps/api/src/ekp_api/schemas.py | 61 +- apps/api/src/ekp_api/worker.py | 51 ++ apps/mcp-server/src/ekp_mcp/server.py | 8 +- docs/api/API_CONTRACT.md | 42 +- docs/demo/DEMO_SCOPE.md | 8 +- docs/development/ADMIN_CONSOLE.md | 10 +- docs/development/ASYNC_INGESTION_SETUP.md | 82 +++ docs/development/REST_API_SETUP.md | 26 +- docs/development/TEST_STRATEGY.md | 5 + docs/plans/ASYNC_INGESTION_MCP_PLAN.md | 8 +- docs/plans/PROGRESS.md | 28 +- docs/plans/TASKS.md | 2 +- docs/plans/V1_EXECUTION_PLAN.md | 2 +- migrations/README.md | 4 + .../20260902_0007_create_ingestion_jobs.py | 174 ++++++ packages/core/src/ekp_core/config.py | 17 + .../core/src/ekp_core/document_chunking.py | 13 +- .../core/src/ekp_core/document_indexing.py | 32 +- .../core/src/ekp_core/document_management.py | 22 +- .../core/src/ekp_core/document_parsing.py | 15 +- packages/core/src/ekp_core/document_upload.py | 23 +- packages/core/src/ekp_core/ingestion_jobs.py | 588 ++++++++++++++++++ packages/domain/src/ekp_domain/__init__.py | 12 + .../domain/src/ekp_domain/ingestion_job.py | 166 +++++ pyproject.toml | 2 + tests/integration/test_async_ingestion.py | 431 +++++++++++++ tests/integration/test_document_migration.py | 37 +- tests/integration/test_document_search_api.py | 69 +- tests/unit/test_celery_queue.py | 57 ++ tests/unit/test_config.py | 21 + uv.lock | 158 +++++ 39 files changed, 2359 insertions(+), 104 deletions(-) create mode 100644 apps/api/src/ekp_api/celery_queue.py create mode 100644 apps/api/src/ekp_api/worker.py create mode 100644 docs/development/ASYNC_INGESTION_SETUP.md create mode 100644 migrations/versions/20260902_0007_create_ingestion_jobs.py create mode 100644 packages/core/src/ekp_core/ingestion_jobs.py create mode 100644 packages/domain/src/ekp_domain/ingestion_job.py create mode 100644 tests/integration/test_async_ingestion.py create mode 100644 tests/unit/test_celery_queue.py diff --git a/.env.example b/.env.example index 37d79c1..ea71749 100644 --- a/.env.example +++ b/.env.example @@ -11,8 +11,13 @@ DATABASE_MAX_OVERFLOW=10 DATABASE_POOL_RECYCLE_SECONDS=600 DATABASE_POOL_TIMEOUT_SECONDS=30 DATABASE_CONNECT_TIMEOUT_SECONDS=30 -REDIS_URL=redis://localhost:6379/0 -RABBITMQ_URL=amqp://guest:guest@localhost:5672// +# DEMO-015 queue. Inject the real credential at runtime; never commit its value. +RABBITMQ_URL= +INGESTION_MAX_ATTEMPTS=3 +INGESTION_RETRY_BACKOFF_SECONDS=2 +INGESTION_RETRY_BACKOFF_MAX_SECONDS=60 +INGESTION_WORKER_LEASE_SECONDS=300 +INGESTION_RECOVERY_INTERVAL_SECONDS=30 OBJECT_STORAGE_PROVIDER=bos OBJECT_STORAGE_PATH=storage diff --git a/Makefile b/Makefile index 8f00577..6d0d600 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install run run-mcp format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down +.PHONY: help install run run-mcp run-worker format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down UV_CACHE_DIR ?= /tmp/ekp-uv-cache export UV_CACHE_DIR @@ -8,6 +8,7 @@ help: @echo " install Install development dependencies" @echo " run Start the development API" @echo " run-mcp Start the authenticated MCP gateway" + @echo " run-worker Start the ingestion worker and recovery scheduler" @echo " format Format code" @echo " format-check Check formatting without modifying files" @echo " lint Run linters" @@ -27,6 +28,8 @@ run: uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}" run-mcp: uv run ekp-mcp +run-worker: + uv run ekp-worker format: uv run ruff format . format-check: diff --git a/README.md b/README.md index fbe2e93..5dcc4d5 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Install dependencies and run the API: ```bash cp .env.example .env make install +make run-worker make run ``` @@ -100,8 +101,11 @@ Chunk metadata, including deterministic IDs, filename, page range, section, index, and strategy version, is stored atomically in canonical relational tables. The lightweight token counter is replaceable so DEMO-006 can inject model-specific behavior. Documents remain `processing` until embedding/indexing completes; pipeline -failures become `failed` without deleting the immutable original. DEMO-009 -exposes this pipeline through the versioned upload endpoint. +failures become `failed` without deleting the immutable original. DEMO-015 now +accepts a source plus durable Job with HTTP `202`, while a Celery/RabbitMQ worker +runs the pipeline. Bounded transient retries clean every derived artifact before +starting the next Attempt; broker redelivery and expired worker leases converge on +OceanBase Job state. See `docs/development/ASYNC_INGESTION_SETUP.md`. DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase. @@ -128,8 +132,8 @@ adapter with the `bce-reranker-base` baseline. The final Top20-to-Top5 pipeline source results are rehydrated from canonical relational chunks and retain both Dense and reranker scores; stale vector content fails closed. See `docs/development/RERANKER_SETUP.md` for configuration and validation limits. -DEMO-009 exposes the completed synchronous pipeline through versioned document -and search REST endpoints under `/api/v1`. Upload, list, detail, direct delete, +DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015 +changes upload to asynchronous acceptance and adds Job status. Upload, list, detail, direct delete, and reranked source search are documented in `docs/development/REST_API_SETUP.md`; provider and storage internals remain outside the public response contract. diff --git a/apps/admin-web/assets/app.js b/apps/admin-web/assets/app.js index aae8090..02a0d1e 100644 --- a/apps/admin-web/assets/app.js +++ b/apps/admin-web/assets/app.js @@ -100,7 +100,7 @@ function initializeUpload() { form.addEventListener("submit", async (event) => { event.preventDefault(); if (!input.files.length) return; - setButtonBusy(button, true, "处理中…"); + setButtonBusy(button, true, "提交中…"); try { const formData = new FormData(); formData.append("file", input.files[0]); @@ -109,7 +109,7 @@ function initializeUpload() { body: formData, }); renderUploadResult(documentData); - showNotice("文档已上传并完成处理。可以前往 Search Lab 检索。", "success"); + showNotice("文档已接收并进入后台处理队列。", "success"); form.reset(); updateFilename(); } catch (error) { diff --git a/apps/admin-web/index.html b/apps/admin-web/index.html index 4376d3c..6241cb4 100644 --- a/apps/admin-web/index.html +++ b/apps/admin-web/index.html @@ -67,7 +67,7 @@

让知识进入检索链路

-

上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将同步完成解析、切分与索引。

+

上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将在后台完成解析、切分与索引。

diff --git a/apps/api/src/ekp_api/celery_queue.py b/apps/api/src/ekp_api/celery_queue.py new file mode 100644 index 0000000..bde1bcc --- /dev/null +++ b/apps/api/src/ekp_api/celery_queue.py @@ -0,0 +1,55 @@ +"""Celery/RabbitMQ adapter behind the queue-neutral ingestion dispatcher.""" + +from uuid import UUID + +from celery import Celery # type: ignore[import-untyped] + +from ekp_core.config import Settings + +INGESTION_TASK_NAME = "ekp.ingestion.run" +RECOVERY_TASK_NAME = "ekp.ingestion.recover" + + +def create_celery_app(settings: Settings) -> Celery: + """Create a secret-safe broker client without opening a connection eagerly.""" + if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip(): + raise ValueError("RABBITMQ_URL is required for asynchronous ingestion") + app = Celery( + "enterprise-knowledge-platform", + broker=settings.rabbitmq_url.get_secret_value(), + ) + app.conf.update( + task_ignore_result=True, + task_acks_late=True, + task_reject_on_worker_lost=True, + worker_prefetch_multiplier=1, + broker_connection_retry_on_startup=True, + timezone="UTC", + enable_utc=True, + task_routes={ + INGESTION_TASK_NAME: {"queue": "ingestion"}, + RECOVERY_TASK_NAME: {"queue": "ingestion"}, + }, + beat_schedule={ + "recover-durable-ingestion-jobs": { + "task": RECOVERY_TASK_NAME, + "schedule": float(settings.ingestion_recovery_interval_seconds), + } + }, + ) + return app + + +class CeleryIngestionDispatcher: + """Publish only a durable Job id; source bytes and state stay canonical.""" + + def __init__(self, app: Celery) -> None: + self._app = app + + def enqueue(self, job_id: UUID) -> None: + """Deliver one idempotent execution hint to the ingestion queue.""" + self._app.send_task( + INGESTION_TASK_NAME, + args=[str(job_id)], + queue="ingestion", + ) diff --git a/apps/api/src/ekp_api/providers.py b/apps/api/src/ekp_api/providers.py index b45f849..453df9f 100644 --- a/apps/api/src/ekp_api/providers.py +++ b/apps/api/src/ekp_api/providers.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from sqlalchemy.orm import Session, sessionmaker +from ekp_api.celery_queue import CeleryIngestionDispatcher, create_celery_app from ekp_core.chunking import StructureAwareChunker from ekp_core.config import Settings from ekp_core.database import create_database_engine, create_session_factory @@ -17,6 +18,12 @@ from ekp_core.document_management import DocumentManagementService from ekp_core.document_parsing import DocumentParsingService from ekp_core.document_upload import DocumentUploadService from ekp_core.index_rebuild import resolve_active_index_profile +from ekp_core.ingestion_jobs import ( + AsyncIngestionSubmissionService, + DerivedDocumentCleanupService, + IngestionJobRunner, + IngestionJobService, +) from ekp_core.search import SearchService from ekp_domain import IndexProfile from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider @@ -42,10 +49,38 @@ class ApiServices: """Business services injected into REST routes without provider leakage.""" ingestion: DocumentIndexingPipelineService + submissions: AsyncIngestionSubmissionService + jobs: IngestionJobService documents: DocumentManagementService search: SearchService +@dataclass(frozen=True, slots=True) +class WorkerServices: + """Provider graph required by Celery without REST search-only services.""" + + runner: IngestionJobRunner + jobs: IngestionJobService + dispatcher: CeleryIngestionDispatcher + + +@dataclass(frozen=True, slots=True) +class _IngestionComponents: + """Shared composition result used by API and independent worker processes.""" + + session_factory: sessionmaker[Session] + storage: ObjectStorage + vector_store: VectorStore + embedding_provider: EmbeddingProvider + index_profile: IndexProfile + upload: DocumentUploadService + parsing: DocumentParsingService + chunking: DocumentChunkingService + indexing: DocumentIndexingService + jobs: IngestionJobService + dispatcher: CeleryIngestionDispatcher + + def create_index_profile( settings: Settings, session_factory: sessionmaker[Session] | None = None, @@ -161,8 +196,31 @@ def create_search_service( ) -def create_api_services(settings: Settings) -> ApiServices: - """Compose the synchronous Demo runtime once for all business endpoints.""" +def create_runtime_search_service(settings: Settings) -> SearchService: + """Compose only read-path providers for the independent MCP Gateway.""" + engine = create_database_engine(settings) + session_factory = create_session_factory(engine) + index_profile = create_index_profile(settings, session_factory) + embedding_provider = create_embedding_provider(settings) + vector_store = create_vector_store(settings) + dense_retrieval = create_dense_retrieval_service( + settings, + embedding_provider, + vector_store, + index_profile, + ) + return create_search_service( + settings, + session_factory, + dense_retrieval, + create_reranker_provider(settings), + ) + + +def _create_ingestion_components(settings: Settings) -> _IngestionComponents: + """Compose provider-neutral ingestion stages and the Celery delivery adapter.""" + if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip(): + raise ProviderConfigurationError("RABBITMQ_URL is required") engine = create_database_engine(settings) session_factory = create_session_factory(engine) storage = create_object_storage(settings) @@ -185,35 +243,96 @@ def create_api_services(settings: Settings) -> ApiServices: max_characters=settings.chunk_max_characters, ), ) - document_pipeline = DocumentPipelineService( - upload_service, - parsing_service, - chunking_service, - ) indexing_service = DocumentIndexingService( session_factory, embedding_provider, vector_store, index_profile, ) - ingestion = DocumentIndexingPipelineService(document_pipeline, indexing_service) + jobs = IngestionJobService(session_factory, settings.ingestion_max_attempts) + dispatcher = CeleryIngestionDispatcher(create_celery_app(settings)) + return _IngestionComponents( + session_factory=session_factory, + storage=storage, + vector_store=vector_store, + embedding_provider=embedding_provider, + index_profile=index_profile, + upload=upload_service, + parsing=parsing_service, + chunking=chunking_service, + indexing=indexing_service, + jobs=jobs, + dispatcher=dispatcher, + ) + + +def _create_runner(settings: Settings, components: _IngestionComponents) -> IngestionJobRunner: + """Compose retry cleanup and execution policy from non-sensitive settings.""" + cleanup = DerivedDocumentCleanupService( + components.session_factory, + components.storage, + components.vector_store, + components.index_profile.profile_id, + ) + return IngestionJobRunner( + components.session_factory, + components.parsing, + components.chunking, + components.indexing, + cleanup, + retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, + retry_backoff_max_seconds=settings.ingestion_retry_backoff_max_seconds, + lease_seconds=settings.ingestion_worker_lease_seconds, + ) + + +def create_worker_services(settings: Settings) -> WorkerServices: + """Build the independent worker runtime from the same application boundaries.""" + components = _create_ingestion_components(settings) + return WorkerServices( + runner=_create_runner(settings, components), + jobs=components.jobs, + dispatcher=components.dispatcher, + ) + + +def create_api_services(settings: Settings) -> ApiServices: + """Compose REST services while ingestion execution remains in workers.""" + components = _create_ingestion_components(settings) + document_pipeline = DocumentPipelineService( + components.upload, + components.parsing, + components.chunking, + ) + ingestion = DocumentIndexingPipelineService(document_pipeline, components.indexing) + submissions = AsyncIngestionSubmissionService( + components.upload, + components.jobs, + components.dispatcher, + ) dense_retrieval = create_dense_retrieval_service( settings, - embedding_provider, - vector_store, - index_profile, + components.embedding_provider, + components.vector_store, + components.index_profile, ) reranker_provider = create_reranker_provider(settings) search = create_search_service( settings, - session_factory, + components.session_factory, dense_retrieval, reranker_provider, ) documents = DocumentManagementService( - session_factory, - storage, - vector_store, - active_index_profile_id=index_profile.profile_id, + components.session_factory, + components.storage, + components.vector_store, + active_index_profile_id=components.index_profile.profile_id, + ) + return ApiServices( + ingestion=ingestion, + submissions=submissions, + jobs=components.jobs, + documents=documents, + search=search, ) - return ApiServices(ingestion=ingestion, documents=documents, search=search) diff --git a/apps/api/src/ekp_api/routes.py b/apps/api/src/ekp_api/routes.py index 682fec8..788501d 100644 --- a/apps/api/src/ekp_api/routes.py +++ b/apps/api/src/ekp_api/routes.py @@ -6,10 +6,17 @@ from uuid import UUID from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services -from ekp_api.schemas import DocumentResponse, SearchRequest, SearchResponse +from ekp_api.schemas import ( + DocumentResponse, + IngestionAcceptedResponse, + IngestionJobResponse, + SearchRequest, + SearchResponse, +) from ekp_core.dense_retrieval import InvalidRetrievalQueryError from ekp_core.document_management import ( DocumentArtifactCleanupError, + DocumentIngestionActiveError, DocumentNotFoundError, ) from ekp_core.document_upload import ( @@ -18,7 +25,7 @@ from ekp_core.document_upload import ( MediaTypeMismatchError, UnsupportedDocumentTypeError, ) -from ekp_parser import DocumentParseError +from ekp_core.ingestion_jobs import IngestionJobNotFoundError router = APIRouter() @@ -44,17 +51,17 @@ Services = Annotated[ApiServices, Depends(get_api_services)] @router.post( "/documents", - response_model=DocumentResponse, - status_code=status.HTTP_201_CREATED, + response_model=IngestionAcceptedResponse, + status_code=status.HTTP_202_ACCEPTED, tags=["documents"], ) def upload_document( services: Services, file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")], -) -> DocumentResponse: - """Upload one file and synchronously run the complete Demo ingestion pipeline.""" +) -> IngestionAcceptedResponse: + """Persist one immutable source and durable Job, then return immediately.""" try: - result = services.ingestion.ingest(file.filename or "", file.content_type, file.file) + submission = services.submissions.submit(file.filename or "", file.content_type, file.file) except DocumentTooLargeError as error: raise HTTPException( status_code=status.HTTP_413_CONTENT_TOO_LARGE, @@ -65,21 +72,30 @@ def upload_document( status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail="unsupported document type or media type", ) from error - except (DocumentUploadError, DocumentParseError) as error: + except DocumentUploadError as error: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, - detail="document could not be accepted or parsed", + detail="document could not be accepted", ) from error except Exception as error: # Provider and persistence exceptions are intentionally not reflected to clients. raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, - detail="document processing failed", + detail="document acceptance failed", ) from error - # Reload after every pipeline stage so server-generated timestamps and the - # final persisted status, not a detached intermediate ORM object, are returned. - document = services.documents.get_document(result.ingestion.document.id) - return DocumentResponse.from_document(document) + return IngestionAcceptedResponse.from_submission(submission) + + +@router.get("/jobs/{job_id}", response_model=IngestionJobResponse, tags=["jobs"]) +def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse: + """Return durable stage, attempt and safe failure state for one Job.""" + try: + job = services.jobs.get(job_id) + except IngestionJobNotFoundError as error: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="job not found" + ) from error + return IngestionJobResponse.from_job(job) @router.get("/documents", response_model=list[DocumentResponse], tags=["documents"]) @@ -115,6 +131,11 @@ def delete_document(document_id: UUID, services: Services) -> Response: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="document not found" ) from error + except DocumentIngestionActiveError as error: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="document ingestion is still active", + ) from error except DocumentArtifactCleanupError as error: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/apps/api/src/ekp_api/schemas.py b/apps/api/src/ekp_api/schemas.py index 3c13e68..d7c9632 100644 --- a/apps/api/src/ekp_api/schemas.py +++ b/apps/api/src/ekp_api/schemas.py @@ -5,8 +5,9 @@ from uuid import UUID from pydantic import BaseModel, ConfigDict, Field +from ekp_core.ingestion_jobs import IngestionSubmission from ekp_core.search import SearchResult -from ekp_domain import Document, DocumentStatus +from ekp_domain import Document, DocumentStatus, IngestionJob, IngestionJobStatus, IngestionStage class DocumentResponse(BaseModel): @@ -29,6 +30,64 @@ class DocumentResponse(BaseModel): return cls.model_validate(document) +class IngestionAcceptedResponse(DocumentResponse): + """Single-file acceptance with the durable Job needed for status polling.""" + + job_id: UUID + job_status: IngestionJobStatus + max_attempts: int + + @classmethod + def from_submission(cls, submission: IngestionSubmission) -> "IngestionAcceptedResponse": + """Flatten identifiers while retaining the existing document fields.""" + document = submission.document + job = submission.job + return cls( + id=document.id, + filename=document.filename, + media_type=document.media_type, + size_bytes=document.size_bytes, + status=document.status, + error_message=document.error_message, + created_at=document.created_at, + updated_at=document.updated_at, + job_id=job.id, + job_status=job.status, + max_attempts=job.max_attempts, + ) + + +class IngestionJobResponse(BaseModel): + """Public queue-independent Job status without provider or broker details.""" + + id: UUID + document_id: UUID + status: IngestionJobStatus + stage: IngestionStage | None + attempt: int + max_attempts: int + error_message: str | None + created_at: datetime + updated_at: datetime + completed_at: datetime | None + + @classmethod + def from_job(cls, job: IngestionJob) -> "IngestionJobResponse": + """Serialize only client-actionable state and safe diagnostics.""" + return cls( + id=job.id, + document_id=job.document_id, + status=job.status, + stage=job.stage, + attempt=job.attempt_count, + max_attempts=job.max_attempts, + error_message=job.error_message, + created_at=job.created_at, + updated_at=job.updated_at, + completed_at=job.completed_at, + ) + + class SearchRequest(BaseModel): """Validated public search input; normalization remains in the core service.""" diff --git a/apps/api/src/ekp_api/worker.py b/apps/api/src/ekp_api/worker.py new file mode 100644 index 0000000..e610459 --- /dev/null +++ b/apps/api/src/ekp_api/worker.py @@ -0,0 +1,51 @@ +"""Celery worker entry point for durable ingestion Jobs.""" + +from functools import lru_cache +from typing import Any +from uuid import UUID + +from ekp_api.celery_queue import ( + INGESTION_TASK_NAME, + RECOVERY_TASK_NAME, + create_celery_app, +) +from ekp_api.providers import WorkerServices, create_worker_services +from ekp_core.config import get_settings +from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs + +_settings = get_settings() +celery_app = create_celery_app(_settings) + + +@lru_cache +def _worker_services() -> WorkerServices: + """Build one provider/service graph per worker process.""" + return create_worker_services(_settings) + + +@celery_app.task( # type: ignore[untyped-decorator] + bind=True, + name=INGESTION_TASK_NAME, + max_retries=None, +) +def run_ingestion(self: Any, job_id: str) -> None: + """Execute one delivery and ask Celery to redeliver only when required. + + Celery delivery retries are unbounded because valid-lease deferrals are not + business Attempts; the durable runner independently enforces max_attempts. + """ + result = _worker_services().runner.run(UUID(job_id)) + if result.action == JobRunAction.RETRY: + raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None) + + +@celery_app.task(name=RECOVERY_TASK_NAME) # type: ignore[untyped-decorator] +def recover_ingestion_jobs() -> int: + """Redeliver due durable Jobs after API/broker or worker interruptions.""" + services = _worker_services() + return dispatch_pending_jobs(services.jobs, services.dispatcher) + + +def main() -> None: + """Run one worker with the embedded Demo recovery scheduler.""" + celery_app.worker_main(["worker", "--loglevel=INFO", "--queues=ingestion", "--beat"]) diff --git a/apps/mcp-server/src/ekp_mcp/server.py b/apps/mcp-server/src/ekp_mcp/server.py index 7aceda7..1d2680b 100644 --- a/apps/mcp-server/src/ekp_mcp/server.py +++ b/apps/mcp-server/src/ekp_mcp/server.py @@ -12,7 +12,7 @@ from mcp.server.transport_security import TransportSecuritySettings from pydantic import Field from starlette.applications import Starlette -from ekp_api.providers import ApiServices, create_api_services +from ekp_api.providers import ApiServices, create_runtime_search_service from ekp_core.config import Settings, get_settings from ekp_core.dense_retrieval import InvalidRetrievalQueryError from ekp_mcp.auth import StaticServiceTokenVerifier, current_mcp_identity @@ -29,8 +29,10 @@ def _create_search_application( settings: Settings, api_services: ApiServices | None, ) -> McpSearchApplication: - services = api_services or create_api_services(settings) - return McpSearchApplication(services.search, settings.mcp_required_scope) + searcher = ( + api_services.search if api_services is not None else create_runtime_search_service(settings) + ) + return McpSearchApplication(searcher, settings.mcp_required_scope) def create_mcp_server( diff --git a/docs/api/API_CONTRACT.md b/docs/api/API_CONTRACT.md index 47bf703..9a273ee 100644 --- a/docs/api/API_CONTRACT.md +++ b/docs/api/API_CONTRACT.md @@ -11,19 +11,20 @@ Use OpenAPI as executable source of truth once implementation starts. Provider-s DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一 版本化为 `/api/v1/*`。 -下表描述当前已实现的同步 Demo 契约;它将在 `DEMO-015` 完成时由后文的异步契约替代。 +`DEMO-015` 已将单文件上传切换为持久化异步 Job;批量提交仍留给 `DEMO-016`。 | Method | Path | 行为 | |---|---|---| -| `POST` | `/api/v1/documents` | multipart 单文件上传并同步执行完整入库流水线 | +| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` | | `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 | | `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 | | `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 | +| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 | | `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 | -上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `201` 和 Document; -Demo 同步流水线完成后状态为 `ready`。Document 响应不暴露 `storage_key`、Provider 配置或 -索引实现。 +上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `202`、Document 字段及 +`job_id`、`job_status=queued`、`max_attempts`;响应不等待 Parse、Clean、Chunk、Embedding 或 +Index。Document 响应不暴露 `storage_key`、Provider 配置或索引实现。 Search 请求: @@ -54,8 +55,9 @@ Search 响应: ``` 公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`, -文档不存在返回 `404`,运行时 Provider 未配置返回 `503`。上游 Provider 或持久化异常只返回 -经过收敛的 `5xx` 信息,不回显凭证、请求正文或厂商响应。 +文档或 Job 不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回 +`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或 +厂商响应。 ## Demo Admin 客户端路由 @@ -72,16 +74,34 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏 集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、 模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。 -## Planned async ingestion contract(DEMO-015/016,尚未实现) +## 异步入库契约(DEMO-015 已实现;DEMO-016 批量部分待实现) | Method | Path | 行为 | |---|---|---| -| `POST` | `/api/v1/documents` | 单文件接收;持久化 Source、Document、Job 后返回 `202` | +| `POST` | `/api/v1/documents` | 已实现:单文件接收并返回 Document/Job 标识 | | `POST` | `/api/v1/ingestion-batches` | multipart 多文件接收,返回批次及逐文件 Document/Job 标识 | | `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新状态 | -| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 | +| `GET` | `/api/v1/jobs/{job_id}` | 已实现:返回持久化 Job 状态 | -单文件与批量上传都不等待解析、Chunk、Embedding 或索引完成。批量响应示意: +单文件响应示意: + +```json +{ + "id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752", + "filename": "退款政策.md", + "media_type": "text/markdown", + "size_bytes": 1024, + "status": "uploaded", + "error_message": null, + "created_at": "2026-09-02T12:00:00Z", + "updated_at": "2026-09-02T12:00:00Z", + "job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e", + "job_status": "queued", + "max_attempts": 3 +} +``` + +DEMO-016 的批量响应示意: ```json { diff --git a/docs/demo/DEMO_SCOPE.md b/docs/demo/DEMO_SCOPE.md index 9f77da5..fca286f 100644 --- a/docs/demo/DEMO_SCOPE.md +++ b/docs/demo/DEMO_SCOPE.md @@ -333,7 +333,7 @@ Search Service 或 `/api/v1/search`,用于验证标准 MCP、identity 边界 ### DEMO-015 异步入库 Job 与自动重试清理 -将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前 +**已完成。** 将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前 删除旧解析产物、Chunk、索引状态和向量数据,但保留不可变原始文件、Document 和 Attempt 历史。清理失败时终止,不在未知脏状态继续执行。通过执行器抽象接入 Celery/RabbitMQ,使 API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重复消息。 @@ -397,8 +397,8 @@ DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收 实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与 `ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和 -任务拆分;`DEMO-012` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,后续严格按 -`DEMO-015`、`DEMO-016`、`DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为 +任务拆分;`DEMO-012`、`DEMO-015` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围, +后续严格按 `DEMO-016`、`DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为 可选项,不因本状态记录自动进入范围。 可选: @@ -547,7 +547,7 @@ docs/demo/DEMO_SCOPE.md 完整 V1 PRD 只允许实现本文件明确列出的 Demo、Architecture、Plan 与 Deploy Task,并且一次只执行一个 -Task。当前严格顺序为 DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。 +Task。当前严格顺序为 DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。 当前公共知识库阶段的业务目标保持为: diff --git a/docs/development/ADMIN_CONSOLE.md b/docs/development/ADMIN_CONSOLE.md index 9e0e4a5..4de4b3e 100644 --- a/docs/development/ADMIN_CONSOLE.md +++ b/docs/development/ADMIN_CONSOLE.md @@ -5,9 +5,10 @@ DEMO-010 在 FastAPI 服务中提供一个轻量、同源的管理客户端。 ## 启动与访问 -完成运行时配置后启动 API: +完成运行时配置后启动 Worker 与 API: ```bash +make run-worker make run ``` @@ -34,7 +35,7 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF ## Demo 操作闭环 1. 在上传页选择一个 PDF、DOCX、TXT 或 Markdown 文件并提交; -2. 等待同步流水线返回 `Ready`,失败时页面显示经过 API 收敛的错误; +2. 接口返回 `202` 后,当前页面显示已接收;在文档列表刷新查看 `Processing/Ready/Failed`; 3. 在文档列表查看状态和安全元数据; 4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和 Rerank Score; @@ -45,8 +46,9 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF - 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时; - API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入; - 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile; -- 本任务不包含 ACL/RBAC、审计、异步任务、Document Version、Compiled Knowledge、MCP、 - Hybrid/BM25 或完整 V1 Admin Console 导航; +- 当前 Console 尚不轮询 Job 详情或展示 Attempt;该状态 UI 与批量上传一起由 DEMO-016 实现; +- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25 + 或完整 V1 Admin Console 导航; - 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。 ## 验证 diff --git a/docs/development/ASYNC_INGESTION_SETUP.md b/docs/development/ASYNC_INGESTION_SETUP.md new file mode 100644 index 0000000..20c5bba --- /dev/null +++ b/docs/development/ASYNC_INGESTION_SETUP.md @@ -0,0 +1,82 @@ +# 异步入库与失败恢复说明 + +## 范围 + +DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker。OceanBase 中的 +`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID, +不携带文件内容、Provider 凭据或业务状态。 + +本任务不提供批量上传和批次 UI;这些能力属于 DEMO-016。 + +## 运行时配置 + +以下值必须通过环境变量或部署 Secret 注入,不得提交真实值: + +```text +RABBITMQ_URL=amqp://:@:5672/ +INGESTION_MAX_ATTEMPTS=3 +INGESTION_RETRY_BACKOFF_SECONDS=2 +INGESTION_RETRY_BACKOFF_MAX_SECONDS=60 +INGESTION_WORKER_LEASE_SECONDS=300 +INGESTION_RECOVERY_INTERVAL_SECONDS=30 +``` + +Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。 +RabbitMQ 用户应只具备指定 vhost/queue 的最小权限。 + +## 启动顺序 + +```bash +uv run alembic upgrade head +make run-worker +make run +``` + +`make run-worker` 在当前 Demo 中同时启动 Worker 和恢复调度器。恢复调度器周期扫描: + +- 尚未成功投递的 `queued` Job; +- 已到重试时间的 `retrying` Job; +- Worker 丢失后租约已过期的 `running` Job。 + +任务投递使用 late acknowledgement、worker-lost reject 和单任务 prefetch。重复消息会先锁定 +Job 行;有效租约内不会创建第二个 Attempt。租约过期后,旧 Attempt 以安全错误关闭,再按重试 +规则恢复。 + +## 重试与清理 + +确定性的内容错误(无法解析、清洗后为空、无 Chunk、派生数据身份不一致)直接进入 `failed`。 +其余运行时/Provider 错误按指数退避重试,默认最多三次总 Attempt。 + +第二次及后续 Attempt 创建前严格执行: + +```text +OceanBase Vector rows +→ ChunkIndexState +→ Chunk +→ parsed JSON object +→ reset Document processing state +``` + +不可变源文件、Document、Job 和已结束 Attempt 始终保留。任一清理步骤失败时不创建新 Attempt, +Job 直接以 `retry cleanup failed` 终止,禁止在未知脏状态继续执行。 + +## API 状态 + +`POST /api/v1/documents` 成功返回 `202` 和 `job_id`。查询: + +```text +GET /api/v1/jobs/{job_id} +``` + +Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;阶段为 +`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或 +返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。 + +## 故障检查 + +1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue; +2. Job 长期为 `running`:检查 Worker,租约到期后恢复调度器会重新投递; +3. Job 为 `retrying`:等待 `next_run_at`,客户端不应重新上传同一文件; +4. Job 为 `failed` 且错误为 `retry cleanup failed`:停止人工重试,先核对向量、关系数据与 BOS + 派生对象的一致性; +5. 不得通过日志打印 `RABBITMQ_URL`、数据库 URL、BOS/模型凭据或原始 Provider 响应。 diff --git a/docs/development/REST_API_SETUP.md b/docs/development/REST_API_SETUP.md index f4245ff..6860054 100644 --- a/docs/development/REST_API_SETUP.md +++ b/docs/development/REST_API_SETUP.md @@ -2,19 +2,21 @@ ## 范围 -DEMO-009 将已经完成的同步入库和检索服务公开为版本化 REST API。API 层只负责输入校验、 +DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步 +Job。API 层只负责输入校验、 错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。 -本任务不包含 Admin Console、ACL/RBAC、异步任务、软删除或 Hybrid Search。 +当前不包含 ACL/RBAC、软删除、批量上传或 Hybrid Search。 ## 启动前提 1. 按 `docs/development/BOS_SETUP.md` 配置 BOS; 2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding; 3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker; -4. 执行 `uv run alembic upgrade head`; -5. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile; -6. 执行 `make run`。 +4. 按 `docs/development/ASYNC_INGESTION_SETUP.md` 配置 RabbitMQ 并启动 Worker; +5. 执行 `uv run alembic upgrade head`; +6. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile; +7. 执行 `make run`。 `/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时 延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。 @@ -25,13 +27,19 @@ Swagger UI:`/api/v1/docs` ## 示例 -上传并同步入库: +上传并异步排队: ```bash curl -F 'file=@./policy.md;type=text/markdown' \ http://127.0.0.1:8000/api/v1/documents ``` +使用响应中的 `job_id` 查询进度: + +```bash +curl http://127.0.0.1:8000/api/v1/jobs/ +``` + 检索: ```bash @@ -40,8 +48,8 @@ curl -H 'Content-Type: application/json' \ http://127.0.0.1:8000/api/v1/search ``` -Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase Upsert 完成。大文 -件或云端调用较慢时,请求耗时会相应增加;异步任务属于 Demo 后置能力。 +上传只等待不可变源文件、Document 与 Job 持久化,成功返回 `202`。Worker 后台执行 Parse、 +Clean、Chunk、Embedding 和 OceanBase Upsert;客户端通过 Job 状态判断何时可检索。 ## 删除一致性 @@ -51,6 +59,8 @@ Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase 2. 删除 OceanBase canonical 关系表中的 ChunkIndexState、Chunk 和 Document; 3. 删除解析 JSON 与不可变原始文件。 +`queued`、`running` 或 `retrying` Job 对应的 Document 返回 `409`,避免删除与 Worker 并发。 + 向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口 返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1 应通过可重试清理任务和审计记录强化。 diff --git a/docs/development/TEST_STRATEGY.md b/docs/development/TEST_STRATEGY.md index 792bafc..581566b 100644 --- a/docs/development/TEST_STRATEGY.md +++ b/docs/development/TEST_STRATEGY.md @@ -11,6 +11,11 @@ Provide deterministic fakes: External integrations use separate integration suites/markers. +Async ingestion tests run with an in-process recording dispatcher and deterministic providers. They +must cover persisted Job/Attempt history, safe error summaries, bounded retry, cleanup-before-retry, +active-lease duplicate delivery, expired-lease recovery, immutable-source preservation and terminal +cleanup failure without requiring a live RabbitMQ broker. + E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval. Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation. diff --git a/docs/plans/ASYNC_INGESTION_MCP_PLAN.md b/docs/plans/ASYNC_INGESTION_MCP_PLAN.md index 175f43a..e67d2e2 100644 --- a/docs/plans/ASYNC_INGESTION_MCP_PLAN.md +++ b/docs/plans/ASYNC_INGESTION_MCP_PLAN.md @@ -8,14 +8,14 @@ | 能力 | 当前实现 | 结论 | |---|---|---| -| 单文件上传 | `POST /api/v1/documents` 接收一个 `file`,并在请求线程内同步完成解析、Chunk、Embedding 和索引 | 已实现,但不满足 V1 异步要求 | +| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 | | 批量上传 | API 与 Admin Console 都只支持一个文件 | 未实现 | | 失败补偿 | 上传元数据失败会删除源对象;解析元数据失败会删除本次解析产物;索引失败会删除当前 Document/Profile 的可能残留向量 | 已有局部补偿,不是自动重试 | -| 自动重试 | 没有持久化 Job、Attempt、调度器、退避策略或自动重新执行入口 | 未实现 | -| 重试前清理 | 影子索引重建能清理目标 Profile 的部分写入;正常上传流水线没有统一清理解析产物、Chunk、索引状态和向量的能力 | 未实现 | +| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 | +| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 | | MCP | 已有 MCP 业务契约和可复用 Search Service/REST API,没有可运行 MCP Gateway 或 WorkBuddy 验证 | 未实现 | -因此,现有失败补偿只能降低一次执行留下半成品的概率,不能满足“失败后自动重新执行,并在重新执行前清理旧数据”。 +PLAN-001 的上述缺口已由 DEMO-015 补齐;批量提交与状态 UI 仍由 DEMO-016 实现。 ## 2. Task 归并结论 diff --git a/docs/plans/PROGRESS.md b/docs/plans/PROGRESS.md index a2ad61c..e1b0021 100644 --- a/docs/plans/PROGRESS.md +++ b/docs/plans/PROGRESS.md @@ -1,12 +1,12 @@ # 项目进度 -当前阶段:公共知识库集群交付规划完成,准备实施异步入库 +当前阶段:异步单文件入库闭环完成,准备实施批量上传 -已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014、ARCH-001~ARCH-006、PLAN-001~PLAN-002 +已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-015、ARCH-001~ARCH-006、PLAN-001~PLAN-002 进行中:无 阻塞:无 -下一任务:`DEMO-015`。随后严格执行 `DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。 +下一任务:`DEMO-016`。随后严格执行 `DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。 Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006` 执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。 @@ -332,3 +332,25 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR - 验证结果:Markdown 相对链接、敏感值模式和 Git diff 检查通过;Ruff Format/Lint、Mypy、 Alembic Head/离线升级通过;Unit + Integration `165 passed`,RAG Evaluation `5 passed`。 - 下一任务:`DEMO-015`。 + +### DEMO-015(已完成) + +- 新增 OceanBase canonical `ingestion_jobs` / `ingestion_attempts` 与 forward-only migration; + Document 唯一 Job、Attempt 编号上限和单一运行中 Attempt 均由数据库约束保护。 +- `POST /api/v1/documents` 改为持久化不可变 Source、Document、Job 后返回 `202`,新增 + `GET /api/v1/jobs/{job_id}`;活动 Job 的 Document 删除返回 `409`,避免与 Worker 竞态。 +- 新增队列无关的 Dispatcher 与 Job Runner,首个适配器使用 Celery/RabbitMQ;消息只包含 Job + ID,启用 late ack、worker-lost reject、单任务 prefetch 与周期性持久化 Job 恢复扫描。 +- 瞬时错误按配置执行指数退避和最多三次总 Attempt;Worker 租约有效时重复投递不创建新 + Attempt,租约过期后关闭旧 Attempt 并恢复。错误记录为固定安全摘要,不保存 Provider 正文。 +- 第二次及后续 Attempt 前依次清理向量、ChunkIndexState、Chunk 与 parsed JSON,再从不可变 + 源文件重建;清理失败直接终止且不创建新 Attempt。Job 创建失败会补偿 Document 与源对象。 +- Admin Console 仅将上传提示调整为“已接收并进入队列”;批量选择、Job 轮询和逐文件状态 UI + 没有提前实现,继续属于 `DEMO-016`。 +- 验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、依赖锁、Alembic Head/离线 MySQL SQL + 与 SQLite schema drift 均通过;Unit + Integration `179 passed`,RAG Evaluation `5 passed`; + Worker 入口确认注册 `ekp.ingestion.run` 与 `ekp.ingestion.recover`。 +- 环境限制:当前工作站没有 Docker/RabbitMQ,也没有持久化注入目标 OceanBase URL,因此未做 + 真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收 + 重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。 +- 下一任务:`DEMO-016`。 diff --git a/docs/plans/TASKS.md b/docs/plans/TASKS.md index 592e6ac..a096eed 100644 --- a/docs/plans/TASKS.md +++ b/docs/plans/TASKS.md @@ -28,7 +28,7 @@ validation. Provider credentials and internal endpoints are runtime inputs and n - [x] PLAN-001 MCP 与异步批量入库现状审计、任务拆分和契约基线 - [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification - [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线 -- [ ] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup +- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup - [ ] DEMO-016 Batch async upload API + per-file Job status + Admin Console - [ ] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker - [ ] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline diff --git a/docs/plans/V1_EXECUTION_PLAN.md b/docs/plans/V1_EXECUTION_PLAN.md index 203879e..f4a33d6 100644 --- a/docs/plans/V1_EXECUTION_PLAN.md +++ b/docs/plans/V1_EXECUTION_PLAN.md @@ -16,7 +16,7 @@ “单一公共知识可见域、企业受控网络、持续可用和一键集群部署”推进,不提前实现用户/权限隔离: 1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛; -2. `DEMO-015`:持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理; +2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理; 3. `DEMO-016`:批量异步上传、批次/Job 查询和 Admin Console 状态展示; 4. `DEPLOY-001`:API/Admin、MCP Gateway、Async Worker 生产镜像; 5. `DEPLOY-002`:Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口; diff --git a/migrations/README.md b/migrations/README.md index ba6fa2e..8d5b044 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -17,3 +17,7 @@ defaults, and primary keys while excluding those three falsely reported object types. Full constraint comparison still runs against SQLite in integration tests; target-cluster checks separately verify HNSW/PRE-FILTER and the single-active generated unique index. + +DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state; +the latter preserves numbered history and uses a generated nullable active marker plus a unique index +to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite. diff --git a/migrations/versions/20260902_0007_create_ingestion_jobs.py b/migrations/versions/20260902_0007_create_ingestion_jobs.py new file mode 100644 index 0000000..247edf0 --- /dev/null +++ b/migrations/versions/20260902_0007_create_ingestion_jobs.py @@ -0,0 +1,174 @@ +"""Create durable ingestion jobs and attempt history. + +Revision ID: 20260902_0007 +Revises: 20260902_0006 +Create Date: 2026-09-02 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "20260902_0007" +down_revision: str | None = "20260902_0006" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Persist queue-independent Job state and bounded execution attempts.""" + op.create_table( + "ingestion_jobs", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("document_id", sa.Uuid(), nullable=False), + sa.Column( + "status", + sa.Enum( + "queued", + "running", + "retrying", + "succeeded", + "failed", + "cancelled", + name="ingestion_job_status", + native_enum=False, + create_constraint=False, + ), + nullable=False, + ), + sa.Column( + "stage", + sa.Enum( + "parsing", + "cleaning", + "chunking", + "embedding", + "indexing", + name="ingestion_stage", + native_enum=False, + create_constraint=False, + ), + nullable=True, + ), + sa.Column("attempt_count", sa.Integer(), nullable=False), + sa.Column("max_attempts", sa.Integer(), nullable=False), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"), + sa.CheckConstraint("max_attempts > 0", name="max_attempts_positive"), + sa.CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"), + sa.CheckConstraint( + "status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')", + name="status_valid", + ), + sa.CheckConstraint( + "stage IS NULL OR stage IN " + "('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')", + name="stage_valid", + ), + sa.ForeignKeyConstraint( + ["document_id"], + ["documents.id"], + name="fk_ingestion_jobs_document_id_documents", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_ingestion_jobs"), + sa.UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"), + ) + op.create_index( + "ix_ingestion_jobs_status_next_run", + "ingestion_jobs", + ["status", "next_run_at"], + unique=False, + ) + op.create_table( + "ingestion_attempts", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("job_id", sa.Uuid(), nullable=False), + sa.Column("attempt_number", sa.Integer(), nullable=False), + sa.Column( + "status", + sa.Enum( + "running", + "succeeded", + "failed", + name="ingestion_attempt_status", + native_enum=False, + create_constraint=False, + ), + nullable=False, + ), + sa.Column( + "stage", + sa.Enum( + "parsing", + "cleaning", + "chunking", + "embedding", + "indexing", + name="ingestion_attempt_stage", + native_enum=False, + create_constraint=False, + ), + nullable=True, + ), + sa.Column("retryable", sa.Boolean(), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column( + "active_marker", + sa.Integer(), + sa.Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"), + nullable=True, + ), + sa.Column( + "started_at", + sa.DateTime(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint("attempt_number > 0", name="attempt_number_positive"), + sa.CheckConstraint( + "status IN ('running', 'succeeded', 'failed')", + name="status_valid", + ), + sa.CheckConstraint( + "stage IS NULL OR stage IN " + "('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')", + name="stage_valid", + ), + sa.ForeignKeyConstraint( + ["job_id"], + ["ingestion_jobs.id"], + name="fk_ingestion_attempts_job_id_ingestion_jobs", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_ingestion_attempts"), + sa.UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"), + ) + op.create_index( + "uq_ingestion_attempts_active", + "ingestion_attempts", + ["job_id", "active_marker"], + unique=True, + ) + + +def downgrade() -> None: + """Reject downgrade because project migrations are forward-only.""" + raise NotImplementedError("Database migrations are forward-only") diff --git a/packages/core/src/ekp_core/config.py b/packages/core/src/ekp_core/config.py index 645b10d..ba27cff 100644 --- a/packages/core/src/ekp_core/config.py +++ b/packages/core/src/ekp_core/config.py @@ -33,6 +33,12 @@ class Settings(BaseSettings): database_pool_recycle_seconds: int = Field(default=600, ge=1) database_pool_timeout_seconds: float = Field(default=30.0, gt=0) database_connect_timeout_seconds: int = Field(default=30, ge=1) + rabbitmq_url: SecretStr | None = None + ingestion_max_attempts: int = Field(default=3, ge=1, le=10) + ingestion_retry_backoff_seconds: int = Field(default=2, ge=1, le=3600) + ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400) + ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400) + ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600) object_storage_provider: Literal["local", "bos"] = "bos" object_storage_path: Path = Path("storage") bos_endpoint: str | None = None @@ -91,6 +97,17 @@ class Settings(BaseSettings): raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens") if self.search_top_k > self.dense_top_n: raise ValueError("search_top_k must not exceed dense_top_n") + if self.ingestion_retry_backoff_max_seconds < self.ingestion_retry_backoff_seconds: + raise ValueError( + "ingestion_retry_backoff_max_seconds must be at least the initial backoff" + ) + longest_provider_timeout = max( + self.bos_timeout_seconds, + self.embedding_timeout_seconds, + self.oceanbase_vector_timeout_seconds, + ) + if self.ingestion_worker_lease_seconds <= longest_provider_timeout: + raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts") if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"): raise ValueError("mcp_path must start with '/' and must not end with '/'") if not self.mcp_allowed_hosts: diff --git a/packages/core/src/ekp_core/document_chunking.py b/packages/core/src/ekp_core/document_chunking.py index b71a646..59c792a 100644 --- a/packages/core/src/ekp_core/document_chunking.py +++ b/packages/core/src/ekp_core/document_chunking.py @@ -1,5 +1,6 @@ """Application services for building and persisting document chunks.""" +from collections.abc import Callable from dataclasses import dataclass from hashlib import sha256 from uuid import UUID, uuid5 @@ -11,7 +12,7 @@ from ekp_core.chunking import ChunkDraft, ChunkingStrategy from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService from ekp_core.document_upload import DocumentUploadService from ekp_core.parsed_document_artifact import deserialize_parsed_document -from ekp_domain import Chunk, Document, DocumentStatus, ParsedDocument +from ekp_domain import Chunk, Document, DocumentStatus, IngestionStage, ParsedDocument from ekp_object_storage import BinarySource, ObjectStorage MAX_CHUNKING_ERROR_LENGTH = 2000 @@ -54,10 +55,16 @@ class DocumentChunkingService: self._storage = storage self._strategy = strategy - def process(self, document_id: UUID) -> tuple[Chunk, ...]: - """Run DEMO-005 while reserving READY for the later index stage.""" + def process( + self, + document_id: UUID, + stage_callback: Callable[[IngestionStage], None] | None = None, + ) -> tuple[Chunk, ...]: + """Build chunks while optionally reporting the durable Job stage.""" filename, parsed_key = self._start_chunking(document_id) try: + if stage_callback is not None: + stage_callback(IngestionStage.CHUNKING) with self._storage.open(parsed_key) as source: parsed = deserialize_parsed_document(source.read()) if parsed.document_id != document_id or parsed.filename != filename: diff --git a/packages/core/src/ekp_core/document_indexing.py b/packages/core/src/ekp_core/document_indexing.py index 2d8100b..594df60 100644 --- a/packages/core/src/ekp_core/document_indexing.py +++ b/packages/core/src/ekp_core/document_indexing.py @@ -1,5 +1,6 @@ """Embedding and vector-index orchestration for canonical document chunks.""" +from collections.abc import Callable from dataclasses import dataclass from uuid import UUID, uuid5 @@ -7,7 +8,14 @@ from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService -from ekp_domain import Chunk, ChunkIndexState, Document, DocumentStatus, IndexProfile +from ekp_domain import ( + Chunk, + ChunkIndexState, + Document, + DocumentStatus, + IndexProfile, + IngestionStage, +) from ekp_embedding import EmbeddingProvider from ekp_object_storage import BinarySource from ekp_vector_store import VectorRecord, VectorStore @@ -67,12 +75,16 @@ class DocumentIndexingService: self._vector_store = vector_store self._index_profile = index_profile - def process(self, document_id: UUID) -> DocumentIndexingResult: - """Index all chunks for one profile, compensating any partial provider write.""" + def process( + self, + document_id: UUID, + stage_callback: Callable[[IngestionStage], None] | None = None, + ) -> DocumentIndexingResult: + """Embed/index chunks while optionally reporting durable Job stages.""" chunks = self._start_indexing(document_id) upsert_attempted = False try: - vectors = self._embed_chunks(chunks) + vectors = self._embed_chunks(chunks, stage_callback) records = tuple( VectorRecord( index_record_id=uuid5( @@ -93,6 +105,8 @@ class DocumentIndexingService: ) for chunk, vector in zip(chunks, vectors, strict=True) ) + if stage_callback is not None: + stage_callback(IngestionStage.INDEXING) upsert_attempted = True self._vector_store.upsert(records) self._record_success(document_id, chunks) @@ -152,13 +166,19 @@ class DocumentIndexingService: session.commit() return chunks - def _embed_chunks(self, chunks: tuple[Chunk, ...]) -> tuple[tuple[float, ...], ...]: - """Respect provider batch limits and validate every returned vector.""" + def _embed_chunks( + self, + chunks: tuple[Chunk, ...], + stage_callback: Callable[[IngestionStage], None] | None, + ) -> tuple[tuple[float, ...], ...]: + """Embed bounded batches and renew an optional Job lease before each call.""" vectors: list[tuple[float, ...]] = [] batch_size = self._embedding_provider.max_batch_size if batch_size <= 0: raise EmbeddingResultError("embedding provider has an invalid batch size") for start in range(0, len(chunks), batch_size): + if stage_callback is not None: + stage_callback(IngestionStage.EMBEDDING) batch = chunks[start : start + batch_size] embedded = self._embedding_provider.embed([chunk.content for chunk in batch]) if len(embedded) != len(batch): diff --git a/packages/core/src/ekp_core/document_management.py b/packages/core/src/ekp_core/document_management.py index 026cd84..36f8bb6 100644 --- a/packages/core/src/ekp_core/document_management.py +++ b/packages/core/src/ekp_core/document_management.py @@ -5,7 +5,7 @@ from uuid import UUID from sqlalchemy import delete, select from sqlalchemy.orm import Session, sessionmaker -from ekp_domain import Chunk, ChunkIndexState, Document +from ekp_domain import Chunk, ChunkIndexState, Document, IngestionJob, IngestionJobStatus from ekp_object_storage import ObjectStorage from ekp_vector_store import VectorStore @@ -18,6 +18,10 @@ class DocumentArtifactCleanupError(RuntimeError): """Raised when deleted metadata leaves an external storage artifact behind.""" +class DocumentIngestionActiveError(RuntimeError): + """Raised when direct deletion would race an active asynchronous Job.""" + + class DocumentManagementService: """Read canonical documents and coordinate the Demo's direct deletion. @@ -63,6 +67,22 @@ class DocumentManagementService: document = session.get(Document, document_id) if document is None: raise DocumentNotFoundError(str(document_id)) + active_job_id = session.scalar( + select(IngestionJob.id) + .where( + IngestionJob.document_id == document_id, + IngestionJob.status.in_( + ( + IngestionJobStatus.QUEUED, + IngestionJobStatus.RUNNING, + IngestionJobStatus.RETRYING, + ) + ), + ) + .limit(1) + ) + if active_job_id is not None: + raise DocumentIngestionActiveError(str(document_id)) source_keys = tuple( key diff --git a/packages/core/src/ekp_core/document_parsing.py b/packages/core/src/ekp_core/document_parsing.py index a5bb067..c36bcb5 100644 --- a/packages/core/src/ekp_core/document_parsing.py +++ b/packages/core/src/ekp_core/document_parsing.py @@ -1,5 +1,6 @@ """Application services for parsing and persisting derived document content.""" +from collections.abc import Callable from io import BytesIO from uuid import UUID @@ -8,7 +9,7 @@ from sqlalchemy.orm import Session, sessionmaker from ekp_core.document_cleaner import clean_parsed_document from ekp_core.document_upload import DocumentUploadService from ekp_core.parsed_document_artifact import serialize_parsed_document -from ekp_domain import Document, DocumentStatus, ParsedDocument +from ekp_domain import Document, DocumentStatus, IngestionStage, ParsedDocument from ekp_object_storage import BinarySource, ObjectStorage from ekp_parser import DocumentParseError, Parser @@ -48,15 +49,23 @@ class DocumentParsingService: self._storage = storage self._parser = parser - def process(self, document_id: UUID) -> ParsedDocument: - """Run DEMO-004 synchronously and persist success or failure state.""" + def process( + self, + document_id: UUID, + stage_callback: Callable[[IngestionStage], None] | None = None, + ) -> ParsedDocument: + """Parse and clean while optionally reporting durable Job stage boundaries.""" filename, source_key = self._start_processing(document_id) parsed_key = f"parsed/{document_id}/document.json" parsed_object_stored = False try: + if stage_callback is not None: + stage_callback(IngestionStage.PARSING) with self._storage.open(source_key) as source: parsed = self._parser.parse(document_id, filename, source) + if stage_callback is not None: + stage_callback(IngestionStage.CLEANING) cleaned = clean_parsed_document(parsed) if not cleaned.blocks: raise EmptyParsedDocumentError( diff --git a/packages/core/src/ekp_core/document_upload.py b/packages/core/src/ekp_core/document_upload.py index 568a726..6899ad5 100644 --- a/packages/core/src/ekp_core/document_upload.py +++ b/packages/core/src/ekp_core/document_upload.py @@ -1,8 +1,9 @@ """Validated, transactional source-document upload application service.""" from pathlib import Path -from uuid import uuid4 +from uuid import UUID, uuid4 +from sqlalchemy import delete from sqlalchemy.orm import Session, sessionmaker from ekp_domain import Document @@ -132,6 +133,26 @@ class DocumentUploadService: return document + def rollback_acceptance(self, document_id: UUID) -> None: + """Compensate a just-uploaded source when durable Job creation fails. + + This narrow method is only valid before parsing starts; normal document + deletion remains coordinated by ``DocumentManagementService``. + """ + with self._session_factory() as session: + document = session.get(Document, document_id) + if document is None: + return + storage_key = document.storage_key + session.execute(delete(Document).where(Document.id == document.id)) + session.commit() + try: + self._storage.delete(storage_key) + except Exception as cleanup_error: + raise UploadRollbackError( + f"job creation failed and source cleanup failed for {storage_key}" + ) from cleanup_error + def _validate_filename(filename: str) -> str: """Accept a plain basename that can be traced without path ambiguity.""" diff --git a/packages/core/src/ekp_core/ingestion_jobs.py b/packages/core/src/ekp_core/ingestion_jobs.py new file mode 100644 index 0000000..94b9f44 --- /dev/null +++ b/packages/core/src/ekp_core/ingestion_jobs.py @@ -0,0 +1,588 @@ +"""Durable asynchronous ingestion orchestration and retry cleanup.""" + +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from math import ceil +from typing import Protocol +from uuid import UUID + +from sqlalchemy import delete, select +from sqlalchemy.orm import Session, sessionmaker + +from ekp_core.document_chunking import ( + DocumentAlreadyChunkedError, + DocumentChunkingService, + DocumentNotParsedError, + EmptyChunkSetError, + ParsedDocumentMismatchError, +) +from ekp_core.document_chunking import ( + DocumentNotFoundError as ChunkingDocumentNotFoundError, +) +from ekp_core.document_indexing import ( + DocumentAlreadyIndexedError, + DocumentHasNoChunksError, + DocumentIndexingService, +) +from ekp_core.document_indexing import ( + DocumentNotFoundError as IndexingDocumentNotFoundError, +) +from ekp_core.document_parsing import ( + DocumentAlreadyParsedError, + DocumentParsingService, + EmptyParsedDocumentError, +) +from ekp_core.document_parsing import ( + DocumentNotFoundError as ParsingDocumentNotFoundError, +) +from ekp_core.document_upload import DocumentUploadService +from ekp_core.parsed_document_artifact import ParsedDocumentArtifactError +from ekp_domain import ( + Chunk, + ChunkIndexState, + Document, + DocumentStatus, + IngestionAttempt, + IngestionAttemptStatus, + IngestionJob, + IngestionJobStatus, + IngestionStage, +) +from ekp_object_storage import BinarySource, ObjectStorage +from ekp_parser import DocumentParseError +from ekp_vector_store import VectorStore + +SAFE_TRANSIENT_ERROR = "temporary ingestion failure" +SAFE_PERMANENT_ERROR = "document content could not be processed" +SAFE_CLEANUP_ERROR = "retry cleanup failed" +SAFE_LEASE_ERROR = "worker execution lease expired" + + +class IngestionJobNotFoundError(LookupError): + """Raised when a requested ingestion Job does not exist.""" + + +class IngestionCleanupError(RuntimeError): + """Raised when derived data cannot be restored to a known clean state.""" + + +class JobDispatcher(Protocol): + """Replaceable message-delivery boundary used by submission services.""" + + def enqueue(self, job_id: UUID) -> None: + """Request asynchronous execution of one already-persisted Job.""" + + +@dataclass(frozen=True, slots=True) +class IngestionSubmission: + """Stable identifiers returned after source and Job persistence.""" + + document: Document + job: IngestionJob + + +class JobRunAction(StrEnum): + """Instruction returned to a queue adapter after one delivery.""" + + COMPLETE = "complete" + RETRY = "retry" + + +@dataclass(frozen=True, slots=True) +class JobRunResult: + """Queue-neutral result for acknowledging or rescheduling a delivery.""" + + action: JobRunAction + status: IngestionJobStatus + retry_delay_seconds: int | None = None + + +@dataclass(frozen=True, slots=True) +class _Claim: + """Internal durable claim outcome used before external provider work.""" + + run: bool + cleanup_required: bool = False + result: JobRunResult | None = None + + +class IngestionJobService: + """Create and query Job state independently from the broker implementation.""" + + def __init__(self, session_factory: sessionmaker[Session], max_attempts: int) -> None: + if max_attempts <= 0: + raise ValueError("max_attempts must be positive") + self._session_factory = session_factory + self._max_attempts = max_attempts + + def create(self, document_id: UUID) -> IngestionJob: + """Create exactly one queued ingestion Job for a canonical Document.""" + with self._session_factory() as session: + job = IngestionJob(document_id=document_id, max_attempts=self._max_attempts) + session.add(job) + session.commit() + return job + + def get(self, job_id: UUID) -> IngestionJob: + """Return one detached Job or a stable not-found error.""" + with self._session_factory() as session: + job = session.get(IngestionJob, job_id) + if job is None: + raise IngestionJobNotFoundError(str(job_id)) + return job + + def list_dispatchable(self, limit: int = 100) -> tuple[UUID, ...]: + """Find due queued/retrying Jobs and expired running leases for recovery.""" + if not 1 <= limit <= 1000: + raise ValueError("dispatch limit must be between 1 and 1000") + now = datetime.now(UTC) + with self._session_factory() as session: + jobs = session.scalars( + select(IngestionJob) + .where( + (IngestionJob.status == IngestionJobStatus.QUEUED) + | ( + (IngestionJob.status == IngestionJobStatus.RETRYING) + & ((IngestionJob.next_run_at.is_(None)) | (IngestionJob.next_run_at <= now)) + ) + | ( + (IngestionJob.status == IngestionJobStatus.RUNNING) + & ( + (IngestionJob.lease_expires_at.is_(None)) + | (IngestionJob.lease_expires_at <= now) + ) + ) + ) + .order_by(IngestionJob.created_at, IngestionJob.id) + .limit(limit) + ).all() + return tuple(job.id for job in jobs) + + +class AsyncIngestionSubmissionService: + """Persist immutable source and Job before best-effort broker delivery.""" + + def __init__( + self, + upload_service: DocumentUploadService, + jobs: IngestionJobService, + dispatcher: JobDispatcher, + ) -> None: + self._upload_service = upload_service + self._jobs = jobs + self._dispatcher = dispatcher + + def submit( + self, + filename: str, + media_type: str | None, + source: BinarySource, + ) -> IngestionSubmission: + """Accept one source and return without executing expensive stages inline. + + Broker delivery is deliberately best effort after the canonical commit. + A periodic dispatcher recovers queued Jobs when the broker was temporarily + unavailable, so the request never treats a message as the source of truth. + """ + document = self._upload_service.upload(filename, media_type, source) + try: + job = self._jobs.create(document.id) + except Exception: + self._upload_service.rollback_acceptance(document.id) + raise + try: + self._dispatcher.enqueue(job.id) + except Exception: + # Keeping the Job queued is intentional. The recovery dispatcher can + # safely redeliver it without asking the client to upload again. + pass + return IngestionSubmission(document=document, job=job) + + +class DerivedDocumentCleanupService: + """Remove only rebuildable derivatives before a retry begins.""" + + def __init__( + self, + session_factory: sessionmaker[Session], + storage: ObjectStorage, + vector_store: VectorStore, + active_index_profile_id: str, + ) -> None: + self._session_factory = session_factory + self._storage = storage + self._vector_store = vector_store + self._active_index_profile_id = active_index_profile_id + + def cleanup(self, document_id: UUID) -> None: + """Clean vector, relational and parsed artifacts while preserving source/Job.""" + try: + with self._session_factory() as session: + document = session.get(Document, document_id) + if document is None: + raise IngestionCleanupError("document metadata is missing") + parsed_key = document.parsed_storage_key + profile_ids = set( + session.scalars( + select(ChunkIndexState.index_profile_id) + .join(Chunk, Chunk.chunk_id == ChunkIndexState.chunk_id) + .where(Chunk.document_id == document_id) + .distinct() + ).all() + ) + profile_ids.add(self._active_index_profile_id) + + # External vector rows are deleted before their canonical evidence. + for profile_id in sorted(profile_ids): + self._vector_store.delete_document(document_id, profile_id) + + with self._session_factory() as session: + chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id) + session.execute( + delete(ChunkIndexState).where(ChunkIndexState.chunk_id.in_(chunk_ids)) + ) + session.execute(delete(Chunk).where(Chunk.document_id == document_id)) + session.commit() + + if parsed_key is not None and self._storage.exists(parsed_key): + self._storage.delete(parsed_key) + + with self._session_factory() as session: + document = session.get(Document, document_id) + if document is None: + raise IngestionCleanupError("document metadata is missing") + document.parsed_storage_key = None + document.status = DocumentStatus.UPLOADED + document.error_message = None + session.commit() + except IngestionCleanupError: + raise + except Exception as error: + raise IngestionCleanupError(SAFE_CLEANUP_ERROR) from error + + +class IngestionJobRunner: + """Execute one durable attempt and return queue-neutral retry instructions.""" + + def __init__( + self, + session_factory: sessionmaker[Session], + parsing: DocumentParsingService, + chunking: DocumentChunkingService, + indexing: DocumentIndexingService, + cleanup: DerivedDocumentCleanupService, + *, + retry_backoff_seconds: int, + retry_backoff_max_seconds: int, + lease_seconds: int, + ) -> None: + if retry_backoff_seconds <= 0 or retry_backoff_max_seconds <= 0 or lease_seconds <= 0: + raise ValueError("retry and lease durations must be positive") + self._session_factory = session_factory + self._parsing = parsing + self._chunking = chunking + self._indexing = indexing + self._cleanup = cleanup + self._retry_backoff_seconds = retry_backoff_seconds + self._retry_backoff_max_seconds = retry_backoff_max_seconds + self._lease_seconds = lease_seconds + + def run(self, job_id: UUID) -> JobRunResult: + """Claim, optionally clean, execute and durably finalize one delivery.""" + claim = self._claim(job_id) + if not claim.run: + assert claim.result is not None + return claim.result + + if claim.cleanup_required: + try: + document_id = self._job_document_id(job_id) + self._cleanup.cleanup(document_id) + except Exception: + self._finish_cleanup_failure(job_id) + return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.FAILED) + + attempt_number, document_id = self._start_attempt(job_id) + + def stage_callback(stage: IngestionStage) -> None: + """Bind this attempt while pipeline stages remain Job-agnostic.""" + self._record_stage(job_id, attempt_number, stage) + + try: + self._parsing.process(document_id, stage_callback=stage_callback) + self._chunking.process(document_id, stage_callback=stage_callback) + self._indexing.process(document_id, stage_callback=stage_callback) + except Exception as error: + return self._finish_failure(job_id, attempt_number, document_id, error) + + self._finish_success(job_id, attempt_number, document_id) + return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.SUCCEEDED) + + def _claim(self, job_id: UUID) -> _Claim: + """Serialize deliveries and recover only after an execution lease expires.""" + now = datetime.now(UTC) + with self._session_factory() as session: + job = session.scalar( + select(IngestionJob).where(IngestionJob.id == job_id).with_for_update() + ) + if job is None: + raise IngestionJobNotFoundError(str(job_id)) + if job.status in { + IngestionJobStatus.SUCCEEDED, + IngestionJobStatus.FAILED, + IngestionJobStatus.CANCELLED, + }: + return _Claim( + run=False, + result=JobRunResult(JobRunAction.COMPLETE, job.status), + ) + if job.status == IngestionJobStatus.RUNNING: + lease_expiry = _as_utc(job.lease_expires_at) + if lease_expiry is not None and lease_expiry > now: + delay = max(1, ceil((lease_expiry - now).total_seconds())) + return _Claim( + run=False, + result=JobRunResult( + JobRunAction.RETRY, + IngestionJobStatus.RUNNING, + delay, + ), + ) + self._expire_active_attempt(session, job, now) + if job.attempt_count >= job.max_attempts: + job.status = IngestionJobStatus.FAILED + job.lease_expires_at = None + job.completed_at = now + document = session.get(Document, job.document_id) + if document is not None: + document.status = DocumentStatus.FAILED + document.error_message = SAFE_LEASE_ERROR + session.commit() + return _Claim( + run=False, + result=JobRunResult(JobRunAction.COMPLETE, job.status), + ) + job.status = IngestionJobStatus.RETRYING + next_run = _as_utc(job.next_run_at) + if ( + job.status == IngestionJobStatus.RETRYING + and next_run is not None + and next_run > now + ): + delay = max(1, ceil((next_run - now).total_seconds())) + return _Claim( + run=False, + result=JobRunResult( + JobRunAction.RETRY, + IngestionJobStatus.RETRYING, + delay, + ), + ) + + cleanup_required = job.attempt_count > 0 + job.status = IngestionJobStatus.RUNNING + job.lease_expires_at = now + timedelta(seconds=self._lease_seconds) + job.next_run_at = None + session.commit() + return _Claim(run=True, cleanup_required=cleanup_required) + + def _expire_active_attempt(self, session: Session, job: IngestionJob, now: datetime) -> None: + """Close an abandoned Attempt before a replacement can be created.""" + attempt = session.scalar( + select(IngestionAttempt) + .where( + IngestionAttempt.job_id == job.id, + IngestionAttempt.status == IngestionAttemptStatus.RUNNING, + ) + .with_for_update() + ) + if attempt is not None: + attempt.status = IngestionAttemptStatus.FAILED + attempt.retryable = True + attempt.error_message = SAFE_LEASE_ERROR + attempt.finished_at = now + job.error_message = SAFE_LEASE_ERROR + + def _job_document_id(self, job_id: UUID) -> UUID: + with self._session_factory() as session: + job = session.get(IngestionJob, job_id) + if job is None: + raise IngestionJobNotFoundError(str(job_id)) + return job.document_id + + def _start_attempt(self, job_id: UUID) -> tuple[int, UUID]: + """Create the next numbered Attempt only after retry cleanup succeeds.""" + now = datetime.now(UTC) + with self._session_factory() as session: + job = session.scalar( + select(IngestionJob).where(IngestionJob.id == job_id).with_for_update() + ) + if job is None: + raise IngestionJobNotFoundError(str(job_id)) + attempt_number = job.attempt_count + 1 + job.attempt_count = attempt_number + job.stage = IngestionStage.PARSING + job.error_message = None + job.lease_expires_at = now + timedelta(seconds=self._lease_seconds) + session.add( + IngestionAttempt( + job_id=job.id, + attempt_number=attempt_number, + stage=IngestionStage.PARSING, + ) + ) + session.commit() + return attempt_number, job.document_id + + def _record_stage(self, job_id: UUID, attempt_number: int, stage: IngestionStage) -> None: + """Persist progress and renew the worker lease at every stage boundary.""" + with self._session_factory() as session: + job = session.get(IngestionJob, job_id) + attempt = session.scalar( + select(IngestionAttempt).where( + IngestionAttempt.job_id == job_id, + IngestionAttempt.attempt_number == attempt_number, + ) + ) + if job is None or attempt is None: + raise IngestionJobNotFoundError(str(job_id)) + job.stage = stage + job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=self._lease_seconds) + attempt.stage = stage + session.commit() + + def _finish_success(self, job_id: UUID, attempt_number: int, document_id: UUID) -> None: + """Atomically close Job/Attempt after the document becomes ready.""" + now = datetime.now(UTC) + with self._session_factory() as session: + job, attempt = self._load_job_attempt(session, job_id, attempt_number) + document = session.get(Document, document_id) + if document is None: + raise IngestionJobNotFoundError(str(job_id)) + attempt.status = IngestionAttemptStatus.SUCCEEDED + attempt.retryable = False + attempt.finished_at = now + job.status = IngestionJobStatus.SUCCEEDED + job.error_message = None + job.lease_expires_at = None + job.completed_at = now + document.status = DocumentStatus.READY + document.error_message = None + session.commit() + + def _finish_failure( + self, + job_id: UUID, + attempt_number: int, + document_id: UUID, + error: Exception, + ) -> JobRunResult: + """Classify safely, bound retry count, and schedule exponential backoff.""" + retryable = _is_retryable(error) + safe_error = SAFE_TRANSIENT_ERROR if retryable else SAFE_PERMANENT_ERROR + now = datetime.now(UTC) + with self._session_factory() as session: + job, attempt = self._load_job_attempt(session, job_id, attempt_number) + document = session.get(Document, document_id) + attempt.status = IngestionAttemptStatus.FAILED + attempt.retryable = retryable + attempt.error_message = safe_error + attempt.finished_at = now + job.error_message = safe_error + job.lease_expires_at = None + if retryable and job.attempt_count < job.max_attempts: + delay = min( + self._retry_backoff_seconds * (2 ** (job.attempt_count - 1)), + self._retry_backoff_max_seconds, + ) + job.status = IngestionJobStatus.RETRYING + job.next_run_at = now + timedelta(seconds=delay) + if document is not None: + document.status = DocumentStatus.PROCESSING + document.error_message = safe_error + session.commit() + return JobRunResult(JobRunAction.RETRY, job.status, delay) + + job.status = IngestionJobStatus.FAILED + job.next_run_at = None + job.completed_at = now + if document is not None: + document.status = DocumentStatus.FAILED + document.error_message = safe_error + session.commit() + return JobRunResult(JobRunAction.COMPLETE, job.status) + + def _finish_cleanup_failure(self, job_id: UUID) -> None: + """Fail closed without creating a new Attempt on unknown dirty state.""" + now = datetime.now(UTC) + with self._session_factory() as session: + job = session.get(IngestionJob, job_id) + if job is None: + raise IngestionJobNotFoundError(str(job_id)) + document = session.get(Document, job.document_id) + job.status = IngestionJobStatus.FAILED + job.error_message = SAFE_CLEANUP_ERROR + job.next_run_at = None + job.lease_expires_at = None + job.completed_at = now + if document is not None: + document.status = DocumentStatus.FAILED + document.error_message = SAFE_CLEANUP_ERROR + session.commit() + + @staticmethod + def _load_job_attempt( + session: Session, job_id: UUID, attempt_number: int + ) -> tuple[IngestionJob, IngestionAttempt]: + job = session.get(IngestionJob, job_id) + attempt = session.scalar( + select(IngestionAttempt).where( + IngestionAttempt.job_id == job_id, + IngestionAttempt.attempt_number == attempt_number, + ) + ) + if job is None or attempt is None: + raise IngestionJobNotFoundError(str(job_id)) + return job, attempt + + +def dispatch_pending_jobs(jobs: IngestionJobService, dispatcher: JobDispatcher) -> int: + """Best-effort redelivery loop used by Celery beat after broker/API failures.""" + dispatched = 0 + for job_id in jobs.list_dispatchable(): + try: + dispatcher.enqueue(job_id) + except Exception: + continue + dispatched += 1 + return dispatched + + +def _is_retryable(error: Exception) -> bool: + """Treat known content/state violations as permanent and providers as transient.""" + permanent_errors = ( + DocumentParseError, + EmptyParsedDocumentError, + EmptyChunkSetError, + ParsedDocumentMismatchError, + ParsedDocumentArtifactError, + DocumentAlreadyParsedError, + DocumentAlreadyChunkedError, + DocumentAlreadyIndexedError, + DocumentNotParsedError, + DocumentHasNoChunksError, + ParsingDocumentNotFoundError, + ChunkingDocumentNotFoundError, + IndexingDocumentNotFoundError, + ) + return not isinstance(error, permanent_errors) + + +def _as_utc(value: datetime | None) -> datetime | None: + """Normalize SQLite's naive DateTime values to the UTC runtime contract.""" + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) diff --git a/packages/domain/src/ekp_domain/__init__.py b/packages/domain/src/ekp_domain/__init__.py index e2c3e2b..054ea88 100644 --- a/packages/domain/src/ekp_domain/__init__.py +++ b/packages/domain/src/ekp_domain/__init__.py @@ -8,6 +8,13 @@ from ekp_domain.index_profile import ( IndexProfileStatus, StoredIndexProfile, ) +from ekp_domain.ingestion_job import ( + IngestionAttempt, + IngestionAttemptStatus, + IngestionJob, + IngestionJobStatus, + IngestionStage, +) from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocument __all__ = [ @@ -18,6 +25,11 @@ __all__ = [ "DocumentStatus", "IndexProfile", "IndexProfileStatus", + "IngestionAttempt", + "IngestionAttemptStatus", + "IngestionJob", + "IngestionJobStatus", + "IngestionStage", "ParsedBlock", "ParsedBlockType", "ParsedDocument", diff --git a/packages/domain/src/ekp_domain/ingestion_job.py b/packages/domain/src/ekp_domain/ingestion_job.py new file mode 100644 index 0000000..ab3b317 --- /dev/null +++ b/packages/domain/src/ekp_domain/ingestion_job.py @@ -0,0 +1,166 @@ +"""Durable ingestion Job and Attempt state for asynchronous processing.""" + +from datetime import datetime +from enum import StrEnum +from uuid import UUID, uuid4 + +from sqlalchemy import ( + Boolean, + CheckConstraint, + Computed, + DateTime, + ForeignKey, + Index, + Integer, + Text, + UniqueConstraint, + Uuid, + func, +) +from sqlalchemy import Enum as SqlEnum +from sqlalchemy.orm import Mapped, mapped_column + +from ekp_domain.document import Base + + +class IngestionJobStatus(StrEnum): + """Public lifecycle states for one document ingestion Job.""" + + QUEUED = "queued" + RUNNING = "running" + RETRYING = "retrying" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + + +class IngestionStage(StrEnum): + """Observable stages within one ingestion attempt.""" + + PARSING = "parsing" + CLEANING = "cleaning" + CHUNKING = "chunking" + EMBEDDING = "embedding" + INDEXING = "indexing" + + +class IngestionAttemptStatus(StrEnum): + """Terminal or active state of one bounded execution attempt.""" + + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + + +def _enum_column(enum_type: type[StrEnum], name: str) -> SqlEnum: + """Build consistent portable string enums for OceanBase and SQLite tests.""" + return SqlEnum( + enum_type, + name=name, + native_enum=False, + create_constraint=False, + validate_strings=True, + values_callable=lambda values: [item.value for item in values], + ) + + +class IngestionJob(Base): + """Canonical orchestration state; broker messages are only delivery hints.""" + + __tablename__ = "ingestion_jobs" + __table_args__ = ( + CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"), + CheckConstraint("max_attempts > 0", name="max_attempts_positive"), + CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"), + CheckConstraint( + "status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')", + name="status_valid", + ), + CheckConstraint( + "stage IS NULL OR stage IN " + "('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')", + name="stage_valid", + ), + UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"), + Index("ix_ingestion_jobs_status_next_run", "status", "next_run_at"), + ) + + id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4) + document_id: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("documents.id", ondelete="CASCADE"), + nullable=False, + ) + status: Mapped[IngestionJobStatus] = mapped_column( + _enum_column(IngestionJobStatus, "ingestion_job_status"), + nullable=False, + default=IngestionJobStatus.QUEUED, + ) + stage: Mapped[IngestionStage | None] = mapped_column( + _enum_column(IngestionStage, "ingestion_stage"), + nullable=True, + ) + attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + max_attempts: Mapped[int] = mapped_column(Integer, nullable=False) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + lease_expires_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() + ) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class IngestionAttempt(Base): + """Immutable-numbered execution history with at most one active row per Job.""" + + __tablename__ = "ingestion_attempts" + __table_args__ = ( + CheckConstraint("attempt_number > 0", name="attempt_number_positive"), + CheckConstraint( + "status IN ('running', 'succeeded', 'failed')", + name="status_valid", + ), + CheckConstraint( + "stage IS NULL OR stage IN " + "('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')", + name="stage_valid", + ), + UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"), + # NULL values may repeat, while every running attempt maps to 1. This + # provides portable partial uniqueness on OceanBase MySQL and SQLite. + Index("uq_ingestion_attempts_active", "job_id", "active_marker", unique=True), + ) + + id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4) + job_id: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("ingestion_jobs.id", ondelete="CASCADE"), + nullable=False, + ) + attempt_number: Mapped[int] = mapped_column(Integer, nullable=False) + status: Mapped[IngestionAttemptStatus] = mapped_column( + _enum_column(IngestionAttemptStatus, "ingestion_attempt_status"), + nullable=False, + default=IngestionAttemptStatus.RUNNING, + ) + stage: Mapped[IngestionStage | None] = mapped_column( + _enum_column(IngestionStage, "ingestion_attempt_stage"), + nullable=True, + ) + retryable: Mapped[bool | None] = mapped_column(Boolean, nullable=True) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + active_marker: Mapped[int | None] = mapped_column( + Integer, + Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"), + nullable=True, + ) + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) diff --git a/pyproject.toml b/pyproject.toml index 8d5b0fa..22ffa23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ requires-python = ">=3.12,<3.14" dependencies = [ "alembic>=1.18,<2", "bce-python-sdk>=0.9.72,<0.10", + "celery>=5.5,<6", "fastapi>=0.115,<1", "mcp>=2,<3", "pymysql>=1.1,<2", @@ -27,6 +28,7 @@ dependencies = [ ekp-index-rebuild = "ekp_api.index_rebuild:main" ekp-live-acceptance = "ekp_api.live_acceptance:main" ekp-mcp = "ekp_mcp.main:main" +ekp-worker = "ekp_api.worker:main" [dependency-groups] dev = [ diff --git a/tests/integration/test_async_ingestion.py b/tests/integration/test_async_ingestion.py new file mode 100644 index 0000000..85d9507 --- /dev/null +++ b/tests/integration/test_async_ingestion.py @@ -0,0 +1,431 @@ +"""Durable Job, bounded retry and cleanup-before-retry integration tests.""" + +from collections.abc import Sequence +from datetime import UTC, datetime, timedelta +from io import BytesIO +from pathlib import Path +from typing import BinaryIO +from uuid import UUID + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, sessionmaker + +from ekp_core.chunking import StructureAwareChunker +from ekp_core.database import create_session_factory +from ekp_core.document_chunking import DocumentChunkingService +from ekp_core.document_indexing import DocumentIndexingService +from ekp_core.document_parsing import DocumentParsingService +from ekp_core.document_upload import DocumentUploadService +from ekp_core.ingestion_jobs import ( + AsyncIngestionSubmissionService, + DerivedDocumentCleanupService, + IngestionJobRunner, + IngestionJobService, + JobRunAction, + dispatch_pending_jobs, +) +from ekp_domain import ( + Base, + Chunk, + Document, + IndexProfile, + IngestionAttempt, + IngestionAttemptStatus, + IngestionJob, + IngestionJobStatus, + ParsedDocument, +) +from ekp_embedding import EmbeddingProviderError, FakeEmbeddingProvider +from ekp_object_storage import LocalObjectStorage +from ekp_parser import ParserRouter +from ekp_vector_store import FakeVectorStore + + +class RecordingDispatcher: + """Capture delivery requests while preserving the dispatcher contract.""" + + def __init__(self, *, fail: bool = False) -> None: + self.fail = fail + self.job_ids: list[UUID] = [] + + def enqueue(self, job_id: UUID) -> None: + if self.fail: + raise RuntimeError("simulated broker outage containing secret=must-not-persist") + self.job_ids.append(job_id) + + +class FailingJobService(IngestionJobService): + """Inject metadata failure after immutable source upload.""" + + def create(self, document_id: UUID) -> IngestionJob: + del document_id + raise RuntimeError("simulated Job persistence failure") + + +class FailOnceEmbeddingProvider: + """Inject one transient provider failure after parse/chunk persistence.""" + + def __init__(self) -> None: + self._delegate = FakeEmbeddingProvider(model_name="fake-bge", dimensions=8) + self.calls = 0 + + @property + def model_name(self) -> str: + return self._delegate.model_name + + @property + def dimensions(self) -> int: + return self._delegate.dimensions + + @property + def max_batch_size(self) -> int: + return self._delegate.max_batch_size + + def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]: + self.calls += 1 + if self.calls == 1: + raise EmbeddingProviderError("temporary failure token=must-not-persist") + return self._delegate.embed(texts) + + +class AlwaysFailEmbeddingProvider(FailOnceEmbeddingProvider): + """Inject a transient error on every attempt to verify the retry bound.""" + + def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]: + del texts + self.calls += 1 + raise EmbeddingProviderError("temporary failure token=must-not-persist") + + +class RetryObservingParser: + """Verify cleanup finished before the second provider execution begins.""" + + def __init__( + self, + session_factory: sessionmaker[Session], + storage: LocalObjectStorage, + vector_store: FakeVectorStore, + ) -> None: + self._delegate = ParserRouter() + self._session_factory = session_factory + self._storage = storage + self._vector_store = vector_store + self.calls = 0 + self.retry_started_clean = False + + def parse( + self, + document_id: UUID, + filename: str, + source: BinaryIO, + ) -> ParsedDocument: + self.calls += 1 + if self.calls == 2: + with self._session_factory() as session: + document = session.get(Document, document_id) + chunks = session.scalars( + select(Chunk).where(Chunk.document_id == document_id) + ).all() + # The source stream is already open here, proving it survived cleanup. + self.retry_started_clean = ( + document is not None + and document.parsed_storage_key is None + and not chunks + and not self._vector_store.records + and self._storage.exists(document.storage_key) + ) + return self._delegate.parse(document_id, filename, source) + + +class CleanupFailingVectorStore(FakeVectorStore): + """Inject a retry cleanup failure without affecting the first attempt.""" + + fail_delete = False + + def delete_document(self, document_id: UUID, index_profile_id: str) -> None: + if self.fail_delete: + raise RuntimeError("cleanup provider failed secret=must-not-persist") + super().delete_document(document_id, index_profile_id) + + +def _build_runner( + tmp_path: Path, + *, + embedding: FailOnceEmbeddingProvider | FakeEmbeddingProvider | None = None, + vector_store: FakeVectorStore | None = None, + max_attempts: int = 3, +) -> tuple[ + DocumentUploadService, + IngestionJobService, + IngestionJobRunner, + sessionmaker[Session], + LocalObjectStorage, + FakeVectorStore, + RetryObservingParser, +]: + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + storage = LocalObjectStorage(tmp_path / "storage") + selected_store = vector_store or FakeVectorStore() + selected_embedding = embedding or FakeEmbeddingProvider(model_name="fake-bge", dimensions=8) + profile = IndexProfile( + profile_id="test-profile", + chunk_strategy_version="structure_v1", + embedding_model=selected_embedding.model_name, + vector_dimension=selected_embedding.dimensions, + ) + parser = RetryObservingParser(session_factory, storage, selected_store) + upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024) + parsing = DocumentParsingService(session_factory, storage, parser) + chunking = DocumentChunkingService( + session_factory, + storage, + StructureAwareChunker(max_tokens=450, overlap_tokens=75), + ) + indexing = DocumentIndexingService( + session_factory, + selected_embedding, + selected_store, + profile, + ) + jobs = IngestionJobService(session_factory, max_attempts=max_attempts) + cleanup = DerivedDocumentCleanupService( + session_factory, + storage, + selected_store, + profile.profile_id, + ) + runner = IngestionJobRunner( + session_factory, + parsing, + chunking, + indexing, + cleanup, + retry_backoff_seconds=1, + retry_backoff_max_seconds=4, + lease_seconds=30, + ) + return upload, jobs, runner, session_factory, storage, selected_store, parser + + +def _submit( + upload: DocumentUploadService, + jobs: IngestionJobService, + payload: bytes = b"# Policy\n\nRefunds take seven days.", +) -> tuple[UUID, UUID, RecordingDispatcher]: + dispatcher = RecordingDispatcher() + result = AsyncIngestionSubmissionService(upload, jobs, dispatcher).submit( + "policy.md", "text/markdown", BytesIO(payload) + ) + return result.document.id, result.job.id, dispatcher + + +def _make_retry_due(session_factory: sessionmaker[Session], job_id: UUID) -> None: + with session_factory() as session: + job = session.get(IngestionJob, job_id) + assert job is not None + job.next_run_at = datetime.now(UTC) - timedelta(seconds=1) + session.commit() + + +def test_transient_failure_cleans_derivatives_before_retry(tmp_path: Path) -> None: + embedding = FailOnceEmbeddingProvider() + upload, jobs, runner, session_factory, storage, vector_store, parser = _build_runner( + tmp_path, embedding=embedding + ) + document_id, job_id, dispatcher = _submit(upload, jobs) + + first = runner.run(job_id) + assert first.action == JobRunAction.RETRY + assert first.retry_delay_seconds == 1 + assert dispatcher.job_ids == [job_id] + with session_factory() as session: + stored_job = session.get(IngestionJob, job_id) + assert stored_job is not None + assert stored_job.status == IngestionJobStatus.RETRYING + assert stored_job.attempt_count == 1 + assert "must-not-persist" not in (stored_job.error_message or "") + + _make_retry_due(session_factory, job_id) + second = runner.run(job_id) + + assert second.status == IngestionJobStatus.SUCCEEDED + assert parser.retry_started_clean + with session_factory() as session: + stored_job = session.get(IngestionJob, job_id) + attempts = session.scalars( + select(IngestionAttempt) + .where(IngestionAttempt.job_id == job_id) + .order_by(IngestionAttempt.attempt_number) + ).all() + assert stored_job is not None + assert stored_job.attempt_count == 2 + assert [attempt.status for attempt in attempts] == [ + IngestionAttemptStatus.FAILED, + IngestionAttemptStatus.SUCCEEDED, + ] + document = session.get(Document, document_id) + assert document is not None + assert storage.exists(document.storage_key) + assert document.parsed_storage_key is not None + assert storage.exists(document.parsed_storage_key) + assert vector_store.records + + +def test_permanent_content_failure_does_not_retry(tmp_path: Path) -> None: + upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path) + _, job_id, _ = _submit(upload, jobs, payload=b" \n\t") + + result = runner.run(job_id) + + assert result.action == JobRunAction.COMPLETE + assert result.status == IngestionJobStatus.FAILED + with session_factory() as session: + job = session.get(IngestionJob, job_id) + assert job is not None + assert job.attempt_count == 1 + assert job.error_message == "document content could not be processed" + + +def test_cleanup_failure_is_terminal_and_preserves_source(tmp_path: Path) -> None: + embedding = FailOnceEmbeddingProvider() + vector_store = CleanupFailingVectorStore() + upload, jobs, runner, session_factory, storage, _, _ = _build_runner( + tmp_path, + embedding=embedding, + vector_store=vector_store, + ) + document_id, job_id, _ = _submit(upload, jobs) + assert runner.run(job_id).action == JobRunAction.RETRY + _make_retry_due(session_factory, job_id) + vector_store.fail_delete = True + + result = runner.run(job_id) + + assert result.status == IngestionJobStatus.FAILED + with session_factory() as session: + job = session.get(IngestionJob, job_id) + assert job is not None + assert job.attempt_count == 1 # Cleanup failed before attempt 2 was created. + assert job.error_message == "retry cleanup failed" + document = session.get(Document, document_id) + assert document is not None + assert storage.exists(document.storage_key) + + +def test_duplicate_delivery_defers_while_active_lease_is_valid(tmp_path: Path) -> None: + upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path) + _, job_id, _ = _submit(upload, jobs) + with session_factory() as session: + job = session.get(IngestionJob, job_id) + assert job is not None + job.status = IngestionJobStatus.RUNNING + job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=30) + session.commit() + + result = runner.run(job_id) + + assert result.action == JobRunAction.RETRY + assert result.status == IngestionJobStatus.RUNNING + with session_factory() as session: + assert session.scalars(select(IngestionAttempt)).all() == [] + + +def test_expired_worker_lease_is_recovered_as_a_new_attempt(tmp_path: Path) -> None: + upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path) + _, job_id, _ = _submit(upload, jobs) + with session_factory() as session: + job = session.get(IngestionJob, job_id) + assert job is not None + job.status = IngestionJobStatus.RUNNING + job.attempt_count = 1 + job.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1) + session.add(IngestionAttempt(job_id=job_id, attempt_number=1)) + session.commit() + + result = runner.run(job_id) + + assert result.status == IngestionJobStatus.SUCCEEDED + with session_factory() as session: + attempts = session.scalars( + select(IngestionAttempt) + .where(IngestionAttempt.job_id == job_id) + .order_by(IngestionAttempt.attempt_number) + ).all() + assert [attempt.status for attempt in attempts] == [ + IngestionAttemptStatus.FAILED, + IngestionAttemptStatus.SUCCEEDED, + ] + assert attempts[0].error_message == "worker execution lease expired" + + +def test_transient_failures_stop_at_configured_attempt_bound(tmp_path: Path) -> None: + embedding = AlwaysFailEmbeddingProvider() + upload, jobs, runner, session_factory, _, _, _ = _build_runner( + tmp_path, + embedding=embedding, + max_attempts=2, + ) + _, job_id, _ = _submit(upload, jobs) + + assert runner.run(job_id).action == JobRunAction.RETRY + _make_retry_due(session_factory, job_id) + result = runner.run(job_id) + + assert result.action == JobRunAction.COMPLETE + assert result.status == IngestionJobStatus.FAILED + assert runner.run(job_id).status == IngestionJobStatus.FAILED + with session_factory() as session: + job = session.get(IngestionJob, job_id) + attempts = session.scalars( + select(IngestionAttempt).where(IngestionAttempt.job_id == job_id) + ).all() + assert job is not None + assert job.attempt_count == job.max_attempts == 2 + assert len(attempts) == 2 + + +def test_recovery_dispatches_persisted_job_after_broker_outage(tmp_path: Path) -> None: + upload, jobs, _, _, _, _, _ = _build_runner(tmp_path) + failing = RecordingDispatcher(fail=True) + submission = AsyncIngestionSubmissionService(upload, jobs, failing).submit( + "policy.md", "text/markdown", BytesIO(b"policy") + ) + recovered = RecordingDispatcher() + + assert dispatch_pending_jobs(jobs, recovered) == 1 + assert recovered.job_ids == [submission.job.id] + + +def test_job_persistence_failure_compensates_document_and_source(tmp_path: Path) -> None: + upload, _, _, session_factory, _, _, _ = _build_runner(tmp_path) + failing_jobs = FailingJobService(session_factory, max_attempts=3) + + with pytest.raises(RuntimeError, match="Job persistence failure"): + AsyncIngestionSubmissionService(upload, failing_jobs, RecordingDispatcher()).submit( + "policy.md", "text/markdown", BytesIO(b"policy") + ) + + with session_factory() as session: + assert session.scalars(select(Document)).all() == [] + assert session.scalars(select(IngestionJob)).all() == [] + assert not [path for path in tmp_path.rglob("*") if path.is_file()] + + +def test_database_rejects_two_active_attempts_for_one_job(tmp_path: Path) -> None: + upload, jobs, _, session_factory, _, _, _ = _build_runner(tmp_path) + _, job_id, _ = _submit(upload, jobs) + + with session_factory() as session: + session.add_all( + [ + IngestionAttempt(job_id=job_id, attempt_number=1), + IngestionAttempt(job_id=job_id, attempt_number=2), + ] + ) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() diff --git a/tests/integration/test_document_migration.py b/tests/integration/test_document_migration.py index 2075d96..0a32b24 100644 --- a/tests/integration/test_document_migration.py +++ b/tests/integration/test_document_migration.py @@ -23,9 +23,14 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None: command.check(config) inspector = inspect(create_engine(database_url)) - assert {"documents", "chunks", "chunk_index_states", "index_profiles"}.issubset( - inspector.get_table_names() - ) + assert { + "documents", + "chunks", + "chunk_index_states", + "index_profiles", + "ingestion_jobs", + "ingestion_attempts", + }.issubset(inspector.get_table_names()) assert "unrelated_provider_table" in inspector.get_table_names() assert {column["name"] for column in inspector.get_columns("documents")} == { "id", @@ -70,3 +75,29 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None: "activated_at", "active_marker", } + assert {column["name"] for column in inspector.get_columns("ingestion_jobs")} == { + "id", + "document_id", + "status", + "stage", + "attempt_count", + "max_attempts", + "error_message", + "next_run_at", + "lease_expires_at", + "created_at", + "updated_at", + "completed_at", + } + assert {column["name"] for column in inspector.get_columns("ingestion_attempts")} == { + "id", + "job_id", + "attempt_number", + "status", + "stage", + "retryable", + "error_message", + "active_marker", + "started_at", + "finished_at", + } diff --git a/tests/integration/test_document_search_api.py b/tests/integration/test_document_search_api.py index b38a9e1..d859361 100644 --- a/tests/integration/test_document_search_api.py +++ b/tests/integration/test_document_search_api.py @@ -19,6 +19,12 @@ from ekp_core.document_indexing import DocumentIndexingPipelineService, Document from ekp_core.document_management import DocumentManagementService from ekp_core.document_parsing import DocumentParsingService from ekp_core.document_upload import DocumentUploadService +from ekp_core.ingestion_jobs import ( + AsyncIngestionSubmissionService, + DerivedDocumentCleanupService, + IngestionJobRunner, + IngestionJobService, +) from ekp_core.search import SearchService from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile from ekp_embedding import FakeEmbeddingProvider @@ -39,6 +45,16 @@ REQUIRED_SOURCE_FIELDS = { } +class RecordingDispatcher: + """Capture durable Job deliveries without starting a real broker.""" + + def __init__(self) -> None: + self.job_ids: list[UUID] = [] + + def enqueue(self, job_id: UUID) -> None: + self.job_ids.append(job_id) + + def _build_api_services( tmp_path: Path, ) -> tuple[ @@ -46,6 +62,7 @@ def _build_api_services( sessionmaker[Session], LocalObjectStorage, FakeVectorStore, + IngestionJobRunner, ]: """Build the production service graph with offline providers and shared SQLite.""" engine = create_engine( @@ -82,6 +99,24 @@ def _build_api_services( profile, ) ingestion = DocumentIndexingPipelineService(pipeline, indexing) + jobs = IngestionJobService(session_factory, max_attempts=3) + dispatcher = RecordingDispatcher() + submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher) + runner = IngestionJobRunner( + session_factory, + parsing, + chunking, + indexing, + DerivedDocumentCleanupService( + session_factory, + storage, + vector_store, + profile.profile_id, + ), + retry_backoff_seconds=1, + retry_backoff_max_seconds=4, + lease_seconds=30, + ) dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20) search = SearchService( session_factory, @@ -96,10 +131,17 @@ def _build_api_services( active_index_profile_id=profile.profile_id, ) return ( - ApiServices(ingestion=ingestion, documents=documents, search=search), + ApiServices( + ingestion=ingestion, + submissions=submissions, + jobs=jobs, + documents=documents, + search=search, + ), session_factory, storage, vector_store, + runner, ) @@ -107,7 +149,7 @@ def test_document_and_search_rest_flow( tmp_path: Path, isolated_settings_environment: None, ) -> None: - services, session_factory, storage, vector_store = _build_api_services(tmp_path) + services, session_factory, storage, vector_store, runner = _build_api_services(tmp_path) app = create_app(Settings(app_env="test"), api_services=services) with TestClient(app) as client: @@ -121,19 +163,33 @@ def test_document_and_search_rest_flow( ) }, ) - assert upload_response.status_code == 201 + assert upload_response.status_code == 202 uploaded = upload_response.json() document_id = UUID(uploaded["id"]) - assert uploaded["status"] == "ready" + job_id = UUID(uploaded["job_id"]) + assert uploaded["status"] == "uploaded" + assert uploaded["job_status"] == "queued" assert uploaded["filename"] == "退款政策.md" assert "storage_key" not in uploaded + queued = client.get(f"/api/v1/jobs/{job_id}") + assert queued.status_code == 200 + assert queued.json()["attempt"] == 0 + assert queued.json()["status"] == "queued" + assert client.delete(f"/api/v1/documents/{document_id}").status_code == 409 + + runner.run(job_id) + list_response = client.get("/api/v1/documents") detail_response = client.get(f"/api/v1/documents/{document_id}") assert list_response.status_code == 200 assert [item["id"] for item in list_response.json()] == [str(document_id)] assert detail_response.status_code == 200 - assert detail_response.json() == uploaded + assert detail_response.json()["status"] == "ready" + completed = client.get(f"/api/v1/jobs/{job_id}").json() + assert completed["status"] == "succeeded" + assert completed["stage"] == "indexing" + assert completed["attempt"] == 1 search_response = client.post( "/api/v1/search", @@ -173,7 +229,7 @@ def test_rest_validation_not_found_and_openapi_contract( tmp_path: Path, isolated_settings_environment: None, ) -> None: - services, _, _, _ = _build_api_services(tmp_path) + services, _, _, _, _ = _build_api_services(tmp_path) app = create_app(Settings(app_env="test"), api_services=services) missing_id = uuid4() @@ -182,6 +238,7 @@ def test_rest_validation_not_found_and_openapi_contract( assert set(openapi["paths"]) >= { "/api/v1/documents", "/api/v1/documents/{document_id}", + "/api/v1/jobs/{job_id}", "/api/v1/search", } assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"} diff --git a/tests/unit/test_celery_queue.py b/tests/unit/test_celery_queue.py new file mode 100644 index 0000000..fb830cb --- /dev/null +++ b/tests/unit/test_celery_queue.py @@ -0,0 +1,57 @@ +"""Unit tests for the Celery adapter without opening a broker connection.""" + +from unittest.mock import Mock +from uuid import uuid4 + +import pytest +from pydantic import SecretStr + +from ekp_api.celery_queue import ( + INGESTION_TASK_NAME, + RECOVERY_TASK_NAME, + CeleryIngestionDispatcher, + create_celery_app, +) +from ekp_core.config import Settings + + +def test_celery_app_enables_late_ack_and_recovery_schedule( + isolated_settings_environment: None, +) -> None: + settings = Settings(rabbitmq_url=SecretStr("amqp://user:test@queue.example/ekp")) + + app = create_celery_app(settings) + + assert app.conf.task_acks_late is True + assert app.conf.task_reject_on_worker_lost is True + assert app.conf.worker_prefetch_multiplier == 1 + assert app.conf.task_routes[INGESTION_TASK_NAME]["queue"] == "ingestion" + assert app.conf.task_routes[RECOVERY_TASK_NAME]["queue"] == "ingestion" + assert app.conf.beat_schedule["recover-durable-ingestion-jobs"]["schedule"] == 30.0 + + +def test_celery_dispatcher_sends_only_job_identifier( + monkeypatch: pytest.MonkeyPatch, + isolated_settings_environment: None, +) -> None: + app = create_celery_app(Settings(rabbitmq_url=SecretStr("amqp://user:test@queue/ekp"))) + send_task = Mock() + monkeypatch.setattr(app, "send_task", send_task) + job_id = uuid4() + + CeleryIngestionDispatcher(app).enqueue(job_id) + + send_task.assert_called_once_with( + INGESTION_TASK_NAME, + args=[str(job_id)], + queue="ingestion", + ) + + +def test_celery_app_requires_runtime_broker_secret( + isolated_settings_environment: None, +) -> None: + with pytest.raises(ValueError, match="RABBITMQ_URL"): + create_celery_app(Settings()) + with pytest.raises(ValueError, match="RABBITMQ_URL"): + create_celery_app(Settings(rabbitmq_url=SecretStr(""))) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 29831c8..faec677 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -20,6 +20,11 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non assert settings.database_pool_recycle_seconds == 600 assert settings.database_pool_timeout_seconds == 30 assert settings.database_connect_timeout_seconds == 30 + assert settings.rabbitmq_url is None + assert settings.ingestion_max_attempts == 3 + assert settings.ingestion_retry_backoff_seconds == 2 + assert settings.ingestion_retry_backoff_max_seconds == 60 + assert settings.ingestion_worker_lease_seconds == 300 assert settings.object_storage_provider == "bos" assert settings.object_storage_path == Path("storage") assert settings.bos_endpoint is None @@ -90,6 +95,7 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) - oceanbase_vector_password=SecretStr("oceanbase-secret"), reranker_api_key=SecretStr("reranker-secret"), mcp_auth_token=SecretStr("mcp-secret"), + rabbitmq_url=SecretStr("amqp://user:rabbit-secret@queue.example/ekp"), ) rendered = repr(settings) @@ -100,6 +106,7 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) - assert "oceanbase-secret" not in rendered assert "reranker-secret" not in rendered assert "mcp-secret" not in rendered + assert "rabbit-secret" not in rendered def test_settings_restrict_sqlite_to_test_environment( @@ -136,3 +143,17 @@ def test_settings_reject_empty_mcp_allowed_hosts( ) -> None: with pytest.raises(ValidationError, match="mcp_allowed_hosts"): Settings(mcp_allowed_hosts=[]) + + +def test_settings_reject_retry_backoff_larger_than_cap( + isolated_settings_environment: None, +) -> None: + with pytest.raises(ValidationError, match="initial backoff"): + Settings(ingestion_retry_backoff_seconds=10, ingestion_retry_backoff_max_seconds=5) + + +def test_settings_require_worker_lease_longer_than_provider_timeout( + isolated_settings_environment: None, +) -> None: + with pytest.raises(ValidationError, match="worker_lease"): + Settings(ingestion_worker_lease_seconds=30, embedding_timeout_seconds=30) diff --git a/uv.lock b/uv.lock index a7b88f8..3aac7f3 100644 --- a/uv.lock +++ b/uv.lock @@ -28,6 +28,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/89/e62cc37b69ad357cc8ecd6e7367f5245f523d3cbb338a66197212bdf6749/alembic-1.19.1-py3-none-any.whl", hash = "sha256:b39018cb3d9413a19cbd54cf3c02ad33998641f0538eb77413a488a21c3e14be", size = 265946, upload-time = "2026-08-08T16:32:03.153Z" }, ] +[[package]] +name = "amqp" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.5" @@ -83,6 +95,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/de/52b763b183e23f4dd73811a2dd2664824ae7fc6ceb1aa8dcd3f1102b1094/bce_python_sdk-0.9.79-py3-none-any.whl", hash = "sha256:71799ac8740505e0759d30873f6f1a478fa8f83aedf425d511f1419a5f30082e", size = 440268, upload-time = "2026-09-01T05:48:50.459Z" }, ] +[[package]] +name = "billiard" +version = "4.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, +] + +[[package]] +name = "celery" +version = "5.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "billiard" }, + { name = "click" }, + { name = "click-didyoumean" }, + { name = "click-plugins" }, + { name = "click-repl" }, + { name = "kombu" }, + { name = "python-dateutil" }, + { name = "tzlocal" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" }, +] + [[package]] name = "cffi" version = "2.1.1" @@ -129,6 +170,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, ] +[[package]] +name = "click-didyoumean" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, +] + +[[package]] +name = "click-plugins" +version = "1.1.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" }, +] + +[[package]] +name = "click-repl" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -249,6 +327,7 @@ source = { editable = "." } dependencies = [ { name = "alembic" }, { name = "bce-python-sdk" }, + { name = "celery" }, { name = "fastapi" }, { name = "mcp" }, { name = "pydantic-settings" }, @@ -274,6 +353,7 @@ dev = [ requires-dist = [ { name = "alembic", specifier = ">=1.18,<2" }, { name = "bce-python-sdk", specifier = ">=0.9.72,<0.10" }, + { name = "celery", specifier = ">=5.5,<6" }, { name = "fastapi", specifier = ">=0.115,<1" }, { name = "mcp", specifier = ">=2,<3" }, { name = "pydantic-settings", specifier = ">=2.7,<3" }, @@ -459,6 +539,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "kombu" +version = "5.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "amqp" }, + { name = "packaging" }, + { name = "tzdata" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" }, +] + [[package]] name = "librt" version = "0.15.0" @@ -740,6 +835,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "prompt-toolkit" +version = "3.0.53" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -949,6 +1056,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + [[package]] name = "python-docx" version = "1.2.0" @@ -1199,6 +1318,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, ] +[[package]] +name = "tzdata" +version = "2026.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, +] + +[[package]] +name = "tzlocal" +version = "5.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/5b/879b2f932adfa7a053c360d50bc896c977fa6426109185f7c12ebdd0cb9d/tzlocal-5.4.4.tar.gz", hash = "sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4", size = 31170, upload-time = "2026-06-29T08:03:40.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/a4/017a7a6cbe387d961a688ec31364ae60a5c4e22c96ae9921b79a947c855d/tzlocal-5.4.4-py3-none-any.whl", hash = "sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15", size = 18115, upload-time = "2026-06-29T08:03:38.666Z" }, +] + [[package]] name = "uvicorn" version = "0.52.4" @@ -1242,6 +1382,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" }, ] +[[package]] +name = "vine" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, +] + [[package]] name = "watchfiles" version = "1.2.0" @@ -1292,6 +1441,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" }, ] +[[package]] +name = "wcwidth" +version = "0.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/57/ed58088fafdf4c55a0ad6bde846502567645424d7ebf325230b9237f4085/wcwidth-0.8.3.tar.gz", hash = "sha256:d128512515fbf4612e0ff21fd6380399210318b7b54a9af59dff8454cf9730eb", size = 1458450, upload-time = "2026-08-28T18:10:06.875Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/0e/57f6bb3024a597b2e8ec4aee710ffe62ddc95af2e2bb1ee7a7abdc22c68c/wcwidth-0.8.3-py3-none-any.whl", hash = "sha256:d5b73dba6158a595ec9370350e7f2637bcac8d6c5e4fde34f30fcffb6103a5e4", size = 331669, upload-time = "2026-08-28T18:10:04.909Z" }, +] + [[package]] name = "websockets" version = "17.1"