diff --git a/.env.example b/.env.example
index 37d79c1..ea71749 100644
--- a/.env.example
+++ b/.env.example
@@ -11,8 +11,13 @@ DATABASE_MAX_OVERFLOW=10
DATABASE_POOL_RECYCLE_SECONDS=600
DATABASE_POOL_TIMEOUT_SECONDS=30
DATABASE_CONNECT_TIMEOUT_SECONDS=30
-REDIS_URL=redis://localhost:6379/0
-RABBITMQ_URL=amqp://guest:guest@localhost:5672//
+# DEMO-015 queue. Inject the real credential at runtime; never commit its value.
+RABBITMQ_URL=
+INGESTION_MAX_ATTEMPTS=3
+INGESTION_RETRY_BACKOFF_SECONDS=2
+INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
+INGESTION_WORKER_LEASE_SECONDS=300
+INGESTION_RECOVERY_INTERVAL_SECONDS=30
OBJECT_STORAGE_PROVIDER=bos
OBJECT_STORAGE_PATH=storage
diff --git a/Makefile b/Makefile
index 8f00577..6d0d600 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: help install run run-mcp format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
+.PHONY: help install run run-mcp run-worker format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
export UV_CACHE_DIR
@@ -8,6 +8,7 @@ help:
@echo " install Install development dependencies"
@echo " run Start the development API"
@echo " run-mcp Start the authenticated MCP gateway"
+ @echo " run-worker Start the ingestion worker and recovery scheduler"
@echo " format Format code"
@echo " format-check Check formatting without modifying files"
@echo " lint Run linters"
@@ -27,6 +28,8 @@ run:
uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}"
run-mcp:
uv run ekp-mcp
+run-worker:
+ uv run ekp-worker
format:
uv run ruff format .
format-check:
diff --git a/README.md b/README.md
index fbe2e93..5dcc4d5 100644
--- a/README.md
+++ b/README.md
@@ -53,6 +53,7 @@ Install dependencies and run the API:
```bash
cp .env.example .env
make install
+make run-worker
make run
```
@@ -100,8 +101,11 @@ Chunk metadata, including deterministic IDs, filename, page range, section,
index, and strategy version, is stored atomically in canonical relational tables. The lightweight
token counter is replaceable so DEMO-006 can inject model-specific behavior.
Documents remain `processing` until embedding/indexing completes; pipeline
-failures become `failed` without deleting the immutable original. DEMO-009
-exposes this pipeline through the versioned upload endpoint.
+failures become `failed` without deleting the immutable original. DEMO-015 now
+accepts a source plus durable Job with HTTP `202`, while a Celery/RabbitMQ worker
+runs the pipeline. Bounded transient retries clean every derived artifact before
+starting the next Attempt; broker redelivery and expired worker leases converge on
+OceanBase Job state. See `docs/development/ASYNC_INGESTION_SETUP.md`.
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a
validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase.
@@ -128,8 +132,8 @@ adapter with the `bce-reranker-base` baseline. The final Top20-to-Top5 pipeline
source results are rehydrated from canonical relational chunks and retain both
Dense and reranker scores; stale vector content fails closed. See
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
-DEMO-009 exposes the completed synchronous pipeline through versioned document
-and search REST endpoints under `/api/v1`. Upload, list, detail, direct delete,
+DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
+changes upload to asynchronous acceptance and adds Job status. Upload, list, detail, direct delete,
and reranked source search are documented in
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
outside the public response contract.
diff --git a/apps/admin-web/assets/app.js b/apps/admin-web/assets/app.js
index aae8090..02a0d1e 100644
--- a/apps/admin-web/assets/app.js
+++ b/apps/admin-web/assets/app.js
@@ -100,7 +100,7 @@ function initializeUpload() {
form.addEventListener("submit", async (event) => {
event.preventDefault();
if (!input.files.length) return;
- setButtonBusy(button, true, "处理中…");
+ setButtonBusy(button, true, "提交中…");
try {
const formData = new FormData();
formData.append("file", input.files[0]);
@@ -109,7 +109,7 @@ function initializeUpload() {
body: formData,
});
renderUploadResult(documentData);
- showNotice("文档已上传并完成处理。可以前往 Search Lab 检索。", "success");
+ showNotice("文档已接收并进入后台处理队列。", "success");
form.reset();
updateFilename();
} catch (error) {
diff --git a/apps/admin-web/index.html b/apps/admin-web/index.html
index 4376d3c..6241cb4 100644
--- a/apps/admin-web/index.html
+++ b/apps/admin-web/index.html
@@ -67,7 +67,7 @@
01 / INGESTION
让知识进入检索链路
-
上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将同步完成解析、切分与索引。
+
上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将在后台完成解析、切分与索引。
diff --git a/apps/api/src/ekp_api/celery_queue.py b/apps/api/src/ekp_api/celery_queue.py
new file mode 100644
index 0000000..bde1bcc
--- /dev/null
+++ b/apps/api/src/ekp_api/celery_queue.py
@@ -0,0 +1,55 @@
+"""Celery/RabbitMQ adapter behind the queue-neutral ingestion dispatcher."""
+
+from uuid import UUID
+
+from celery import Celery # type: ignore[import-untyped]
+
+from ekp_core.config import Settings
+
+INGESTION_TASK_NAME = "ekp.ingestion.run"
+RECOVERY_TASK_NAME = "ekp.ingestion.recover"
+
+
+def create_celery_app(settings: Settings) -> Celery:
+ """Create a secret-safe broker client without opening a connection eagerly."""
+ if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
+ raise ValueError("RABBITMQ_URL is required for asynchronous ingestion")
+ app = Celery(
+ "enterprise-knowledge-platform",
+ broker=settings.rabbitmq_url.get_secret_value(),
+ )
+ app.conf.update(
+ task_ignore_result=True,
+ task_acks_late=True,
+ task_reject_on_worker_lost=True,
+ worker_prefetch_multiplier=1,
+ broker_connection_retry_on_startup=True,
+ timezone="UTC",
+ enable_utc=True,
+ task_routes={
+ INGESTION_TASK_NAME: {"queue": "ingestion"},
+ RECOVERY_TASK_NAME: {"queue": "ingestion"},
+ },
+ beat_schedule={
+ "recover-durable-ingestion-jobs": {
+ "task": RECOVERY_TASK_NAME,
+ "schedule": float(settings.ingestion_recovery_interval_seconds),
+ }
+ },
+ )
+ return app
+
+
+class CeleryIngestionDispatcher:
+ """Publish only a durable Job id; source bytes and state stay canonical."""
+
+ def __init__(self, app: Celery) -> None:
+ self._app = app
+
+ def enqueue(self, job_id: UUID) -> None:
+ """Deliver one idempotent execution hint to the ingestion queue."""
+ self._app.send_task(
+ INGESTION_TASK_NAME,
+ args=[str(job_id)],
+ queue="ingestion",
+ )
diff --git a/apps/api/src/ekp_api/providers.py b/apps/api/src/ekp_api/providers.py
index b45f849..453df9f 100644
--- a/apps/api/src/ekp_api/providers.py
+++ b/apps/api/src/ekp_api/providers.py
@@ -4,6 +4,7 @@ from dataclasses import dataclass
from sqlalchemy.orm import Session, sessionmaker
+from ekp_api.celery_queue import CeleryIngestionDispatcher, create_celery_app
from ekp_core.chunking import StructureAwareChunker
from ekp_core.config import Settings
from ekp_core.database import create_database_engine, create_session_factory
@@ -17,6 +18,12 @@ from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.index_rebuild import resolve_active_index_profile
+from ekp_core.ingestion_jobs import (
+ AsyncIngestionSubmissionService,
+ DerivedDocumentCleanupService,
+ IngestionJobRunner,
+ IngestionJobService,
+)
from ekp_core.search import SearchService
from ekp_domain import IndexProfile
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
@@ -42,10 +49,38 @@ class ApiServices:
"""Business services injected into REST routes without provider leakage."""
ingestion: DocumentIndexingPipelineService
+ submissions: AsyncIngestionSubmissionService
+ jobs: IngestionJobService
documents: DocumentManagementService
search: SearchService
+@dataclass(frozen=True, slots=True)
+class WorkerServices:
+ """Provider graph required by Celery without REST search-only services."""
+
+ runner: IngestionJobRunner
+ jobs: IngestionJobService
+ dispatcher: CeleryIngestionDispatcher
+
+
+@dataclass(frozen=True, slots=True)
+class _IngestionComponents:
+ """Shared composition result used by API and independent worker processes."""
+
+ session_factory: sessionmaker[Session]
+ storage: ObjectStorage
+ vector_store: VectorStore
+ embedding_provider: EmbeddingProvider
+ index_profile: IndexProfile
+ upload: DocumentUploadService
+ parsing: DocumentParsingService
+ chunking: DocumentChunkingService
+ indexing: DocumentIndexingService
+ jobs: IngestionJobService
+ dispatcher: CeleryIngestionDispatcher
+
+
def create_index_profile(
settings: Settings,
session_factory: sessionmaker[Session] | None = None,
@@ -161,8 +196,31 @@ def create_search_service(
)
-def create_api_services(settings: Settings) -> ApiServices:
- """Compose the synchronous Demo runtime once for all business endpoints."""
+def create_runtime_search_service(settings: Settings) -> SearchService:
+ """Compose only read-path providers for the independent MCP Gateway."""
+ engine = create_database_engine(settings)
+ session_factory = create_session_factory(engine)
+ index_profile = create_index_profile(settings, session_factory)
+ embedding_provider = create_embedding_provider(settings)
+ vector_store = create_vector_store(settings)
+ dense_retrieval = create_dense_retrieval_service(
+ settings,
+ embedding_provider,
+ vector_store,
+ index_profile,
+ )
+ return create_search_service(
+ settings,
+ session_factory,
+ dense_retrieval,
+ create_reranker_provider(settings),
+ )
+
+
+def _create_ingestion_components(settings: Settings) -> _IngestionComponents:
+ """Compose provider-neutral ingestion stages and the Celery delivery adapter."""
+ if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
+ raise ProviderConfigurationError("RABBITMQ_URL is required")
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
storage = create_object_storage(settings)
@@ -185,35 +243,96 @@ def create_api_services(settings: Settings) -> ApiServices:
max_characters=settings.chunk_max_characters,
),
)
- document_pipeline = DocumentPipelineService(
- upload_service,
- parsing_service,
- chunking_service,
- )
indexing_service = DocumentIndexingService(
session_factory,
embedding_provider,
vector_store,
index_profile,
)
- ingestion = DocumentIndexingPipelineService(document_pipeline, indexing_service)
+ jobs = IngestionJobService(session_factory, settings.ingestion_max_attempts)
+ dispatcher = CeleryIngestionDispatcher(create_celery_app(settings))
+ return _IngestionComponents(
+ session_factory=session_factory,
+ storage=storage,
+ vector_store=vector_store,
+ embedding_provider=embedding_provider,
+ index_profile=index_profile,
+ upload=upload_service,
+ parsing=parsing_service,
+ chunking=chunking_service,
+ indexing=indexing_service,
+ jobs=jobs,
+ dispatcher=dispatcher,
+ )
+
+
+def _create_runner(settings: Settings, components: _IngestionComponents) -> IngestionJobRunner:
+ """Compose retry cleanup and execution policy from non-sensitive settings."""
+ cleanup = DerivedDocumentCleanupService(
+ components.session_factory,
+ components.storage,
+ components.vector_store,
+ components.index_profile.profile_id,
+ )
+ return IngestionJobRunner(
+ components.session_factory,
+ components.parsing,
+ components.chunking,
+ components.indexing,
+ cleanup,
+ retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+ retry_backoff_max_seconds=settings.ingestion_retry_backoff_max_seconds,
+ lease_seconds=settings.ingestion_worker_lease_seconds,
+ )
+
+
+def create_worker_services(settings: Settings) -> WorkerServices:
+ """Build the independent worker runtime from the same application boundaries."""
+ components = _create_ingestion_components(settings)
+ return WorkerServices(
+ runner=_create_runner(settings, components),
+ jobs=components.jobs,
+ dispatcher=components.dispatcher,
+ )
+
+
+def create_api_services(settings: Settings) -> ApiServices:
+ """Compose REST services while ingestion execution remains in workers."""
+ components = _create_ingestion_components(settings)
+ document_pipeline = DocumentPipelineService(
+ components.upload,
+ components.parsing,
+ components.chunking,
+ )
+ ingestion = DocumentIndexingPipelineService(document_pipeline, components.indexing)
+ submissions = AsyncIngestionSubmissionService(
+ components.upload,
+ components.jobs,
+ components.dispatcher,
+ )
dense_retrieval = create_dense_retrieval_service(
settings,
- embedding_provider,
- vector_store,
- index_profile,
+ components.embedding_provider,
+ components.vector_store,
+ components.index_profile,
)
reranker_provider = create_reranker_provider(settings)
search = create_search_service(
settings,
- session_factory,
+ components.session_factory,
dense_retrieval,
reranker_provider,
)
documents = DocumentManagementService(
- session_factory,
- storage,
- vector_store,
- active_index_profile_id=index_profile.profile_id,
+ components.session_factory,
+ components.storage,
+ components.vector_store,
+ active_index_profile_id=components.index_profile.profile_id,
+ )
+ return ApiServices(
+ ingestion=ingestion,
+ submissions=submissions,
+ jobs=components.jobs,
+ documents=documents,
+ search=search,
)
- return ApiServices(ingestion=ingestion, documents=documents, search=search)
diff --git a/apps/api/src/ekp_api/routes.py b/apps/api/src/ekp_api/routes.py
index 682fec8..788501d 100644
--- a/apps/api/src/ekp_api/routes.py
+++ b/apps/api/src/ekp_api/routes.py
@@ -6,10 +6,17 @@ from uuid import UUID
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services
-from ekp_api.schemas import DocumentResponse, SearchRequest, SearchResponse
+from ekp_api.schemas import (
+ DocumentResponse,
+ IngestionAcceptedResponse,
+ IngestionJobResponse,
+ SearchRequest,
+ SearchResponse,
+)
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
from ekp_core.document_management import (
DocumentArtifactCleanupError,
+ DocumentIngestionActiveError,
DocumentNotFoundError,
)
from ekp_core.document_upload import (
@@ -18,7 +25,7 @@ from ekp_core.document_upload import (
MediaTypeMismatchError,
UnsupportedDocumentTypeError,
)
-from ekp_parser import DocumentParseError
+from ekp_core.ingestion_jobs import IngestionJobNotFoundError
router = APIRouter()
@@ -44,17 +51,17 @@ Services = Annotated[ApiServices, Depends(get_api_services)]
@router.post(
"/documents",
- response_model=DocumentResponse,
- status_code=status.HTTP_201_CREATED,
+ response_model=IngestionAcceptedResponse,
+ status_code=status.HTTP_202_ACCEPTED,
tags=["documents"],
)
def upload_document(
services: Services,
file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")],
-) -> DocumentResponse:
- """Upload one file and synchronously run the complete Demo ingestion pipeline."""
+) -> IngestionAcceptedResponse:
+ """Persist one immutable source and durable Job, then return immediately."""
try:
- result = services.ingestion.ingest(file.filename or "", file.content_type, file.file)
+ submission = services.submissions.submit(file.filename or "", file.content_type, file.file)
except DocumentTooLargeError as error:
raise HTTPException(
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
@@ -65,21 +72,30 @@ def upload_document(
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
detail="unsupported document type or media type",
) from error
- except (DocumentUploadError, DocumentParseError) as error:
+ except DocumentUploadError as error:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
- detail="document could not be accepted or parsed",
+ detail="document could not be accepted",
) from error
except Exception as error:
# Provider and persistence exceptions are intentionally not reflected to clients.
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
- detail="document processing failed",
+ detail="document acceptance failed",
) from error
- # Reload after every pipeline stage so server-generated timestamps and the
- # final persisted status, not a detached intermediate ORM object, are returned.
- document = services.documents.get_document(result.ingestion.document.id)
- return DocumentResponse.from_document(document)
+ return IngestionAcceptedResponse.from_submission(submission)
+
+
+@router.get("/jobs/{job_id}", response_model=IngestionJobResponse, tags=["jobs"])
+def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
+ """Return durable stage, attempt and safe failure state for one Job."""
+ try:
+ job = services.jobs.get(job_id)
+ except IngestionJobNotFoundError as error:
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND, detail="job not found"
+ ) from error
+ return IngestionJobResponse.from_job(job)
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
@@ -115,6 +131,11 @@ def delete_document(document_id: UUID, services: Services) -> Response:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
) from error
+ except DocumentIngestionActiveError as error:
+ raise HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail="document ingestion is still active",
+ ) from error
except DocumentArtifactCleanupError as error:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
diff --git a/apps/api/src/ekp_api/schemas.py b/apps/api/src/ekp_api/schemas.py
index 3c13e68..d7c9632 100644
--- a/apps/api/src/ekp_api/schemas.py
+++ b/apps/api/src/ekp_api/schemas.py
@@ -5,8 +5,9 @@ from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
+from ekp_core.ingestion_jobs import IngestionSubmission
from ekp_core.search import SearchResult
-from ekp_domain import Document, DocumentStatus
+from ekp_domain import Document, DocumentStatus, IngestionJob, IngestionJobStatus, IngestionStage
class DocumentResponse(BaseModel):
@@ -29,6 +30,64 @@ class DocumentResponse(BaseModel):
return cls.model_validate(document)
+class IngestionAcceptedResponse(DocumentResponse):
+ """Single-file acceptance with the durable Job needed for status polling."""
+
+ job_id: UUID
+ job_status: IngestionJobStatus
+ max_attempts: int
+
+ @classmethod
+ def from_submission(cls, submission: IngestionSubmission) -> "IngestionAcceptedResponse":
+ """Flatten identifiers while retaining the existing document fields."""
+ document = submission.document
+ job = submission.job
+ return cls(
+ id=document.id,
+ filename=document.filename,
+ media_type=document.media_type,
+ size_bytes=document.size_bytes,
+ status=document.status,
+ error_message=document.error_message,
+ created_at=document.created_at,
+ updated_at=document.updated_at,
+ job_id=job.id,
+ job_status=job.status,
+ max_attempts=job.max_attempts,
+ )
+
+
+class IngestionJobResponse(BaseModel):
+ """Public queue-independent Job status without provider or broker details."""
+
+ id: UUID
+ document_id: UUID
+ status: IngestionJobStatus
+ stage: IngestionStage | None
+ attempt: int
+ max_attempts: int
+ error_message: str | None
+ created_at: datetime
+ updated_at: datetime
+ completed_at: datetime | None
+
+ @classmethod
+ def from_job(cls, job: IngestionJob) -> "IngestionJobResponse":
+ """Serialize only client-actionable state and safe diagnostics."""
+ return cls(
+ id=job.id,
+ document_id=job.document_id,
+ status=job.status,
+ stage=job.stage,
+ attempt=job.attempt_count,
+ max_attempts=job.max_attempts,
+ error_message=job.error_message,
+ created_at=job.created_at,
+ updated_at=job.updated_at,
+ completed_at=job.completed_at,
+ )
+
+
class SearchRequest(BaseModel):
"""Validated public search input; normalization remains in the core service."""
diff --git a/apps/api/src/ekp_api/worker.py b/apps/api/src/ekp_api/worker.py
new file mode 100644
index 0000000..e610459
--- /dev/null
+++ b/apps/api/src/ekp_api/worker.py
@@ -0,0 +1,51 @@
+"""Celery worker entry point for durable ingestion Jobs."""
+
+from functools import lru_cache
+from typing import Any
+from uuid import UUID
+
+from ekp_api.celery_queue import (
+ INGESTION_TASK_NAME,
+ RECOVERY_TASK_NAME,
+ create_celery_app,
+)
+from ekp_api.providers import WorkerServices, create_worker_services
+from ekp_core.config import get_settings
+from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs
+
+_settings = get_settings()
+celery_app = create_celery_app(_settings)
+
+
+@lru_cache
+def _worker_services() -> WorkerServices:
+ """Build one provider/service graph per worker process."""
+ return create_worker_services(_settings)
+
+
+@celery_app.task( # type: ignore[untyped-decorator]
+ bind=True,
+ name=INGESTION_TASK_NAME,
+ max_retries=None,
+)
+def run_ingestion(self: Any, job_id: str) -> None:
+ """Execute one delivery and ask Celery to redeliver only when required.
+
+ Celery delivery retries are unbounded because valid-lease deferrals are not
+ business Attempts; the durable runner independently enforces max_attempts.
+ """
+ result = _worker_services().runner.run(UUID(job_id))
+ if result.action == JobRunAction.RETRY:
+ raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None)
+
+
+@celery_app.task(name=RECOVERY_TASK_NAME) # type: ignore[untyped-decorator]
+def recover_ingestion_jobs() -> int:
+ """Redeliver due durable Jobs after API/broker or worker interruptions."""
+ services = _worker_services()
+ return dispatch_pending_jobs(services.jobs, services.dispatcher)
+
+
+def main() -> None:
+ """Run one worker with the embedded Demo recovery scheduler."""
+ celery_app.worker_main(["worker", "--loglevel=INFO", "--queues=ingestion", "--beat"])
diff --git a/apps/mcp-server/src/ekp_mcp/server.py b/apps/mcp-server/src/ekp_mcp/server.py
index 7aceda7..1d2680b 100644
--- a/apps/mcp-server/src/ekp_mcp/server.py
+++ b/apps/mcp-server/src/ekp_mcp/server.py
@@ -12,7 +12,7 @@ from mcp.server.transport_security import TransportSecuritySettings
from pydantic import Field
from starlette.applications import Starlette
-from ekp_api.providers import ApiServices, create_api_services
+from ekp_api.providers import ApiServices, create_runtime_search_service
from ekp_core.config import Settings, get_settings
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
from ekp_mcp.auth import StaticServiceTokenVerifier, current_mcp_identity
@@ -29,8 +29,10 @@ def _create_search_application(
settings: Settings,
api_services: ApiServices | None,
) -> McpSearchApplication:
- services = api_services or create_api_services(settings)
- return McpSearchApplication(services.search, settings.mcp_required_scope)
+ searcher = (
+ api_services.search if api_services is not None else create_runtime_search_service(settings)
+ )
+ return McpSearchApplication(searcher, settings.mcp_required_scope)
def create_mcp_server(
diff --git a/docs/api/API_CONTRACT.md b/docs/api/API_CONTRACT.md
index 47bf703..9a273ee 100644
--- a/docs/api/API_CONTRACT.md
+++ b/docs/api/API_CONTRACT.md
@@ -11,19 +11,20 @@ Use OpenAPI as executable source of truth once implementation starts. Provider-s
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
版本化为 `/api/v1/*`。
-下表描述当前已实现的同步 Demo 契约;它将在 `DEMO-015` 完成时由后文的异步契约替代。
+`DEMO-015` 已将单文件上传切换为持久化异步 Job;批量提交仍留给 `DEMO-016`。
| Method | Path | 行为 |
|---|---|---|
-| `POST` | `/api/v1/documents` | multipart 单文件上传并同步执行完整入库流水线 |
+| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
+| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
| `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 |
-上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `201` 和 Document;
-Demo 同步流水线完成后状态为 `ready`。Document 响应不暴露 `storage_key`、Provider 配置或
-索引实现。
+上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `202`、Document 字段及
+`job_id`、`job_status=queued`、`max_attempts`;响应不等待 Parse、Clean、Chunk、Embedding 或
+Index。Document 响应不暴露 `storage_key`、Provider 配置或索引实现。
Search 请求:
@@ -54,8 +55,9 @@ Search 响应:
```
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
-文档不存在返回 `404`,运行时 Provider 未配置返回 `503`。上游 Provider 或持久化异常只返回
-经过收敛的 `5xx` 信息,不回显凭证、请求正文或厂商响应。
+文档或 Job 不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
+`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
+厂商响应。
## Demo Admin 客户端路由
@@ -72,16 +74,34 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
-## Planned async ingestion contract(DEMO-015/016,尚未实现)
+## 异步入库契约(DEMO-015 已实现;DEMO-016 批量部分待实现)
| Method | Path | 行为 |
|---|---|---|
-| `POST` | `/api/v1/documents` | 单文件接收;持久化 Source、Document、Job 后返回 `202` |
+| `POST` | `/api/v1/documents` | 已实现:单文件接收并返回 Document/Job 标识 |
| `POST` | `/api/v1/ingestion-batches` | multipart 多文件接收,返回批次及逐文件 Document/Job 标识 |
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新状态 |
-| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
+| `GET` | `/api/v1/jobs/{job_id}` | 已实现:返回持久化 Job 状态 |
-单文件与批量上传都不等待解析、Chunk、Embedding 或索引完成。批量响应示意:
+单文件响应示意:
+
+```json
+{
+ "id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
+ "filename": "退款政策.md",
+ "media_type": "text/markdown",
+ "size_bytes": 1024,
+ "status": "uploaded",
+ "error_message": null,
+ "created_at": "2026-09-02T12:00:00Z",
+ "updated_at": "2026-09-02T12:00:00Z",
+ "job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
+ "job_status": "queued",
+ "max_attempts": 3
+}
+```
+
+DEMO-016 的批量响应示意:
```json
{
diff --git a/docs/demo/DEMO_SCOPE.md b/docs/demo/DEMO_SCOPE.md
index 9f77da5..fca286f 100644
--- a/docs/demo/DEMO_SCOPE.md
+++ b/docs/demo/DEMO_SCOPE.md
@@ -333,7 +333,7 @@ Search Service 或 `/api/v1/search`,用于验证标准 MCP、identity 边界
### DEMO-015 异步入库 Job 与自动重试清理
-将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
+**已完成。** 将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
删除旧解析产物、Chunk、索引状态和向量数据,但保留不可变原始文件、Document 和 Attempt
历史。清理失败时终止,不在未知脏状态继续执行。通过执行器抽象接入 Celery/RabbitMQ,使
API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重复消息。
@@ -397,8 +397,8 @@ DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
-任务拆分;`DEMO-012` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,后续严格按
-`DEMO-015`、`DEMO-016`、`DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
+任务拆分;`DEMO-012`、`DEMO-015` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
+后续严格按 `DEMO-016`、`DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
可选项,不因本状态记录自动进入范围。
可选:
@@ -547,7 +547,7 @@ docs/demo/DEMO_SCOPE.md
完整 V1 PRD
只允许实现本文件明确列出的 Demo、Architecture、Plan 与 Deploy Task,并且一次只执行一个
-Task。当前严格顺序为 DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
+Task。当前严格顺序为 DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
当前公共知识库阶段的业务目标保持为:
diff --git a/docs/development/ADMIN_CONSOLE.md b/docs/development/ADMIN_CONSOLE.md
index 9e0e4a5..4de4b3e 100644
--- a/docs/development/ADMIN_CONSOLE.md
+++ b/docs/development/ADMIN_CONSOLE.md
@@ -5,9 +5,10 @@ DEMO-010 在 FastAPI 服务中提供一个轻量、同源的管理客户端。
## 启动与访问
-完成运行时配置后启动 API:
+完成运行时配置后启动 Worker 与 API:
```bash
+make run-worker
make run
```
@@ -34,7 +35,7 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
## Demo 操作闭环
1. 在上传页选择一个 PDF、DOCX、TXT 或 Markdown 文件并提交;
-2. 等待同步流水线返回 `Ready`,失败时页面显示经过 API 收敛的错误;
+2. 接口返回 `202` 后,当前页面显示已接收;在文档列表刷新查看 `Processing/Ready/Failed`;
3. 在文档列表查看状态和安全元数据;
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
Rerank Score;
@@ -45,8 +46,9 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
-- 本任务不包含 ACL/RBAC、审计、异步任务、Document Version、Compiled Knowledge、MCP、
- Hybrid/BM25 或完整 V1 Admin Console 导航;
+- 当前 Console 尚不轮询 Job 详情或展示 Attempt;该状态 UI 与批量上传一起由 DEMO-016 实现;
+- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
+ 或完整 V1 Admin Console 导航;
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。
## 验证
diff --git a/docs/development/ASYNC_INGESTION_SETUP.md b/docs/development/ASYNC_INGESTION_SETUP.md
new file mode 100644
index 0000000..20c5bba
--- /dev/null
+++ b/docs/development/ASYNC_INGESTION_SETUP.md
@@ -0,0 +1,82 @@
+# 异步入库与失败恢复说明
+
+## 范围
+
+DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker。OceanBase 中的
+`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
+不携带文件内容、Provider 凭据或业务状态。
+
+本任务不提供批量上传和批次 UI;这些能力属于 DEMO-016。
+
+## 运行时配置
+
+以下值必须通过环境变量或部署 Secret 注入,不得提交真实值:
+
+```text
+RABBITMQ_URL=amqp://:@:5672/
+INGESTION_MAX_ATTEMPTS=3
+INGESTION_RETRY_BACKOFF_SECONDS=2
+INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
+INGESTION_WORKER_LEASE_SECONDS=300
+INGESTION_RECOVERY_INTERVAL_SECONDS=30
+```
+
+Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
+RabbitMQ 用户应只具备指定 vhost/queue 的最小权限。
+
+## 启动顺序
+
+```bash
+uv run alembic upgrade head
+make run-worker
+make run
+```
+
+`make run-worker` 在当前 Demo 中同时启动 Worker 和恢复调度器。恢复调度器周期扫描:
+
+- 尚未成功投递的 `queued` Job;
+- 已到重试时间的 `retrying` Job;
+- Worker 丢失后租约已过期的 `running` Job。
+
+任务投递使用 late acknowledgement、worker-lost reject 和单任务 prefetch。重复消息会先锁定
+Job 行;有效租约内不会创建第二个 Attempt。租约过期后,旧 Attempt 以安全错误关闭,再按重试
+规则恢复。
+
+## 重试与清理
+
+确定性的内容错误(无法解析、清洗后为空、无 Chunk、派生数据身份不一致)直接进入 `failed`。
+其余运行时/Provider 错误按指数退避重试,默认最多三次总 Attempt。
+
+第二次及后续 Attempt 创建前严格执行:
+
+```text
+OceanBase Vector rows
+→ ChunkIndexState
+→ Chunk
+→ parsed JSON object
+→ reset Document processing state
+```
+
+不可变源文件、Document、Job 和已结束 Attempt 始终保留。任一清理步骤失败时不创建新 Attempt,
+Job 直接以 `retry cleanup failed` 终止,禁止在未知脏状态继续执行。
+
+## API 状态
+
+`POST /api/v1/documents` 成功返回 `202` 和 `job_id`。查询:
+
+```text
+GET /api/v1/jobs/{job_id}
+```
+
+Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;阶段为
+`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
+返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
+
+## 故障检查
+
+1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;
+2. Job 长期为 `running`:检查 Worker,租约到期后恢复调度器会重新投递;
+3. Job 为 `retrying`:等待 `next_run_at`,客户端不应重新上传同一文件;
+4. Job 为 `failed` 且错误为 `retry cleanup failed`:停止人工重试,先核对向量、关系数据与 BOS
+ 派生对象的一致性;
+5. 不得通过日志打印 `RABBITMQ_URL`、数据库 URL、BOS/模型凭据或原始 Provider 响应。
diff --git a/docs/development/REST_API_SETUP.md b/docs/development/REST_API_SETUP.md
index f4245ff..6860054 100644
--- a/docs/development/REST_API_SETUP.md
+++ b/docs/development/REST_API_SETUP.md
@@ -2,19 +2,21 @@
## 范围
-DEMO-009 将已经完成的同步入库和检索服务公开为版本化 REST API。API 层只负责输入校验、
+DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
+Job。API 层只负责输入校验、
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
-本任务不包含 Admin Console、ACL/RBAC、异步任务、软删除或 Hybrid Search。
+当前不包含 ACL/RBAC、软删除、批量上传或 Hybrid Search。
## 启动前提
1. 按 `docs/development/BOS_SETUP.md` 配置 BOS;
2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding;
3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
-4. 执行 `uv run alembic upgrade head`;
-5. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
-6. 执行 `make run`。
+4. 按 `docs/development/ASYNC_INGESTION_SETUP.md` 配置 RabbitMQ 并启动 Worker;
+5. 执行 `uv run alembic upgrade head`;
+6. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
+7. 执行 `make run`。
`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时
延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。
@@ -25,13 +27,19 @@ Swagger UI:`/api/v1/docs`
## 示例
-上传并同步入库:
+上传并异步排队:
```bash
curl -F 'file=@./policy.md;type=text/markdown' \
http://127.0.0.1:8000/api/v1/documents
```
+使用响应中的 `job_id` 查询进度:
+
+```bash
+curl http://127.0.0.1:8000/api/v1/jobs/
+```
+
检索:
```bash
@@ -40,8 +48,8 @@ curl -H 'Content-Type: application/json' \
http://127.0.0.1:8000/api/v1/search
```
-Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase Upsert 完成。大文
-件或云端调用较慢时,请求耗时会相应增加;异步任务属于 Demo 后置能力。
+上传只等待不可变源文件、Document 与 Job 持久化,成功返回 `202`。Worker 后台执行 Parse、
+Clean、Chunk、Embedding 和 OceanBase Upsert;客户端通过 Job 状态判断何时可检索。
## 删除一致性
@@ -51,6 +59,8 @@ Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase
2. 删除 OceanBase canonical 关系表中的 ChunkIndexState、Chunk 和 Document;
3. 删除解析 JSON 与不可变原始文件。
+`queued`、`running` 或 `retrying` Job 对应的 Document 返回 `409`,避免删除与 Worker 并发。
+
向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口
返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1
应通过可重试清理任务和审计记录强化。
diff --git a/docs/development/TEST_STRATEGY.md b/docs/development/TEST_STRATEGY.md
index 792bafc..581566b 100644
--- a/docs/development/TEST_STRATEGY.md
+++ b/docs/development/TEST_STRATEGY.md
@@ -11,6 +11,11 @@ Provide deterministic fakes:
External integrations use separate integration suites/markers.
+Async ingestion tests run with an in-process recording dispatcher and deterministic providers. They
+must cover persisted Job/Attempt history, safe error summaries, bounded retry, cleanup-before-retry,
+active-lease duplicate delivery, expired-lease recovery, immutable-source preservation and terminal
+cleanup failure without requiring a live RabbitMQ broker.
+
E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval.
Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation.
diff --git a/docs/plans/ASYNC_INGESTION_MCP_PLAN.md b/docs/plans/ASYNC_INGESTION_MCP_PLAN.md
index 175f43a..e67d2e2 100644
--- a/docs/plans/ASYNC_INGESTION_MCP_PLAN.md
+++ b/docs/plans/ASYNC_INGESTION_MCP_PLAN.md
@@ -8,14 +8,14 @@
| 能力 | 当前实现 | 结论 |
|---|---|---|
-| 单文件上传 | `POST /api/v1/documents` 接收一个 `file`,并在请求线程内同步完成解析、Chunk、Embedding 和索引 | 已实现,但不满足 V1 异步要求 |
+| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
| 批量上传 | API 与 Admin Console 都只支持一个文件 | 未实现 |
| 失败补偿 | 上传元数据失败会删除源对象;解析元数据失败会删除本次解析产物;索引失败会删除当前 Document/Profile 的可能残留向量 | 已有局部补偿,不是自动重试 |
-| 自动重试 | 没有持久化 Job、Attempt、调度器、退避策略或自动重新执行入口 | 未实现 |
-| 重试前清理 | 影子索引重建能清理目标 Profile 的部分写入;正常上传流水线没有统一清理解析产物、Chunk、索引状态和向量的能力 | 未实现 |
+| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
+| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
| MCP | 已有 MCP 业务契约和可复用 Search Service/REST API,没有可运行 MCP Gateway 或 WorkBuddy 验证 | 未实现 |
-因此,现有失败补偿只能降低一次执行留下半成品的概率,不能满足“失败后自动重新执行,并在重新执行前清理旧数据”。
+PLAN-001 的上述缺口已由 DEMO-015 补齐;批量提交与状态 UI 仍由 DEMO-016 实现。
## 2. Task 归并结论
diff --git a/docs/plans/PROGRESS.md b/docs/plans/PROGRESS.md
index a2ad61c..e1b0021 100644
--- a/docs/plans/PROGRESS.md
+++ b/docs/plans/PROGRESS.md
@@ -1,12 +1,12 @@
# 项目进度
-当前阶段:公共知识库集群交付规划完成,准备实施异步入库
+当前阶段:异步单文件入库闭环完成,准备实施批量上传
-已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014、ARCH-001~ARCH-006、PLAN-001~PLAN-002
+已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-015、ARCH-001~ARCH-006、PLAN-001~PLAN-002
进行中:无
阻塞:无
-下一任务:`DEMO-015`。随后严格执行 `DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
+下一任务:`DEMO-016`。随后严格执行 `DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
@@ -332,3 +332,25 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
- 验证结果:Markdown 相对链接、敏感值模式和 Git diff 检查通过;Ruff Format/Lint、Mypy、
Alembic Head/离线升级通过;Unit + Integration `165 passed`,RAG Evaluation `5 passed`。
- 下一任务:`DEMO-015`。
+
+### DEMO-015(已完成)
+
+- 新增 OceanBase canonical `ingestion_jobs` / `ingestion_attempts` 与 forward-only migration;
+ Document 唯一 Job、Attempt 编号上限和单一运行中 Attempt 均由数据库约束保护。
+- `POST /api/v1/documents` 改为持久化不可变 Source、Document、Job 后返回 `202`,新增
+ `GET /api/v1/jobs/{job_id}`;活动 Job 的 Document 删除返回 `409`,避免与 Worker 竞态。
+- 新增队列无关的 Dispatcher 与 Job Runner,首个适配器使用 Celery/RabbitMQ;消息只包含 Job
+ ID,启用 late ack、worker-lost reject、单任务 prefetch 与周期性持久化 Job 恢复扫描。
+- 瞬时错误按配置执行指数退避和最多三次总 Attempt;Worker 租约有效时重复投递不创建新
+ Attempt,租约过期后关闭旧 Attempt 并恢复。错误记录为固定安全摘要,不保存 Provider 正文。
+- 第二次及后续 Attempt 前依次清理向量、ChunkIndexState、Chunk 与 parsed JSON,再从不可变
+ 源文件重建;清理失败直接终止且不创建新 Attempt。Job 创建失败会补偿 Document 与源对象。
+- Admin Console 仅将上传提示调整为“已接收并进入队列”;批量选择、Job 轮询和逐文件状态 UI
+ 没有提前实现,继续属于 `DEMO-016`。
+- 验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、依赖锁、Alembic Head/离线 MySQL SQL
+ 与 SQLite schema drift 均通过;Unit + Integration `179 passed`,RAG Evaluation `5 passed`;
+ Worker 入口确认注册 `ekp.ingestion.run` 与 `ekp.ingestion.recover`。
+- 环境限制:当前工作站没有 Docker/RabbitMQ,也没有持久化注入目标 OceanBase URL,因此未做
+ 真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
+ 重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
+- 下一任务:`DEMO-016`。
diff --git a/docs/plans/TASKS.md b/docs/plans/TASKS.md
index 592e6ac..a096eed 100644
--- a/docs/plans/TASKS.md
+++ b/docs/plans/TASKS.md
@@ -28,7 +28,7 @@ validation. Provider credentials and internal endpoints are runtime inputs and n
- [x] PLAN-001 MCP 与异步批量入库现状审计、任务拆分和契约基线
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
-- [ ] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
+- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
- [ ] DEMO-016 Batch async upload API + per-file Job status + Admin Console
- [ ] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
- [ ] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
diff --git a/docs/plans/V1_EXECUTION_PLAN.md b/docs/plans/V1_EXECUTION_PLAN.md
index 203879e..f4a33d6 100644
--- a/docs/plans/V1_EXECUTION_PLAN.md
+++ b/docs/plans/V1_EXECUTION_PLAN.md
@@ -16,7 +16,7 @@
“单一公共知识可见域、企业受控网络、持续可用和一键集群部署”推进,不提前实现用户/权限隔离:
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
-2. `DEMO-015`:持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
+2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
3. `DEMO-016`:批量异步上传、批次/Job 查询和 Admin Console 状态展示;
4. `DEPLOY-001`:API/Admin、MCP Gateway、Async Worker 生产镜像;
5. `DEPLOY-002`:Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
diff --git a/migrations/README.md b/migrations/README.md
index ba6fa2e..8d5b044 100644
--- a/migrations/README.md
+++ b/migrations/README.md
@@ -17,3 +17,7 @@ defaults, and primary keys while excluding those three falsely reported object
types. Full constraint comparison still runs against SQLite in integration
tests; target-cluster checks separately verify HNSW/PRE-FILTER and the
single-active generated unique index.
+
+DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
+the latter preserves numbered history and uses a generated nullable active marker plus a unique index
+to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.
diff --git a/migrations/versions/20260902_0007_create_ingestion_jobs.py b/migrations/versions/20260902_0007_create_ingestion_jobs.py
new file mode 100644
index 0000000..247edf0
--- /dev/null
+++ b/migrations/versions/20260902_0007_create_ingestion_jobs.py
@@ -0,0 +1,174 @@
+"""Create durable ingestion jobs and attempt history.
+
+Revision ID: 20260902_0007
+Revises: 20260902_0006
+Create Date: 2026-09-02
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "20260902_0007"
+down_revision: str | None = "20260902_0006"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ """Persist queue-independent Job state and bounded execution attempts."""
+ op.create_table(
+ "ingestion_jobs",
+ sa.Column("id", sa.Uuid(), nullable=False),
+ sa.Column("document_id", sa.Uuid(), nullable=False),
+ sa.Column(
+ "status",
+ sa.Enum(
+ "queued",
+ "running",
+ "retrying",
+ "succeeded",
+ "failed",
+ "cancelled",
+ name="ingestion_job_status",
+ native_enum=False,
+ create_constraint=False,
+ ),
+ nullable=False,
+ ),
+ sa.Column(
+ "stage",
+ sa.Enum(
+ "parsing",
+ "cleaning",
+ "chunking",
+ "embedding",
+ "indexing",
+ name="ingestion_stage",
+ native_enum=False,
+ create_constraint=False,
+ ),
+ nullable=True,
+ ),
+ sa.Column("attempt_count", sa.Integer(), nullable=False),
+ sa.Column("max_attempts", sa.Integer(), nullable=False),
+ sa.Column("error_message", sa.Text(), nullable=True),
+ sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("CURRENT_TIMESTAMP"),
+ nullable=False,
+ ),
+ sa.Column(
+ "updated_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("CURRENT_TIMESTAMP"),
+ nullable=False,
+ ),
+ sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+ sa.CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
+ sa.CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
+ sa.CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
+ sa.CheckConstraint(
+ "status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
+ name="status_valid",
+ ),
+ sa.CheckConstraint(
+ "stage IS NULL OR stage IN "
+ "('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
+ name="stage_valid",
+ ),
+ sa.ForeignKeyConstraint(
+ ["document_id"],
+ ["documents.id"],
+ name="fk_ingestion_jobs_document_id_documents",
+ ondelete="CASCADE",
+ ),
+ sa.PrimaryKeyConstraint("id", name="pk_ingestion_jobs"),
+ sa.UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
+ )
+ op.create_index(
+ "ix_ingestion_jobs_status_next_run",
+ "ingestion_jobs",
+ ["status", "next_run_at"],
+ unique=False,
+ )
+ op.create_table(
+ "ingestion_attempts",
+ sa.Column("id", sa.Uuid(), nullable=False),
+ sa.Column("job_id", sa.Uuid(), nullable=False),
+ sa.Column("attempt_number", sa.Integer(), nullable=False),
+ sa.Column(
+ "status",
+ sa.Enum(
+ "running",
+ "succeeded",
+ "failed",
+ name="ingestion_attempt_status",
+ native_enum=False,
+ create_constraint=False,
+ ),
+ nullable=False,
+ ),
+ sa.Column(
+ "stage",
+ sa.Enum(
+ "parsing",
+ "cleaning",
+ "chunking",
+ "embedding",
+ "indexing",
+ name="ingestion_attempt_stage",
+ native_enum=False,
+ create_constraint=False,
+ ),
+ nullable=True,
+ ),
+ sa.Column("retryable", sa.Boolean(), nullable=True),
+ sa.Column("error_message", sa.Text(), nullable=True),
+ sa.Column(
+ "active_marker",
+ sa.Integer(),
+ sa.Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
+ nullable=True,
+ ),
+ sa.Column(
+ "started_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("CURRENT_TIMESTAMP"),
+ nullable=False,
+ ),
+ sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+ sa.CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
+ sa.CheckConstraint(
+ "status IN ('running', 'succeeded', 'failed')",
+ name="status_valid",
+ ),
+ sa.CheckConstraint(
+ "stage IS NULL OR stage IN "
+ "('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
+ name="stage_valid",
+ ),
+ sa.ForeignKeyConstraint(
+ ["job_id"],
+ ["ingestion_jobs.id"],
+ name="fk_ingestion_attempts_job_id_ingestion_jobs",
+ ondelete="CASCADE",
+ ),
+ sa.PrimaryKeyConstraint("id", name="pk_ingestion_attempts"),
+ sa.UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
+ )
+ op.create_index(
+ "uq_ingestion_attempts_active",
+ "ingestion_attempts",
+ ["job_id", "active_marker"],
+ unique=True,
+ )
+
+
+def downgrade() -> None:
+ """Reject downgrade because project migrations are forward-only."""
+ raise NotImplementedError("Database migrations are forward-only")
diff --git a/packages/core/src/ekp_core/config.py b/packages/core/src/ekp_core/config.py
index 645b10d..ba27cff 100644
--- a/packages/core/src/ekp_core/config.py
+++ b/packages/core/src/ekp_core/config.py
@@ -33,6 +33,12 @@ class Settings(BaseSettings):
database_pool_recycle_seconds: int = Field(default=600, ge=1)
database_pool_timeout_seconds: float = Field(default=30.0, gt=0)
database_connect_timeout_seconds: int = Field(default=30, ge=1)
+ rabbitmq_url: SecretStr | None = None
+ ingestion_max_attempts: int = Field(default=3, ge=1, le=10)
+ ingestion_retry_backoff_seconds: int = Field(default=2, ge=1, le=3600)
+ ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
+ ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
+ ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
object_storage_provider: Literal["local", "bos"] = "bos"
object_storage_path: Path = Path("storage")
bos_endpoint: str | None = None
@@ -91,6 +97,17 @@ class Settings(BaseSettings):
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
if self.search_top_k > self.dense_top_n:
raise ValueError("search_top_k must not exceed dense_top_n")
+ if self.ingestion_retry_backoff_max_seconds < self.ingestion_retry_backoff_seconds:
+ raise ValueError(
+ "ingestion_retry_backoff_max_seconds must be at least the initial backoff"
+ )
+ longest_provider_timeout = max(
+ self.bos_timeout_seconds,
+ self.embedding_timeout_seconds,
+ self.oceanbase_vector_timeout_seconds,
+ )
+ if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
+ raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
raise ValueError("mcp_path must start with '/' and must not end with '/'")
if not self.mcp_allowed_hosts:
diff --git a/packages/core/src/ekp_core/document_chunking.py b/packages/core/src/ekp_core/document_chunking.py
index b71a646..59c792a 100644
--- a/packages/core/src/ekp_core/document_chunking.py
+++ b/packages/core/src/ekp_core/document_chunking.py
@@ -1,5 +1,6 @@
"""Application services for building and persisting document chunks."""
+from collections.abc import Callable
from dataclasses import dataclass
from hashlib import sha256
from uuid import UUID, uuid5
@@ -11,7 +12,7 @@ from ekp_core.chunking import ChunkDraft, ChunkingStrategy
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import deserialize_parsed_document
-from ekp_domain import Chunk, Document, DocumentStatus, ParsedDocument
+from ekp_domain import Chunk, Document, DocumentStatus, IngestionStage, ParsedDocument
from ekp_object_storage import BinarySource, ObjectStorage
MAX_CHUNKING_ERROR_LENGTH = 2000
@@ -54,10 +55,16 @@ class DocumentChunkingService:
self._storage = storage
self._strategy = strategy
- def process(self, document_id: UUID) -> tuple[Chunk, ...]:
- """Run DEMO-005 while reserving READY for the later index stage."""
+ def process(
+ self,
+ document_id: UUID,
+ stage_callback: Callable[[IngestionStage], None] | None = None,
+ ) -> tuple[Chunk, ...]:
+ """Build chunks while optionally reporting the durable Job stage."""
filename, parsed_key = self._start_chunking(document_id)
try:
+ if stage_callback is not None:
+ stage_callback(IngestionStage.CHUNKING)
with self._storage.open(parsed_key) as source:
parsed = deserialize_parsed_document(source.read())
if parsed.document_id != document_id or parsed.filename != filename:
diff --git a/packages/core/src/ekp_core/document_indexing.py b/packages/core/src/ekp_core/document_indexing.py
index 2d8100b..594df60 100644
--- a/packages/core/src/ekp_core/document_indexing.py
+++ b/packages/core/src/ekp_core/document_indexing.py
@@ -1,5 +1,6 @@
"""Embedding and vector-index orchestration for canonical document chunks."""
+from collections.abc import Callable
from dataclasses import dataclass
from uuid import UUID, uuid5
@@ -7,7 +8,14 @@ from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService
-from ekp_domain import Chunk, ChunkIndexState, Document, DocumentStatus, IndexProfile
+from ekp_domain import (
+ Chunk,
+ ChunkIndexState,
+ Document,
+ DocumentStatus,
+ IndexProfile,
+ IngestionStage,
+)
from ekp_embedding import EmbeddingProvider
from ekp_object_storage import BinarySource
from ekp_vector_store import VectorRecord, VectorStore
@@ -67,12 +75,16 @@ class DocumentIndexingService:
self._vector_store = vector_store
self._index_profile = index_profile
- def process(self, document_id: UUID) -> DocumentIndexingResult:
- """Index all chunks for one profile, compensating any partial provider write."""
+ def process(
+ self,
+ document_id: UUID,
+ stage_callback: Callable[[IngestionStage], None] | None = None,
+ ) -> DocumentIndexingResult:
+ """Embed/index chunks while optionally reporting durable Job stages."""
chunks = self._start_indexing(document_id)
upsert_attempted = False
try:
- vectors = self._embed_chunks(chunks)
+ vectors = self._embed_chunks(chunks, stage_callback)
records = tuple(
VectorRecord(
index_record_id=uuid5(
@@ -93,6 +105,8 @@ class DocumentIndexingService:
)
for chunk, vector in zip(chunks, vectors, strict=True)
)
+ if stage_callback is not None:
+ stage_callback(IngestionStage.INDEXING)
upsert_attempted = True
self._vector_store.upsert(records)
self._record_success(document_id, chunks)
@@ -152,13 +166,19 @@ class DocumentIndexingService:
session.commit()
return chunks
- def _embed_chunks(self, chunks: tuple[Chunk, ...]) -> tuple[tuple[float, ...], ...]:
- """Respect provider batch limits and validate every returned vector."""
+ def _embed_chunks(
+ self,
+ chunks: tuple[Chunk, ...],
+ stage_callback: Callable[[IngestionStage], None] | None,
+ ) -> tuple[tuple[float, ...], ...]:
+ """Embed bounded batches and renew an optional Job lease before each call."""
vectors: list[tuple[float, ...]] = []
batch_size = self._embedding_provider.max_batch_size
if batch_size <= 0:
raise EmbeddingResultError("embedding provider has an invalid batch size")
for start in range(0, len(chunks), batch_size):
+ if stage_callback is not None:
+ stage_callback(IngestionStage.EMBEDDING)
batch = chunks[start : start + batch_size]
embedded = self._embedding_provider.embed([chunk.content for chunk in batch])
if len(embedded) != len(batch):
diff --git a/packages/core/src/ekp_core/document_management.py b/packages/core/src/ekp_core/document_management.py
index 026cd84..36f8bb6 100644
--- a/packages/core/src/ekp_core/document_management.py
+++ b/packages/core/src/ekp_core/document_management.py
@@ -5,7 +5,7 @@ from uuid import UUID
from sqlalchemy import delete, select
from sqlalchemy.orm import Session, sessionmaker
-from ekp_domain import Chunk, ChunkIndexState, Document
+from ekp_domain import Chunk, ChunkIndexState, Document, IngestionJob, IngestionJobStatus
from ekp_object_storage import ObjectStorage
from ekp_vector_store import VectorStore
@@ -18,6 +18,10 @@ class DocumentArtifactCleanupError(RuntimeError):
"""Raised when deleted metadata leaves an external storage artifact behind."""
+class DocumentIngestionActiveError(RuntimeError):
+ """Raised when direct deletion would race an active asynchronous Job."""
+
+
class DocumentManagementService:
"""Read canonical documents and coordinate the Demo's direct deletion.
@@ -63,6 +67,22 @@ class DocumentManagementService:
document = session.get(Document, document_id)
if document is None:
raise DocumentNotFoundError(str(document_id))
+ active_job_id = session.scalar(
+ select(IngestionJob.id)
+ .where(
+ IngestionJob.document_id == document_id,
+ IngestionJob.status.in_(
+ (
+ IngestionJobStatus.QUEUED,
+ IngestionJobStatus.RUNNING,
+ IngestionJobStatus.RETRYING,
+ )
+ ),
+ )
+ .limit(1)
+ )
+ if active_job_id is not None:
+ raise DocumentIngestionActiveError(str(document_id))
source_keys = tuple(
key
diff --git a/packages/core/src/ekp_core/document_parsing.py b/packages/core/src/ekp_core/document_parsing.py
index a5bb067..c36bcb5 100644
--- a/packages/core/src/ekp_core/document_parsing.py
+++ b/packages/core/src/ekp_core/document_parsing.py
@@ -1,5 +1,6 @@
"""Application services for parsing and persisting derived document content."""
+from collections.abc import Callable
from io import BytesIO
from uuid import UUID
@@ -8,7 +9,7 @@ from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_cleaner import clean_parsed_document
from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import serialize_parsed_document
-from ekp_domain import Document, DocumentStatus, ParsedDocument
+from ekp_domain import Document, DocumentStatus, IngestionStage, ParsedDocument
from ekp_object_storage import BinarySource, ObjectStorage
from ekp_parser import DocumentParseError, Parser
@@ -48,15 +49,23 @@ class DocumentParsingService:
self._storage = storage
self._parser = parser
- def process(self, document_id: UUID) -> ParsedDocument:
- """Run DEMO-004 synchronously and persist success or failure state."""
+ def process(
+ self,
+ document_id: UUID,
+ stage_callback: Callable[[IngestionStage], None] | None = None,
+ ) -> ParsedDocument:
+ """Parse and clean while optionally reporting durable Job stage boundaries."""
filename, source_key = self._start_processing(document_id)
parsed_key = f"parsed/{document_id}/document.json"
parsed_object_stored = False
try:
+ if stage_callback is not None:
+ stage_callback(IngestionStage.PARSING)
with self._storage.open(source_key) as source:
parsed = self._parser.parse(document_id, filename, source)
+ if stage_callback is not None:
+ stage_callback(IngestionStage.CLEANING)
cleaned = clean_parsed_document(parsed)
if not cleaned.blocks:
raise EmptyParsedDocumentError(
diff --git a/packages/core/src/ekp_core/document_upload.py b/packages/core/src/ekp_core/document_upload.py
index 568a726..6899ad5 100644
--- a/packages/core/src/ekp_core/document_upload.py
+++ b/packages/core/src/ekp_core/document_upload.py
@@ -1,8 +1,9 @@
"""Validated, transactional source-document upload application service."""
from pathlib import Path
-from uuid import uuid4
+from uuid import UUID, uuid4
+from sqlalchemy import delete
from sqlalchemy.orm import Session, sessionmaker
from ekp_domain import Document
@@ -132,6 +133,26 @@ class DocumentUploadService:
return document
+ def rollback_acceptance(self, document_id: UUID) -> None:
+ """Compensate a just-uploaded source when durable Job creation fails.
+
+ This narrow method is only valid before parsing starts; normal document
+ deletion remains coordinated by ``DocumentManagementService``.
+ """
+ with self._session_factory() as session:
+ document = session.get(Document, document_id)
+ if document is None:
+ return
+ storage_key = document.storage_key
+ session.execute(delete(Document).where(Document.id == document.id))
+ session.commit()
+ try:
+ self._storage.delete(storage_key)
+ except Exception as cleanup_error:
+ raise UploadRollbackError(
+ f"job creation failed and source cleanup failed for {storage_key}"
+ ) from cleanup_error
+
def _validate_filename(filename: str) -> str:
"""Accept a plain basename that can be traced without path ambiguity."""
diff --git a/packages/core/src/ekp_core/ingestion_jobs.py b/packages/core/src/ekp_core/ingestion_jobs.py
new file mode 100644
index 0000000..94b9f44
--- /dev/null
+++ b/packages/core/src/ekp_core/ingestion_jobs.py
@@ -0,0 +1,588 @@
+"""Durable asynchronous ingestion orchestration and retry cleanup."""
+
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from enum import StrEnum
+from math import ceil
+from typing import Protocol
+from uuid import UUID
+
+from sqlalchemy import delete, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from ekp_core.document_chunking import (
+ DocumentAlreadyChunkedError,
+ DocumentChunkingService,
+ DocumentNotParsedError,
+ EmptyChunkSetError,
+ ParsedDocumentMismatchError,
+)
+from ekp_core.document_chunking import (
+ DocumentNotFoundError as ChunkingDocumentNotFoundError,
+)
+from ekp_core.document_indexing import (
+ DocumentAlreadyIndexedError,
+ DocumentHasNoChunksError,
+ DocumentIndexingService,
+)
+from ekp_core.document_indexing import (
+ DocumentNotFoundError as IndexingDocumentNotFoundError,
+)
+from ekp_core.document_parsing import (
+ DocumentAlreadyParsedError,
+ DocumentParsingService,
+ EmptyParsedDocumentError,
+)
+from ekp_core.document_parsing import (
+ DocumentNotFoundError as ParsingDocumentNotFoundError,
+)
+from ekp_core.document_upload import DocumentUploadService
+from ekp_core.parsed_document_artifact import ParsedDocumentArtifactError
+from ekp_domain import (
+ Chunk,
+ ChunkIndexState,
+ Document,
+ DocumentStatus,
+ IngestionAttempt,
+ IngestionAttemptStatus,
+ IngestionJob,
+ IngestionJobStatus,
+ IngestionStage,
+)
+from ekp_object_storage import BinarySource, ObjectStorage
+from ekp_parser import DocumentParseError
+from ekp_vector_store import VectorStore
+
+SAFE_TRANSIENT_ERROR = "temporary ingestion failure"
+SAFE_PERMANENT_ERROR = "document content could not be processed"
+SAFE_CLEANUP_ERROR = "retry cleanup failed"
+SAFE_LEASE_ERROR = "worker execution lease expired"
+
+
+class IngestionJobNotFoundError(LookupError):
+ """Raised when a requested ingestion Job does not exist."""
+
+
+class IngestionCleanupError(RuntimeError):
+ """Raised when derived data cannot be restored to a known clean state."""
+
+
+class JobDispatcher(Protocol):
+ """Replaceable message-delivery boundary used by submission services."""
+
+ def enqueue(self, job_id: UUID) -> None:
+ """Request asynchronous execution of one already-persisted Job."""
+
+
+@dataclass(frozen=True, slots=True)
+class IngestionSubmission:
+ """Stable identifiers returned after source and Job persistence."""
+
+ document: Document
+ job: IngestionJob
+
+
+class JobRunAction(StrEnum):
+ """Instruction returned to a queue adapter after one delivery."""
+
+ COMPLETE = "complete"
+ RETRY = "retry"
+
+
+@dataclass(frozen=True, slots=True)
+class JobRunResult:
+ """Queue-neutral result for acknowledging or rescheduling a delivery."""
+
+ action: JobRunAction
+ status: IngestionJobStatus
+ retry_delay_seconds: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class _Claim:
+ """Internal durable claim outcome used before external provider work."""
+
+ run: bool
+ cleanup_required: bool = False
+ result: JobRunResult | None = None
+
+
+class IngestionJobService:
+ """Create and query Job state independently from the broker implementation."""
+
+ def __init__(self, session_factory: sessionmaker[Session], max_attempts: int) -> None:
+ if max_attempts <= 0:
+ raise ValueError("max_attempts must be positive")
+ self._session_factory = session_factory
+ self._max_attempts = max_attempts
+
+ def create(self, document_id: UUID) -> IngestionJob:
+ """Create exactly one queued ingestion Job for a canonical Document."""
+ with self._session_factory() as session:
+ job = IngestionJob(document_id=document_id, max_attempts=self._max_attempts)
+ session.add(job)
+ session.commit()
+ return job
+
+ def get(self, job_id: UUID) -> IngestionJob:
+ """Return one detached Job or a stable not-found error."""
+ with self._session_factory() as session:
+ job = session.get(IngestionJob, job_id)
+ if job is None:
+ raise IngestionJobNotFoundError(str(job_id))
+ return job
+
+ def list_dispatchable(self, limit: int = 100) -> tuple[UUID, ...]:
+ """Find due queued/retrying Jobs and expired running leases for recovery."""
+ if not 1 <= limit <= 1000:
+ raise ValueError("dispatch limit must be between 1 and 1000")
+ now = datetime.now(UTC)
+ with self._session_factory() as session:
+ jobs = session.scalars(
+ select(IngestionJob)
+ .where(
+ (IngestionJob.status == IngestionJobStatus.QUEUED)
+ | (
+ (IngestionJob.status == IngestionJobStatus.RETRYING)
+ & ((IngestionJob.next_run_at.is_(None)) | (IngestionJob.next_run_at <= now))
+ )
+ | (
+ (IngestionJob.status == IngestionJobStatus.RUNNING)
+ & (
+ (IngestionJob.lease_expires_at.is_(None))
+ | (IngestionJob.lease_expires_at <= now)
+ )
+ )
+ )
+ .order_by(IngestionJob.created_at, IngestionJob.id)
+ .limit(limit)
+ ).all()
+ return tuple(job.id for job in jobs)
+
+
+class AsyncIngestionSubmissionService:
+ """Persist immutable source and Job before best-effort broker delivery."""
+
+ def __init__(
+ self,
+ upload_service: DocumentUploadService,
+ jobs: IngestionJobService,
+ dispatcher: JobDispatcher,
+ ) -> None:
+ self._upload_service = upload_service
+ self._jobs = jobs
+ self._dispatcher = dispatcher
+
+ def submit(
+ self,
+ filename: str,
+ media_type: str | None,
+ source: BinarySource,
+ ) -> IngestionSubmission:
+ """Accept one source and return without executing expensive stages inline.
+
+ Broker delivery is deliberately best effort after the canonical commit.
+ A periodic dispatcher recovers queued Jobs when the broker was temporarily
+ unavailable, so the request never treats a message as the source of truth.
+ """
+ document = self._upload_service.upload(filename, media_type, source)
+ try:
+ job = self._jobs.create(document.id)
+ except Exception:
+ self._upload_service.rollback_acceptance(document.id)
+ raise
+ try:
+ self._dispatcher.enqueue(job.id)
+ except Exception:
+ # Keeping the Job queued is intentional. The recovery dispatcher can
+ # safely redeliver it without asking the client to upload again.
+ pass
+ return IngestionSubmission(document=document, job=job)
+
+
+class DerivedDocumentCleanupService:
+ """Remove only rebuildable derivatives before a retry begins."""
+
+ def __init__(
+ self,
+ session_factory: sessionmaker[Session],
+ storage: ObjectStorage,
+ vector_store: VectorStore,
+ active_index_profile_id: str,
+ ) -> None:
+ self._session_factory = session_factory
+ self._storage = storage
+ self._vector_store = vector_store
+ self._active_index_profile_id = active_index_profile_id
+
+ def cleanup(self, document_id: UUID) -> None:
+ """Clean vector, relational and parsed artifacts while preserving source/Job."""
+ try:
+ with self._session_factory() as session:
+ document = session.get(Document, document_id)
+ if document is None:
+ raise IngestionCleanupError("document metadata is missing")
+ parsed_key = document.parsed_storage_key
+ profile_ids = set(
+ session.scalars(
+ select(ChunkIndexState.index_profile_id)
+ .join(Chunk, Chunk.chunk_id == ChunkIndexState.chunk_id)
+ .where(Chunk.document_id == document_id)
+ .distinct()
+ ).all()
+ )
+ profile_ids.add(self._active_index_profile_id)
+
+ # External vector rows are deleted before their canonical evidence.
+ for profile_id in sorted(profile_ids):
+ self._vector_store.delete_document(document_id, profile_id)
+
+ with self._session_factory() as session:
+ chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
+ session.execute(
+ delete(ChunkIndexState).where(ChunkIndexState.chunk_id.in_(chunk_ids))
+ )
+ session.execute(delete(Chunk).where(Chunk.document_id == document_id))
+ session.commit()
+
+ if parsed_key is not None and self._storage.exists(parsed_key):
+ self._storage.delete(parsed_key)
+
+ with self._session_factory() as session:
+ document = session.get(Document, document_id)
+ if document is None:
+ raise IngestionCleanupError("document metadata is missing")
+ document.parsed_storage_key = None
+ document.status = DocumentStatus.UPLOADED
+ document.error_message = None
+ session.commit()
+ except IngestionCleanupError:
+ raise
+ except Exception as error:
+ raise IngestionCleanupError(SAFE_CLEANUP_ERROR) from error
+
+
+class IngestionJobRunner:
+ """Execute one durable attempt and return queue-neutral retry instructions."""
+
+ def __init__(
+ self,
+ session_factory: sessionmaker[Session],
+ parsing: DocumentParsingService,
+ chunking: DocumentChunkingService,
+ indexing: DocumentIndexingService,
+ cleanup: DerivedDocumentCleanupService,
+ *,
+ retry_backoff_seconds: int,
+ retry_backoff_max_seconds: int,
+ lease_seconds: int,
+ ) -> None:
+ if retry_backoff_seconds <= 0 or retry_backoff_max_seconds <= 0 or lease_seconds <= 0:
+ raise ValueError("retry and lease durations must be positive")
+ self._session_factory = session_factory
+ self._parsing = parsing
+ self._chunking = chunking
+ self._indexing = indexing
+ self._cleanup = cleanup
+ self._retry_backoff_seconds = retry_backoff_seconds
+ self._retry_backoff_max_seconds = retry_backoff_max_seconds
+ self._lease_seconds = lease_seconds
+
+ def run(self, job_id: UUID) -> JobRunResult:
+ """Claim, optionally clean, execute and durably finalize one delivery."""
+ claim = self._claim(job_id)
+ if not claim.run:
+ assert claim.result is not None
+ return claim.result
+
+ if claim.cleanup_required:
+ try:
+ document_id = self._job_document_id(job_id)
+ self._cleanup.cleanup(document_id)
+ except Exception:
+ self._finish_cleanup_failure(job_id)
+ return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.FAILED)
+
+ attempt_number, document_id = self._start_attempt(job_id)
+
+ def stage_callback(stage: IngestionStage) -> None:
+ """Bind this attempt while pipeline stages remain Job-agnostic."""
+ self._record_stage(job_id, attempt_number, stage)
+
+ try:
+ self._parsing.process(document_id, stage_callback=stage_callback)
+ self._chunking.process(document_id, stage_callback=stage_callback)
+ self._indexing.process(document_id, stage_callback=stage_callback)
+ except Exception as error:
+ return self._finish_failure(job_id, attempt_number, document_id, error)
+
+ self._finish_success(job_id, attempt_number, document_id)
+ return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.SUCCEEDED)
+
+ def _claim(self, job_id: UUID) -> _Claim:
+ """Serialize deliveries and recover only after an execution lease expires."""
+ now = datetime.now(UTC)
+ with self._session_factory() as session:
+ job = session.scalar(
+ select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
+ )
+ if job is None:
+ raise IngestionJobNotFoundError(str(job_id))
+ if job.status in {
+ IngestionJobStatus.SUCCEEDED,
+ IngestionJobStatus.FAILED,
+ IngestionJobStatus.CANCELLED,
+ }:
+ return _Claim(
+ run=False,
+ result=JobRunResult(JobRunAction.COMPLETE, job.status),
+ )
+ if job.status == IngestionJobStatus.RUNNING:
+ lease_expiry = _as_utc(job.lease_expires_at)
+ if lease_expiry is not None and lease_expiry > now:
+ delay = max(1, ceil((lease_expiry - now).total_seconds()))
+ return _Claim(
+ run=False,
+ result=JobRunResult(
+ JobRunAction.RETRY,
+ IngestionJobStatus.RUNNING,
+ delay,
+ ),
+ )
+ self._expire_active_attempt(session, job, now)
+ if job.attempt_count >= job.max_attempts:
+ job.status = IngestionJobStatus.FAILED
+ job.lease_expires_at = None
+ job.completed_at = now
+ document = session.get(Document, job.document_id)
+ if document is not None:
+ document.status = DocumentStatus.FAILED
+ document.error_message = SAFE_LEASE_ERROR
+ session.commit()
+ return _Claim(
+ run=False,
+ result=JobRunResult(JobRunAction.COMPLETE, job.status),
+ )
+ job.status = IngestionJobStatus.RETRYING
+ next_run = _as_utc(job.next_run_at)
+ if (
+ job.status == IngestionJobStatus.RETRYING
+ and next_run is not None
+ and next_run > now
+ ):
+ delay = max(1, ceil((next_run - now).total_seconds()))
+ return _Claim(
+ run=False,
+ result=JobRunResult(
+ JobRunAction.RETRY,
+ IngestionJobStatus.RETRYING,
+ delay,
+ ),
+ )
+
+ cleanup_required = job.attempt_count > 0
+ job.status = IngestionJobStatus.RUNNING
+ job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
+ job.next_run_at = None
+ session.commit()
+ return _Claim(run=True, cleanup_required=cleanup_required)
+
+ def _expire_active_attempt(self, session: Session, job: IngestionJob, now: datetime) -> None:
+ """Close an abandoned Attempt before a replacement can be created."""
+ attempt = session.scalar(
+ select(IngestionAttempt)
+ .where(
+ IngestionAttempt.job_id == job.id,
+ IngestionAttempt.status == IngestionAttemptStatus.RUNNING,
+ )
+ .with_for_update()
+ )
+ if attempt is not None:
+ attempt.status = IngestionAttemptStatus.FAILED
+ attempt.retryable = True
+ attempt.error_message = SAFE_LEASE_ERROR
+ attempt.finished_at = now
+ job.error_message = SAFE_LEASE_ERROR
+
+ def _job_document_id(self, job_id: UUID) -> UUID:
+ with self._session_factory() as session:
+ job = session.get(IngestionJob, job_id)
+ if job is None:
+ raise IngestionJobNotFoundError(str(job_id))
+ return job.document_id
+
+ def _start_attempt(self, job_id: UUID) -> tuple[int, UUID]:
+ """Create the next numbered Attempt only after retry cleanup succeeds."""
+ now = datetime.now(UTC)
+ with self._session_factory() as session:
+ job = session.scalar(
+ select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
+ )
+ if job is None:
+ raise IngestionJobNotFoundError(str(job_id))
+ attempt_number = job.attempt_count + 1
+ job.attempt_count = attempt_number
+ job.stage = IngestionStage.PARSING
+ job.error_message = None
+ job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
+ session.add(
+ IngestionAttempt(
+ job_id=job.id,
+ attempt_number=attempt_number,
+ stage=IngestionStage.PARSING,
+ )
+ )
+ session.commit()
+ return attempt_number, job.document_id
+
+ def _record_stage(self, job_id: UUID, attempt_number: int, stage: IngestionStage) -> None:
+ """Persist progress and renew the worker lease at every stage boundary."""
+ with self._session_factory() as session:
+ job = session.get(IngestionJob, job_id)
+ attempt = session.scalar(
+ select(IngestionAttempt).where(
+ IngestionAttempt.job_id == job_id,
+ IngestionAttempt.attempt_number == attempt_number,
+ )
+ )
+ if job is None or attempt is None:
+ raise IngestionJobNotFoundError(str(job_id))
+ job.stage = stage
+ job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=self._lease_seconds)
+ attempt.stage = stage
+ session.commit()
+
+ def _finish_success(self, job_id: UUID, attempt_number: int, document_id: UUID) -> None:
+ """Atomically close Job/Attempt after the document becomes ready."""
+ now = datetime.now(UTC)
+ with self._session_factory() as session:
+ job, attempt = self._load_job_attempt(session, job_id, attempt_number)
+ document = session.get(Document, document_id)
+ if document is None:
+ raise IngestionJobNotFoundError(str(job_id))
+ attempt.status = IngestionAttemptStatus.SUCCEEDED
+ attempt.retryable = False
+ attempt.finished_at = now
+ job.status = IngestionJobStatus.SUCCEEDED
+ job.error_message = None
+ job.lease_expires_at = None
+ job.completed_at = now
+ document.status = DocumentStatus.READY
+ document.error_message = None
+ session.commit()
+
+ def _finish_failure(
+ self,
+ job_id: UUID,
+ attempt_number: int,
+ document_id: UUID,
+ error: Exception,
+ ) -> JobRunResult:
+ """Classify safely, bound retry count, and schedule exponential backoff."""
+ retryable = _is_retryable(error)
+ safe_error = SAFE_TRANSIENT_ERROR if retryable else SAFE_PERMANENT_ERROR
+ now = datetime.now(UTC)
+ with self._session_factory() as session:
+ job, attempt = self._load_job_attempt(session, job_id, attempt_number)
+ document = session.get(Document, document_id)
+ attempt.status = IngestionAttemptStatus.FAILED
+ attempt.retryable = retryable
+ attempt.error_message = safe_error
+ attempt.finished_at = now
+ job.error_message = safe_error
+ job.lease_expires_at = None
+ if retryable and job.attempt_count < job.max_attempts:
+ delay = min(
+ self._retry_backoff_seconds * (2 ** (job.attempt_count - 1)),
+ self._retry_backoff_max_seconds,
+ )
+ job.status = IngestionJobStatus.RETRYING
+ job.next_run_at = now + timedelta(seconds=delay)
+ if document is not None:
+ document.status = DocumentStatus.PROCESSING
+ document.error_message = safe_error
+ session.commit()
+ return JobRunResult(JobRunAction.RETRY, job.status, delay)
+
+ job.status = IngestionJobStatus.FAILED
+ job.next_run_at = None
+ job.completed_at = now
+ if document is not None:
+ document.status = DocumentStatus.FAILED
+ document.error_message = safe_error
+ session.commit()
+ return JobRunResult(JobRunAction.COMPLETE, job.status)
+
+ def _finish_cleanup_failure(self, job_id: UUID) -> None:
+ """Fail closed without creating a new Attempt on unknown dirty state."""
+ now = datetime.now(UTC)
+ with self._session_factory() as session:
+ job = session.get(IngestionJob, job_id)
+ if job is None:
+ raise IngestionJobNotFoundError(str(job_id))
+ document = session.get(Document, job.document_id)
+ job.status = IngestionJobStatus.FAILED
+ job.error_message = SAFE_CLEANUP_ERROR
+ job.next_run_at = None
+ job.lease_expires_at = None
+ job.completed_at = now
+ if document is not None:
+ document.status = DocumentStatus.FAILED
+ document.error_message = SAFE_CLEANUP_ERROR
+ session.commit()
+
+ @staticmethod
+ def _load_job_attempt(
+ session: Session, job_id: UUID, attempt_number: int
+ ) -> tuple[IngestionJob, IngestionAttempt]:
+ job = session.get(IngestionJob, job_id)
+ attempt = session.scalar(
+ select(IngestionAttempt).where(
+ IngestionAttempt.job_id == job_id,
+ IngestionAttempt.attempt_number == attempt_number,
+ )
+ )
+ if job is None or attempt is None:
+ raise IngestionJobNotFoundError(str(job_id))
+ return job, attempt
+
+
+def dispatch_pending_jobs(jobs: IngestionJobService, dispatcher: JobDispatcher) -> int:
+ """Best-effort redelivery loop used by Celery beat after broker/API failures."""
+ dispatched = 0
+ for job_id in jobs.list_dispatchable():
+ try:
+ dispatcher.enqueue(job_id)
+ except Exception:
+ continue
+ dispatched += 1
+ return dispatched
+
+
+def _is_retryable(error: Exception) -> bool:
+ """Treat known content/state violations as permanent and providers as transient."""
+ permanent_errors = (
+ DocumentParseError,
+ EmptyParsedDocumentError,
+ EmptyChunkSetError,
+ ParsedDocumentMismatchError,
+ ParsedDocumentArtifactError,
+ DocumentAlreadyParsedError,
+ DocumentAlreadyChunkedError,
+ DocumentAlreadyIndexedError,
+ DocumentNotParsedError,
+ DocumentHasNoChunksError,
+ ParsingDocumentNotFoundError,
+ ChunkingDocumentNotFoundError,
+ IndexingDocumentNotFoundError,
+ )
+ return not isinstance(error, permanent_errors)
+
+
+def _as_utc(value: datetime | None) -> datetime | None:
+ """Normalize SQLite's naive DateTime values to the UTC runtime contract."""
+ if value is None:
+ return None
+ if value.tzinfo is None:
+ return value.replace(tzinfo=UTC)
+ return value.astimezone(UTC)
diff --git a/packages/domain/src/ekp_domain/__init__.py b/packages/domain/src/ekp_domain/__init__.py
index e2c3e2b..054ea88 100644
--- a/packages/domain/src/ekp_domain/__init__.py
+++ b/packages/domain/src/ekp_domain/__init__.py
@@ -8,6 +8,13 @@ from ekp_domain.index_profile import (
IndexProfileStatus,
StoredIndexProfile,
)
+from ekp_domain.ingestion_job import (
+ IngestionAttempt,
+ IngestionAttemptStatus,
+ IngestionJob,
+ IngestionJobStatus,
+ IngestionStage,
+)
from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocument
__all__ = [
@@ -18,6 +25,11 @@ __all__ = [
"DocumentStatus",
"IndexProfile",
"IndexProfileStatus",
+ "IngestionAttempt",
+ "IngestionAttemptStatus",
+ "IngestionJob",
+ "IngestionJobStatus",
+ "IngestionStage",
"ParsedBlock",
"ParsedBlockType",
"ParsedDocument",
diff --git a/packages/domain/src/ekp_domain/ingestion_job.py b/packages/domain/src/ekp_domain/ingestion_job.py
new file mode 100644
index 0000000..ab3b317
--- /dev/null
+++ b/packages/domain/src/ekp_domain/ingestion_job.py
@@ -0,0 +1,166 @@
+"""Durable ingestion Job and Attempt state for asynchronous processing."""
+
+from datetime import datetime
+from enum import StrEnum
+from uuid import UUID, uuid4
+
+from sqlalchemy import (
+ Boolean,
+ CheckConstraint,
+ Computed,
+ DateTime,
+ ForeignKey,
+ Index,
+ Integer,
+ Text,
+ UniqueConstraint,
+ Uuid,
+ func,
+)
+from sqlalchemy import Enum as SqlEnum
+from sqlalchemy.orm import Mapped, mapped_column
+
+from ekp_domain.document import Base
+
+
+class IngestionJobStatus(StrEnum):
+ """Public lifecycle states for one document ingestion Job."""
+
+ QUEUED = "queued"
+ RUNNING = "running"
+ RETRYING = "retrying"
+ SUCCEEDED = "succeeded"
+ FAILED = "failed"
+ CANCELLED = "cancelled"
+
+
+class IngestionStage(StrEnum):
+ """Observable stages within one ingestion attempt."""
+
+ PARSING = "parsing"
+ CLEANING = "cleaning"
+ CHUNKING = "chunking"
+ EMBEDDING = "embedding"
+ INDEXING = "indexing"
+
+
+class IngestionAttemptStatus(StrEnum):
+ """Terminal or active state of one bounded execution attempt."""
+
+ RUNNING = "running"
+ SUCCEEDED = "succeeded"
+ FAILED = "failed"
+
+
+def _enum_column(enum_type: type[StrEnum], name: str) -> SqlEnum:
+ """Build consistent portable string enums for OceanBase and SQLite tests."""
+ return SqlEnum(
+ enum_type,
+ name=name,
+ native_enum=False,
+ create_constraint=False,
+ validate_strings=True,
+ values_callable=lambda values: [item.value for item in values],
+ )
+
+
+class IngestionJob(Base):
+ """Canonical orchestration state; broker messages are only delivery hints."""
+
+ __tablename__ = "ingestion_jobs"
+ __table_args__ = (
+ CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
+ CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
+ CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
+ CheckConstraint(
+ "status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
+ name="status_valid",
+ ),
+ CheckConstraint(
+ "stage IS NULL OR stage IN "
+ "('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
+ name="stage_valid",
+ ),
+ UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
+ Index("ix_ingestion_jobs_status_next_run", "status", "next_run_at"),
+ )
+
+ id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
+ document_id: Mapped[UUID] = mapped_column(
+ Uuid(as_uuid=True),
+ ForeignKey("documents.id", ondelete="CASCADE"),
+ nullable=False,
+ )
+ status: Mapped[IngestionJobStatus] = mapped_column(
+ _enum_column(IngestionJobStatus, "ingestion_job_status"),
+ nullable=False,
+ default=IngestionJobStatus.QUEUED,
+ )
+ stage: Mapped[IngestionStage | None] = mapped_column(
+ _enum_column(IngestionStage, "ingestion_stage"),
+ nullable=True,
+ )
+ attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
+ error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+ next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+ lease_expires_at: Mapped[datetime | None] = mapped_column(
+ DateTime(timezone=True), nullable=True
+ )
+ created_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False, server_default=func.now()
+ )
+ updated_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
+ )
+ completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+
+class IngestionAttempt(Base):
+ """Immutable-numbered execution history with at most one active row per Job."""
+
+ __tablename__ = "ingestion_attempts"
+ __table_args__ = (
+ CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
+ CheckConstraint(
+ "status IN ('running', 'succeeded', 'failed')",
+ name="status_valid",
+ ),
+ CheckConstraint(
+ "stage IS NULL OR stage IN "
+ "('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
+ name="stage_valid",
+ ),
+ UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
+ # NULL values may repeat, while every running attempt maps to 1. This
+ # provides portable partial uniqueness on OceanBase MySQL and SQLite.
+ Index("uq_ingestion_attempts_active", "job_id", "active_marker", unique=True),
+ )
+
+ id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
+ job_id: Mapped[UUID] = mapped_column(
+ Uuid(as_uuid=True),
+ ForeignKey("ingestion_jobs.id", ondelete="CASCADE"),
+ nullable=False,
+ )
+ attempt_number: Mapped[int] = mapped_column(Integer, nullable=False)
+ status: Mapped[IngestionAttemptStatus] = mapped_column(
+ _enum_column(IngestionAttemptStatus, "ingestion_attempt_status"),
+ nullable=False,
+ default=IngestionAttemptStatus.RUNNING,
+ )
+ stage: Mapped[IngestionStage | None] = mapped_column(
+ _enum_column(IngestionStage, "ingestion_attempt_stage"),
+ nullable=True,
+ )
+ retryable: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
+ error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+ active_marker: Mapped[int | None] = mapped_column(
+ Integer,
+ Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
+ nullable=True,
+ )
+ started_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False, server_default=func.now()
+ )
+ finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
diff --git a/pyproject.toml b/pyproject.toml
index 8d5b0fa..22ffa23 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,6 +11,7 @@ requires-python = ">=3.12,<3.14"
dependencies = [
"alembic>=1.18,<2",
"bce-python-sdk>=0.9.72,<0.10",
+ "celery>=5.5,<6",
"fastapi>=0.115,<1",
"mcp>=2,<3",
"pymysql>=1.1,<2",
@@ -27,6 +28,7 @@ dependencies = [
ekp-index-rebuild = "ekp_api.index_rebuild:main"
ekp-live-acceptance = "ekp_api.live_acceptance:main"
ekp-mcp = "ekp_mcp.main:main"
+ekp-worker = "ekp_api.worker:main"
[dependency-groups]
dev = [
diff --git a/tests/integration/test_async_ingestion.py b/tests/integration/test_async_ingestion.py
new file mode 100644
index 0000000..85d9507
--- /dev/null
+++ b/tests/integration/test_async_ingestion.py
@@ -0,0 +1,431 @@
+"""Durable Job, bounded retry and cleanup-before-retry integration tests."""
+
+from collections.abc import Sequence
+from datetime import UTC, datetime, timedelta
+from io import BytesIO
+from pathlib import Path
+from typing import BinaryIO
+from uuid import UUID
+
+import pytest
+from sqlalchemy import create_engine, select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session, sessionmaker
+
+from ekp_core.chunking import StructureAwareChunker
+from ekp_core.database import create_session_factory
+from ekp_core.document_chunking import DocumentChunkingService
+from ekp_core.document_indexing import DocumentIndexingService
+from ekp_core.document_parsing import DocumentParsingService
+from ekp_core.document_upload import DocumentUploadService
+from ekp_core.ingestion_jobs import (
+ AsyncIngestionSubmissionService,
+ DerivedDocumentCleanupService,
+ IngestionJobRunner,
+ IngestionJobService,
+ JobRunAction,
+ dispatch_pending_jobs,
+)
+from ekp_domain import (
+ Base,
+ Chunk,
+ Document,
+ IndexProfile,
+ IngestionAttempt,
+ IngestionAttemptStatus,
+ IngestionJob,
+ IngestionJobStatus,
+ ParsedDocument,
+)
+from ekp_embedding import EmbeddingProviderError, FakeEmbeddingProvider
+from ekp_object_storage import LocalObjectStorage
+from ekp_parser import ParserRouter
+from ekp_vector_store import FakeVectorStore
+
+
+class RecordingDispatcher:
+ """Capture delivery requests while preserving the dispatcher contract."""
+
+ def __init__(self, *, fail: bool = False) -> None:
+ self.fail = fail
+ self.job_ids: list[UUID] = []
+
+ def enqueue(self, job_id: UUID) -> None:
+ if self.fail:
+ raise RuntimeError("simulated broker outage containing secret=must-not-persist")
+ self.job_ids.append(job_id)
+
+
+class FailingJobService(IngestionJobService):
+ """Inject metadata failure after immutable source upload."""
+
+ def create(self, document_id: UUID) -> IngestionJob:
+ del document_id
+ raise RuntimeError("simulated Job persistence failure")
+
+
+class FailOnceEmbeddingProvider:
+ """Inject one transient provider failure after parse/chunk persistence."""
+
+ def __init__(self) -> None:
+ self._delegate = FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
+ self.calls = 0
+
+ @property
+ def model_name(self) -> str:
+ return self._delegate.model_name
+
+ @property
+ def dimensions(self) -> int:
+ return self._delegate.dimensions
+
+ @property
+ def max_batch_size(self) -> int:
+ return self._delegate.max_batch_size
+
+ def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
+ self.calls += 1
+ if self.calls == 1:
+ raise EmbeddingProviderError("temporary failure token=must-not-persist")
+ return self._delegate.embed(texts)
+
+
+class AlwaysFailEmbeddingProvider(FailOnceEmbeddingProvider):
+ """Inject a transient error on every attempt to verify the retry bound."""
+
+ def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
+ del texts
+ self.calls += 1
+ raise EmbeddingProviderError("temporary failure token=must-not-persist")
+
+
+class RetryObservingParser:
+ """Verify cleanup finished before the second provider execution begins."""
+
+ def __init__(
+ self,
+ session_factory: sessionmaker[Session],
+ storage: LocalObjectStorage,
+ vector_store: FakeVectorStore,
+ ) -> None:
+ self._delegate = ParserRouter()
+ self._session_factory = session_factory
+ self._storage = storage
+ self._vector_store = vector_store
+ self.calls = 0
+ self.retry_started_clean = False
+
+ def parse(
+ self,
+ document_id: UUID,
+ filename: str,
+ source: BinaryIO,
+ ) -> ParsedDocument:
+ self.calls += 1
+ if self.calls == 2:
+ with self._session_factory() as session:
+ document = session.get(Document, document_id)
+ chunks = session.scalars(
+ select(Chunk).where(Chunk.document_id == document_id)
+ ).all()
+ # The source stream is already open here, proving it survived cleanup.
+ self.retry_started_clean = (
+ document is not None
+ and document.parsed_storage_key is None
+ and not chunks
+ and not self._vector_store.records
+ and self._storage.exists(document.storage_key)
+ )
+ return self._delegate.parse(document_id, filename, source)
+
+
+class CleanupFailingVectorStore(FakeVectorStore):
+ """Inject a retry cleanup failure without affecting the first attempt."""
+
+ fail_delete = False
+
+ def delete_document(self, document_id: UUID, index_profile_id: str) -> None:
+ if self.fail_delete:
+ raise RuntimeError("cleanup provider failed secret=must-not-persist")
+ super().delete_document(document_id, index_profile_id)
+
+
+def _build_runner(
+ tmp_path: Path,
+ *,
+ embedding: FailOnceEmbeddingProvider | FakeEmbeddingProvider | None = None,
+ vector_store: FakeVectorStore | None = None,
+ max_attempts: int = 3,
+) -> tuple[
+ DocumentUploadService,
+ IngestionJobService,
+ IngestionJobRunner,
+ sessionmaker[Session],
+ LocalObjectStorage,
+ FakeVectorStore,
+ RetryObservingParser,
+]:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ session_factory = create_session_factory(engine)
+ storage = LocalObjectStorage(tmp_path / "storage")
+ selected_store = vector_store or FakeVectorStore()
+ selected_embedding = embedding or FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
+ profile = IndexProfile(
+ profile_id="test-profile",
+ chunk_strategy_version="structure_v1",
+ embedding_model=selected_embedding.model_name,
+ vector_dimension=selected_embedding.dimensions,
+ )
+ parser = RetryObservingParser(session_factory, storage, selected_store)
+ upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
+ parsing = DocumentParsingService(session_factory, storage, parser)
+ chunking = DocumentChunkingService(
+ session_factory,
+ storage,
+ StructureAwareChunker(max_tokens=450, overlap_tokens=75),
+ )
+ indexing = DocumentIndexingService(
+ session_factory,
+ selected_embedding,
+ selected_store,
+ profile,
+ )
+ jobs = IngestionJobService(session_factory, max_attempts=max_attempts)
+ cleanup = DerivedDocumentCleanupService(
+ session_factory,
+ storage,
+ selected_store,
+ profile.profile_id,
+ )
+ runner = IngestionJobRunner(
+ session_factory,
+ parsing,
+ chunking,
+ indexing,
+ cleanup,
+ retry_backoff_seconds=1,
+ retry_backoff_max_seconds=4,
+ lease_seconds=30,
+ )
+ return upload, jobs, runner, session_factory, storage, selected_store, parser
+
+
+def _submit(
+ upload: DocumentUploadService,
+ jobs: IngestionJobService,
+ payload: bytes = b"# Policy\n\nRefunds take seven days.",
+) -> tuple[UUID, UUID, RecordingDispatcher]:
+ dispatcher = RecordingDispatcher()
+ result = AsyncIngestionSubmissionService(upload, jobs, dispatcher).submit(
+ "policy.md", "text/markdown", BytesIO(payload)
+ )
+ return result.document.id, result.job.id, dispatcher
+
+
+def _make_retry_due(session_factory: sessionmaker[Session], job_id: UUID) -> None:
+ with session_factory() as session:
+ job = session.get(IngestionJob, job_id)
+ assert job is not None
+ job.next_run_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+
+
+def test_transient_failure_cleans_derivatives_before_retry(tmp_path: Path) -> None:
+ embedding = FailOnceEmbeddingProvider()
+ upload, jobs, runner, session_factory, storage, vector_store, parser = _build_runner(
+ tmp_path, embedding=embedding
+ )
+ document_id, job_id, dispatcher = _submit(upload, jobs)
+
+ first = runner.run(job_id)
+ assert first.action == JobRunAction.RETRY
+ assert first.retry_delay_seconds == 1
+ assert dispatcher.job_ids == [job_id]
+ with session_factory() as session:
+ stored_job = session.get(IngestionJob, job_id)
+ assert stored_job is not None
+ assert stored_job.status == IngestionJobStatus.RETRYING
+ assert stored_job.attempt_count == 1
+ assert "must-not-persist" not in (stored_job.error_message or "")
+
+ _make_retry_due(session_factory, job_id)
+ second = runner.run(job_id)
+
+ assert second.status == IngestionJobStatus.SUCCEEDED
+ assert parser.retry_started_clean
+ with session_factory() as session:
+ stored_job = session.get(IngestionJob, job_id)
+ attempts = session.scalars(
+ select(IngestionAttempt)
+ .where(IngestionAttempt.job_id == job_id)
+ .order_by(IngestionAttempt.attempt_number)
+ ).all()
+ assert stored_job is not None
+ assert stored_job.attempt_count == 2
+ assert [attempt.status for attempt in attempts] == [
+ IngestionAttemptStatus.FAILED,
+ IngestionAttemptStatus.SUCCEEDED,
+ ]
+ document = session.get(Document, document_id)
+ assert document is not None
+ assert storage.exists(document.storage_key)
+ assert document.parsed_storage_key is not None
+ assert storage.exists(document.parsed_storage_key)
+ assert vector_store.records
+
+
+def test_permanent_content_failure_does_not_retry(tmp_path: Path) -> None:
+ upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
+ _, job_id, _ = _submit(upload, jobs, payload=b" \n\t")
+
+ result = runner.run(job_id)
+
+ assert result.action == JobRunAction.COMPLETE
+ assert result.status == IngestionJobStatus.FAILED
+ with session_factory() as session:
+ job = session.get(IngestionJob, job_id)
+ assert job is not None
+ assert job.attempt_count == 1
+ assert job.error_message == "document content could not be processed"
+
+
+def test_cleanup_failure_is_terminal_and_preserves_source(tmp_path: Path) -> None:
+ embedding = FailOnceEmbeddingProvider()
+ vector_store = CleanupFailingVectorStore()
+ upload, jobs, runner, session_factory, storage, _, _ = _build_runner(
+ tmp_path,
+ embedding=embedding,
+ vector_store=vector_store,
+ )
+ document_id, job_id, _ = _submit(upload, jobs)
+ assert runner.run(job_id).action == JobRunAction.RETRY
+ _make_retry_due(session_factory, job_id)
+ vector_store.fail_delete = True
+
+ result = runner.run(job_id)
+
+ assert result.status == IngestionJobStatus.FAILED
+ with session_factory() as session:
+ job = session.get(IngestionJob, job_id)
+ assert job is not None
+ assert job.attempt_count == 1 # Cleanup failed before attempt 2 was created.
+ assert job.error_message == "retry cleanup failed"
+ document = session.get(Document, document_id)
+ assert document is not None
+ assert storage.exists(document.storage_key)
+
+
+def test_duplicate_delivery_defers_while_active_lease_is_valid(tmp_path: Path) -> None:
+ upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
+ _, job_id, _ = _submit(upload, jobs)
+ with session_factory() as session:
+ job = session.get(IngestionJob, job_id)
+ assert job is not None
+ job.status = IngestionJobStatus.RUNNING
+ job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=30)
+ session.commit()
+
+ result = runner.run(job_id)
+
+ assert result.action == JobRunAction.RETRY
+ assert result.status == IngestionJobStatus.RUNNING
+ with session_factory() as session:
+ assert session.scalars(select(IngestionAttempt)).all() == []
+
+
+def test_expired_worker_lease_is_recovered_as_a_new_attempt(tmp_path: Path) -> None:
+ upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
+ _, job_id, _ = _submit(upload, jobs)
+ with session_factory() as session:
+ job = session.get(IngestionJob, job_id)
+ assert job is not None
+ job.status = IngestionJobStatus.RUNNING
+ job.attempt_count = 1
+ job.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.add(IngestionAttempt(job_id=job_id, attempt_number=1))
+ session.commit()
+
+ result = runner.run(job_id)
+
+ assert result.status == IngestionJobStatus.SUCCEEDED
+ with session_factory() as session:
+ attempts = session.scalars(
+ select(IngestionAttempt)
+ .where(IngestionAttempt.job_id == job_id)
+ .order_by(IngestionAttempt.attempt_number)
+ ).all()
+ assert [attempt.status for attempt in attempts] == [
+ IngestionAttemptStatus.FAILED,
+ IngestionAttemptStatus.SUCCEEDED,
+ ]
+ assert attempts[0].error_message == "worker execution lease expired"
+
+
+def test_transient_failures_stop_at_configured_attempt_bound(tmp_path: Path) -> None:
+ embedding = AlwaysFailEmbeddingProvider()
+ upload, jobs, runner, session_factory, _, _, _ = _build_runner(
+ tmp_path,
+ embedding=embedding,
+ max_attempts=2,
+ )
+ _, job_id, _ = _submit(upload, jobs)
+
+ assert runner.run(job_id).action == JobRunAction.RETRY
+ _make_retry_due(session_factory, job_id)
+ result = runner.run(job_id)
+
+ assert result.action == JobRunAction.COMPLETE
+ assert result.status == IngestionJobStatus.FAILED
+ assert runner.run(job_id).status == IngestionJobStatus.FAILED
+ with session_factory() as session:
+ job = session.get(IngestionJob, job_id)
+ attempts = session.scalars(
+ select(IngestionAttempt).where(IngestionAttempt.job_id == job_id)
+ ).all()
+ assert job is not None
+ assert job.attempt_count == job.max_attempts == 2
+ assert len(attempts) == 2
+
+
+def test_recovery_dispatches_persisted_job_after_broker_outage(tmp_path: Path) -> None:
+ upload, jobs, _, _, _, _, _ = _build_runner(tmp_path)
+ failing = RecordingDispatcher(fail=True)
+ submission = AsyncIngestionSubmissionService(upload, jobs, failing).submit(
+ "policy.md", "text/markdown", BytesIO(b"policy")
+ )
+ recovered = RecordingDispatcher()
+
+ assert dispatch_pending_jobs(jobs, recovered) == 1
+ assert recovered.job_ids == [submission.job.id]
+
+
+def test_job_persistence_failure_compensates_document_and_source(tmp_path: Path) -> None:
+ upload, _, _, session_factory, _, _, _ = _build_runner(tmp_path)
+ failing_jobs = FailingJobService(session_factory, max_attempts=3)
+
+ with pytest.raises(RuntimeError, match="Job persistence failure"):
+ AsyncIngestionSubmissionService(upload, failing_jobs, RecordingDispatcher()).submit(
+ "policy.md", "text/markdown", BytesIO(b"policy")
+ )
+
+ with session_factory() as session:
+ assert session.scalars(select(Document)).all() == []
+ assert session.scalars(select(IngestionJob)).all() == []
+ assert not [path for path in tmp_path.rglob("*") if path.is_file()]
+
+
+def test_database_rejects_two_active_attempts_for_one_job(tmp_path: Path) -> None:
+ upload, jobs, _, session_factory, _, _, _ = _build_runner(tmp_path)
+ _, job_id, _ = _submit(upload, jobs)
+
+ with session_factory() as session:
+ session.add_all(
+ [
+ IngestionAttempt(job_id=job_id, attempt_number=1),
+ IngestionAttempt(job_id=job_id, attempt_number=2),
+ ]
+ )
+ with pytest.raises(IntegrityError):
+ session.commit()
+ session.rollback()
diff --git a/tests/integration/test_document_migration.py b/tests/integration/test_document_migration.py
index 2075d96..0a32b24 100644
--- a/tests/integration/test_document_migration.py
+++ b/tests/integration/test_document_migration.py
@@ -23,9 +23,14 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
command.check(config)
inspector = inspect(create_engine(database_url))
- assert {"documents", "chunks", "chunk_index_states", "index_profiles"}.issubset(
- inspector.get_table_names()
- )
+ assert {
+ "documents",
+ "chunks",
+ "chunk_index_states",
+ "index_profiles",
+ "ingestion_jobs",
+ "ingestion_attempts",
+ }.issubset(inspector.get_table_names())
assert "unrelated_provider_table" in inspector.get_table_names()
assert {column["name"] for column in inspector.get_columns("documents")} == {
"id",
@@ -70,3 +75,29 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
"activated_at",
"active_marker",
}
+ assert {column["name"] for column in inspector.get_columns("ingestion_jobs")} == {
+ "id",
+ "document_id",
+ "status",
+ "stage",
+ "attempt_count",
+ "max_attempts",
+ "error_message",
+ "next_run_at",
+ "lease_expires_at",
+ "created_at",
+ "updated_at",
+ "completed_at",
+ }
+ assert {column["name"] for column in inspector.get_columns("ingestion_attempts")} == {
+ "id",
+ "job_id",
+ "attempt_number",
+ "status",
+ "stage",
+ "retryable",
+ "error_message",
+ "active_marker",
+ "started_at",
+ "finished_at",
+ }
diff --git a/tests/integration/test_document_search_api.py b/tests/integration/test_document_search_api.py
index b38a9e1..d859361 100644
--- a/tests/integration/test_document_search_api.py
+++ b/tests/integration/test_document_search_api.py
@@ -19,6 +19,12 @@ from ekp_core.document_indexing import DocumentIndexingPipelineService, Document
from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
+from ekp_core.ingestion_jobs import (
+ AsyncIngestionSubmissionService,
+ DerivedDocumentCleanupService,
+ IngestionJobRunner,
+ IngestionJobService,
+)
from ekp_core.search import SearchService
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile
from ekp_embedding import FakeEmbeddingProvider
@@ -39,6 +45,16 @@ REQUIRED_SOURCE_FIELDS = {
}
+class RecordingDispatcher:
+ """Capture durable Job deliveries without starting a real broker."""
+
+ def __init__(self) -> None:
+ self.job_ids: list[UUID] = []
+
+ def enqueue(self, job_id: UUID) -> None:
+ self.job_ids.append(job_id)
+
+
def _build_api_services(
tmp_path: Path,
) -> tuple[
@@ -46,6 +62,7 @@ def _build_api_services(
sessionmaker[Session],
LocalObjectStorage,
FakeVectorStore,
+ IngestionJobRunner,
]:
"""Build the production service graph with offline providers and shared SQLite."""
engine = create_engine(
@@ -82,6 +99,24 @@ def _build_api_services(
profile,
)
ingestion = DocumentIndexingPipelineService(pipeline, indexing)
+ jobs = IngestionJobService(session_factory, max_attempts=3)
+ dispatcher = RecordingDispatcher()
+ submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher)
+ runner = IngestionJobRunner(
+ session_factory,
+ parsing,
+ chunking,
+ indexing,
+ DerivedDocumentCleanupService(
+ session_factory,
+ storage,
+ vector_store,
+ profile.profile_id,
+ ),
+ retry_backoff_seconds=1,
+ retry_backoff_max_seconds=4,
+ lease_seconds=30,
+ )
dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20)
search = SearchService(
session_factory,
@@ -96,10 +131,17 @@ def _build_api_services(
active_index_profile_id=profile.profile_id,
)
return (
- ApiServices(ingestion=ingestion, documents=documents, search=search),
+ ApiServices(
+ ingestion=ingestion,
+ submissions=submissions,
+ jobs=jobs,
+ documents=documents,
+ search=search,
+ ),
session_factory,
storage,
vector_store,
+ runner,
)
@@ -107,7 +149,7 @@ def test_document_and_search_rest_flow(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
- services, session_factory, storage, vector_store = _build_api_services(tmp_path)
+ services, session_factory, storage, vector_store, runner = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
@@ -121,19 +163,33 @@ def test_document_and_search_rest_flow(
)
},
)
- assert upload_response.status_code == 201
+ assert upload_response.status_code == 202
uploaded = upload_response.json()
document_id = UUID(uploaded["id"])
- assert uploaded["status"] == "ready"
+ job_id = UUID(uploaded["job_id"])
+ assert uploaded["status"] == "uploaded"
+ assert uploaded["job_status"] == "queued"
assert uploaded["filename"] == "退款政策.md"
assert "storage_key" not in uploaded
+ queued = client.get(f"/api/v1/jobs/{job_id}")
+ assert queued.status_code == 200
+ assert queued.json()["attempt"] == 0
+ assert queued.json()["status"] == "queued"
+ assert client.delete(f"/api/v1/documents/{document_id}").status_code == 409
+
+ runner.run(job_id)
+
list_response = client.get("/api/v1/documents")
detail_response = client.get(f"/api/v1/documents/{document_id}")
assert list_response.status_code == 200
assert [item["id"] for item in list_response.json()] == [str(document_id)]
assert detail_response.status_code == 200
- assert detail_response.json() == uploaded
+ assert detail_response.json()["status"] == "ready"
+ completed = client.get(f"/api/v1/jobs/{job_id}").json()
+ assert completed["status"] == "succeeded"
+ assert completed["stage"] == "indexing"
+ assert completed["attempt"] == 1
search_response = client.post(
"/api/v1/search",
@@ -173,7 +229,7 @@ def test_rest_validation_not_found_and_openapi_contract(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
- services, _, _, _ = _build_api_services(tmp_path)
+ services, _, _, _, _ = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
missing_id = uuid4()
@@ -182,6 +238,7 @@ def test_rest_validation_not_found_and_openapi_contract(
assert set(openapi["paths"]) >= {
"/api/v1/documents",
"/api/v1/documents/{document_id}",
+ "/api/v1/jobs/{job_id}",
"/api/v1/search",
}
assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"}
diff --git a/tests/unit/test_celery_queue.py b/tests/unit/test_celery_queue.py
new file mode 100644
index 0000000..fb830cb
--- /dev/null
+++ b/tests/unit/test_celery_queue.py
@@ -0,0 +1,57 @@
+"""Unit tests for the Celery adapter without opening a broker connection."""
+
+from unittest.mock import Mock
+from uuid import uuid4
+
+import pytest
+from pydantic import SecretStr
+
+from ekp_api.celery_queue import (
+ INGESTION_TASK_NAME,
+ RECOVERY_TASK_NAME,
+ CeleryIngestionDispatcher,
+ create_celery_app,
+)
+from ekp_core.config import Settings
+
+
+def test_celery_app_enables_late_ack_and_recovery_schedule(
+ isolated_settings_environment: None,
+) -> None:
+ settings = Settings(rabbitmq_url=SecretStr("amqp://user:test@queue.example/ekp"))
+
+ app = create_celery_app(settings)
+
+ assert app.conf.task_acks_late is True
+ assert app.conf.task_reject_on_worker_lost is True
+ assert app.conf.worker_prefetch_multiplier == 1
+ assert app.conf.task_routes[INGESTION_TASK_NAME]["queue"] == "ingestion"
+ assert app.conf.task_routes[RECOVERY_TASK_NAME]["queue"] == "ingestion"
+ assert app.conf.beat_schedule["recover-durable-ingestion-jobs"]["schedule"] == 30.0
+
+
+def test_celery_dispatcher_sends_only_job_identifier(
+ monkeypatch: pytest.MonkeyPatch,
+ isolated_settings_environment: None,
+) -> None:
+ app = create_celery_app(Settings(rabbitmq_url=SecretStr("amqp://user:test@queue/ekp")))
+ send_task = Mock()
+ monkeypatch.setattr(app, "send_task", send_task)
+ job_id = uuid4()
+
+ CeleryIngestionDispatcher(app).enqueue(job_id)
+
+ send_task.assert_called_once_with(
+ INGESTION_TASK_NAME,
+ args=[str(job_id)],
+ queue="ingestion",
+ )
+
+
+def test_celery_app_requires_runtime_broker_secret(
+ isolated_settings_environment: None,
+) -> None:
+ with pytest.raises(ValueError, match="RABBITMQ_URL"):
+ create_celery_app(Settings())
+ with pytest.raises(ValueError, match="RABBITMQ_URL"):
+ create_celery_app(Settings(rabbitmq_url=SecretStr("")))
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 29831c8..faec677 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -20,6 +20,11 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
assert settings.database_pool_recycle_seconds == 600
assert settings.database_pool_timeout_seconds == 30
assert settings.database_connect_timeout_seconds == 30
+ assert settings.rabbitmq_url is None
+ assert settings.ingestion_max_attempts == 3
+ assert settings.ingestion_retry_backoff_seconds == 2
+ assert settings.ingestion_retry_backoff_max_seconds == 60
+ assert settings.ingestion_worker_lease_seconds == 300
assert settings.object_storage_provider == "bos"
assert settings.object_storage_path == Path("storage")
assert settings.bos_endpoint is None
@@ -90,6 +95,7 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) -
oceanbase_vector_password=SecretStr("oceanbase-secret"),
reranker_api_key=SecretStr("reranker-secret"),
mcp_auth_token=SecretStr("mcp-secret"),
+ rabbitmq_url=SecretStr("amqp://user:rabbit-secret@queue.example/ekp"),
)
rendered = repr(settings)
@@ -100,6 +106,7 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) -
assert "oceanbase-secret" not in rendered
assert "reranker-secret" not in rendered
assert "mcp-secret" not in rendered
+ assert "rabbit-secret" not in rendered
def test_settings_restrict_sqlite_to_test_environment(
@@ -136,3 +143,17 @@ def test_settings_reject_empty_mcp_allowed_hosts(
) -> None:
with pytest.raises(ValidationError, match="mcp_allowed_hosts"):
Settings(mcp_allowed_hosts=[])
+
+
+def test_settings_reject_retry_backoff_larger_than_cap(
+ isolated_settings_environment: None,
+) -> None:
+ with pytest.raises(ValidationError, match="initial backoff"):
+ Settings(ingestion_retry_backoff_seconds=10, ingestion_retry_backoff_max_seconds=5)
+
+
+def test_settings_require_worker_lease_longer_than_provider_timeout(
+ isolated_settings_environment: None,
+) -> None:
+ with pytest.raises(ValidationError, match="worker_lease"):
+ Settings(ingestion_worker_lease_seconds=30, embedding_timeout_seconds=30)
diff --git a/uv.lock b/uv.lock
index a7b88f8..3aac7f3 100644
--- a/uv.lock
+++ b/uv.lock
@@ -28,6 +28,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/20/89/e62cc37b69ad357cc8ecd6e7367f5245f523d3cbb338a66197212bdf6749/alembic-1.19.1-py3-none-any.whl", hash = "sha256:b39018cb3d9413a19cbd54cf3c02ad33998641f0538eb77413a488a21c3e14be", size = 265946, upload-time = "2026-08-08T16:32:03.153Z" },
]
+[[package]]
+name = "amqp"
+version = "5.3.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "vine" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" },
+]
+
[[package]]
name = "annotated-doc"
version = "0.0.5"
@@ -83,6 +95,35 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ef/de/52b763b183e23f4dd73811a2dd2664824ae7fc6ceb1aa8dcd3f1102b1094/bce_python_sdk-0.9.79-py3-none-any.whl", hash = "sha256:71799ac8740505e0759d30873f6f1a478fa8f83aedf425d511f1419a5f30082e", size = 440268, upload-time = "2026-09-01T05:48:50.459Z" },
]
+[[package]]
+name = "billiard"
+version = "4.2.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" },
+]
+
+[[package]]
+name = "celery"
+version = "5.6.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "billiard" },
+ { name = "click" },
+ { name = "click-didyoumean" },
+ { name = "click-plugins" },
+ { name = "click-repl" },
+ { name = "kombu" },
+ { name = "python-dateutil" },
+ { name = "tzlocal" },
+ { name = "vine" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" },
+]
+
[[package]]
name = "cffi"
version = "2.1.1"
@@ -129,6 +170,43 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" },
]
+[[package]]
+name = "click-didyoumean"
+version = "0.3.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "click" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" },
+]
+
+[[package]]
+name = "click-plugins"
+version = "1.1.1.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "click" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" },
+]
+
+[[package]]
+name = "click-repl"
+version = "0.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "click" },
+ { name = "prompt-toolkit" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" },
+]
+
[[package]]
name = "colorama"
version = "0.4.6"
@@ -249,6 +327,7 @@ source = { editable = "." }
dependencies = [
{ name = "alembic" },
{ name = "bce-python-sdk" },
+ { name = "celery" },
{ name = "fastapi" },
{ name = "mcp" },
{ name = "pydantic-settings" },
@@ -274,6 +353,7 @@ dev = [
requires-dist = [
{ name = "alembic", specifier = ">=1.18,<2" },
{ name = "bce-python-sdk", specifier = ">=0.9.72,<0.10" },
+ { name = "celery", specifier = ">=5.5,<6" },
{ name = "fastapi", specifier = ">=0.115,<1" },
{ name = "mcp", specifier = ">=2,<3" },
{ name = "pydantic-settings", specifier = ">=2.7,<3" },
@@ -459,6 +539,21 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" },
]
+[[package]]
+name = "kombu"
+version = "5.6.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "amqp" },
+ { name = "packaging" },
+ { name = "tzdata" },
+ { name = "vine" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" },
+]
+
[[package]]
name = "librt"
version = "0.15.0"
@@ -740,6 +835,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
]
+[[package]]
+name = "prompt-toolkit"
+version = "3.0.53"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "wcwidth" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" },
+]
+
[[package]]
name = "pycparser"
version = "3.0"
@@ -949,6 +1056,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" },
]
+[[package]]
+name = "python-dateutil"
+version = "2.9.0.post0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "six" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
+]
+
[[package]]
name = "python-docx"
version = "1.2.0"
@@ -1199,6 +1318,27 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" },
]
+[[package]]
+name = "tzdata"
+version = "2026.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" },
+]
+
+[[package]]
+name = "tzlocal"
+version = "5.4.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "tzdata", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/81/5b/879b2f932adfa7a053c360d50bc896c977fa6426109185f7c12ebdd0cb9d/tzlocal-5.4.4.tar.gz", hash = "sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4", size = 31170, upload-time = "2026-06-29T08:03:40.026Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9e/a4/017a7a6cbe387d961a688ec31364ae60a5c4e22c96ae9921b79a947c855d/tzlocal-5.4.4-py3-none-any.whl", hash = "sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15", size = 18115, upload-time = "2026-06-29T08:03:38.666Z" },
+]
+
[[package]]
name = "uvicorn"
version = "0.52.4"
@@ -1242,6 +1382,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" },
]
+[[package]]
+name = "vine"
+version = "5.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" },
+]
+
[[package]]
name = "watchfiles"
version = "1.2.0"
@@ -1292,6 +1441,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" },
]
+[[package]]
+name = "wcwidth"
+version = "0.8.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/36/57/ed58088fafdf4c55a0ad6bde846502567645424d7ebf325230b9237f4085/wcwidth-0.8.3.tar.gz", hash = "sha256:d128512515fbf4612e0ff21fd6380399210318b7b54a9af59dff8454cf9730eb", size = 1458450, upload-time = "2026-08-28T18:10:06.875Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/c4/0e/57f6bb3024a597b2e8ec4aee710ffe62ddc95af2e2bb1ee7a7abdc22c68c/wcwidth-0.8.3-py3-none-any.whl", hash = "sha256:d5b73dba6158a595ec9370350e7f2637bcac8d6c5e4fde34f30fcffb6103a5e4", size = 331669, upload-time = "2026-08-28T18:10:04.909Z" },
+]
+
[[package]]
name = "websockets"
version = "17.1"