feat: add durable async ingestion jobs

This commit is contained in:
d-robotics 2026-09-02 20:18:58 +08:00
parent 4d73465888
commit 7938c0b9cf
39 changed files with 2359 additions and 104 deletions

View File

@ -11,8 +11,13 @@ DATABASE_MAX_OVERFLOW=10
DATABASE_POOL_RECYCLE_SECONDS=600
DATABASE_POOL_TIMEOUT_SECONDS=30
DATABASE_CONNECT_TIMEOUT_SECONDS=30
REDIS_URL=redis://localhost:6379/0
RABBITMQ_URL=amqp://guest:guest@localhost:5672//
# DEMO-015 queue. Inject the real credential at runtime; never commit its value.
RABBITMQ_URL=
INGESTION_MAX_ATTEMPTS=3
INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
OBJECT_STORAGE_PROVIDER=bos
OBJECT_STORAGE_PATH=storage

View File

@ -1,4 +1,4 @@
.PHONY: help install run run-mcp format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
.PHONY: help install run run-mcp run-worker format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
export UV_CACHE_DIR
@ -8,6 +8,7 @@ help:
@echo " install Install development dependencies"
@echo " run Start the development API"
@echo " run-mcp Start the authenticated MCP gateway"
@echo " run-worker Start the ingestion worker and recovery scheduler"
@echo " format Format code"
@echo " format-check Check formatting without modifying files"
@echo " lint Run linters"
@ -27,6 +28,8 @@ run:
uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}"
run-mcp:
uv run ekp-mcp
run-worker:
uv run ekp-worker
format:
uv run ruff format .
format-check:

View File

@ -53,6 +53,7 @@ Install dependencies and run the API:
```bash
cp .env.example .env
make install
make run-worker
make run
```
@ -100,8 +101,11 @@ Chunk metadata, including deterministic IDs, filename, page range, section,
index, and strategy version, is stored atomically in canonical relational tables. The lightweight
token counter is replaceable so DEMO-006 can inject model-specific behavior.
Documents remain `processing` until embedding/indexing completes; pipeline
failures become `failed` without deleting the immutable original. DEMO-009
exposes this pipeline through the versioned upload endpoint.
failures become `failed` without deleting the immutable original. DEMO-015 now
accepts a source plus durable Job with HTTP `202`, while a Celery/RabbitMQ worker
runs the pipeline. Bounded transient retries clean every derived artifact before
starting the next Attempt; broker redelivery and expired worker leases converge on
OceanBase Job state. See `docs/development/ASYNC_INGESTION_SETUP.md`.
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a
validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase.
@ -128,8 +132,8 @@ adapter with the `bce-reranker-base` baseline. The final Top20-to-Top5 pipeline
source results are rehydrated from canonical relational chunks and retain both
Dense and reranker scores; stale vector content fails closed. See
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
DEMO-009 exposes the completed synchronous pipeline through versioned document
and search REST endpoints under `/api/v1`. Upload, list, detail, direct delete,
DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
changes upload to asynchronous acceptance and adds Job status. Upload, list, detail, direct delete,
and reranked source search are documented in
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
outside the public response contract.

View File

@ -100,7 +100,7 @@ function initializeUpload() {
form.addEventListener("submit", async (event) => {
event.preventDefault();
if (!input.files.length) return;
setButtonBusy(button, true, "处理中…");
setButtonBusy(button, true, "提交中…");
try {
const formData = new FormData();
formData.append("file", input.files[0]);
@ -109,7 +109,7 @@ function initializeUpload() {
body: formData,
});
renderUploadResult(documentData);
showNotice("文档已上传并完成处理。可以前往 Search Lab 检索。", "success");
showNotice("文档已接收并进入后台处理队列。", "success");
form.reset();
updateFilename();
} catch (error) {

View File

@ -67,7 +67,7 @@
<div>
<p class="section-label">01 / INGESTION</p>
<h2>让知识进入检索链路</h2>
<p>上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将同步完成解析、切分与索引。</p>
<p>上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将在后台完成解析、切分与索引。</p>
</div>
</div>

View File

@ -0,0 +1,55 @@
"""Celery/RabbitMQ adapter behind the queue-neutral ingestion dispatcher."""
from uuid import UUID
from celery import Celery # type: ignore[import-untyped]
from ekp_core.config import Settings
INGESTION_TASK_NAME = "ekp.ingestion.run"
RECOVERY_TASK_NAME = "ekp.ingestion.recover"
def create_celery_app(settings: Settings) -> Celery:
"""Create a secret-safe broker client without opening a connection eagerly."""
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
raise ValueError("RABBITMQ_URL is required for asynchronous ingestion")
app = Celery(
"enterprise-knowledge-platform",
broker=settings.rabbitmq_url.get_secret_value(),
)
app.conf.update(
task_ignore_result=True,
task_acks_late=True,
task_reject_on_worker_lost=True,
worker_prefetch_multiplier=1,
broker_connection_retry_on_startup=True,
timezone="UTC",
enable_utc=True,
task_routes={
INGESTION_TASK_NAME: {"queue": "ingestion"},
RECOVERY_TASK_NAME: {"queue": "ingestion"},
},
beat_schedule={
"recover-durable-ingestion-jobs": {
"task": RECOVERY_TASK_NAME,
"schedule": float(settings.ingestion_recovery_interval_seconds),
}
},
)
return app
class CeleryIngestionDispatcher:
"""Publish only a durable Job id; source bytes and state stay canonical."""
def __init__(self, app: Celery) -> None:
self._app = app
def enqueue(self, job_id: UUID) -> None:
"""Deliver one idempotent execution hint to the ingestion queue."""
self._app.send_task(
INGESTION_TASK_NAME,
args=[str(job_id)],
queue="ingestion",
)

View File

@ -4,6 +4,7 @@ from dataclasses import dataclass
from sqlalchemy.orm import Session, sessionmaker
from ekp_api.celery_queue import CeleryIngestionDispatcher, create_celery_app
from ekp_core.chunking import StructureAwareChunker
from ekp_core.config import Settings
from ekp_core.database import create_database_engine, create_session_factory
@ -17,6 +18,12 @@ from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.index_rebuild import resolve_active_index_profile
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
)
from ekp_core.search import SearchService
from ekp_domain import IndexProfile
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
@ -42,10 +49,38 @@ class ApiServices:
"""Business services injected into REST routes without provider leakage."""
ingestion: DocumentIndexingPipelineService
submissions: AsyncIngestionSubmissionService
jobs: IngestionJobService
documents: DocumentManagementService
search: SearchService
@dataclass(frozen=True, slots=True)
class WorkerServices:
"""Provider graph required by Celery without REST search-only services."""
runner: IngestionJobRunner
jobs: IngestionJobService
dispatcher: CeleryIngestionDispatcher
@dataclass(frozen=True, slots=True)
class _IngestionComponents:
"""Shared composition result used by API and independent worker processes."""
session_factory: sessionmaker[Session]
storage: ObjectStorage
vector_store: VectorStore
embedding_provider: EmbeddingProvider
index_profile: IndexProfile
upload: DocumentUploadService
parsing: DocumentParsingService
chunking: DocumentChunkingService
indexing: DocumentIndexingService
jobs: IngestionJobService
dispatcher: CeleryIngestionDispatcher
def create_index_profile(
settings: Settings,
session_factory: sessionmaker[Session] | None = None,
@ -161,8 +196,31 @@ def create_search_service(
)
def create_api_services(settings: Settings) -> ApiServices:
"""Compose the synchronous Demo runtime once for all business endpoints."""
def create_runtime_search_service(settings: Settings) -> SearchService:
"""Compose only read-path providers for the independent MCP Gateway."""
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
index_profile = create_index_profile(settings, session_factory)
embedding_provider = create_embedding_provider(settings)
vector_store = create_vector_store(settings)
dense_retrieval = create_dense_retrieval_service(
settings,
embedding_provider,
vector_store,
index_profile,
)
return create_search_service(
settings,
session_factory,
dense_retrieval,
create_reranker_provider(settings),
)
def _create_ingestion_components(settings: Settings) -> _IngestionComponents:
"""Compose provider-neutral ingestion stages and the Celery delivery adapter."""
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
raise ProviderConfigurationError("RABBITMQ_URL is required")
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
storage = create_object_storage(settings)
@ -185,35 +243,96 @@ def create_api_services(settings: Settings) -> ApiServices:
max_characters=settings.chunk_max_characters,
),
)
document_pipeline = DocumentPipelineService(
upload_service,
parsing_service,
chunking_service,
)
indexing_service = DocumentIndexingService(
session_factory,
embedding_provider,
vector_store,
index_profile,
)
ingestion = DocumentIndexingPipelineService(document_pipeline, indexing_service)
jobs = IngestionJobService(session_factory, settings.ingestion_max_attempts)
dispatcher = CeleryIngestionDispatcher(create_celery_app(settings))
return _IngestionComponents(
session_factory=session_factory,
storage=storage,
vector_store=vector_store,
embedding_provider=embedding_provider,
index_profile=index_profile,
upload=upload_service,
parsing=parsing_service,
chunking=chunking_service,
indexing=indexing_service,
jobs=jobs,
dispatcher=dispatcher,
)
def _create_runner(settings: Settings, components: _IngestionComponents) -> IngestionJobRunner:
"""Compose retry cleanup and execution policy from non-sensitive settings."""
cleanup = DerivedDocumentCleanupService(
components.session_factory,
components.storage,
components.vector_store,
components.index_profile.profile_id,
)
return IngestionJobRunner(
components.session_factory,
components.parsing,
components.chunking,
components.indexing,
cleanup,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
retry_backoff_max_seconds=settings.ingestion_retry_backoff_max_seconds,
lease_seconds=settings.ingestion_worker_lease_seconds,
)
def create_worker_services(settings: Settings) -> WorkerServices:
"""Build the independent worker runtime from the same application boundaries."""
components = _create_ingestion_components(settings)
return WorkerServices(
runner=_create_runner(settings, components),
jobs=components.jobs,
dispatcher=components.dispatcher,
)
def create_api_services(settings: Settings) -> ApiServices:
"""Compose REST services while ingestion execution remains in workers."""
components = _create_ingestion_components(settings)
document_pipeline = DocumentPipelineService(
components.upload,
components.parsing,
components.chunking,
)
ingestion = DocumentIndexingPipelineService(document_pipeline, components.indexing)
submissions = AsyncIngestionSubmissionService(
components.upload,
components.jobs,
components.dispatcher,
)
dense_retrieval = create_dense_retrieval_service(
settings,
embedding_provider,
vector_store,
index_profile,
components.embedding_provider,
components.vector_store,
components.index_profile,
)
reranker_provider = create_reranker_provider(settings)
search = create_search_service(
settings,
session_factory,
components.session_factory,
dense_retrieval,
reranker_provider,
)
documents = DocumentManagementService(
session_factory,
storage,
vector_store,
active_index_profile_id=index_profile.profile_id,
components.session_factory,
components.storage,
components.vector_store,
active_index_profile_id=components.index_profile.profile_id,
)
return ApiServices(
ingestion=ingestion,
submissions=submissions,
jobs=components.jobs,
documents=documents,
search=search,
)
return ApiServices(ingestion=ingestion, documents=documents, search=search)

View File

@ -6,10 +6,17 @@ from uuid import UUID
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services
from ekp_api.schemas import DocumentResponse, SearchRequest, SearchResponse
from ekp_api.schemas import (
DocumentResponse,
IngestionAcceptedResponse,
IngestionJobResponse,
SearchRequest,
SearchResponse,
)
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
from ekp_core.document_management import (
DocumentArtifactCleanupError,
DocumentIngestionActiveError,
DocumentNotFoundError,
)
from ekp_core.document_upload import (
@ -18,7 +25,7 @@ from ekp_core.document_upload import (
MediaTypeMismatchError,
UnsupportedDocumentTypeError,
)
from ekp_parser import DocumentParseError
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
router = APIRouter()
@ -44,17 +51,17 @@ Services = Annotated[ApiServices, Depends(get_api_services)]
@router.post(
"/documents",
response_model=DocumentResponse,
status_code=status.HTTP_201_CREATED,
response_model=IngestionAcceptedResponse,
status_code=status.HTTP_202_ACCEPTED,
tags=["documents"],
)
def upload_document(
services: Services,
file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")],
) -> DocumentResponse:
"""Upload one file and synchronously run the complete Demo ingestion pipeline."""
) -> IngestionAcceptedResponse:
"""Persist one immutable source and durable Job, then return immediately."""
try:
result = services.ingestion.ingest(file.filename or "", file.content_type, file.file)
submission = services.submissions.submit(file.filename or "", file.content_type, file.file)
except DocumentTooLargeError as error:
raise HTTPException(
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
@ -65,21 +72,30 @@ def upload_document(
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
detail="unsupported document type or media type",
) from error
except (DocumentUploadError, DocumentParseError) as error:
except DocumentUploadError as error:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail="document could not be accepted or parsed",
detail="document could not be accepted",
) from error
except Exception as error:
# Provider and persistence exceptions are intentionally not reflected to clients.
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="document processing failed",
detail="document acceptance failed",
) from error
# Reload after every pipeline stage so server-generated timestamps and the
# final persisted status, not a detached intermediate ORM object, are returned.
document = services.documents.get_document(result.ingestion.document.id)
return DocumentResponse.from_document(document)
return IngestionAcceptedResponse.from_submission(submission)
@router.get("/jobs/{job_id}", response_model=IngestionJobResponse, tags=["jobs"])
def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
"""Return durable stage, attempt and safe failure state for one Job."""
try:
job = services.jobs.get(job_id)
except IngestionJobNotFoundError as error:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="job not found"
) from error
return IngestionJobResponse.from_job(job)
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
@ -115,6 +131,11 @@ def delete_document(document_id: UUID, services: Services) -> Response:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
) from error
except DocumentIngestionActiveError as error:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="document ingestion is still active",
) from error
except DocumentArtifactCleanupError as error:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,

View File

@ -5,8 +5,9 @@ from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
from ekp_core.ingestion_jobs import IngestionSubmission
from ekp_core.search import SearchResult
from ekp_domain import Document, DocumentStatus
from ekp_domain import Document, DocumentStatus, IngestionJob, IngestionJobStatus, IngestionStage
class DocumentResponse(BaseModel):
@ -29,6 +30,64 @@ class DocumentResponse(BaseModel):
return cls.model_validate(document)
class IngestionAcceptedResponse(DocumentResponse):
"""Single-file acceptance with the durable Job needed for status polling."""
job_id: UUID
job_status: IngestionJobStatus
max_attempts: int
@classmethod
def from_submission(cls, submission: IngestionSubmission) -> "IngestionAcceptedResponse":
"""Flatten identifiers while retaining the existing document fields."""
document = submission.document
job = submission.job
return cls(
id=document.id,
filename=document.filename,
media_type=document.media_type,
size_bytes=document.size_bytes,
status=document.status,
error_message=document.error_message,
created_at=document.created_at,
updated_at=document.updated_at,
job_id=job.id,
job_status=job.status,
max_attempts=job.max_attempts,
)
class IngestionJobResponse(BaseModel):
"""Public queue-independent Job status without provider or broker details."""
id: UUID
document_id: UUID
status: IngestionJobStatus
stage: IngestionStage | None
attempt: int
max_attempts: int
error_message: str | None
created_at: datetime
updated_at: datetime
completed_at: datetime | None
@classmethod
def from_job(cls, job: IngestionJob) -> "IngestionJobResponse":
"""Serialize only client-actionable state and safe diagnostics."""
return cls(
id=job.id,
document_id=job.document_id,
status=job.status,
stage=job.stage,
attempt=job.attempt_count,
max_attempts=job.max_attempts,
error_message=job.error_message,
created_at=job.created_at,
updated_at=job.updated_at,
completed_at=job.completed_at,
)
class SearchRequest(BaseModel):
"""Validated public search input; normalization remains in the core service."""

View File

@ -0,0 +1,51 @@
"""Celery worker entry point for durable ingestion Jobs."""
from functools import lru_cache
from typing import Any
from uuid import UUID
from ekp_api.celery_queue import (
INGESTION_TASK_NAME,
RECOVERY_TASK_NAME,
create_celery_app,
)
from ekp_api.providers import WorkerServices, create_worker_services
from ekp_core.config import get_settings
from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs
_settings = get_settings()
celery_app = create_celery_app(_settings)
@lru_cache
def _worker_services() -> WorkerServices:
"""Build one provider/service graph per worker process."""
return create_worker_services(_settings)
@celery_app.task( # type: ignore[untyped-decorator]
bind=True,
name=INGESTION_TASK_NAME,
max_retries=None,
)
def run_ingestion(self: Any, job_id: str) -> None:
"""Execute one delivery and ask Celery to redeliver only when required.
Celery delivery retries are unbounded because valid-lease deferrals are not
business Attempts; the durable runner independently enforces max_attempts.
"""
result = _worker_services().runner.run(UUID(job_id))
if result.action == JobRunAction.RETRY:
raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None)
@celery_app.task(name=RECOVERY_TASK_NAME) # type: ignore[untyped-decorator]
def recover_ingestion_jobs() -> int:
"""Redeliver due durable Jobs after API/broker or worker interruptions."""
services = _worker_services()
return dispatch_pending_jobs(services.jobs, services.dispatcher)
def main() -> None:
"""Run one worker with the embedded Demo recovery scheduler."""
celery_app.worker_main(["worker", "--loglevel=INFO", "--queues=ingestion", "--beat"])

View File

@ -12,7 +12,7 @@ from mcp.server.transport_security import TransportSecuritySettings
from pydantic import Field
from starlette.applications import Starlette
from ekp_api.providers import ApiServices, create_api_services
from ekp_api.providers import ApiServices, create_runtime_search_service
from ekp_core.config import Settings, get_settings
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
from ekp_mcp.auth import StaticServiceTokenVerifier, current_mcp_identity
@ -29,8 +29,10 @@ def _create_search_application(
settings: Settings,
api_services: ApiServices | None,
) -> McpSearchApplication:
services = api_services or create_api_services(settings)
return McpSearchApplication(services.search, settings.mcp_required_scope)
searcher = (
api_services.search if api_services is not None else create_runtime_search_service(settings)
)
return McpSearchApplication(searcher, settings.mcp_required_scope)
def create_mcp_server(

View File

@ -11,19 +11,20 @@ Use OpenAPI as executable source of truth once implementation starts. Provider-s
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
版本化为 `/api/v1/*`。
下表描述当前已实现的同步 Demo 契约;它将在 `DEMO-015` 完成时由后文的异步契约替代。
`DEMO-015` 已将单文件上传切换为持久化异步 Job;批量提交仍留给 `DEMO-016`。
| Method | Path | 行为 |
|---|---|---|
| `POST` | `/api/v1/documents` | multipart 单文件上传并同步执行完整入库流水线 |
| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
| `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 |
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `201` 和 Document;
Demo 同步流水线完成后状态为 `ready`。Document 响应不暴露 `storage_key`、Provider 配置或
索引实现。
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `202`、Document 字段及
`job_id`、`job_status=queued`、`max_attempts`;响应不等待 Parse、Clean、Chunk、Embedding 或
Index。Document 响应不暴露 `storage_key`、Provider 配置或索引实现。
Search 请求:
@ -54,8 +55,9 @@ Search 响应:
```
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
文档不存在返回 `404`,运行时 Provider 未配置返回 `503`。上游 Provider 或持久化异常只返回
经过收敛的 `5xx` 信息,不回显凭证、请求正文或厂商响应。
文档或 Job 不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
厂商响应。
## Demo Admin 客户端路由
@ -72,16 +74,34 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
## Planned async ingestion contract(DEMO-015/016,尚未实现)
## 异步入库契约(DEMO-015 已实现;DEMO-016 批量部分待实现)
| Method | Path | 行为 |
|---|---|---|
| `POST` | `/api/v1/documents` | 单文件接收;持久化 Source、Document、Job 后返回 `202` |
| `POST` | `/api/v1/documents` | 已实现:单文件接收并返回 Document/Job 标识 |
| `POST` | `/api/v1/ingestion-batches` | multipart 多文件接收,返回批次及逐文件 Document/Job 标识 |
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新状态 |
| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
| `GET` | `/api/v1/jobs/{job_id}` | 已实现:返回持久化 Job 状态 |
单文件与批量上传都不等待解析、Chunk、Embedding 或索引完成。批量响应示意:
单文件响应示意:
```json
{
"id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
"filename": "退款政策.md",
"media_type": "text/markdown",
"size_bytes": 1024,
"status": "uploaded",
"error_message": null,
"created_at": "2026-09-02T12:00:00Z",
"updated_at": "2026-09-02T12:00:00Z",
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
"job_status": "queued",
"max_attempts": 3
}
```
DEMO-016 的批量响应示意:
```json
{

View File

@ -333,7 +333,7 @@ Search Service 或 `/api/v1/search`,用于验证标准 MCP、identity 边界
### DEMO-015 异步入库 Job 与自动重试清理
将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
**已完成。** 将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
删除旧解析产物、Chunk、索引状态和向量数据,但保留不可变原始文件、Document 和 Attempt
历史。清理失败时终止,不在未知脏状态继续执行。通过执行器抽象接入 Celery/RabbitMQ,使
API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重复消息。
@ -397,8 +397,8 @@ DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
任务拆分;`DEMO-012` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,后续严格按
`DEMO-015`、`DEMO-016`、`DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
任务拆分;`DEMO-012`、`DEMO-015` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
后续严格按 `DEMO-016`、`DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
可选项,不因本状态记录自动进入范围。
可选:
@ -547,7 +547,7 @@ docs/demo/DEMO_SCOPE.md
完整 V1 PRD
只允许实现本文件明确列出的 Demo、Architecture、Plan 与 Deploy Task,并且一次只执行一个
Task。当前严格顺序为 DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
Task。当前严格顺序为 DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
当前公共知识库阶段的业务目标保持为:

View File

@ -5,9 +5,10 @@ DEMO-010 在 FastAPI 服务中提供一个轻量、同源的管理客户端。
## 启动与访问
完成运行时配置后启动 API:
完成运行时配置后启动 Worker 与 API:
```bash
make run-worker
make run
```
@ -34,7 +35,7 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
## Demo 操作闭环
1. 在上传页选择一个 PDF、DOCX、TXT 或 Markdown 文件并提交;
2. 等待同步流水线返回 `Ready`,失败时页面显示经过 API 收敛的错误;
2. 接口返回 `202` 后,当前页面显示已接收;在文档列表刷新查看 `Processing/Ready/Failed`;
3. 在文档列表查看状态和安全元数据;
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
Rerank Score;
@ -45,8 +46,9 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
- 本任务不包含 ACL/RBAC、审计、异步任务、Document Version、Compiled Knowledge、MCP、
Hybrid/BM25 或完整 V1 Admin Console 导航;
- 当前 Console 尚不轮询 Job 详情或展示 Attempt;该状态 UI 与批量上传一起由 DEMO-016 实现;
- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
或完整 V1 Admin Console 导航;
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。
## 验证

View File

@ -0,0 +1,82 @@
# 异步入库与失败恢复说明
## 范围
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker。OceanBase 中的
`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
不携带文件内容、Provider 凭据或业务状态。
本任务不提供批量上传和批次 UI;这些能力属于 DEMO-016。
## 运行时配置
以下值必须通过环境变量或部署 Secret 注入,不得提交真实值:
```text
RABBITMQ_URL=amqp://<user>:<password>@<host>:5672/<vhost>
INGESTION_MAX_ATTEMPTS=3
INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
```
Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
RabbitMQ 用户应只具备指定 vhost/queue 的最小权限。
## 启动顺序
```bash
uv run alembic upgrade head
make run-worker
make run
```
`make run-worker` 在当前 Demo 中同时启动 Worker 和恢复调度器。恢复调度器周期扫描:
- 尚未成功投递的 `queued` Job;
- 已到重试时间的 `retrying` Job;
- Worker 丢失后租约已过期的 `running` Job。
任务投递使用 late acknowledgement、worker-lost reject 和单任务 prefetch。重复消息会先锁定
Job 行;有效租约内不会创建第二个 Attempt。租约过期后,旧 Attempt 以安全错误关闭,再按重试
规则恢复。
## 重试与清理
确定性的内容错误(无法解析、清洗后为空、无 Chunk、派生数据身份不一致)直接进入 `failed`。
其余运行时/Provider 错误按指数退避重试,默认最多三次总 Attempt。
第二次及后续 Attempt 创建前严格执行:
```text
OceanBase Vector rows
→ ChunkIndexState
→ Chunk
→ parsed JSON object
→ reset Document processing state
```
不可变源文件、Document、Job 和已结束 Attempt 始终保留。任一清理步骤失败时不创建新 Attempt,
Job 直接以 `retry cleanup failed` 终止,禁止在未知脏状态继续执行。
## API 状态
`POST /api/v1/documents` 成功返回 `202` 和 `job_id`。查询:
```text
GET /api/v1/jobs/{job_id}
```
Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;阶段为
`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
## 故障检查
1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;
2. Job 长期为 `running`:检查 Worker,租约到期后恢复调度器会重新投递;
3. Job 为 `retrying`:等待 `next_run_at`,客户端不应重新上传同一文件;
4. Job 为 `failed` 且错误为 `retry cleanup failed`:停止人工重试,先核对向量、关系数据与 BOS
派生对象的一致性;
5. 不得通过日志打印 `RABBITMQ_URL`、数据库 URL、BOS/模型凭据或原始 Provider 响应。

View File

@ -2,19 +2,21 @@
## 范围
DEMO-009 将已经完成的同步入库和检索服务公开为版本化 REST API。API 层只负责输入校验、
DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
Job。API 层只负责输入校验、
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
本任务不包含 Admin Console、ACL/RBAC、异步任务、软删除或 Hybrid Search。
当前不包含 ACL/RBAC、软删除、批量上传或 Hybrid Search。
## 启动前提
1. 按 `docs/development/BOS_SETUP.md` 配置 BOS;
2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding;
3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
4. 执行 `uv run alembic upgrade head`;
5. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
6. 执行 `make run`。
4. 按 `docs/development/ASYNC_INGESTION_SETUP.md` 配置 RabbitMQ 并启动 Worker;
5. 执行 `uv run alembic upgrade head`;
6. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
7. 执行 `make run`。
`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时
延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。
@ -25,13 +27,19 @@ Swagger UI:`/api/v1/docs`
## 示例
上传并同步入库:
上传并异步排队:
```bash
curl -F 'file=@./policy.md;type=text/markdown' \
http://127.0.0.1:8000/api/v1/documents
```
使用响应中的 `job_id` 查询进度:
```bash
curl http://127.0.0.1:8000/api/v1/jobs/<job-id>
```
检索:
```bash
@ -40,8 +48,8 @@ curl -H 'Content-Type: application/json' \
http://127.0.0.1:8000/api/v1/search
```
Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase Upsert 完成。大文
件或云端调用较慢时,请求耗时会相应增加;异步任务属于 Demo 后置能力。
上传只等待不可变源文件、Document 与 Job 持久化,成功返回 `202`。Worker 后台执行 Parse、
Clean、Chunk、Embedding 和 OceanBase Upsert;客户端通过 Job 状态判断何时可检索。
## 删除一致性
@ -51,6 +59,8 @@ Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase
2. 删除 OceanBase canonical 关系表中的 ChunkIndexState、Chunk 和 Document;
3. 删除解析 JSON 与不可变原始文件。
`queued`、`running` 或 `retrying` Job 对应的 Document 返回 `409`,避免删除与 Worker 并发。
向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口
返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1
应通过可重试清理任务和审计记录强化。

View File

@ -11,6 +11,11 @@ Provide deterministic fakes:
External integrations use separate integration suites/markers.
Async ingestion tests run with an in-process recording dispatcher and deterministic providers. They
must cover persisted Job/Attempt history, safe error summaries, bounded retry, cleanup-before-retry,
active-lease duplicate delivery, expired-lease recovery, immutable-source preservation and terminal
cleanup failure without requiring a live RabbitMQ broker.
E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval.
Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation.

View File

@ -8,14 +8,14 @@
| 能力 | 当前实现 | 结论 |
|---|---|---|
| 单文件上传 | `POST /api/v1/documents` 接收一个 `file`,并在请求线程内同步完成解析、Chunk、Embedding 和索引 | 已实现,但不满足 V1 异步要求 |
| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
| 批量上传 | API 与 Admin Console 都只支持一个文件 | 未实现 |
| 失败补偿 | 上传元数据失败会删除源对象;解析元数据失败会删除本次解析产物;索引失败会删除当前 Document/Profile 的可能残留向量 | 已有局部补偿,不是自动重试 |
| 自动重试 | 没有持久化 Job、Attempt、调度器、退避策略或自动重新执行入口 | 未实现 |
| 重试前清理 | 影子索引重建能清理目标 Profile 的部分写入;正常上传流水线没有统一清理解析产物、Chunk、索引状态和向量的能力 | 未实现 |
| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
| MCP | 已有 MCP 业务契约和可复用 Search Service/REST API,没有可运行 MCP Gateway 或 WorkBuddy 验证 | 未实现 |
因此,现有失败补偿只能降低一次执行留下半成品的概率,不能满足“失败后自动重新执行,并在重新执行前清理旧数据”。
PLAN-001 的上述缺口已由 DEMO-015 补齐;批量提交与状态 UI 仍由 DEMO-016 实现。
## 2. Task 归并结论

View File

@ -1,12 +1,12 @@
# 项目进度
当前阶段:公共知识库集群交付规划完成,准备实施异步入库
当前阶段:异步单文件入库闭环完成,准备实施批量上传
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014、ARCH-001~ARCH-006、PLAN-001~PLAN-002
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-015、ARCH-001~ARCH-006、PLAN-001~PLAN-002
进行中:无
阻塞:无
下一任务:`DEMO-015`。随后严格执行 `DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
下一任务:`DEMO-016`。随后严格执行 `DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
@ -332,3 +332,25 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
- 验证结果:Markdown 相对链接、敏感值模式和 Git diff 检查通过;Ruff Format/Lint、Mypy、
Alembic Head/离线升级通过;Unit + Integration `165 passed`,RAG Evaluation `5 passed`。
- 下一任务:`DEMO-015`。
### DEMO-015(已完成)
- 新增 OceanBase canonical `ingestion_jobs` / `ingestion_attempts` 与 forward-only migration;
Document 唯一 Job、Attempt 编号上限和单一运行中 Attempt 均由数据库约束保护。
- `POST /api/v1/documents` 改为持久化不可变 Source、Document、Job 后返回 `202`,新增
`GET /api/v1/jobs/{job_id}`;活动 Job 的 Document 删除返回 `409`,避免与 Worker 竞态。
- 新增队列无关的 Dispatcher 与 Job Runner,首个适配器使用 Celery/RabbitMQ;消息只包含 Job
ID,启用 late ack、worker-lost reject、单任务 prefetch 与周期性持久化 Job 恢复扫描。
- 瞬时错误按配置执行指数退避和最多三次总 Attempt;Worker 租约有效时重复投递不创建新
Attempt,租约过期后关闭旧 Attempt 并恢复。错误记录为固定安全摘要,不保存 Provider 正文。
- 第二次及后续 Attempt 前依次清理向量、ChunkIndexState、Chunk 与 parsed JSON,再从不可变
源文件重建;清理失败直接终止且不创建新 Attempt。Job 创建失败会补偿 Document 与源对象。
- Admin Console 仅将上传提示调整为“已接收并进入队列”;批量选择、Job 轮询和逐文件状态 UI
没有提前实现,继续属于 `DEMO-016`。
- 验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、依赖锁、Alembic Head/离线 MySQL SQL
与 SQLite schema drift 均通过;Unit + Integration `179 passed`,RAG Evaluation `5 passed`;
Worker 入口确认注册 `ekp.ingestion.run` 与 `ekp.ingestion.recover`。
- 环境限制:当前工作站没有 Docker/RabbitMQ,也没有持久化注入目标 OceanBase URL,因此未做
真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
- 下一任务:`DEMO-016`。

View File

@ -28,7 +28,7 @@ validation. Provider credentials and internal endpoints are runtime inputs and n
- [x] PLAN-001 MCP 与异步批量入库现状审计、任务拆分和契约基线
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
- [ ] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
- [ ] DEMO-016 Batch async upload API + per-file Job status + Admin Console
- [ ] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
- [ ] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline

View File

@ -16,7 +16,7 @@
“单一公共知识可见域、企业受控网络、持续可用和一键集群部署”推进,不提前实现用户/权限隔离:
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
2. `DEMO-015`:持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
3. `DEMO-016`:批量异步上传、批次/Job 查询和 Admin Console 状态展示;
4. `DEPLOY-001`:API/Admin、MCP Gateway、Async Worker 生产镜像;
5. `DEPLOY-002`:Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;

View File

@ -17,3 +17,7 @@ defaults, and primary keys while excluding those three falsely reported object
types. Full constraint comparison still runs against SQLite in integration
tests; target-cluster checks separately verify HNSW/PRE-FILTER and the
single-active generated unique index.
DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
the latter preserves numbered history and uses a generated nullable active marker plus a unique index
to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.

View File

@ -0,0 +1,174 @@
"""Create durable ingestion jobs and attempt history.
Revision ID: 20260902_0007
Revises: 20260902_0006
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0007"
down_revision: str | None = "20260902_0006"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Persist queue-independent Job state and bounded execution attempts."""
op.create_table(
"ingestion_jobs",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("document_id", sa.Uuid(), nullable=False),
sa.Column(
"status",
sa.Enum(
"queued",
"running",
"retrying",
"succeeded",
"failed",
"cancelled",
name="ingestion_job_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column(
"stage",
sa.Enum(
"parsing",
"cleaning",
"chunking",
"embedding",
"indexing",
name="ingestion_stage",
native_enum=False,
create_constraint=False,
),
nullable=True,
),
sa.Column("attempt_count", sa.Integer(), nullable=False),
sa.Column("max_attempts", sa.Integer(), nullable=False),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
sa.CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
sa.CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
sa.CheckConstraint(
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
name="status_valid",
),
sa.CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
sa.ForeignKeyConstraint(
["document_id"],
["documents.id"],
name="fk_ingestion_jobs_document_id_documents",
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_jobs"),
sa.UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
)
op.create_index(
"ix_ingestion_jobs_status_next_run",
"ingestion_jobs",
["status", "next_run_at"],
unique=False,
)
op.create_table(
"ingestion_attempts",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("job_id", sa.Uuid(), nullable=False),
sa.Column("attempt_number", sa.Integer(), nullable=False),
sa.Column(
"status",
sa.Enum(
"running",
"succeeded",
"failed",
name="ingestion_attempt_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column(
"stage",
sa.Enum(
"parsing",
"cleaning",
"chunking",
"embedding",
"indexing",
name="ingestion_attempt_stage",
native_enum=False,
create_constraint=False,
),
nullable=True,
),
sa.Column("retryable", sa.Boolean(), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column(
"active_marker",
sa.Integer(),
sa.Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
nullable=True,
),
sa.Column(
"started_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
sa.CheckConstraint(
"status IN ('running', 'succeeded', 'failed')",
name="status_valid",
),
sa.CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
sa.ForeignKeyConstraint(
["job_id"],
["ingestion_jobs.id"],
name="fk_ingestion_attempts_job_id_ingestion_jobs",
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_attempts"),
sa.UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
)
op.create_index(
"uq_ingestion_attempts_active",
"ingestion_attempts",
["job_id", "active_marker"],
unique=True,
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -33,6 +33,12 @@ class Settings(BaseSettings):
database_pool_recycle_seconds: int = Field(default=600, ge=1)
database_pool_timeout_seconds: float = Field(default=30.0, gt=0)
database_connect_timeout_seconds: int = Field(default=30, ge=1)
rabbitmq_url: SecretStr | None = None
ingestion_max_attempts: int = Field(default=3, ge=1, le=10)
ingestion_retry_backoff_seconds: int = Field(default=2, ge=1, le=3600)
ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
object_storage_provider: Literal["local", "bos"] = "bos"
object_storage_path: Path = Path("storage")
bos_endpoint: str | None = None
@ -91,6 +97,17 @@ class Settings(BaseSettings):
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
if self.search_top_k > self.dense_top_n:
raise ValueError("search_top_k must not exceed dense_top_n")
if self.ingestion_retry_backoff_max_seconds < self.ingestion_retry_backoff_seconds:
raise ValueError(
"ingestion_retry_backoff_max_seconds must be at least the initial backoff"
)
longest_provider_timeout = max(
self.bos_timeout_seconds,
self.embedding_timeout_seconds,
self.oceanbase_vector_timeout_seconds,
)
if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
raise ValueError("mcp_path must start with '/' and must not end with '/'")
if not self.mcp_allowed_hosts:

View File

@ -1,5 +1,6 @@
"""Application services for building and persisting document chunks."""
from collections.abc import Callable
from dataclasses import dataclass
from hashlib import sha256
from uuid import UUID, uuid5
@ -11,7 +12,7 @@ from ekp_core.chunking import ChunkDraft, ChunkingStrategy
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import deserialize_parsed_document
from ekp_domain import Chunk, Document, DocumentStatus, ParsedDocument
from ekp_domain import Chunk, Document, DocumentStatus, IngestionStage, ParsedDocument
from ekp_object_storage import BinarySource, ObjectStorage
MAX_CHUNKING_ERROR_LENGTH = 2000
@ -54,10 +55,16 @@ class DocumentChunkingService:
self._storage = storage
self._strategy = strategy
def process(self, document_id: UUID) -> tuple[Chunk, ...]:
"""Run DEMO-005 while reserving READY for the later index stage."""
def process(
self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> tuple[Chunk, ...]:
"""Build chunks while optionally reporting the durable Job stage."""
filename, parsed_key = self._start_chunking(document_id)
try:
if stage_callback is not None:
stage_callback(IngestionStage.CHUNKING)
with self._storage.open(parsed_key) as source:
parsed = deserialize_parsed_document(source.read())
if parsed.document_id != document_id or parsed.filename != filename:

View File

@ -1,5 +1,6 @@
"""Embedding and vector-index orchestration for canonical document chunks."""
from collections.abc import Callable
from dataclasses import dataclass
from uuid import UUID, uuid5
@ -7,7 +8,14 @@ from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService
from ekp_domain import Chunk, ChunkIndexState, Document, DocumentStatus, IndexProfile
from ekp_domain import (
Chunk,
ChunkIndexState,
Document,
DocumentStatus,
IndexProfile,
IngestionStage,
)
from ekp_embedding import EmbeddingProvider
from ekp_object_storage import BinarySource
from ekp_vector_store import VectorRecord, VectorStore
@ -67,12 +75,16 @@ class DocumentIndexingService:
self._vector_store = vector_store
self._index_profile = index_profile
def process(self, document_id: UUID) -> DocumentIndexingResult:
"""Index all chunks for one profile, compensating any partial provider write."""
def process(
self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> DocumentIndexingResult:
"""Embed/index chunks while optionally reporting durable Job stages."""
chunks = self._start_indexing(document_id)
upsert_attempted = False
try:
vectors = self._embed_chunks(chunks)
vectors = self._embed_chunks(chunks, stage_callback)
records = tuple(
VectorRecord(
index_record_id=uuid5(
@ -93,6 +105,8 @@ class DocumentIndexingService:
)
for chunk, vector in zip(chunks, vectors, strict=True)
)
if stage_callback is not None:
stage_callback(IngestionStage.INDEXING)
upsert_attempted = True
self._vector_store.upsert(records)
self._record_success(document_id, chunks)
@ -152,13 +166,19 @@ class DocumentIndexingService:
session.commit()
return chunks
def _embed_chunks(self, chunks: tuple[Chunk, ...]) -> tuple[tuple[float, ...], ...]:
"""Respect provider batch limits and validate every returned vector."""
def _embed_chunks(
self,
chunks: tuple[Chunk, ...],
stage_callback: Callable[[IngestionStage], None] | None,
) -> tuple[tuple[float, ...], ...]:
"""Embed bounded batches and renew an optional Job lease before each call."""
vectors: list[tuple[float, ...]] = []
batch_size = self._embedding_provider.max_batch_size
if batch_size <= 0:
raise EmbeddingResultError("embedding provider has an invalid batch size")
for start in range(0, len(chunks), batch_size):
if stage_callback is not None:
stage_callback(IngestionStage.EMBEDDING)
batch = chunks[start : start + batch_size]
embedded = self._embedding_provider.embed([chunk.content for chunk in batch])
if len(embedded) != len(batch):

View File

@ -5,7 +5,7 @@ from uuid import UUID
from sqlalchemy import delete, select
from sqlalchemy.orm import Session, sessionmaker
from ekp_domain import Chunk, ChunkIndexState, Document
from ekp_domain import Chunk, ChunkIndexState, Document, IngestionJob, IngestionJobStatus
from ekp_object_storage import ObjectStorage
from ekp_vector_store import VectorStore
@ -18,6 +18,10 @@ class DocumentArtifactCleanupError(RuntimeError):
"""Raised when deleted metadata leaves an external storage artifact behind."""
class DocumentIngestionActiveError(RuntimeError):
"""Raised when direct deletion would race an active asynchronous Job."""
class DocumentManagementService:
"""Read canonical documents and coordinate the Demo's direct deletion.
@ -63,6 +67,22 @@ class DocumentManagementService:
document = session.get(Document, document_id)
if document is None:
raise DocumentNotFoundError(str(document_id))
active_job_id = session.scalar(
select(IngestionJob.id)
.where(
IngestionJob.document_id == document_id,
IngestionJob.status.in_(
(
IngestionJobStatus.QUEUED,
IngestionJobStatus.RUNNING,
IngestionJobStatus.RETRYING,
)
),
)
.limit(1)
)
if active_job_id is not None:
raise DocumentIngestionActiveError(str(document_id))
source_keys = tuple(
key

View File

@ -1,5 +1,6 @@
"""Application services for parsing and persisting derived document content."""
from collections.abc import Callable
from io import BytesIO
from uuid import UUID
@ -8,7 +9,7 @@ from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_cleaner import clean_parsed_document
from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import serialize_parsed_document
from ekp_domain import Document, DocumentStatus, ParsedDocument
from ekp_domain import Document, DocumentStatus, IngestionStage, ParsedDocument
from ekp_object_storage import BinarySource, ObjectStorage
from ekp_parser import DocumentParseError, Parser
@ -48,15 +49,23 @@ class DocumentParsingService:
self._storage = storage
self._parser = parser
def process(self, document_id: UUID) -> ParsedDocument:
"""Run DEMO-004 synchronously and persist success or failure state."""
def process(
self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> ParsedDocument:
"""Parse and clean while optionally reporting durable Job stage boundaries."""
filename, source_key = self._start_processing(document_id)
parsed_key = f"parsed/{document_id}/document.json"
parsed_object_stored = False
try:
if stage_callback is not None:
stage_callback(IngestionStage.PARSING)
with self._storage.open(source_key) as source:
parsed = self._parser.parse(document_id, filename, source)
if stage_callback is not None:
stage_callback(IngestionStage.CLEANING)
cleaned = clean_parsed_document(parsed)
if not cleaned.blocks:
raise EmptyParsedDocumentError(

View File

@ -1,8 +1,9 @@
"""Validated, transactional source-document upload application service."""
from pathlib import Path
from uuid import uuid4
from uuid import UUID, uuid4
from sqlalchemy import delete
from sqlalchemy.orm import Session, sessionmaker
from ekp_domain import Document
@ -132,6 +133,26 @@ class DocumentUploadService:
return document
def rollback_acceptance(self, document_id: UUID) -> None:
"""Compensate a just-uploaded source when durable Job creation fails.
This narrow method is only valid before parsing starts; normal document
deletion remains coordinated by ``DocumentManagementService``.
"""
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
return
storage_key = document.storage_key
session.execute(delete(Document).where(Document.id == document.id))
session.commit()
try:
self._storage.delete(storage_key)
except Exception as cleanup_error:
raise UploadRollbackError(
f"job creation failed and source cleanup failed for {storage_key}"
) from cleanup_error
def _validate_filename(filename: str) -> str:
"""Accept a plain basename that can be traced without path ambiguity."""

View File

@ -0,0 +1,588 @@
"""Durable asynchronous ingestion orchestration and retry cleanup."""
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from enum import StrEnum
from math import ceil
from typing import Protocol
from uuid import UUID
from sqlalchemy import delete, select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_chunking import (
DocumentAlreadyChunkedError,
DocumentChunkingService,
DocumentNotParsedError,
EmptyChunkSetError,
ParsedDocumentMismatchError,
)
from ekp_core.document_chunking import (
DocumentNotFoundError as ChunkingDocumentNotFoundError,
)
from ekp_core.document_indexing import (
DocumentAlreadyIndexedError,
DocumentHasNoChunksError,
DocumentIndexingService,
)
from ekp_core.document_indexing import (
DocumentNotFoundError as IndexingDocumentNotFoundError,
)
from ekp_core.document_parsing import (
DocumentAlreadyParsedError,
DocumentParsingService,
EmptyParsedDocumentError,
)
from ekp_core.document_parsing import (
DocumentNotFoundError as ParsingDocumentNotFoundError,
)
from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import ParsedDocumentArtifactError
from ekp_domain import (
Chunk,
ChunkIndexState,
Document,
DocumentStatus,
IngestionAttempt,
IngestionAttemptStatus,
IngestionJob,
IngestionJobStatus,
IngestionStage,
)
from ekp_object_storage import BinarySource, ObjectStorage
from ekp_parser import DocumentParseError
from ekp_vector_store import VectorStore
SAFE_TRANSIENT_ERROR = "temporary ingestion failure"
SAFE_PERMANENT_ERROR = "document content could not be processed"
SAFE_CLEANUP_ERROR = "retry cleanup failed"
SAFE_LEASE_ERROR = "worker execution lease expired"
class IngestionJobNotFoundError(LookupError):
"""Raised when a requested ingestion Job does not exist."""
class IngestionCleanupError(RuntimeError):
"""Raised when derived data cannot be restored to a known clean state."""
class JobDispatcher(Protocol):
"""Replaceable message-delivery boundary used by submission services."""
def enqueue(self, job_id: UUID) -> None:
"""Request asynchronous execution of one already-persisted Job."""
@dataclass(frozen=True, slots=True)
class IngestionSubmission:
"""Stable identifiers returned after source and Job persistence."""
document: Document
job: IngestionJob
class JobRunAction(StrEnum):
"""Instruction returned to a queue adapter after one delivery."""
COMPLETE = "complete"
RETRY = "retry"
@dataclass(frozen=True, slots=True)
class JobRunResult:
"""Queue-neutral result for acknowledging or rescheduling a delivery."""
action: JobRunAction
status: IngestionJobStatus
retry_delay_seconds: int | None = None
@dataclass(frozen=True, slots=True)
class _Claim:
"""Internal durable claim outcome used before external provider work."""
run: bool
cleanup_required: bool = False
result: JobRunResult | None = None
class IngestionJobService:
"""Create and query Job state independently from the broker implementation."""
def __init__(self, session_factory: sessionmaker[Session], max_attempts: int) -> None:
if max_attempts <= 0:
raise ValueError("max_attempts must be positive")
self._session_factory = session_factory
self._max_attempts = max_attempts
def create(self, document_id: UUID) -> IngestionJob:
"""Create exactly one queued ingestion Job for a canonical Document."""
with self._session_factory() as session:
job = IngestionJob(document_id=document_id, max_attempts=self._max_attempts)
session.add(job)
session.commit()
return job
def get(self, job_id: UUID) -> IngestionJob:
"""Return one detached Job or a stable not-found error."""
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
return job
def list_dispatchable(self, limit: int = 100) -> tuple[UUID, ...]:
"""Find due queued/retrying Jobs and expired running leases for recovery."""
if not 1 <= limit <= 1000:
raise ValueError("dispatch limit must be between 1 and 1000")
now = datetime.now(UTC)
with self._session_factory() as session:
jobs = session.scalars(
select(IngestionJob)
.where(
(IngestionJob.status == IngestionJobStatus.QUEUED)
| (
(IngestionJob.status == IngestionJobStatus.RETRYING)
& ((IngestionJob.next_run_at.is_(None)) | (IngestionJob.next_run_at <= now))
)
| (
(IngestionJob.status == IngestionJobStatus.RUNNING)
& (
(IngestionJob.lease_expires_at.is_(None))
| (IngestionJob.lease_expires_at <= now)
)
)
)
.order_by(IngestionJob.created_at, IngestionJob.id)
.limit(limit)
).all()
return tuple(job.id for job in jobs)
class AsyncIngestionSubmissionService:
"""Persist immutable source and Job before best-effort broker delivery."""
def __init__(
self,
upload_service: DocumentUploadService,
jobs: IngestionJobService,
dispatcher: JobDispatcher,
) -> None:
self._upload_service = upload_service
self._jobs = jobs
self._dispatcher = dispatcher
def submit(
self,
filename: str,
media_type: str | None,
source: BinarySource,
) -> IngestionSubmission:
"""Accept one source and return without executing expensive stages inline.
Broker delivery is deliberately best effort after the canonical commit.
A periodic dispatcher recovers queued Jobs when the broker was temporarily
unavailable, so the request never treats a message as the source of truth.
"""
document = self._upload_service.upload(filename, media_type, source)
try:
job = self._jobs.create(document.id)
except Exception:
self._upload_service.rollback_acceptance(document.id)
raise
try:
self._dispatcher.enqueue(job.id)
except Exception:
# Keeping the Job queued is intentional. The recovery dispatcher can
# safely redeliver it without asking the client to upload again.
pass
return IngestionSubmission(document=document, job=job)
class DerivedDocumentCleanupService:
"""Remove only rebuildable derivatives before a retry begins."""
def __init__(
self,
session_factory: sessionmaker[Session],
storage: ObjectStorage,
vector_store: VectorStore,
active_index_profile_id: str,
) -> None:
self._session_factory = session_factory
self._storage = storage
self._vector_store = vector_store
self._active_index_profile_id = active_index_profile_id
def cleanup(self, document_id: UUID) -> None:
"""Clean vector, relational and parsed artifacts while preserving source/Job."""
try:
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise IngestionCleanupError("document metadata is missing")
parsed_key = document.parsed_storage_key
profile_ids = set(
session.scalars(
select(ChunkIndexState.index_profile_id)
.join(Chunk, Chunk.chunk_id == ChunkIndexState.chunk_id)
.where(Chunk.document_id == document_id)
.distinct()
).all()
)
profile_ids.add(self._active_index_profile_id)
# External vector rows are deleted before their canonical evidence.
for profile_id in sorted(profile_ids):
self._vector_store.delete_document(document_id, profile_id)
with self._session_factory() as session:
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
session.execute(
delete(ChunkIndexState).where(ChunkIndexState.chunk_id.in_(chunk_ids))
)
session.execute(delete(Chunk).where(Chunk.document_id == document_id))
session.commit()
if parsed_key is not None and self._storage.exists(parsed_key):
self._storage.delete(parsed_key)
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise IngestionCleanupError("document metadata is missing")
document.parsed_storage_key = None
document.status = DocumentStatus.UPLOADED
document.error_message = None
session.commit()
except IngestionCleanupError:
raise
except Exception as error:
raise IngestionCleanupError(SAFE_CLEANUP_ERROR) from error
class IngestionJobRunner:
"""Execute one durable attempt and return queue-neutral retry instructions."""
def __init__(
self,
session_factory: sessionmaker[Session],
parsing: DocumentParsingService,
chunking: DocumentChunkingService,
indexing: DocumentIndexingService,
cleanup: DerivedDocumentCleanupService,
*,
retry_backoff_seconds: int,
retry_backoff_max_seconds: int,
lease_seconds: int,
) -> None:
if retry_backoff_seconds <= 0 or retry_backoff_max_seconds <= 0 or lease_seconds <= 0:
raise ValueError("retry and lease durations must be positive")
self._session_factory = session_factory
self._parsing = parsing
self._chunking = chunking
self._indexing = indexing
self._cleanup = cleanup
self._retry_backoff_seconds = retry_backoff_seconds
self._retry_backoff_max_seconds = retry_backoff_max_seconds
self._lease_seconds = lease_seconds
def run(self, job_id: UUID) -> JobRunResult:
"""Claim, optionally clean, execute and durably finalize one delivery."""
claim = self._claim(job_id)
if not claim.run:
assert claim.result is not None
return claim.result
if claim.cleanup_required:
try:
document_id = self._job_document_id(job_id)
self._cleanup.cleanup(document_id)
except Exception:
self._finish_cleanup_failure(job_id)
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.FAILED)
attempt_number, document_id = self._start_attempt(job_id)
def stage_callback(stage: IngestionStage) -> None:
"""Bind this attempt while pipeline stages remain Job-agnostic."""
self._record_stage(job_id, attempt_number, stage)
try:
self._parsing.process(document_id, stage_callback=stage_callback)
self._chunking.process(document_id, stage_callback=stage_callback)
self._indexing.process(document_id, stage_callback=stage_callback)
except Exception as error:
return self._finish_failure(job_id, attempt_number, document_id, error)
self._finish_success(job_id, attempt_number, document_id)
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.SUCCEEDED)
def _claim(self, job_id: UUID) -> _Claim:
"""Serialize deliveries and recover only after an execution lease expires."""
now = datetime.now(UTC)
with self._session_factory() as session:
job = session.scalar(
select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
if job.status in {
IngestionJobStatus.SUCCEEDED,
IngestionJobStatus.FAILED,
IngestionJobStatus.CANCELLED,
}:
return _Claim(
run=False,
result=JobRunResult(JobRunAction.COMPLETE, job.status),
)
if job.status == IngestionJobStatus.RUNNING:
lease_expiry = _as_utc(job.lease_expires_at)
if lease_expiry is not None and lease_expiry > now:
delay = max(1, ceil((lease_expiry - now).total_seconds()))
return _Claim(
run=False,
result=JobRunResult(
JobRunAction.RETRY,
IngestionJobStatus.RUNNING,
delay,
),
)
self._expire_active_attempt(session, job, now)
if job.attempt_count >= job.max_attempts:
job.status = IngestionJobStatus.FAILED
job.lease_expires_at = None
job.completed_at = now
document = session.get(Document, job.document_id)
if document is not None:
document.status = DocumentStatus.FAILED
document.error_message = SAFE_LEASE_ERROR
session.commit()
return _Claim(
run=False,
result=JobRunResult(JobRunAction.COMPLETE, job.status),
)
job.status = IngestionJobStatus.RETRYING
next_run = _as_utc(job.next_run_at)
if (
job.status == IngestionJobStatus.RETRYING
and next_run is not None
and next_run > now
):
delay = max(1, ceil((next_run - now).total_seconds()))
return _Claim(
run=False,
result=JobRunResult(
JobRunAction.RETRY,
IngestionJobStatus.RETRYING,
delay,
),
)
cleanup_required = job.attempt_count > 0
job.status = IngestionJobStatus.RUNNING
job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
job.next_run_at = None
session.commit()
return _Claim(run=True, cleanup_required=cleanup_required)
def _expire_active_attempt(self, session: Session, job: IngestionJob, now: datetime) -> None:
"""Close an abandoned Attempt before a replacement can be created."""
attempt = session.scalar(
select(IngestionAttempt)
.where(
IngestionAttempt.job_id == job.id,
IngestionAttempt.status == IngestionAttemptStatus.RUNNING,
)
.with_for_update()
)
if attempt is not None:
attempt.status = IngestionAttemptStatus.FAILED
attempt.retryable = True
attempt.error_message = SAFE_LEASE_ERROR
attempt.finished_at = now
job.error_message = SAFE_LEASE_ERROR
def _job_document_id(self, job_id: UUID) -> UUID:
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
return job.document_id
def _start_attempt(self, job_id: UUID) -> tuple[int, UUID]:
"""Create the next numbered Attempt only after retry cleanup succeeds."""
now = datetime.now(UTC)
with self._session_factory() as session:
job = session.scalar(
select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
attempt_number = job.attempt_count + 1
job.attempt_count = attempt_number
job.stage = IngestionStage.PARSING
job.error_message = None
job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
session.add(
IngestionAttempt(
job_id=job.id,
attempt_number=attempt_number,
stage=IngestionStage.PARSING,
)
)
session.commit()
return attempt_number, job.document_id
def _record_stage(self, job_id: UUID, attempt_number: int, stage: IngestionStage) -> None:
"""Persist progress and renew the worker lease at every stage boundary."""
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
attempt = session.scalar(
select(IngestionAttempt).where(
IngestionAttempt.job_id == job_id,
IngestionAttempt.attempt_number == attempt_number,
)
)
if job is None or attempt is None:
raise IngestionJobNotFoundError(str(job_id))
job.stage = stage
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=self._lease_seconds)
attempt.stage = stage
session.commit()
def _finish_success(self, job_id: UUID, attempt_number: int, document_id: UUID) -> None:
"""Atomically close Job/Attempt after the document becomes ready."""
now = datetime.now(UTC)
with self._session_factory() as session:
job, attempt = self._load_job_attempt(session, job_id, attempt_number)
document = session.get(Document, document_id)
if document is None:
raise IngestionJobNotFoundError(str(job_id))
attempt.status = IngestionAttemptStatus.SUCCEEDED
attempt.retryable = False
attempt.finished_at = now
job.status = IngestionJobStatus.SUCCEEDED
job.error_message = None
job.lease_expires_at = None
job.completed_at = now
document.status = DocumentStatus.READY
document.error_message = None
session.commit()
def _finish_failure(
self,
job_id: UUID,
attempt_number: int,
document_id: UUID,
error: Exception,
) -> JobRunResult:
"""Classify safely, bound retry count, and schedule exponential backoff."""
retryable = _is_retryable(error)
safe_error = SAFE_TRANSIENT_ERROR if retryable else SAFE_PERMANENT_ERROR
now = datetime.now(UTC)
with self._session_factory() as session:
job, attempt = self._load_job_attempt(session, job_id, attempt_number)
document = session.get(Document, document_id)
attempt.status = IngestionAttemptStatus.FAILED
attempt.retryable = retryable
attempt.error_message = safe_error
attempt.finished_at = now
job.error_message = safe_error
job.lease_expires_at = None
if retryable and job.attempt_count < job.max_attempts:
delay = min(
self._retry_backoff_seconds * (2 ** (job.attempt_count - 1)),
self._retry_backoff_max_seconds,
)
job.status = IngestionJobStatus.RETRYING
job.next_run_at = now + timedelta(seconds=delay)
if document is not None:
document.status = DocumentStatus.PROCESSING
document.error_message = safe_error
session.commit()
return JobRunResult(JobRunAction.RETRY, job.status, delay)
job.status = IngestionJobStatus.FAILED
job.next_run_at = None
job.completed_at = now
if document is not None:
document.status = DocumentStatus.FAILED
document.error_message = safe_error
session.commit()
return JobRunResult(JobRunAction.COMPLETE, job.status)
def _finish_cleanup_failure(self, job_id: UUID) -> None:
"""Fail closed without creating a new Attempt on unknown dirty state."""
now = datetime.now(UTC)
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
document = session.get(Document, job.document_id)
job.status = IngestionJobStatus.FAILED
job.error_message = SAFE_CLEANUP_ERROR
job.next_run_at = None
job.lease_expires_at = None
job.completed_at = now
if document is not None:
document.status = DocumentStatus.FAILED
document.error_message = SAFE_CLEANUP_ERROR
session.commit()
@staticmethod
def _load_job_attempt(
session: Session, job_id: UUID, attempt_number: int
) -> tuple[IngestionJob, IngestionAttempt]:
job = session.get(IngestionJob, job_id)
attempt = session.scalar(
select(IngestionAttempt).where(
IngestionAttempt.job_id == job_id,
IngestionAttempt.attempt_number == attempt_number,
)
)
if job is None or attempt is None:
raise IngestionJobNotFoundError(str(job_id))
return job, attempt
def dispatch_pending_jobs(jobs: IngestionJobService, dispatcher: JobDispatcher) -> int:
"""Best-effort redelivery loop used by Celery beat after broker/API failures."""
dispatched = 0
for job_id in jobs.list_dispatchable():
try:
dispatcher.enqueue(job_id)
except Exception:
continue
dispatched += 1
return dispatched
def _is_retryable(error: Exception) -> bool:
"""Treat known content/state violations as permanent and providers as transient."""
permanent_errors = (
DocumentParseError,
EmptyParsedDocumentError,
EmptyChunkSetError,
ParsedDocumentMismatchError,
ParsedDocumentArtifactError,
DocumentAlreadyParsedError,
DocumentAlreadyChunkedError,
DocumentAlreadyIndexedError,
DocumentNotParsedError,
DocumentHasNoChunksError,
ParsingDocumentNotFoundError,
ChunkingDocumentNotFoundError,
IndexingDocumentNotFoundError,
)
return not isinstance(error, permanent_errors)
def _as_utc(value: datetime | None) -> datetime | None:
"""Normalize SQLite's naive DateTime values to the UTC runtime contract."""
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)

View File

@ -8,6 +8,13 @@ from ekp_domain.index_profile import (
IndexProfileStatus,
StoredIndexProfile,
)
from ekp_domain.ingestion_job import (
IngestionAttempt,
IngestionAttemptStatus,
IngestionJob,
IngestionJobStatus,
IngestionStage,
)
from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocument
__all__ = [
@ -18,6 +25,11 @@ __all__ = [
"DocumentStatus",
"IndexProfile",
"IndexProfileStatus",
"IngestionAttempt",
"IngestionAttemptStatus",
"IngestionJob",
"IngestionJobStatus",
"IngestionStage",
"ParsedBlock",
"ParsedBlockType",
"ParsedDocument",

View File

@ -0,0 +1,166 @@
"""Durable ingestion Job and Attempt state for asynchronous processing."""
from datetime import datetime
from enum import StrEnum
from uuid import UUID, uuid4
from sqlalchemy import (
Boolean,
CheckConstraint,
Computed,
DateTime,
ForeignKey,
Index,
Integer,
Text,
UniqueConstraint,
Uuid,
func,
)
from sqlalchemy import Enum as SqlEnum
from sqlalchemy.orm import Mapped, mapped_column
from ekp_domain.document import Base
class IngestionJobStatus(StrEnum):
"""Public lifecycle states for one document ingestion Job."""
QUEUED = "queued"
RUNNING = "running"
RETRYING = "retrying"
SUCCEEDED = "succeeded"
FAILED = "failed"
CANCELLED = "cancelled"
class IngestionStage(StrEnum):
"""Observable stages within one ingestion attempt."""
PARSING = "parsing"
CLEANING = "cleaning"
CHUNKING = "chunking"
EMBEDDING = "embedding"
INDEXING = "indexing"
class IngestionAttemptStatus(StrEnum):
"""Terminal or active state of one bounded execution attempt."""
RUNNING = "running"
SUCCEEDED = "succeeded"
FAILED = "failed"
def _enum_column(enum_type: type[StrEnum], name: str) -> SqlEnum:
"""Build consistent portable string enums for OceanBase and SQLite tests."""
return SqlEnum(
enum_type,
name=name,
native_enum=False,
create_constraint=False,
validate_strings=True,
values_callable=lambda values: [item.value for item in values],
)
class IngestionJob(Base):
"""Canonical orchestration state; broker messages are only delivery hints."""
__tablename__ = "ingestion_jobs"
__table_args__ = (
CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
CheckConstraint(
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
name="status_valid",
),
CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
Index("ix_ingestion_jobs_status_next_run", "status", "next_run_at"),
)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
document_id: Mapped[UUID] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("documents.id", ondelete="CASCADE"),
nullable=False,
)
status: Mapped[IngestionJobStatus] = mapped_column(
_enum_column(IngestionJobStatus, "ingestion_job_status"),
nullable=False,
default=IngestionJobStatus.QUEUED,
)
stage: Mapped[IngestionStage | None] = mapped_column(
_enum_column(IngestionStage, "ingestion_stage"),
nullable=True,
)
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
lease_expires_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
)
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class IngestionAttempt(Base):
"""Immutable-numbered execution history with at most one active row per Job."""
__tablename__ = "ingestion_attempts"
__table_args__ = (
CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
CheckConstraint(
"status IN ('running', 'succeeded', 'failed')",
name="status_valid",
),
CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
# NULL values may repeat, while every running attempt maps to 1. This
# provides portable partial uniqueness on OceanBase MySQL and SQLite.
Index("uq_ingestion_attempts_active", "job_id", "active_marker", unique=True),
)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
job_id: Mapped[UUID] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("ingestion_jobs.id", ondelete="CASCADE"),
nullable=False,
)
attempt_number: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[IngestionAttemptStatus] = mapped_column(
_enum_column(IngestionAttemptStatus, "ingestion_attempt_status"),
nullable=False,
default=IngestionAttemptStatus.RUNNING,
)
stage: Mapped[IngestionStage | None] = mapped_column(
_enum_column(IngestionStage, "ingestion_attempt_stage"),
nullable=True,
)
retryable: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
active_marker: Mapped[int | None] = mapped_column(
Integer,
Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
nullable=True,
)
started_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

View File

@ -11,6 +11,7 @@ requires-python = ">=3.12,<3.14"
dependencies = [
"alembic>=1.18,<2",
"bce-python-sdk>=0.9.72,<0.10",
"celery>=5.5,<6",
"fastapi>=0.115,<1",
"mcp>=2,<3",
"pymysql>=1.1,<2",
@ -27,6 +28,7 @@ dependencies = [
ekp-index-rebuild = "ekp_api.index_rebuild:main"
ekp-live-acceptance = "ekp_api.live_acceptance:main"
ekp-mcp = "ekp_mcp.main:main"
ekp-worker = "ekp_api.worker:main"
[dependency-groups]
dev = [

View File

@ -0,0 +1,431 @@
"""Durable Job, bounded retry and cleanup-before-retry integration tests."""
from collections.abc import Sequence
from datetime import UTC, datetime, timedelta
from io import BytesIO
from pathlib import Path
from typing import BinaryIO
from uuid import UUID
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.chunking import StructureAwareChunker
from ekp_core.database import create_session_factory
from ekp_core.document_chunking import DocumentChunkingService
from ekp_core.document_indexing import DocumentIndexingService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
JobRunAction,
dispatch_pending_jobs,
)
from ekp_domain import (
Base,
Chunk,
Document,
IndexProfile,
IngestionAttempt,
IngestionAttemptStatus,
IngestionJob,
IngestionJobStatus,
ParsedDocument,
)
from ekp_embedding import EmbeddingProviderError, FakeEmbeddingProvider
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_vector_store import FakeVectorStore
class RecordingDispatcher:
"""Capture delivery requests while preserving the dispatcher contract."""
def __init__(self, *, fail: bool = False) -> None:
self.fail = fail
self.job_ids: list[UUID] = []
def enqueue(self, job_id: UUID) -> None:
if self.fail:
raise RuntimeError("simulated broker outage containing secret=must-not-persist")
self.job_ids.append(job_id)
class FailingJobService(IngestionJobService):
"""Inject metadata failure after immutable source upload."""
def create(self, document_id: UUID) -> IngestionJob:
del document_id
raise RuntimeError("simulated Job persistence failure")
class FailOnceEmbeddingProvider:
"""Inject one transient provider failure after parse/chunk persistence."""
def __init__(self) -> None:
self._delegate = FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
self.calls = 0
@property
def model_name(self) -> str:
return self._delegate.model_name
@property
def dimensions(self) -> int:
return self._delegate.dimensions
@property
def max_batch_size(self) -> int:
return self._delegate.max_batch_size
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
self.calls += 1
if self.calls == 1:
raise EmbeddingProviderError("temporary failure token=must-not-persist")
return self._delegate.embed(texts)
class AlwaysFailEmbeddingProvider(FailOnceEmbeddingProvider):
"""Inject a transient error on every attempt to verify the retry bound."""
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
del texts
self.calls += 1
raise EmbeddingProviderError("temporary failure token=must-not-persist")
class RetryObservingParser:
"""Verify cleanup finished before the second provider execution begins."""
def __init__(
self,
session_factory: sessionmaker[Session],
storage: LocalObjectStorage,
vector_store: FakeVectorStore,
) -> None:
self._delegate = ParserRouter()
self._session_factory = session_factory
self._storage = storage
self._vector_store = vector_store
self.calls = 0
self.retry_started_clean = False
def parse(
self,
document_id: UUID,
filename: str,
source: BinaryIO,
) -> ParsedDocument:
self.calls += 1
if self.calls == 2:
with self._session_factory() as session:
document = session.get(Document, document_id)
chunks = session.scalars(
select(Chunk).where(Chunk.document_id == document_id)
).all()
# The source stream is already open here, proving it survived cleanup.
self.retry_started_clean = (
document is not None
and document.parsed_storage_key is None
and not chunks
and not self._vector_store.records
and self._storage.exists(document.storage_key)
)
return self._delegate.parse(document_id, filename, source)
class CleanupFailingVectorStore(FakeVectorStore):
"""Inject a retry cleanup failure without affecting the first attempt."""
fail_delete = False
def delete_document(self, document_id: UUID, index_profile_id: str) -> None:
if self.fail_delete:
raise RuntimeError("cleanup provider failed secret=must-not-persist")
super().delete_document(document_id, index_profile_id)
def _build_runner(
tmp_path: Path,
*,
embedding: FailOnceEmbeddingProvider | FakeEmbeddingProvider | None = None,
vector_store: FakeVectorStore | None = None,
max_attempts: int = 3,
) -> tuple[
DocumentUploadService,
IngestionJobService,
IngestionJobRunner,
sessionmaker[Session],
LocalObjectStorage,
FakeVectorStore,
RetryObservingParser,
]:
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
selected_store = vector_store or FakeVectorStore()
selected_embedding = embedding or FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
profile = IndexProfile(
profile_id="test-profile",
chunk_strategy_version="structure_v1",
embedding_model=selected_embedding.model_name,
vector_dimension=selected_embedding.dimensions,
)
parser = RetryObservingParser(session_factory, storage, selected_store)
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, parser)
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
)
indexing = DocumentIndexingService(
session_factory,
selected_embedding,
selected_store,
profile,
)
jobs = IngestionJobService(session_factory, max_attempts=max_attempts)
cleanup = DerivedDocumentCleanupService(
session_factory,
storage,
selected_store,
profile.profile_id,
)
runner = IngestionJobRunner(
session_factory,
parsing,
chunking,
indexing,
cleanup,
retry_backoff_seconds=1,
retry_backoff_max_seconds=4,
lease_seconds=30,
)
return upload, jobs, runner, session_factory, storage, selected_store, parser
def _submit(
upload: DocumentUploadService,
jobs: IngestionJobService,
payload: bytes = b"# Policy\n\nRefunds take seven days.",
) -> tuple[UUID, UUID, RecordingDispatcher]:
dispatcher = RecordingDispatcher()
result = AsyncIngestionSubmissionService(upload, jobs, dispatcher).submit(
"policy.md", "text/markdown", BytesIO(payload)
)
return result.document.id, result.job.id, dispatcher
def _make_retry_due(session_factory: sessionmaker[Session], job_id: UUID) -> None:
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
job.next_run_at = datetime.now(UTC) - timedelta(seconds=1)
session.commit()
def test_transient_failure_cleans_derivatives_before_retry(tmp_path: Path) -> None:
embedding = FailOnceEmbeddingProvider()
upload, jobs, runner, session_factory, storage, vector_store, parser = _build_runner(
tmp_path, embedding=embedding
)
document_id, job_id, dispatcher = _submit(upload, jobs)
first = runner.run(job_id)
assert first.action == JobRunAction.RETRY
assert first.retry_delay_seconds == 1
assert dispatcher.job_ids == [job_id]
with session_factory() as session:
stored_job = session.get(IngestionJob, job_id)
assert stored_job is not None
assert stored_job.status == IngestionJobStatus.RETRYING
assert stored_job.attempt_count == 1
assert "must-not-persist" not in (stored_job.error_message or "")
_make_retry_due(session_factory, job_id)
second = runner.run(job_id)
assert second.status == IngestionJobStatus.SUCCEEDED
assert parser.retry_started_clean
with session_factory() as session:
stored_job = session.get(IngestionJob, job_id)
attempts = session.scalars(
select(IngestionAttempt)
.where(IngestionAttempt.job_id == job_id)
.order_by(IngestionAttempt.attempt_number)
).all()
assert stored_job is not None
assert stored_job.attempt_count == 2
assert [attempt.status for attempt in attempts] == [
IngestionAttemptStatus.FAILED,
IngestionAttemptStatus.SUCCEEDED,
]
document = session.get(Document, document_id)
assert document is not None
assert storage.exists(document.storage_key)
assert document.parsed_storage_key is not None
assert storage.exists(document.parsed_storage_key)
assert vector_store.records
def test_permanent_content_failure_does_not_retry(tmp_path: Path) -> None:
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs, payload=b" \n\t")
result = runner.run(job_id)
assert result.action == JobRunAction.COMPLETE
assert result.status == IngestionJobStatus.FAILED
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
assert job.attempt_count == 1
assert job.error_message == "document content could not be processed"
def test_cleanup_failure_is_terminal_and_preserves_source(tmp_path: Path) -> None:
embedding = FailOnceEmbeddingProvider()
vector_store = CleanupFailingVectorStore()
upload, jobs, runner, session_factory, storage, _, _ = _build_runner(
tmp_path,
embedding=embedding,
vector_store=vector_store,
)
document_id, job_id, _ = _submit(upload, jobs)
assert runner.run(job_id).action == JobRunAction.RETRY
_make_retry_due(session_factory, job_id)
vector_store.fail_delete = True
result = runner.run(job_id)
assert result.status == IngestionJobStatus.FAILED
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
assert job.attempt_count == 1 # Cleanup failed before attempt 2 was created.
assert job.error_message == "retry cleanup failed"
document = session.get(Document, document_id)
assert document is not None
assert storage.exists(document.storage_key)
def test_duplicate_delivery_defers_while_active_lease_is_valid(tmp_path: Path) -> None:
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs)
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
job.status = IngestionJobStatus.RUNNING
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=30)
session.commit()
result = runner.run(job_id)
assert result.action == JobRunAction.RETRY
assert result.status == IngestionJobStatus.RUNNING
with session_factory() as session:
assert session.scalars(select(IngestionAttempt)).all() == []
def test_expired_worker_lease_is_recovered_as_a_new_attempt(tmp_path: Path) -> None:
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs)
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
job.status = IngestionJobStatus.RUNNING
job.attempt_count = 1
job.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
session.add(IngestionAttempt(job_id=job_id, attempt_number=1))
session.commit()
result = runner.run(job_id)
assert result.status == IngestionJobStatus.SUCCEEDED
with session_factory() as session:
attempts = session.scalars(
select(IngestionAttempt)
.where(IngestionAttempt.job_id == job_id)
.order_by(IngestionAttempt.attempt_number)
).all()
assert [attempt.status for attempt in attempts] == [
IngestionAttemptStatus.FAILED,
IngestionAttemptStatus.SUCCEEDED,
]
assert attempts[0].error_message == "worker execution lease expired"
def test_transient_failures_stop_at_configured_attempt_bound(tmp_path: Path) -> None:
embedding = AlwaysFailEmbeddingProvider()
upload, jobs, runner, session_factory, _, _, _ = _build_runner(
tmp_path,
embedding=embedding,
max_attempts=2,
)
_, job_id, _ = _submit(upload, jobs)
assert runner.run(job_id).action == JobRunAction.RETRY
_make_retry_due(session_factory, job_id)
result = runner.run(job_id)
assert result.action == JobRunAction.COMPLETE
assert result.status == IngestionJobStatus.FAILED
assert runner.run(job_id).status == IngestionJobStatus.FAILED
with session_factory() as session:
job = session.get(IngestionJob, job_id)
attempts = session.scalars(
select(IngestionAttempt).where(IngestionAttempt.job_id == job_id)
).all()
assert job is not None
assert job.attempt_count == job.max_attempts == 2
assert len(attempts) == 2
def test_recovery_dispatches_persisted_job_after_broker_outage(tmp_path: Path) -> None:
upload, jobs, _, _, _, _, _ = _build_runner(tmp_path)
failing = RecordingDispatcher(fail=True)
submission = AsyncIngestionSubmissionService(upload, jobs, failing).submit(
"policy.md", "text/markdown", BytesIO(b"policy")
)
recovered = RecordingDispatcher()
assert dispatch_pending_jobs(jobs, recovered) == 1
assert recovered.job_ids == [submission.job.id]
def test_job_persistence_failure_compensates_document_and_source(tmp_path: Path) -> None:
upload, _, _, session_factory, _, _, _ = _build_runner(tmp_path)
failing_jobs = FailingJobService(session_factory, max_attempts=3)
with pytest.raises(RuntimeError, match="Job persistence failure"):
AsyncIngestionSubmissionService(upload, failing_jobs, RecordingDispatcher()).submit(
"policy.md", "text/markdown", BytesIO(b"policy")
)
with session_factory() as session:
assert session.scalars(select(Document)).all() == []
assert session.scalars(select(IngestionJob)).all() == []
assert not [path for path in tmp_path.rglob("*") if path.is_file()]
def test_database_rejects_two_active_attempts_for_one_job(tmp_path: Path) -> None:
upload, jobs, _, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs)
with session_factory() as session:
session.add_all(
[
IngestionAttempt(job_id=job_id, attempt_number=1),
IngestionAttempt(job_id=job_id, attempt_number=2),
]
)
with pytest.raises(IntegrityError):
session.commit()
session.rollback()

View File

@ -23,9 +23,14 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
command.check(config)
inspector = inspect(create_engine(database_url))
assert {"documents", "chunks", "chunk_index_states", "index_profiles"}.issubset(
inspector.get_table_names()
)
assert {
"documents",
"chunks",
"chunk_index_states",
"index_profiles",
"ingestion_jobs",
"ingestion_attempts",
}.issubset(inspector.get_table_names())
assert "unrelated_provider_table" in inspector.get_table_names()
assert {column["name"] for column in inspector.get_columns("documents")} == {
"id",
@ -70,3 +75,29 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
"activated_at",
"active_marker",
}
assert {column["name"] for column in inspector.get_columns("ingestion_jobs")} == {
"id",
"document_id",
"status",
"stage",
"attempt_count",
"max_attempts",
"error_message",
"next_run_at",
"lease_expires_at",
"created_at",
"updated_at",
"completed_at",
}
assert {column["name"] for column in inspector.get_columns("ingestion_attempts")} == {
"id",
"job_id",
"attempt_number",
"status",
"stage",
"retryable",
"error_message",
"active_marker",
"started_at",
"finished_at",
}

View File

@ -19,6 +19,12 @@ from ekp_core.document_indexing import DocumentIndexingPipelineService, Document
from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
)
from ekp_core.search import SearchService
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile
from ekp_embedding import FakeEmbeddingProvider
@ -39,6 +45,16 @@ REQUIRED_SOURCE_FIELDS = {
}
class RecordingDispatcher:
"""Capture durable Job deliveries without starting a real broker."""
def __init__(self) -> None:
self.job_ids: list[UUID] = []
def enqueue(self, job_id: UUID) -> None:
self.job_ids.append(job_id)
def _build_api_services(
tmp_path: Path,
) -> tuple[
@ -46,6 +62,7 @@ def _build_api_services(
sessionmaker[Session],
LocalObjectStorage,
FakeVectorStore,
IngestionJobRunner,
]:
"""Build the production service graph with offline providers and shared SQLite."""
engine = create_engine(
@ -82,6 +99,24 @@ def _build_api_services(
profile,
)
ingestion = DocumentIndexingPipelineService(pipeline, indexing)
jobs = IngestionJobService(session_factory, max_attempts=3)
dispatcher = RecordingDispatcher()
submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher)
runner = IngestionJobRunner(
session_factory,
parsing,
chunking,
indexing,
DerivedDocumentCleanupService(
session_factory,
storage,
vector_store,
profile.profile_id,
),
retry_backoff_seconds=1,
retry_backoff_max_seconds=4,
lease_seconds=30,
)
dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20)
search = SearchService(
session_factory,
@ -96,10 +131,17 @@ def _build_api_services(
active_index_profile_id=profile.profile_id,
)
return (
ApiServices(ingestion=ingestion, documents=documents, search=search),
ApiServices(
ingestion=ingestion,
submissions=submissions,
jobs=jobs,
documents=documents,
search=search,
),
session_factory,
storage,
vector_store,
runner,
)
@ -107,7 +149,7 @@ def test_document_and_search_rest_flow(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
services, session_factory, storage, vector_store = _build_api_services(tmp_path)
services, session_factory, storage, vector_store, runner = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
@ -121,19 +163,33 @@ def test_document_and_search_rest_flow(
)
},
)
assert upload_response.status_code == 201
assert upload_response.status_code == 202
uploaded = upload_response.json()
document_id = UUID(uploaded["id"])
assert uploaded["status"] == "ready"
job_id = UUID(uploaded["job_id"])
assert uploaded["status"] == "uploaded"
assert uploaded["job_status"] == "queued"
assert uploaded["filename"] == "退款政策.md"
assert "storage_key" not in uploaded
queued = client.get(f"/api/v1/jobs/{job_id}")
assert queued.status_code == 200
assert queued.json()["attempt"] == 0
assert queued.json()["status"] == "queued"
assert client.delete(f"/api/v1/documents/{document_id}").status_code == 409
runner.run(job_id)
list_response = client.get("/api/v1/documents")
detail_response = client.get(f"/api/v1/documents/{document_id}")
assert list_response.status_code == 200
assert [item["id"] for item in list_response.json()] == [str(document_id)]
assert detail_response.status_code == 200
assert detail_response.json() == uploaded
assert detail_response.json()["status"] == "ready"
completed = client.get(f"/api/v1/jobs/{job_id}").json()
assert completed["status"] == "succeeded"
assert completed["stage"] == "indexing"
assert completed["attempt"] == 1
search_response = client.post(
"/api/v1/search",
@ -173,7 +229,7 @@ def test_rest_validation_not_found_and_openapi_contract(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
services, _, _, _ = _build_api_services(tmp_path)
services, _, _, _, _ = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
missing_id = uuid4()
@ -182,6 +238,7 @@ def test_rest_validation_not_found_and_openapi_contract(
assert set(openapi["paths"]) >= {
"/api/v1/documents",
"/api/v1/documents/{document_id}",
"/api/v1/jobs/{job_id}",
"/api/v1/search",
}
assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"}

View File

@ -0,0 +1,57 @@
"""Unit tests for the Celery adapter without opening a broker connection."""
from unittest.mock import Mock
from uuid import uuid4
import pytest
from pydantic import SecretStr
from ekp_api.celery_queue import (
INGESTION_TASK_NAME,
RECOVERY_TASK_NAME,
CeleryIngestionDispatcher,
create_celery_app,
)
from ekp_core.config import Settings
def test_celery_app_enables_late_ack_and_recovery_schedule(
isolated_settings_environment: None,
) -> None:
settings = Settings(rabbitmq_url=SecretStr("amqp://user:test@queue.example/ekp"))
app = create_celery_app(settings)
assert app.conf.task_acks_late is True
assert app.conf.task_reject_on_worker_lost is True
assert app.conf.worker_prefetch_multiplier == 1
assert app.conf.task_routes[INGESTION_TASK_NAME]["queue"] == "ingestion"
assert app.conf.task_routes[RECOVERY_TASK_NAME]["queue"] == "ingestion"
assert app.conf.beat_schedule["recover-durable-ingestion-jobs"]["schedule"] == 30.0
def test_celery_dispatcher_sends_only_job_identifier(
monkeypatch: pytest.MonkeyPatch,
isolated_settings_environment: None,
) -> None:
app = create_celery_app(Settings(rabbitmq_url=SecretStr("amqp://user:test@queue/ekp")))
send_task = Mock()
monkeypatch.setattr(app, "send_task", send_task)
job_id = uuid4()
CeleryIngestionDispatcher(app).enqueue(job_id)
send_task.assert_called_once_with(
INGESTION_TASK_NAME,
args=[str(job_id)],
queue="ingestion",
)
def test_celery_app_requires_runtime_broker_secret(
isolated_settings_environment: None,
) -> None:
with pytest.raises(ValueError, match="RABBITMQ_URL"):
create_celery_app(Settings())
with pytest.raises(ValueError, match="RABBITMQ_URL"):
create_celery_app(Settings(rabbitmq_url=SecretStr("")))

View File

@ -20,6 +20,11 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
assert settings.database_pool_recycle_seconds == 600
assert settings.database_pool_timeout_seconds == 30
assert settings.database_connect_timeout_seconds == 30
assert settings.rabbitmq_url is None
assert settings.ingestion_max_attempts == 3
assert settings.ingestion_retry_backoff_seconds == 2
assert settings.ingestion_retry_backoff_max_seconds == 60
assert settings.ingestion_worker_lease_seconds == 300
assert settings.object_storage_provider == "bos"
assert settings.object_storage_path == Path("storage")
assert settings.bos_endpoint is None
@ -90,6 +95,7 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) -
oceanbase_vector_password=SecretStr("oceanbase-secret"),
reranker_api_key=SecretStr("reranker-secret"),
mcp_auth_token=SecretStr("mcp-secret"),
rabbitmq_url=SecretStr("amqp://user:rabbit-secret@queue.example/ekp"),
)
rendered = repr(settings)
@ -100,6 +106,7 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) -
assert "oceanbase-secret" not in rendered
assert "reranker-secret" not in rendered
assert "mcp-secret" not in rendered
assert "rabbit-secret" not in rendered
def test_settings_restrict_sqlite_to_test_environment(
@ -136,3 +143,17 @@ def test_settings_reject_empty_mcp_allowed_hosts(
) -> None:
with pytest.raises(ValidationError, match="mcp_allowed_hosts"):
Settings(mcp_allowed_hosts=[])
def test_settings_reject_retry_backoff_larger_than_cap(
isolated_settings_environment: None,
) -> None:
with pytest.raises(ValidationError, match="initial backoff"):
Settings(ingestion_retry_backoff_seconds=10, ingestion_retry_backoff_max_seconds=5)
def test_settings_require_worker_lease_longer_than_provider_timeout(
isolated_settings_environment: None,
) -> None:
with pytest.raises(ValidationError, match="worker_lease"):
Settings(ingestion_worker_lease_seconds=30, embedding_timeout_seconds=30)

158
uv.lock generated
View File

@ -28,6 +28,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/20/89/e62cc37b69ad357cc8ecd6e7367f5245f523d3cbb338a66197212bdf6749/alembic-1.19.1-py3-none-any.whl", hash = "sha256:b39018cb3d9413a19cbd54cf3c02ad33998641f0538eb77413a488a21c3e14be", size = 265946, upload-time = "2026-08-08T16:32:03.153Z" },
]
[[package]]
name = "amqp"
version = "5.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "vine" },
]
sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" },
]
[[package]]
name = "annotated-doc"
version = "0.0.5"
@ -83,6 +95,35 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ef/de/52b763b183e23f4dd73811a2dd2664824ae7fc6ceb1aa8dcd3f1102b1094/bce_python_sdk-0.9.79-py3-none-any.whl", hash = "sha256:71799ac8740505e0759d30873f6f1a478fa8f83aedf425d511f1419a5f30082e", size = 440268, upload-time = "2026-09-01T05:48:50.459Z" },
]
[[package]]
name = "billiard"
version = "4.2.4"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" },
]
[[package]]
name = "celery"
version = "5.6.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "billiard" },
{ name = "click" },
{ name = "click-didyoumean" },
{ name = "click-plugins" },
{ name = "click-repl" },
{ name = "kombu" },
{ name = "python-dateutil" },
{ name = "tzlocal" },
{ name = "vine" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" },
]
[[package]]
name = "cffi"
version = "2.1.1"
@ -129,6 +170,43 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" },
]
[[package]]
name = "click-didyoumean"
version = "0.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
]
sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" },
]
[[package]]
name = "click-plugins"
version = "1.1.1.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" },
]
[[package]]
name = "click-repl"
version = "0.3.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
{ name = "prompt-toolkit" },
]
sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" },
]
[[package]]
name = "colorama"
version = "0.4.6"
@ -249,6 +327,7 @@ source = { editable = "." }
dependencies = [
{ name = "alembic" },
{ name = "bce-python-sdk" },
{ name = "celery" },
{ name = "fastapi" },
{ name = "mcp" },
{ name = "pydantic-settings" },
@ -274,6 +353,7 @@ dev = [
requires-dist = [
{ name = "alembic", specifier = ">=1.18,<2" },
{ name = "bce-python-sdk", specifier = ">=0.9.72,<0.10" },
{ name = "celery", specifier = ">=5.5,<6" },
{ name = "fastapi", specifier = ">=0.115,<1" },
{ name = "mcp", specifier = ">=2,<3" },
{ name = "pydantic-settings", specifier = ">=2.7,<3" },
@ -459,6 +539,21 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" },
]
[[package]]
name = "kombu"
version = "5.6.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "amqp" },
{ name = "packaging" },
{ name = "tzdata" },
{ name = "vine" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" },
]
[[package]]
name = "librt"
version = "0.15.0"
@ -740,6 +835,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
]
[[package]]
name = "prompt-toolkit"
version = "3.0.53"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "wcwidth" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" },
]
[[package]]
name = "pycparser"
version = "3.0"
@ -949,6 +1056,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" },
]
[[package]]
name = "python-dateutil"
version = "2.9.0.post0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "six" },
]
sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
]
[[package]]
name = "python-docx"
version = "1.2.0"
@ -1199,6 +1318,27 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" },
]
[[package]]
name = "tzdata"
version = "2026.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" },
]
[[package]]
name = "tzlocal"
version = "5.4.4"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "tzdata", marker = "sys_platform == 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/81/5b/879b2f932adfa7a053c360d50bc896c977fa6426109185f7c12ebdd0cb9d/tzlocal-5.4.4.tar.gz", hash = "sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4", size = 31170, upload-time = "2026-06-29T08:03:40.026Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9e/a4/017a7a6cbe387d961a688ec31364ae60a5c4e22c96ae9921b79a947c855d/tzlocal-5.4.4-py3-none-any.whl", hash = "sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15", size = 18115, upload-time = "2026-06-29T08:03:38.666Z" },
]
[[package]]
name = "uvicorn"
version = "0.52.4"
@ -1242,6 +1382,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" },
]
[[package]]
name = "vine"
version = "5.1.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" },
]
[[package]]
name = "watchfiles"
version = "1.2.0"
@ -1292,6 +1441,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" },
]
[[package]]
name = "wcwidth"
version = "0.8.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/36/57/ed58088fafdf4c55a0ad6bde846502567645424d7ebf325230b9237f4085/wcwidth-0.8.3.tar.gz", hash = "sha256:d128512515fbf4612e0ff21fd6380399210318b7b54a9af59dff8454cf9730eb", size = 1458450, upload-time = "2026-08-28T18:10:06.875Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/c4/0e/57f6bb3024a597b2e8ec4aee710ffe62ddc95af2e2bb1ee7a7abdc22c68c/wcwidth-0.8.3-py3-none-any.whl", hash = "sha256:d5b73dba6158a595ec9370350e7f2637bcac8d6c5e4fde34f30fcffb6103a5e4", size = 331669, upload-time = "2026-08-28T18:10:04.909Z" },
]
[[package]]
name = "websockets"
version = "17.1"