From 0fad825eb0c0080f86cb531a923ba9934b172baf Mon Sep 17 00:00:00 2001 From: d-robotics Date: Wed, 2 Sep 2026 20:42:52 +0800 Subject: [PATCH] feat: add batch async ingestion --- .env.example | 2 + README.md | 3 +- apps/admin-web/assets/app.js | 106 ++++++- apps/admin-web/assets/styles.css | 55 ++++ apps/admin-web/index.html | 20 +- apps/api/src/ekp_api/providers.py | 9 + apps/api/src/ekp_api/routes.py | 57 ++++ apps/api/src/ekp_api/schemas.py | 84 +++++- docs/api/API_CONTRACT.md | 31 +- docs/architecture/ARCHITECTURE.md | 4 + docs/demo/DEMO_SCOPE.md | 6 +- docs/development/ADMIN_CONSOLE.md | 11 +- docs/development/ASYNC_INGESTION_SETUP.md | 19 +- docs/development/REST_API_SETUP.md | 14 +- docs/plans/ASYNC_INGESTION_MCP_PLAN.md | 14 +- docs/plans/PROGRESS.md | 29 +- docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md | 3 +- docs/plans/TASKS.md | 2 +- docs/plans/V1_EXECUTION_PLAN.md | 2 +- docs/product/PRD.md | 3 + migrations/README.md | 4 + .../20260902_0008_create_ingestion_batches.py | 109 +++++++ packages/core/src/ekp_core/config.py | 4 + .../core/src/ekp_core/ingestion_batches.py | 270 ++++++++++++++++++ packages/domain/src/ekp_domain/__init__.py | 8 + .../domain/src/ekp_domain/ingestion_batch.py | 96 +++++++ tests/integration/test_admin_console.py | 4 +- tests/integration/test_document_migration.py | 19 ++ tests/integration/test_document_search_api.py | 160 ++++++++++- tests/unit/test_config.py | 9 + 30 files changed, 1098 insertions(+), 59 deletions(-) create mode 100644 migrations/versions/20260902_0008_create_ingestion_batches.py create mode 100644 packages/core/src/ekp_core/ingestion_batches.py create mode 100644 packages/domain/src/ekp_domain/ingestion_batch.py diff --git a/.env.example b/.env.example index ea71749..f21033e 100644 --- a/.env.example +++ b/.env.example @@ -18,6 +18,8 @@ INGESTION_RETRY_BACKOFF_SECONDS=2 INGESTION_RETRY_BACKOFF_MAX_SECONDS=60 INGESTION_WORKER_LEASE_SECONDS=300 INGESTION_RECOVERY_INTERVAL_SECONDS=30 +BATCH_MAX_FILES=20 +BATCH_MAX_TOTAL_BYTES=209715200 OBJECT_STORAGE_PROVIDER=bos OBJECT_STORAGE_PATH=storage diff --git a/README.md b/README.md index 5dcc4d5..5956082 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,8 @@ source results are rehydrated from canonical relational chunks and retain both Dense and reranker scores; stale vector content fails closed. See `docs/development/RERANKER_SETUP.md` for configuration and validation limits. DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015 -changes upload to asynchronous acceptance and adds Job status. Upload, list, detail, direct delete, +changes upload to asynchronous acceptance and adds Job status. DEMO-016 adds bounded multi-file +submission, batch/per-file status, and automatic Admin Console polling. Upload, list, detail, direct delete, and reranked source search are documented in `docs/development/REST_API_SETUP.md`; provider and storage internals remain outside the public response contract. diff --git a/apps/admin-web/assets/app.js b/apps/admin-web/assets/app.js index 02a0d1e..a73b43a 100644 --- a/apps/admin-web/assets/app.js +++ b/apps/admin-web/assets/app.js @@ -1,12 +1,33 @@ -// DEMO-010 stays API First: this client owns presentation state only and sends -// every document/search operation through the versioned public REST contract. +// The Admin Console owns presentation state only. Batch acceptance, Job state, +// and retrieval remain behind the versioned public REST contract. const API_BASE = "/api/v1"; +const BATCH_POLL_INTERVAL_MS = 1500; +const terminalBatchStatuses = new Set(["completed", "partial_failure", "failed"]); +let batchPollTimer; +let activeBatchId; const statusLabels = { uploaded: "Uploaded", processing: "Processing", ready: "Ready", failed: "Failed", + pending: "Pending", + accepted: "Accepted", + rejected: "Rejected", + queued: "Queued", + running: "Running", + retrying: "Retrying", + succeeded: "Succeeded", + completed: "Completed", + partial_failure: "Partial failure", +}; + +const stageLabels = { + parsing: "解析", + cleaning: "清理", + chunking: "切分", + embedding: "向量化", + indexing: "索引", }; const pageTitles = { @@ -93,23 +114,37 @@ function initializeUpload() { const button = document.querySelector("#upload-button"); const updateFilename = () => { - selected.textContent = input.files[0]?.name ?? "尚未选择文件"; + const files = Array.from(input.files); + if (!files.length) { + selected.textContent = "尚未选择文件"; + } else if (files.length === 1) { + selected.textContent = files[0].name; + } else { + selected.textContent = `已选择 ${files.length} 个文件`; + } }; input.addEventListener("change", updateFilename); form.addEventListener("submit", async (event) => { event.preventDefault(); if (!input.files.length) return; + window.clearTimeout(batchPollTimer); + activeBatchId = undefined; setButtonBusy(button, true, "提交中…"); try { const formData = new FormData(); - formData.append("file", input.files[0]); - const documentData = await requestJson("/documents", { + Array.from(input.files).forEach((file) => formData.append("files", file)); + const batch = await requestJson("/ingestion-batches", { method: "POST", body: formData, }); - renderUploadResult(documentData); - showNotice("文档已接收并进入后台处理队列。", "success"); + activeBatchId = batch.batch_id; + renderBatchResult(batch); + scheduleBatchPoll(batch); + showNotice( + `批次已接收:${batch.accepted_files} 个进入后台队列,${batch.rejected_files} 个被拒绝。`, + batch.rejected_files ? "error" : "success", + ); form.reset(); updateFilename(); } catch (error) { @@ -120,18 +155,62 @@ function initializeUpload() { }); } -function renderUploadResult(documentData) { +function scheduleBatchPoll(batch) { + window.clearTimeout(batchPollTimer); + if (batch.batch_id !== activeBatchId) return; + if (terminalBatchStatuses.has(batch.status)) return; + batchPollTimer = window.setTimeout(async () => { + try { + const latest = await requestJson(`/ingestion-batches/${batch.batch_id}`); + renderBatchResult(latest); + scheduleBatchPoll(latest); + } catch (error) { + showNotice(`批次状态刷新失败:${error.message}`, "error"); + scheduleBatchPoll(batch); + } + }, BATCH_POLL_INTERVAL_MS); +} + +function renderBatchResult(batch) { const card = document.querySelector("#upload-result"); - document.querySelector("#upload-filename").textContent = documentData.filename; - document.querySelector("#upload-type").textContent = documentData.media_type; - document.querySelector("#upload-size").textContent = formatSize(documentData.size_bytes); - document.querySelector("#upload-time").textContent = formatDate(documentData.created_at); - document.querySelector("#upload-status").replaceWith(statusPill(documentData.status)); + document.querySelector("#upload-batch-title").textContent = + `批次 ${batch.batch_id.slice(0, 8)} · ${formatDate(batch.created_at)}`; + document.querySelector("#upload-total").textContent = `${batch.total_files} 个`; + document.querySelector("#upload-completed").textContent = + `${batch.completed_files} / ${batch.total_files}`; + document.querySelector("#upload-accepted").textContent = + `${batch.accepted_files} / ${batch.rejected_files}`; + document.querySelector("#upload-status").replaceWith(statusPill(batch.status)); const status = card.querySelector(".status-pill"); status.id = "upload-status"; + const items = document.querySelector("#batch-items"); + items.replaceChildren(...batch.items.map(batchItem)); card.hidden = false; } +function batchItem(item) { + const row = document.createElement("article"); + row.className = "batch-item"; + + const order = document.createElement("span"); + order.className = "batch-item-order"; + order.textContent = String(item.item_index + 1).padStart(2, "0"); + + const body = document.createElement("div"); + body.className = "batch-item-body"; + const name = document.createElement("strong"); + name.textContent = item.filename; + const detail = document.createElement("span"); + const stage = item.stage ? ` · ${stageLabels[item.stage] ?? item.stage}` : ""; + const attempts = item.max_attempts == null ? "" : ` · 尝试 ${item.attempt}/${item.max_attempts}`; + detail.textContent = item.error_message ?? `后台任务${stage}${attempts}`; + body.append(name, detail); + + const state = item.job_status ?? item.acceptance_status; + row.append(order, body, statusPill(state)); + return row; +} + async function loadDocuments() { const refresh = document.querySelector("#refresh-documents"); setButtonBusy(refresh, true, "加载中…"); @@ -329,3 +408,4 @@ initializeUpload(); initializeSearch(); document.querySelector("#refresh-documents").addEventListener("click", loadDocuments); if (page === "documents") loadDocuments(); +window.addEventListener("pagehide", () => window.clearTimeout(batchPollTimer)); diff --git a/apps/admin-web/assets/styles.css b/apps/admin-web/assets/styles.css index d693fe7..f88d561 100644 --- a/apps/admin-web/assets/styles.css +++ b/apps/admin-web/assets/styles.css @@ -533,6 +533,50 @@ a { border-block: 1px solid var(--line); } +.batch-items { + display: grid; + gap: 8px; + margin: -8px 0 24px; +} + +.batch-item { + display: grid; + grid-template-columns: 34px minmax(0, 1fr) auto; + gap: 12px; + align-items: center; + min-height: 64px; + padding: 11px 13px; + border: 1px solid var(--line); + border-radius: 3px; + background: #fafaf8; +} + +.batch-item-order { + color: var(--brand); + font-family: var(--data-font); + font-size: 10px; + font-weight: 850; +} + +.batch-item-body { + display: grid; + min-width: 0; + gap: 5px; +} + +.batch-item-body strong { + overflow: hidden; + font-size: 12px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.batch-item-body span { + overflow-wrap: anywhere; + color: var(--muted); + font-size: 10px; +} + dt { color: var(--muted); font-family: var(--data-font); @@ -655,6 +699,15 @@ tbody tr:hover { .status-processing { background: #fff0ce; color: var(--warning); } .status-failed { background: #fbe6e3; color: var(--danger); } .status-uploaded { background: var(--brand-soft); color: var(--brand-deep); } +.status-pending, +.status-queued, +.status-running, +.status-retrying, +.status-accepted { background: #fff0ce; color: var(--warning); } +.status-succeeded, +.status-completed { background: #e5f3eb; color: var(--success); } +.status-rejected, +.status-partial_failure { background: #fbe6e3; color: var(--danger); } .empty-state { min-height: 240px; @@ -958,6 +1011,8 @@ select:focus { .search-controls { grid-template-columns: minmax(0, 1fr) 90px; } .search-controls textarea { grid-column: 1 / -1; } .metadata-grid { grid-template-columns: 1fr; } + .batch-item { grid-template-columns: 28px minmax(0, 1fr); } + .batch-item .status-pill { grid-column: 2; } } @media (max-width: 540px) { diff --git a/apps/admin-web/index.html b/apps/admin-web/index.html index 6241cb4..2be10fe 100644 --- a/apps/admin-web/index.html +++ b/apps/admin-web/index.html @@ -67,7 +67,7 @@

让知识进入检索链路

-

上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将在后台完成解析、切分与索引。

+

批量提交 PDF、DOCX、TXT 或 Markdown 文件,系统将逐个接收并在后台独立完成解析、切分与索引。

@@ -75,14 +75,15 @@ @@ -90,15 +91,16 @@ diff --git a/apps/api/src/ekp_api/providers.py b/apps/api/src/ekp_api/providers.py index 453df9f..7f25a90 100644 --- a/apps/api/src/ekp_api/providers.py +++ b/apps/api/src/ekp_api/providers.py @@ -18,6 +18,7 @@ from ekp_core.document_management import DocumentManagementService from ekp_core.document_parsing import DocumentParsingService from ekp_core.document_upload import DocumentUploadService from ekp_core.index_rebuild import resolve_active_index_profile +from ekp_core.ingestion_batches import IngestionBatchService from ekp_core.ingestion_jobs import ( AsyncIngestionSubmissionService, DerivedDocumentCleanupService, @@ -50,6 +51,7 @@ class ApiServices: ingestion: DocumentIndexingPipelineService submissions: AsyncIngestionSubmissionService + batches: IngestionBatchService jobs: IngestionJobService documents: DocumentManagementService search: SearchService @@ -310,6 +312,12 @@ def create_api_services(settings: Settings) -> ApiServices: components.jobs, components.dispatcher, ) + batches = IngestionBatchService( + components.session_factory, + submissions, + max_files=settings.batch_max_files, + max_total_bytes=settings.batch_max_total_bytes, + ) dense_retrieval = create_dense_retrieval_service( settings, components.embedding_provider, @@ -332,6 +340,7 @@ def create_api_services(settings: Settings) -> ApiServices: return ApiServices( ingestion=ingestion, submissions=submissions, + batches=batches, jobs=components.jobs, documents=documents, search=search, diff --git a/apps/api/src/ekp_api/routes.py b/apps/api/src/ekp_api/routes.py index 788501d..b12b481 100644 --- a/apps/api/src/ekp_api/routes.py +++ b/apps/api/src/ekp_api/routes.py @@ -9,6 +9,7 @@ from ekp_api.providers import ApiServices, ProviderConfigurationError, create_ap from ekp_api.schemas import ( DocumentResponse, IngestionAcceptedResponse, + IngestionBatchResponse, IngestionJobResponse, SearchRequest, SearchResponse, @@ -25,6 +26,7 @@ from ekp_core.document_upload import ( MediaTypeMismatchError, UnsupportedDocumentTypeError, ) +from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundError, BatchSource from ekp_core.ingestion_jobs import IngestionJobNotFoundError router = APIRouter() @@ -98,6 +100,61 @@ def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse: return IngestionJobResponse.from_job(job) +@router.post( + "/ingestion-batches", + response_model=IngestionBatchResponse, + status_code=status.HTTP_202_ACCEPTED, + tags=["jobs"], +) +def upload_ingestion_batch( + services: Services, + files: Annotated[ + list[UploadFile], + File(description="One or more PDF, DOCX, TXT, or Markdown files"), + ], +) -> IngestionBatchResponse: + """Accept a bounded batch while isolating every file's result.""" + try: + view = services.batches.submit( + tuple( + BatchSource( + filename=file.filename or "", + media_type=file.content_type, + source=file.file, + ) + for file in files + ) + ) + except BatchFileCountLimitError as error: + raise HTTPException( + status_code=status.HTTP_413_CONTENT_TOO_LARGE, + detail="batch file count exceeds the configured limit", + ) from error + except Exception as error: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="batch could not be accepted", + ) from error + return IngestionBatchResponse.from_view(view) + + +@router.get( + "/ingestion-batches/{batch_id}", + response_model=IngestionBatchResponse, + tags=["jobs"], +) +def get_ingestion_batch(batch_id: UUID, services: Services) -> IngestionBatchResponse: + """Return aggregate and per-file status for Admin Console polling.""" + try: + view = services.batches.get(batch_id) + except BatchNotFoundError as error: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="ingestion batch not found", + ) from error + return IngestionBatchResponse.from_view(view) + + @router.get("/documents", response_model=list[DocumentResponse], tags=["documents"]) def list_documents(services: Services) -> list[DocumentResponse]: """List canonical document metadata for the Demo admin page.""" diff --git a/apps/api/src/ekp_api/schemas.py b/apps/api/src/ekp_api/schemas.py index d7c9632..ca17ce1 100644 --- a/apps/api/src/ekp_api/schemas.py +++ b/apps/api/src/ekp_api/schemas.py @@ -5,9 +5,17 @@ from uuid import UUID from pydantic import BaseModel, ConfigDict, Field +from ekp_core.ingestion_batches import BatchAggregateStatus, BatchItemView, BatchView from ekp_core.ingestion_jobs import IngestionSubmission from ekp_core.search import SearchResult -from ekp_domain import Document, DocumentStatus, IngestionJob, IngestionJobStatus, IngestionStage +from ekp_domain import ( + BatchItemAcceptanceStatus, + Document, + DocumentStatus, + IngestionJob, + IngestionJobStatus, + IngestionStage, +) class DocumentResponse(BaseModel): @@ -88,6 +96,80 @@ class IngestionJobResponse(BaseModel): ) +class IngestionBatchItemResponse(BaseModel): + """Per-file acceptance and latest asynchronous processing state.""" + + item_index: int + filename: str + acceptance_status: BatchItemAcceptanceStatus + document_id: UUID | None + job_id: UUID | None + job_status: IngestionJobStatus | None + stage: IngestionStage | None + attempt: int + max_attempts: int | None + error_message: str | None + + @classmethod + def from_view(cls, view: BatchItemView) -> "IngestionBatchItemResponse": + job = view.job + return cls( + item_index=view.item.item_index, + filename=view.item.filename, + acceptance_status=view.item.acceptance_status, + document_id=view.item.document_id, + job_id=view.item.job_id, + job_status=job.status if job is not None else None, + stage=job.stage if job is not None else None, + attempt=job.attempt_count if job is not None else 0, + max_attempts=job.max_attempts if job is not None else None, + error_message=(job.error_message if job is not None else view.item.error_message), + ) + + +class IngestionBatchResponse(BaseModel): + """Batch aggregate with ordered independently processed files.""" + + batch_id: UUID + status: BatchAggregateStatus + total_files: int + accepted_files: int + rejected_files: int + completed_files: int + created_at: datetime + items: list[IngestionBatchItemResponse] + + @classmethod + def from_view(cls, view: BatchView) -> "IngestionBatchResponse": + items = [IngestionBatchItemResponse.from_view(item) for item in view.items] + accepted = sum( + item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED for item in items + ) + rejected = sum( + item.acceptance_status == BatchItemAcceptanceStatus.REJECTED for item in items + ) + completed = sum( + item.acceptance_status == BatchItemAcceptanceStatus.REJECTED + or item.job_status + in { + IngestionJobStatus.SUCCEEDED, + IngestionJobStatus.FAILED, + IngestionJobStatus.CANCELLED, + } + for item in items + ) + return cls( + batch_id=view.batch.id, + status=view.status, + total_files=view.batch.total_files, + accepted_files=accepted, + rejected_files=rejected, + completed_files=completed, + created_at=view.batch.created_at, + items=items, + ) + + class SearchRequest(BaseModel): """Validated public search input; normalization remains in the core service.""" diff --git a/docs/api/API_CONTRACT.md b/docs/api/API_CONTRACT.md index 9a273ee..2ba44ce 100644 --- a/docs/api/API_CONTRACT.md +++ b/docs/api/API_CONTRACT.md @@ -11,11 +11,13 @@ Use OpenAPI as executable source of truth once implementation starts. Provider-s DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一 版本化为 `/api/v1/*`。 -`DEMO-015` 已将单文件上传切换为持久化异步 Job;批量提交仍留给 `DEMO-016`。 +`DEMO-015` 已将单文件上传切换为持久化异步 Job;`DEMO-016` 已增加批量提交和聚合查询。 | Method | Path | 行为 | |---|---|---| | `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` | +| `POST` | `/api/v1/ingestion-batches` | multipart `files` 多文件接收;返回批次与逐文件状态 | +| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新 Job 状态 | | `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 | | `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 | | `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 | @@ -55,7 +57,7 @@ Search 响应: ``` 公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`, -文档或 Job 不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回 +文档、Job 或批次不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回 `503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或 厂商响应。 @@ -74,7 +76,7 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏 集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、 模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。 -## 异步入库契约(DEMO-015 已实现;DEMO-016 批量部分待实现) +## 异步入库契约(DEMO-015~016 已实现) | Method | Path | 行为 | |---|---|---| @@ -101,17 +103,29 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏 } ``` -DEMO-016 的批量响应示意: +批量字段名为 `files`,成功返回 `202`。响应示意: ```json { "batch_id": "d21f8012-4c4e-4bb4-89af-e11c0f9f470e", + "status": "processing", + "total_files": 2, + "accepted_files": 1, + "rejected_files": 1, + "completed_files": 1, + "created_at": "2026-09-02T12:00:00Z", "items": [ { + "item_index": 0, "filename": "退款政策.md", + "acceptance_status": "accepted", "document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752", "job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e", - "status": "queued" + "job_status": "queued", + "stage": null, + "attempt": 0, + "max_attempts": 3, + "error_message": null } ] } @@ -119,8 +133,11 @@ DEMO-016 的批量响应示意: Job 公开状态为 `queued | running | retrying | succeeded | failed | cancelled`,并通过 `stage` 表示 `parsing | cleaning | chunking | embedding | indexing`。错误字段只返回有界、安全摘要。 -批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚其他文件。文件数量、单文件大小和 -请求总大小均由服务端配置限制。 +批次状态为 `processing | completed | partial_failure | failed`;逐文件接收状态为 +`pending | accepted | rejected`。批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚 +其他文件。文件数量、单文件大小和请求总大小分别由 `BATCH_MAX_FILES`、`MAX_UPLOAD_BYTES` +和 `BATCH_MAX_TOTAL_BYTES` 限制。数量超限返回 `413`;类型、大小或内容不合规作为该文件的 +`rejected` 安全结果返回,其他合规文件继续接收。 自动重试是服务端 Job 行为,不要求客户端再次上传,也不新增公开手动重试接口。每次重试前 清理派生数据但保留原始文件和稳定的 Document ID;达到次数上限或清理失败后进入 `failed`。 diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.md index db41147..a510d66 100644 --- a/docs/architecture/ARCHITECTURE.md +++ b/docs/architecture/ARCHITECTURE.md @@ -179,6 +179,10 @@ Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Fail API 只负责校验并持久化不可变源文件、Document 和 Ingestion Job,然后返回 `202`。批次是 提交与聚合查询边界,不是跨文件数据库事务;每个文件独立排队、执行和失败。 +DEMO-016 以 canonical `ingestion_batches` / `ingestion_batch_items` 保存输入顺序和接收引用; +处理阶段、Attempt 与最终结果继续以 `ingestion_jobs` / `ingestion_attempts` 为事实源,批次表 +不复制 Worker 状态。 + ```text Batch Request ├─ Source A + Document A + Job A ─→ Worker diff --git a/docs/demo/DEMO_SCOPE.md b/docs/demo/DEMO_SCOPE.md index fca286f..d467284 100644 --- a/docs/demo/DEMO_SCOPE.md +++ b/docs/demo/DEMO_SCOPE.md @@ -340,7 +340,7 @@ API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重 ### DEMO-016 批量异步上传 -在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和 +**已完成。** 在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和 Admin Console 状态展示。批次内文件相互隔离,不包含目录上传、压缩包解包或断点续传。 --- @@ -397,8 +397,8 @@ DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收 实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与 `ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和 -任务拆分;`DEMO-012`、`DEMO-015` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围, -后续严格按 `DEMO-016`、`DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为 +任务拆分;`DEMO-012`、`DEMO-015~016` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围, +后续严格按 `DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为 可选项,不因本状态记录自动进入范围。 可选: diff --git a/docs/development/ADMIN_CONSOLE.md b/docs/development/ADMIN_CONSOLE.md index 4de4b3e..fb27b9a 100644 --- a/docs/development/ADMIN_CONSOLE.md +++ b/docs/development/ADMIN_CONSOLE.md @@ -16,7 +16,7 @@ make run | 页面 | 地址 | 对应业务接口 | |---|---|---| -| 文档上传 | `/admin/upload` | `POST /api/v1/documents` | +| 文档上传 | `/admin/upload` | `POST/GET /api/v1/ingestion-batches*` | | 文档列表与详情 | `/admin/documents` | `GET/DELETE /api/v1/documents*` | | Search Lab | `/admin/search` | `POST /api/v1/search` | @@ -34,9 +34,9 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF ## Demo 操作闭环 -1. 在上传页选择一个 PDF、DOCX、TXT 或 Markdown 文件并提交; -2. 接口返回 `202` 后,当前页面显示已接收;在文档列表刷新查看 `Processing/Ready/Failed`; -3. 在文档列表查看状态和安全元数据; +1. 在上传页选择一个或多个 PDF、DOCX、TXT 或 Markdown 文件并提交; +2. 接口返回 `202` 后,页面自动轮询批次,逐文件显示接收状态、处理阶段、尝试次数和安全错误; +3. 批次进入 `Completed`、`Partial failure` 或 `Failed` 后停止轮询,再在文档列表查看元数据; 4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和 Rerank Score; 5. 如需清理 Demo 数据,在文档列表确认后直接删除。 @@ -46,7 +46,8 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF - 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时; - API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入; - 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile; -- 当前 Console 尚不轮询 Job 详情或展示 Attempt;该状态 UI 与批量上传一起由 DEMO-016 实现; +- Console 只轮询当前批次的聚合接口,不读取 Provider/队列细节,也不提供人工重试按钮; +- 批量上传不包含目录上传、压缩包解包、断点续传或跨批次去重; - 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25 或完整 V1 Admin Console 导航; - 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。 diff --git a/docs/development/ASYNC_INGESTION_SETUP.md b/docs/development/ASYNC_INGESTION_SETUP.md index 20c5bba..ca81bf8 100644 --- a/docs/development/ASYNC_INGESTION_SETUP.md +++ b/docs/development/ASYNC_INGESTION_SETUP.md @@ -2,11 +2,12 @@ ## 范围 -DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker。OceanBase 中的 +DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker,DEMO-016 在其上增加逐文件 +隔离的批量接收和状态聚合。OceanBase 中的 `ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID, 不携带文件内容、Provider 凭据或业务状态。 -本任务不提供批量上传和批次 UI;这些能力属于 DEMO-016。 +批次只聚合接收与 Job 状态,不改变每个文件的独立执行和重试边界。 ## 运行时配置 @@ -19,6 +20,8 @@ INGESTION_RETRY_BACKOFF_SECONDS=2 INGESTION_RETRY_BACKOFF_MAX_SECONDS=60 INGESTION_WORKER_LEASE_SECONDS=300 INGESTION_RECOVERY_INTERVAL_SECONDS=30 +BATCH_MAX_FILES=20 +BATCH_MAX_TOTAL_BYTES=209715200 ``` Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。 @@ -72,6 +75,18 @@ Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`; `parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或 返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。 +批量提交与查询: + +```text +POST /api/v1/ingestion-batches # multipart 字段 files,可重复 +GET /api/v1/ingestion-batches/{batch_id} +``` + +批次创建后逐文件返回 `accepted` 或 `rejected`;已接收文件继续通过上述 Job 状态异步推进。 +批次聚合状态为 `processing | completed | partial_failure | failed`。请求文件数超过 +`BATCH_MAX_FILES` 时整体返回 `413`;单文件或共享总字节预算超限只拒绝当前文件,不删除此前 +已接收文件。默认总预算 200 MiB,且配置不得小于单文件 `MAX_UPLOAD_BYTES`。 + ## 故障检查 1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue; diff --git a/docs/development/REST_API_SETUP.md b/docs/development/REST_API_SETUP.md index 6860054..aaf8704 100644 --- a/docs/development/REST_API_SETUP.md +++ b/docs/development/REST_API_SETUP.md @@ -3,10 +3,10 @@ ## 范围 DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步 -Job。API 层只负责输入校验、 +Job,DEMO-016 增加批量接收与聚合查询。API 层只负责输入校验、 错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。 -当前不包含 ACL/RBAC、软删除、批量上传或 Hybrid Search。 +当前不包含 ACL/RBAC、软删除、目录/压缩包上传、断点续传或 Hybrid Search。 ## 启动前提 @@ -40,6 +40,16 @@ curl -F 'file=@./policy.md;type=text/markdown' \ curl http://127.0.0.1:8000/api/v1/jobs/ ``` +批量上传并查询聚合进度: + +```bash +curl -F 'files=@./policy.md;type=text/markdown' \ + -F 'files=@./manual.pdf;type=application/pdf' \ + http://127.0.0.1:8000/api/v1/ingestion-batches + +curl http://127.0.0.1:8000/api/v1/ingestion-batches/ +``` + 检索: ```bash diff --git a/docs/plans/ASYNC_INGESTION_MCP_PLAN.md b/docs/plans/ASYNC_INGESTION_MCP_PLAN.md index e67d2e2..89d5d5e 100644 --- a/docs/plans/ASYNC_INGESTION_MCP_PLAN.md +++ b/docs/plans/ASYNC_INGESTION_MCP_PLAN.md @@ -9,13 +9,13 @@ | 能力 | 当前实现 | 结论 | |---|---|---| | 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 | -| 批量上传 | API 与 Admin Console 都只支持一个文件 | 未实现 | -| 失败补偿 | 上传元数据失败会删除源对象;解析元数据失败会删除本次解析产物;索引失败会删除当前 Document/Profile 的可能残留向量 | 已有局部补偿,不是自动重试 | +| 批量上传 | 批次 API、逐文件 Job 状态和 Admin Console 自动轮询 | DEMO-016 已实现 | +| 失败补偿 | 阶段内补偿 + 重试前清理全部派生数据,保留不可变源和执行历史 | DEMO-015 已实现 | | 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 | | 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 | -| MCP | 已有 MCP 业务契约和可复用 Search Service/REST API,没有可运行 MCP Gateway 或 WorkBuddy 验证 | 未实现 | +| MCP | 独立 Gateway、认证边界、`search_knowledge` 与 WorkBuddy 协议级验收 | DEMO-012 已实现 | -PLAN-001 的上述缺口已由 DEMO-015 补齐;批量提交与状态 UI 仍由 DEMO-016 实现。 +PLAN-001 的异步执行缺口已由 DEMO-015 补齐,批量提交与状态 UI 已由 DEMO-016 补齐。 ## 2. Task 归并结论 @@ -51,7 +51,7 @@ PLAN-001 的上述缺口已由 DEMO-015 补齐;批量提交与状态 UI 仍由 该 Task 只完成公共知识库入库所需的 Job/Retry 与队列基线,不扩展到完整审计系统、 Compilation、OCR、跨区域高可用或通用企业级 Job 平台。 -### DEMO-016:批量异步上传与状态展示(新增) +### DEMO-016:批量异步上传与状态展示(已完成) 建立在 DEMO-015 之上,不与 Job 基础合并: @@ -79,8 +79,8 @@ PLAN-002(已完成) 推荐严格按以下顺序执行并逐项闭环: 1. `PLAN-002`(已完成) -2. `DEMO-015` -3. `DEMO-016` +2. `DEMO-015`(已完成) +3. `DEMO-016`(已完成) 4. `DEPLOY-001` 5. `DEPLOY-002` 6. `DEPLOY-003` diff --git a/docs/plans/PROGRESS.md b/docs/plans/PROGRESS.md index e1b0021..50ee46c 100644 --- a/docs/plans/PROGRESS.md +++ b/docs/plans/PROGRESS.md @@ -1,12 +1,12 @@ # 项目进度 -当前阶段:异步单文件入库闭环完成,准备实施批量上传 +当前阶段:批量异步上传闭环完成,准备交付生产镜像 -已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-015、ARCH-001~ARCH-006、PLAN-001~PLAN-002 +已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002 进行中:无 阻塞:无 -下一任务:`DEMO-016`。随后严格执行 `DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。 +下一任务:`DEPLOY-001`。随后严格执行 `DEPLOY-002 → DEPLOY-003`。 Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006` 执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。 @@ -354,3 +354,26 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR 真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收 重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。 - 下一任务:`DEMO-016`。 + +### DEMO-016(已完成) + +- 新增 canonical `ingestion_batches` / `ingestion_batch_items` 及 forward-only migration,保存 + 输入顺序、逐文件接收结果与 Document/Job 引用;异步阶段和最终结果仍以 Job 为事实源, + 批次不是跨文件事务。 +- 新增 `POST /api/v1/ingestion-batches` 与 `GET /api/v1/ingestion-batches/{batch_id}`;批量 + 请求字段为 `files`,成功返回 `202`、聚合状态和逐文件接收/Job 状态。类型、大小或内容错误 + 只拒绝当前文件,错误映射为固定安全摘要,其他文件继续接收。 +- 新增 `BATCH_MAX_FILES` 与 `BATCH_MAX_TOTAL_BYTES`,复用 `MAX_UPLOAD_BYTES` 约束单文件; + 上传流共享总字节预算,不把整批文件读入内存。数量超限在创建批次前返回 `413`。 +- Admin Console 支持多选提交,显示当前批次的接收数量、完成数量、逐文件阶段、Attempt 与安全 + 错误,并每 1.5 秒自动轮询;进入 `completed`、`partial_failure` 或 `failed` 后停止轮询。 + DOM 数据继续只通过文本节点渲染。 +- 集成测试覆盖顺序稳定、部分拒绝不回滚、数量/单文件/批次总大小、全成功/部分失败聚合、 + 未知批次、OpenAPI 和前端契约;迁移 drift 增加两张新表。Ruff Format/Lint、Mypy、 + JavaScript 语法、依赖锁、Alembic Head/离线升级均通过;Unit + Integration `183 passed`, + RAG Evaluation `5 passed`。 +- 浏览器验收确认桌面布局和 390px 窄屏布局正常、无根页面横向溢出、无控制台警告或错误; + 本地页面不注入云端凭据,批次结果行为由确定性 API 集成测试验证。 +- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续 + 复用 DEMO-015,Accepted ADR 未变更。 +- 下一任务:`DEPLOY-001`。 diff --git a/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md b/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md index e76ddfc..cde092b 100644 --- a/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md +++ b/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md @@ -82,7 +82,8 @@ Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务 ### DEMO-016 -基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console 状态展示。 +**已完成。** 基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console +状态展示;下一交付门槛为 DEPLOY-001 的生产镜像。 ### DEPLOY-001 diff --git a/docs/plans/TASKS.md b/docs/plans/TASKS.md index a096eed..ed2a828 100644 --- a/docs/plans/TASKS.md +++ b/docs/plans/TASKS.md @@ -29,7 +29,7 @@ validation. Provider credentials and internal endpoints are runtime inputs and n - [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification - [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线 - [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup -- [ ] DEMO-016 Batch async upload API + per-file Job status + Admin Console +- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console - [ ] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker - [ ] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline - [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance diff --git a/docs/plans/V1_EXECUTION_PLAN.md b/docs/plans/V1_EXECUTION_PLAN.md index f4a33d6..20b26ee 100644 --- a/docs/plans/V1_EXECUTION_PLAN.md +++ b/docs/plans/V1_EXECUTION_PLAN.md @@ -17,7 +17,7 @@ 1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛; 2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理; -3. `DEMO-016`:批量异步上传、批次/Job 查询和 Admin Console 状态展示; +3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示; 4. `DEPLOY-001`:API/Admin、MCP Gateway、Async Worker 生产镜像; 5. `DEPLOY-002`:Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口; 6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。 diff --git a/docs/product/PRD.md b/docs/product/PRD.md index ef561be..576db92 100644 --- a/docs/product/PRD.md +++ b/docs/product/PRD.md @@ -465,6 +465,9 @@ Document processing 必须异步执行。 源文件与 Document/Job 成功持久化后返回 `202 Accepted`。批量上传中的每个文件拥有独立的 Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批次内其他已接收文件。 +当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、 +断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。 + **REQ-INGEST-004** 展示处理阶段: diff --git a/migrations/README.md b/migrations/README.md index 8d5b044..8188767 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -21,3 +21,7 @@ single-active generated unique index. DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state; the latter preserves numbered history and uses a generated nullable active marker plus a unique index to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite. + +DEMO-016 adds `ingestion_batches` and `ingestion_batch_items`. They retain input order and per-file +acceptance references while Job rows remain the processing state source of truth; a batch is an aggregate, +not a cross-file transaction. diff --git a/migrations/versions/20260902_0008_create_ingestion_batches.py b/migrations/versions/20260902_0008_create_ingestion_batches.py new file mode 100644 index 0000000..ba22b8f --- /dev/null +++ b/migrations/versions/20260902_0008_create_ingestion_batches.py @@ -0,0 +1,109 @@ +"""Create ingestion batch aggregation metadata. + +Revision ID: 20260902_0008 +Revises: 20260902_0007 +Create Date: 2026-09-02 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "20260902_0008" +down_revision: str | None = "20260902_0007" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Persist ordered per-file acceptance without making a batch transactional.""" + op.create_table( + "ingestion_batches", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("total_files", sa.Integer(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.CheckConstraint("total_files > 0", name="total_files_positive"), + sa.PrimaryKeyConstraint("id", name="pk_ingestion_batches"), + ) + op.create_table( + "ingestion_batch_items", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("batch_id", sa.Uuid(), nullable=False), + sa.Column("item_index", sa.Integer(), nullable=False), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column( + "acceptance_status", + sa.Enum( + "pending", + "accepted", + "rejected", + name="batch_item_acceptance_status", + native_enum=False, + create_constraint=False, + ), + nullable=False, + ), + sa.Column("document_id", sa.Uuid(), nullable=True), + sa.Column("job_id", sa.Uuid(), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.CheckConstraint("item_index >= 0", name="item_index_non_negative"), + sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"), + sa.CheckConstraint( + "acceptance_status IN ('pending', 'accepted', 'rejected')", + name="acceptance_status_valid", + ), + sa.ForeignKeyConstraint( + ["batch_id"], + ["ingestion_batches.id"], + name="fk_ingestion_batch_items_batch_id_ingestion_batches", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["document_id"], + ["documents.id"], + name="fk_ingestion_batch_items_document_id_documents", + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["job_id"], + ["ingestion_jobs.id"], + name="fk_ingestion_batch_items_job_id_ingestion_jobs", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_ingestion_batch_items"), + ) + op.create_index( + "uq_ingestion_batch_items_order", + "ingestion_batch_items", + ["batch_id", "item_index"], + unique=True, + ) + op.create_index( + "uq_ingestion_batch_items_job_id", + "ingestion_batch_items", + ["job_id"], + unique=True, + ) + + +def downgrade() -> None: + """Reject downgrade because project migrations are forward-only.""" + raise NotImplementedError("Database migrations are forward-only") diff --git a/packages/core/src/ekp_core/config.py b/packages/core/src/ekp_core/config.py index ba27cff..b8034b4 100644 --- a/packages/core/src/ekp_core/config.py +++ b/packages/core/src/ekp_core/config.py @@ -39,6 +39,8 @@ class Settings(BaseSettings): ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400) ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400) ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600) + batch_max_files: int = Field(default=20, ge=1, le=100) + batch_max_total_bytes: int = Field(default=200 * 1024 * 1024, gt=0) object_storage_provider: Literal["local", "bos"] = "bos" object_storage_path: Path = Path("storage") bos_endpoint: str | None = None @@ -108,6 +110,8 @@ class Settings(BaseSettings): ) if self.ingestion_worker_lease_seconds <= longest_provider_timeout: raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts") + if self.batch_max_total_bytes < self.max_upload_bytes: + raise ValueError("batch_max_total_bytes must be at least max_upload_bytes") if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"): raise ValueError("mcp_path must start with '/' and must not end with '/'") if not self.mcp_allowed_hosts: diff --git a/packages/core/src/ekp_core/ingestion_batches.py b/packages/core/src/ekp_core/ingestion_batches.py new file mode 100644 index 0000000..4e4da62 --- /dev/null +++ b/packages/core/src/ekp_core/ingestion_batches.py @@ -0,0 +1,270 @@ +"""Batch submission and aggregate status over independent ingestion Jobs.""" + +from dataclasses import dataclass +from enum import StrEnum +from typing import Protocol +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.orm import Session, sessionmaker + +from ekp_core.document_upload import ( + DocumentTooLargeError, + DocumentUploadError, + EmptyDocumentError, + MediaTypeMismatchError, + UnsupportedDocumentTypeError, +) +from ekp_core.ingestion_jobs import AsyncIngestionSubmissionService +from ekp_domain import ( + BatchItemAcceptanceStatus, + IngestionBatch, + IngestionBatchItem, + IngestionJob, + IngestionJobStatus, +) +from ekp_object_storage import BinarySource + +SAFE_UNSUPPORTED_ERROR = "unsupported document type or media type" +SAFE_FILE_SIZE_ERROR = "document exceeds the configured upload limit" +SAFE_BATCH_SIZE_ERROR = "batch exceeds the configured total upload limit" +SAFE_EMPTY_ERROR = "document must contain content" +SAFE_ACCEPTANCE_ERROR = "document could not be accepted" + + +class BatchNotFoundError(LookupError): + """Raised when a requested ingestion batch does not exist.""" + + +class BatchFileCountLimitError(ValueError): + """Raised before persistence when the multipart file count is invalid.""" + + +class BatchAggregateStatus(StrEnum): + """Computed batch state derived from item acceptance and durable Jobs.""" + + PROCESSING = "processing" + COMPLETED = "completed" + PARTIAL_FAILURE = "partial_failure" + FAILED = "failed" + + +class BatchTotalSizeLimitError(ValueError): + """Raised while streaming the file that crosses the shared batch budget.""" + + +class _ReadableSource(Protocol): + def read(self, size: int = -1, /) -> bytes: + """Read bytes from an uploaded file.""" + + +@dataclass(frozen=True, slots=True) +class BatchSource: + """Provider-neutral input for one ordered multipart file.""" + + filename: str + media_type: str | None + source: _ReadableSource + + +@dataclass(frozen=True, slots=True) +class BatchItemView: + """Detached batch item plus its current optional Job state.""" + + item: IngestionBatchItem + job: IngestionJob | None + + +@dataclass(frozen=True, slots=True) +class BatchView: + """Detached aggregate used by REST without exposing persistence sessions.""" + + batch: IngestionBatch + items: tuple[BatchItemView, ...] + + @property + def status(self) -> BatchAggregateStatus: + """Aggregate terminal results without copying Job state into batch rows.""" + if any( + view.item.acceptance_status == BatchItemAcceptanceStatus.PENDING + or ( + view.item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED + and ( + view.job is None + or view.job.status + in { + IngestionJobStatus.QUEUED, + IngestionJobStatus.RUNNING, + IngestionJobStatus.RETRYING, + } + ) + ) + for view in self.items + ): + return BatchAggregateStatus.PROCESSING + succeeded = sum( + view.job is not None and view.job.status == IngestionJobStatus.SUCCEEDED + for view in self.items + ) + failed = len(self.items) - succeeded + if failed == 0: + return BatchAggregateStatus.COMPLETED + if succeeded == 0: + return BatchAggregateStatus.FAILED + return BatchAggregateStatus.PARTIAL_FAILURE + + +class _BatchByteBudget: + """Count bytes across independently streamed files without buffering them.""" + + def __init__(self, limit_bytes: int) -> None: + self._limit_bytes = limit_bytes + self._consumed_bytes = 0 + + def wrap(self, source: _ReadableSource) -> BinarySource: + return _BatchLimitedSource(source, self) + + def consume(self, size_bytes: int) -> None: + self._consumed_bytes += size_bytes + if self._consumed_bytes > self._limit_bytes: + raise BatchTotalSizeLimitError(SAFE_BATCH_SIZE_ERROR) + + +class _BatchLimitedSource: + """Apply one shared request budget through the ObjectStorage stream contract.""" + + def __init__(self, source: _ReadableSource, budget: _BatchByteBudget) -> None: + self._source = source + self._budget = budget + + def read(self, size: int = -1, /) -> bytes: + chunk = self._source.read(size) + self._budget.consume(len(chunk)) + return chunk + + +class IngestionBatchService: + """Accept files independently and persist ordered aggregate status.""" + + def __init__( + self, + session_factory: sessionmaker[Session], + submissions: AsyncIngestionSubmissionService, + *, + max_files: int, + max_total_bytes: int, + ) -> None: + if max_files <= 0 or max_total_bytes <= 0: + raise ValueError("batch limits must be positive") + self._session_factory = session_factory + self._submissions = submissions + self._max_files = max_files + self._max_total_bytes = max_total_bytes + + def submit(self, sources: tuple[BatchSource, ...]) -> BatchView: + """Create a batch and independently accept or reject every ordered file.""" + if not sources or len(sources) > self._max_files: + raise BatchFileCountLimitError( + f"batch must contain between 1 and {self._max_files} files" + ) + batch_id, item_ids = self._create_pending_batch(sources) + budget = _BatchByteBudget(self._max_total_bytes) + for item_id, source in zip(item_ids, sources, strict=True): + try: + submission = self._submissions.submit( + source.filename, + source.media_type, + budget.wrap(source.source), + ) + except Exception as error: + self._record_rejected(item_id, _safe_acceptance_error(error)) + continue + self._record_accepted( + item_id, + document_id=submission.document.id, + job_id=submission.job.id, + ) + return self.get(batch_id) + + def get(self, batch_id: UUID) -> BatchView: + """Return ordered items and fresh Job state for polling clients.""" + with self._session_factory() as session: + batch = session.get(IngestionBatch, batch_id) + if batch is None: + raise BatchNotFoundError(str(batch_id)) + items = tuple( + session.scalars( + select(IngestionBatchItem) + .where(IngestionBatchItem.batch_id == batch_id) + .order_by(IngestionBatchItem.item_index) + ).all() + ) + views = tuple( + BatchItemView( + item=item, + job=session.get(IngestionJob, item.job_id) if item.job_id is not None else None, + ) + for item in items + ) + return BatchView(batch=batch, items=views) + + def _create_pending_batch( + self, sources: tuple[BatchSource, ...] + ) -> tuple[UUID, tuple[UUID, ...]]: + """Persist every input position before starting external storage writes.""" + with self._session_factory() as session: + batch = IngestionBatch(total_files=len(sources)) + session.add(batch) + session.flush() + items = tuple( + IngestionBatchItem( + batch_id=batch.id, + item_index=index, + filename=_display_filename(source.filename), + ) + for index, source in enumerate(sources) + ) + session.add_all(items) + session.commit() + return batch.id, tuple(item.id for item in items) + + def _record_accepted(self, item_id: UUID, *, document_id: UUID, job_id: UUID) -> None: + with self._session_factory() as session: + item = session.get(IngestionBatchItem, item_id) + if item is None: + raise BatchNotFoundError(str(item_id)) + item.acceptance_status = BatchItemAcceptanceStatus.ACCEPTED + item.document_id = document_id + item.job_id = job_id + item.error_message = None + session.commit() + + def _record_rejected(self, item_id: UUID, error_message: str) -> None: + with self._session_factory() as session: + item = session.get(IngestionBatchItem, item_id) + if item is None: + raise BatchNotFoundError(str(item_id)) + item.acceptance_status = BatchItemAcceptanceStatus.REJECTED + item.error_message = error_message + session.commit() + + +def _display_filename(filename: str) -> str: + """Store a bounded safe label even when the upload filename is invalid.""" + display = filename.replace("\x00", "�").strip() + return display[:255] or "" + + +def _safe_acceptance_error(error: Exception) -> str: + """Map per-file failures without persisting exception text or credentials.""" + if isinstance(error, (UnsupportedDocumentTypeError, MediaTypeMismatchError)): + return SAFE_UNSUPPORTED_ERROR + if isinstance(error, DocumentTooLargeError): + return SAFE_FILE_SIZE_ERROR + if isinstance(error, BatchTotalSizeLimitError): + return SAFE_BATCH_SIZE_ERROR + if isinstance(error, EmptyDocumentError): + return SAFE_EMPTY_ERROR + if isinstance(error, DocumentUploadError): + return SAFE_ACCEPTANCE_ERROR + return SAFE_ACCEPTANCE_ERROR diff --git a/packages/domain/src/ekp_domain/__init__.py b/packages/domain/src/ekp_domain/__init__.py index 054ea88..563fc39 100644 --- a/packages/domain/src/ekp_domain/__init__.py +++ b/packages/domain/src/ekp_domain/__init__.py @@ -8,6 +8,11 @@ from ekp_domain.index_profile import ( IndexProfileStatus, StoredIndexProfile, ) +from ekp_domain.ingestion_batch import ( + BatchItemAcceptanceStatus, + IngestionBatch, + IngestionBatchItem, +) from ekp_domain.ingestion_job import ( IngestionAttempt, IngestionAttemptStatus, @@ -19,6 +24,7 @@ from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocum __all__ = [ "Base", + "BatchItemAcceptanceStatus", "Chunk", "ChunkIndexState", "Document", @@ -30,6 +36,8 @@ __all__ = [ "IngestionJob", "IngestionJobStatus", "IngestionStage", + "IngestionBatch", + "IngestionBatchItem", "ParsedBlock", "ParsedBlockType", "ParsedDocument", diff --git a/packages/domain/src/ekp_domain/ingestion_batch.py b/packages/domain/src/ekp_domain/ingestion_batch.py new file mode 100644 index 0000000..1b5795c --- /dev/null +++ b/packages/domain/src/ekp_domain/ingestion_batch.py @@ -0,0 +1,96 @@ +"""Canonical batch submission metadata for per-file ingestion tracking.""" + +from datetime import datetime +from enum import StrEnum +from uuid import UUID, uuid4 + +from sqlalchemy import ( + CheckConstraint, + DateTime, + ForeignKey, + Index, + Integer, + String, + Text, + Uuid, + func, +) +from sqlalchemy import Enum as SqlEnum +from sqlalchemy.orm import Mapped, mapped_column + +from ekp_domain.document import Base + + +class BatchItemAcceptanceStatus(StrEnum): + """Source acceptance state, independent from its asynchronous Job state.""" + + PENDING = "pending" + ACCEPTED = "accepted" + REJECTED = "rejected" + + +class IngestionBatch(Base): + """Stable aggregation identifier for one bounded multipart submission.""" + + __tablename__ = "ingestion_batches" + __table_args__ = (CheckConstraint("total_files > 0", name="total_files_positive"),) + + id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4) + total_files: Mapped[int] = mapped_column(Integer, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class IngestionBatchItem(Base): + """One ordered input and its optional accepted Document/Job references.""" + + __tablename__ = "ingestion_batch_items" + __table_args__ = ( + CheckConstraint("item_index >= 0", name="item_index_non_negative"), + CheckConstraint("length(filename) > 0", name="filename_not_empty"), + CheckConstraint( + "acceptance_status IN ('pending', 'accepted', 'rejected')", + name="acceptance_status_valid", + ), + Index("uq_ingestion_batch_items_order", "batch_id", "item_index", unique=True), + Index("uq_ingestion_batch_items_job_id", "job_id", unique=True), + ) + + id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4) + batch_id: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("ingestion_batches.id", ondelete="CASCADE"), + nullable=False, + ) + item_index: Mapped[int] = mapped_column(Integer, nullable=False) + filename: Mapped[str] = mapped_column(String(255), nullable=False) + acceptance_status: Mapped[BatchItemAcceptanceStatus] = mapped_column( + SqlEnum( + BatchItemAcceptanceStatus, + name="batch_item_acceptance_status", + native_enum=False, + create_constraint=False, + validate_strings=True, + values_callable=lambda values: [item.value for item in values], + ), + nullable=False, + default=BatchItemAcceptanceStatus.PENDING, + ) + document_id: Mapped[UUID | None] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("documents.id", ondelete="SET NULL"), + nullable=True, + ) + job_id: Mapped[UUID | None] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("ingestion_jobs.id", ondelete="SET NULL"), + nullable=True, + ) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() + ) diff --git a/tests/integration/test_admin_console.py b/tests/integration/test_admin_console.py index 077271f..e92e9e9 100644 --- a/tests/integration/test_admin_console.py +++ b/tests/integration/test_admin_console.py @@ -43,7 +43,9 @@ def test_admin_console_contains_only_demo_pages_and_versioned_api_calls( assert all(label not in html for label in ("权限管理", "任务管理", "Compiled Knowledge")) assert 'const API_BASE = "/api/v1"' in script assert "innerHTML" not in script - assert all(endpoint in script for endpoint in ('"/documents"', '"/search"')) + assert all(endpoint in script for endpoint in ('"/ingestion-batches"', '"/search"')) + assert "multiple" in html + assert "BATCH_POLL_INTERVAL_MS" in script assert all(field in script for field in ("Page", "Section", "Dense Score", "Rerank Score")) assert "--brand: #FF3C00" in stylesheet assert "prefers-reduced-motion" in stylesheet diff --git a/tests/integration/test_document_migration.py b/tests/integration/test_document_migration.py index 0a32b24..d02c09e 100644 --- a/tests/integration/test_document_migration.py +++ b/tests/integration/test_document_migration.py @@ -30,6 +30,8 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None: "index_profiles", "ingestion_jobs", "ingestion_attempts", + "ingestion_batches", + "ingestion_batch_items", }.issubset(inspector.get_table_names()) assert "unrelated_provider_table" in inspector.get_table_names() assert {column["name"] for column in inspector.get_columns("documents")} == { @@ -101,3 +103,20 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None: "started_at", "finished_at", } + assert {column["name"] for column in inspector.get_columns("ingestion_batches")} == { + "id", + "total_files", + "created_at", + } + assert {column["name"] for column in inspector.get_columns("ingestion_batch_items")} == { + "id", + "batch_id", + "item_index", + "filename", + "acceptance_status", + "document_id", + "job_id", + "error_message", + "created_at", + "updated_at", + } diff --git a/tests/integration/test_document_search_api.py b/tests/integration/test_document_search_api.py index d859361..0c83df8 100644 --- a/tests/integration/test_document_search_api.py +++ b/tests/integration/test_document_search_api.py @@ -19,6 +19,7 @@ from ekp_core.document_indexing import DocumentIndexingPipelineService, Document from ekp_core.document_management import DocumentManagementService from ekp_core.document_parsing import DocumentParsingService from ekp_core.document_upload import DocumentUploadService +from ekp_core.ingestion_batches import IngestionBatchService from ekp_core.ingestion_jobs import ( AsyncIngestionSubmissionService, DerivedDocumentCleanupService, @@ -26,7 +27,7 @@ from ekp_core.ingestion_jobs import ( IngestionJobService, ) from ekp_core.search import SearchService -from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile +from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile, IngestionBatch from ekp_embedding import FakeEmbeddingProvider from ekp_object_storage import LocalObjectStorage from ekp_parser import ParserRouter @@ -57,6 +58,10 @@ class RecordingDispatcher: def _build_api_services( tmp_path: Path, + *, + max_upload_bytes: int = 1024 * 1024, + batch_max_files: int = 20, + batch_max_total_bytes: int = 2 * 1024 * 1024, ) -> tuple[ ApiServices, sessionmaker[Session], @@ -84,7 +89,11 @@ def _build_api_services( embedding_model=embedding.model_name, vector_dimension=embedding.dimensions, ) - upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024) + upload = DocumentUploadService( + session_factory, + storage, + max_upload_bytes=max_upload_bytes, + ) parsing = DocumentParsingService(session_factory, storage, ParserRouter()) chunking = DocumentChunkingService( session_factory, @@ -102,6 +111,12 @@ def _build_api_services( jobs = IngestionJobService(session_factory, max_attempts=3) dispatcher = RecordingDispatcher() submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher) + batches = IngestionBatchService( + session_factory, + submissions, + max_files=batch_max_files, + max_total_bytes=batch_max_total_bytes, + ) runner = IngestionJobRunner( session_factory, parsing, @@ -134,6 +149,7 @@ def _build_api_services( ApiServices( ingestion=ingestion, submissions=submissions, + batches=batches, jobs=jobs, documents=documents, search=search, @@ -238,6 +254,8 @@ def test_rest_validation_not_found_and_openapi_contract( assert set(openapi["paths"]) >= { "/api/v1/documents", "/api/v1/documents/{document_id}", + "/api/v1/ingestion-batches", + "/api/v1/ingestion-batches/{batch_id}", "/api/v1/jobs/{job_id}", "/api/v1/search", } @@ -261,3 +279,141 @@ def test_rest_validation_not_found_and_openapi_contract( assert boolean_top_k.status_code == 422 assert missing_get.status_code == 404 assert missing_delete.status_code == 404 + + +def test_batch_upload_isolates_files_and_reports_terminal_status( + tmp_path: Path, + isolated_settings_environment: None, +) -> None: + """A rejected file must not roll back valid siblings in the same batch.""" + services, _, _, _, runner = _build_api_services(tmp_path) + app = create_app(Settings(app_env="test"), api_services=services) + + with TestClient(app) as client: + accepted = client.post( + "/api/v1/ingestion-batches", + files=[ + ("files", ("alpha.md", b"# Alpha\n\nFirst policy.", "text/markdown")), + ("files", ("blocked.csv", b"secret=must-not-persist", "text/csv")), + ("files", ("beta.txt", b"Second policy.", "text/plain")), + ], + ) + + assert accepted.status_code == 202 + body = accepted.json() + assert body["status"] == "processing" + assert body["total_files"] == 3 + assert body["accepted_files"] == 2 + assert body["rejected_files"] == 1 + assert body["completed_files"] == 1 + assert [item["filename"] for item in body["items"]] == [ + "alpha.md", + "blocked.csv", + "beta.txt", + ] + assert [item["acceptance_status"] for item in body["items"]] == [ + "accepted", + "rejected", + "accepted", + ] + assert body["items"][1]["error_message"] == ("unsupported document type or media type") + assert "must-not-persist" not in str(body) + + for item in body["items"]: + if item["job_id"] is not None: + runner.run(UUID(item["job_id"])) + + terminal = client.get(f"/api/v1/ingestion-batches/{body['batch_id']}") + + assert terminal.status_code == 200 + terminal_body = terminal.json() + assert terminal_body["status"] == "partial_failure" + assert terminal_body["completed_files"] == 3 + assert [item["job_status"] for item in terminal_body["items"]] == [ + "succeeded", + None, + "succeeded", + ] + + +def test_batch_upload_enforces_count_and_shared_byte_limits( + tmp_path: Path, + isolated_settings_environment: None, +) -> None: + """Request limits are bounded while already accepted siblings remain durable.""" + services, session_factory, _, _, _ = _build_api_services( + tmp_path, + max_upload_bytes=10, + batch_max_files=2, + batch_max_total_bytes=15, + ) + app = create_app(Settings(app_env="test"), api_services=services) + + with TestClient(app) as client: + too_many = client.post( + "/api/v1/ingestion-batches", + files=[ + ("files", ("one.txt", b"1", "text/plain")), + ("files", ("two.txt", b"2", "text/plain")), + ("files", ("three.txt", b"3", "text/plain")), + ], + ) + bounded = client.post( + "/api/v1/ingestion-batches", + files=[ + ("files", ("first.txt", b"1234567890", "text/plain")), + ("files", ("second.txt", b"abcdefghij", "text/plain")), + ], + ) + oversized = client.post( + "/api/v1/ingestion-batches", + files=[("files", ("large.txt", b"12345678901", "text/plain"))], + ) + + assert too_many.status_code == 413 + assert too_many.json()["detail"] == "batch file count exceeds the configured limit" + assert bounded.status_code == 202 + body = bounded.json() + assert body["accepted_files"] == 1 + assert body["rejected_files"] == 1 + assert body["items"][1]["error_message"] == ("batch exceeds the configured total upload limit") + assert oversized.status_code == 202 + assert oversized.json()["status"] == "failed" + assert oversized.json()["accepted_files"] == 0 + assert oversized.json()["items"][0]["error_message"] == ( + "document exceeds the configured upload limit" + ) + + # A count-rejected request creates no batch; the two accepted requests do. + with session_factory() as session: + assert len(session.scalars(select(IngestionBatch)).all()) == 2 + + +def test_batch_upload_completed_and_missing_contract( + tmp_path: Path, + isolated_settings_environment: None, +) -> None: + """All-success batches become completed and unknown IDs stay explicit.""" + services, _, _, _, runner = _build_api_services(tmp_path) + app = create_app(Settings(app_env="test"), api_services=services) + missing_id = uuid4() + + with TestClient(app) as client: + accepted = client.post( + "/api/v1/ingestion-batches", + files=[ + ("files", ("one.md", b"# One\n\nContent.", "text/markdown")), + ("files", ("two.txt", b"More content.", "text/plain")), + ], + ).json() + for item in accepted["items"]: + runner.run(UUID(item["job_id"])) + + completed = client.get(f"/api/v1/ingestion-batches/{accepted['batch_id']}") + missing = client.get(f"/api/v1/ingestion-batches/{missing_id}") + + assert completed.status_code == 200 + assert completed.json()["status"] == "completed" + assert completed.json()["completed_files"] == 2 + assert missing.status_code == 404 + assert missing.json()["detail"] == "ingestion batch not found" diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index faec677..6e66ed7 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -25,6 +25,8 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non assert settings.ingestion_retry_backoff_seconds == 2 assert settings.ingestion_retry_backoff_max_seconds == 60 assert settings.ingestion_worker_lease_seconds == 300 + assert settings.batch_max_files == 20 + assert settings.batch_max_total_bytes == 200 * 1024 * 1024 assert settings.object_storage_provider == "bos" assert settings.object_storage_path == Path("storage") assert settings.bos_endpoint is None @@ -157,3 +159,10 @@ def test_settings_require_worker_lease_longer_than_provider_timeout( ) -> None: with pytest.raises(ValidationError, match="worker_lease"): Settings(ingestion_worker_lease_seconds=30, embedding_timeout_seconds=30) + + +def test_settings_require_batch_limit_to_fit_one_file( + isolated_settings_environment: None, +) -> None: + with pytest.raises(ValidationError, match="batch_max_total_bytes"): + Settings(max_upload_bytes=100, batch_max_total_bytes=99)