diff --git a/.env.example b/.env.example
index ea71749..f21033e 100644
--- a/.env.example
+++ b/.env.example
@@ -18,6 +18,8 @@ INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
+BATCH_MAX_FILES=20
+BATCH_MAX_TOTAL_BYTES=209715200
OBJECT_STORAGE_PROVIDER=bos
OBJECT_STORAGE_PATH=storage
diff --git a/README.md b/README.md
index 5dcc4d5..5956082 100644
--- a/README.md
+++ b/README.md
@@ -133,7 +133,8 @@ source results are rehydrated from canonical relational chunks and retain both
Dense and reranker scores; stale vector content fails closed. See
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
-changes upload to asynchronous acceptance and adds Job status. Upload, list, detail, direct delete,
+changes upload to asynchronous acceptance and adds Job status. DEMO-016 adds bounded multi-file
+submission, batch/per-file status, and automatic Admin Console polling. Upload, list, detail, direct delete,
and reranked source search are documented in
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
outside the public response contract.
diff --git a/apps/admin-web/assets/app.js b/apps/admin-web/assets/app.js
index 02a0d1e..a73b43a 100644
--- a/apps/admin-web/assets/app.js
+++ b/apps/admin-web/assets/app.js
@@ -1,12 +1,33 @@
-// DEMO-010 stays API First: this client owns presentation state only and sends
-// every document/search operation through the versioned public REST contract.
+// The Admin Console owns presentation state only. Batch acceptance, Job state,
+// and retrieval remain behind the versioned public REST contract.
const API_BASE = "/api/v1";
+const BATCH_POLL_INTERVAL_MS = 1500;
+const terminalBatchStatuses = new Set(["completed", "partial_failure", "failed"]);
+let batchPollTimer;
+let activeBatchId;
const statusLabels = {
uploaded: "Uploaded",
processing: "Processing",
ready: "Ready",
failed: "Failed",
+ pending: "Pending",
+ accepted: "Accepted",
+ rejected: "Rejected",
+ queued: "Queued",
+ running: "Running",
+ retrying: "Retrying",
+ succeeded: "Succeeded",
+ completed: "Completed",
+ partial_failure: "Partial failure",
+};
+
+const stageLabels = {
+ parsing: "解析",
+ cleaning: "清理",
+ chunking: "切分",
+ embedding: "向量化",
+ indexing: "索引",
};
const pageTitles = {
@@ -93,23 +114,37 @@ function initializeUpload() {
const button = document.querySelector("#upload-button");
const updateFilename = () => {
- selected.textContent = input.files[0]?.name ?? "尚未选择文件";
+ const files = Array.from(input.files);
+ if (!files.length) {
+ selected.textContent = "尚未选择文件";
+ } else if (files.length === 1) {
+ selected.textContent = files[0].name;
+ } else {
+ selected.textContent = `已选择 ${files.length} 个文件`;
+ }
};
input.addEventListener("change", updateFilename);
form.addEventListener("submit", async (event) => {
event.preventDefault();
if (!input.files.length) return;
+ window.clearTimeout(batchPollTimer);
+ activeBatchId = undefined;
setButtonBusy(button, true, "提交中…");
try {
const formData = new FormData();
- formData.append("file", input.files[0]);
- const documentData = await requestJson("/documents", {
+ Array.from(input.files).forEach((file) => formData.append("files", file));
+ const batch = await requestJson("/ingestion-batches", {
method: "POST",
body: formData,
});
- renderUploadResult(documentData);
- showNotice("文档已接收并进入后台处理队列。", "success");
+ activeBatchId = batch.batch_id;
+ renderBatchResult(batch);
+ scheduleBatchPoll(batch);
+ showNotice(
+ `批次已接收:${batch.accepted_files} 个进入后台队列,${batch.rejected_files} 个被拒绝。`,
+ batch.rejected_files ? "error" : "success",
+ );
form.reset();
updateFilename();
} catch (error) {
@@ -120,18 +155,62 @@ function initializeUpload() {
});
}
-function renderUploadResult(documentData) {
+function scheduleBatchPoll(batch) {
+ window.clearTimeout(batchPollTimer);
+ if (batch.batch_id !== activeBatchId) return;
+ if (terminalBatchStatuses.has(batch.status)) return;
+ batchPollTimer = window.setTimeout(async () => {
+ try {
+ const latest = await requestJson(`/ingestion-batches/${batch.batch_id}`);
+ renderBatchResult(latest);
+ scheduleBatchPoll(latest);
+ } catch (error) {
+ showNotice(`批次状态刷新失败:${error.message}`, "error");
+ scheduleBatchPoll(batch);
+ }
+ }, BATCH_POLL_INTERVAL_MS);
+}
+
+function renderBatchResult(batch) {
const card = document.querySelector("#upload-result");
- document.querySelector("#upload-filename").textContent = documentData.filename;
- document.querySelector("#upload-type").textContent = documentData.media_type;
- document.querySelector("#upload-size").textContent = formatSize(documentData.size_bytes);
- document.querySelector("#upload-time").textContent = formatDate(documentData.created_at);
- document.querySelector("#upload-status").replaceWith(statusPill(documentData.status));
+ document.querySelector("#upload-batch-title").textContent =
+ `批次 ${batch.batch_id.slice(0, 8)} · ${formatDate(batch.created_at)}`;
+ document.querySelector("#upload-total").textContent = `${batch.total_files} 个`;
+ document.querySelector("#upload-completed").textContent =
+ `${batch.completed_files} / ${batch.total_files}`;
+ document.querySelector("#upload-accepted").textContent =
+ `${batch.accepted_files} / ${batch.rejected_files}`;
+ document.querySelector("#upload-status").replaceWith(statusPill(batch.status));
const status = card.querySelector(".status-pill");
status.id = "upload-status";
+ const items = document.querySelector("#batch-items");
+ items.replaceChildren(...batch.items.map(batchItem));
card.hidden = false;
}
+function batchItem(item) {
+ const row = document.createElement("article");
+ row.className = "batch-item";
+
+ const order = document.createElement("span");
+ order.className = "batch-item-order";
+ order.textContent = String(item.item_index + 1).padStart(2, "0");
+
+ const body = document.createElement("div");
+ body.className = "batch-item-body";
+ const name = document.createElement("strong");
+ name.textContent = item.filename;
+ const detail = document.createElement("span");
+ const stage = item.stage ? ` · ${stageLabels[item.stage] ?? item.stage}` : "";
+ const attempts = item.max_attempts == null ? "" : ` · 尝试 ${item.attempt}/${item.max_attempts}`;
+ detail.textContent = item.error_message ?? `后台任务${stage}${attempts}`;
+ body.append(name, detail);
+
+ const state = item.job_status ?? item.acceptance_status;
+ row.append(order, body, statusPill(state));
+ return row;
+}
+
async function loadDocuments() {
const refresh = document.querySelector("#refresh-documents");
setButtonBusy(refresh, true, "加载中…");
@@ -329,3 +408,4 @@ initializeUpload();
initializeSearch();
document.querySelector("#refresh-documents").addEventListener("click", loadDocuments);
if (page === "documents") loadDocuments();
+window.addEventListener("pagehide", () => window.clearTimeout(batchPollTimer));
diff --git a/apps/admin-web/assets/styles.css b/apps/admin-web/assets/styles.css
index d693fe7..f88d561 100644
--- a/apps/admin-web/assets/styles.css
+++ b/apps/admin-web/assets/styles.css
@@ -533,6 +533,50 @@ a {
border-block: 1px solid var(--line);
}
+.batch-items {
+ display: grid;
+ gap: 8px;
+ margin: -8px 0 24px;
+}
+
+.batch-item {
+ display: grid;
+ grid-template-columns: 34px minmax(0, 1fr) auto;
+ gap: 12px;
+ align-items: center;
+ min-height: 64px;
+ padding: 11px 13px;
+ border: 1px solid var(--line);
+ border-radius: 3px;
+ background: #fafaf8;
+}
+
+.batch-item-order {
+ color: var(--brand);
+ font-family: var(--data-font);
+ font-size: 10px;
+ font-weight: 850;
+}
+
+.batch-item-body {
+ display: grid;
+ min-width: 0;
+ gap: 5px;
+}
+
+.batch-item-body strong {
+ overflow: hidden;
+ font-size: 12px;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
+.batch-item-body span {
+ overflow-wrap: anywhere;
+ color: var(--muted);
+ font-size: 10px;
+}
+
dt {
color: var(--muted);
font-family: var(--data-font);
@@ -655,6 +699,15 @@ tbody tr:hover {
.status-processing { background: #fff0ce; color: var(--warning); }
.status-failed { background: #fbe6e3; color: var(--danger); }
.status-uploaded { background: var(--brand-soft); color: var(--brand-deep); }
+.status-pending,
+.status-queued,
+.status-running,
+.status-retrying,
+.status-accepted { background: #fff0ce; color: var(--warning); }
+.status-succeeded,
+.status-completed { background: #e5f3eb; color: var(--success); }
+.status-rejected,
+.status-partial_failure { background: #fbe6e3; color: var(--danger); }
.empty-state {
min-height: 240px;
@@ -958,6 +1011,8 @@ select:focus {
.search-controls { grid-template-columns: minmax(0, 1fr) 90px; }
.search-controls textarea { grid-column: 1 / -1; }
.metadata-grid { grid-template-columns: 1fr; }
+ .batch-item { grid-template-columns: 28px minmax(0, 1fr); }
+ .batch-item .status-pill { grid-column: 2; }
}
@media (max-width: 540px) {
diff --git a/apps/admin-web/index.html b/apps/admin-web/index.html
index 6241cb4..2be10fe 100644
--- a/apps/admin-web/index.html
+++ b/apps/admin-web/index.html
@@ -67,7 +67,7 @@
01 / INGESTION
让知识进入检索链路
-
上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将在后台完成解析、切分与索引。
+
批量提交 PDF、DOCX、TXT 或 Markdown 文件,系统将逐个接收并在后台独立完成解析、切分与索引。
@@ -75,14 +75,15 @@
@@ -90,15 +91,16 @@
-
LATEST SIGNAL
-
+
LATEST BATCH
+
- - 文件类型
- - 文件大小
- - 上传时间
+ - 总文件
+ - 已完成
+ - 接收 / 拒绝
+
前往 Search Lab 检索 →
diff --git a/apps/api/src/ekp_api/providers.py b/apps/api/src/ekp_api/providers.py
index 453df9f..7f25a90 100644
--- a/apps/api/src/ekp_api/providers.py
+++ b/apps/api/src/ekp_api/providers.py
@@ -18,6 +18,7 @@ from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.index_rebuild import resolve_active_index_profile
+from ekp_core.ingestion_batches import IngestionBatchService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
@@ -50,6 +51,7 @@ class ApiServices:
ingestion: DocumentIndexingPipelineService
submissions: AsyncIngestionSubmissionService
+ batches: IngestionBatchService
jobs: IngestionJobService
documents: DocumentManagementService
search: SearchService
@@ -310,6 +312,12 @@ def create_api_services(settings: Settings) -> ApiServices:
components.jobs,
components.dispatcher,
)
+ batches = IngestionBatchService(
+ components.session_factory,
+ submissions,
+ max_files=settings.batch_max_files,
+ max_total_bytes=settings.batch_max_total_bytes,
+ )
dense_retrieval = create_dense_retrieval_service(
settings,
components.embedding_provider,
@@ -332,6 +340,7 @@ def create_api_services(settings: Settings) -> ApiServices:
return ApiServices(
ingestion=ingestion,
submissions=submissions,
+ batches=batches,
jobs=components.jobs,
documents=documents,
search=search,
diff --git a/apps/api/src/ekp_api/routes.py b/apps/api/src/ekp_api/routes.py
index 788501d..b12b481 100644
--- a/apps/api/src/ekp_api/routes.py
+++ b/apps/api/src/ekp_api/routes.py
@@ -9,6 +9,7 @@ from ekp_api.providers import ApiServices, ProviderConfigurationError, create_ap
from ekp_api.schemas import (
DocumentResponse,
IngestionAcceptedResponse,
+ IngestionBatchResponse,
IngestionJobResponse,
SearchRequest,
SearchResponse,
@@ -25,6 +26,7 @@ from ekp_core.document_upload import (
MediaTypeMismatchError,
UnsupportedDocumentTypeError,
)
+from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundError, BatchSource
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
router = APIRouter()
@@ -98,6 +100,61 @@ def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
return IngestionJobResponse.from_job(job)
+@router.post(
+ "/ingestion-batches",
+ response_model=IngestionBatchResponse,
+ status_code=status.HTTP_202_ACCEPTED,
+ tags=["jobs"],
+)
+def upload_ingestion_batch(
+ services: Services,
+ files: Annotated[
+ list[UploadFile],
+ File(description="One or more PDF, DOCX, TXT, or Markdown files"),
+ ],
+) -> IngestionBatchResponse:
+ """Accept a bounded batch while isolating every file's result."""
+ try:
+ view = services.batches.submit(
+ tuple(
+ BatchSource(
+ filename=file.filename or "",
+ media_type=file.content_type,
+ source=file.file,
+ )
+ for file in files
+ )
+ )
+ except BatchFileCountLimitError as error:
+ raise HTTPException(
+ status_code=status.HTTP_413_CONTENT_TOO_LARGE,
+ detail="batch file count exceeds the configured limit",
+ ) from error
+ except Exception as error:
+ raise HTTPException(
+ status_code=status.HTTP_502_BAD_GATEWAY,
+ detail="batch could not be accepted",
+ ) from error
+ return IngestionBatchResponse.from_view(view)
+
+
+@router.get(
+ "/ingestion-batches/{batch_id}",
+ response_model=IngestionBatchResponse,
+ tags=["jobs"],
+)
+def get_ingestion_batch(batch_id: UUID, services: Services) -> IngestionBatchResponse:
+ """Return aggregate and per-file status for Admin Console polling."""
+ try:
+ view = services.batches.get(batch_id)
+ except BatchNotFoundError as error:
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail="ingestion batch not found",
+ ) from error
+ return IngestionBatchResponse.from_view(view)
+
+
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
def list_documents(services: Services) -> list[DocumentResponse]:
"""List canonical document metadata for the Demo admin page."""
diff --git a/apps/api/src/ekp_api/schemas.py b/apps/api/src/ekp_api/schemas.py
index d7c9632..ca17ce1 100644
--- a/apps/api/src/ekp_api/schemas.py
+++ b/apps/api/src/ekp_api/schemas.py
@@ -5,9 +5,17 @@ from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
+from ekp_core.ingestion_batches import BatchAggregateStatus, BatchItemView, BatchView
from ekp_core.ingestion_jobs import IngestionSubmission
from ekp_core.search import SearchResult
-from ekp_domain import Document, DocumentStatus, IngestionJob, IngestionJobStatus, IngestionStage
+from ekp_domain import (
+ BatchItemAcceptanceStatus,
+ Document,
+ DocumentStatus,
+ IngestionJob,
+ IngestionJobStatus,
+ IngestionStage,
+)
class DocumentResponse(BaseModel):
@@ -88,6 +96,80 @@ class IngestionJobResponse(BaseModel):
)
+class IngestionBatchItemResponse(BaseModel):
+ """Per-file acceptance and latest asynchronous processing state."""
+
+ item_index: int
+ filename: str
+ acceptance_status: BatchItemAcceptanceStatus
+ document_id: UUID | None
+ job_id: UUID | None
+ job_status: IngestionJobStatus | None
+ stage: IngestionStage | None
+ attempt: int
+ max_attempts: int | None
+ error_message: str | None
+
+ @classmethod
+ def from_view(cls, view: BatchItemView) -> "IngestionBatchItemResponse":
+ job = view.job
+ return cls(
+ item_index=view.item.item_index,
+ filename=view.item.filename,
+ acceptance_status=view.item.acceptance_status,
+ document_id=view.item.document_id,
+ job_id=view.item.job_id,
+ job_status=job.status if job is not None else None,
+ stage=job.stage if job is not None else None,
+ attempt=job.attempt_count if job is not None else 0,
+ max_attempts=job.max_attempts if job is not None else None,
+ error_message=(job.error_message if job is not None else view.item.error_message),
+ )
+
+
+class IngestionBatchResponse(BaseModel):
+ """Batch aggregate with ordered independently processed files."""
+
+ batch_id: UUID
+ status: BatchAggregateStatus
+ total_files: int
+ accepted_files: int
+ rejected_files: int
+ completed_files: int
+ created_at: datetime
+ items: list[IngestionBatchItemResponse]
+
+ @classmethod
+ def from_view(cls, view: BatchView) -> "IngestionBatchResponse":
+ items = [IngestionBatchItemResponse.from_view(item) for item in view.items]
+ accepted = sum(
+ item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED for item in items
+ )
+ rejected = sum(
+ item.acceptance_status == BatchItemAcceptanceStatus.REJECTED for item in items
+ )
+ completed = sum(
+ item.acceptance_status == BatchItemAcceptanceStatus.REJECTED
+ or item.job_status
+ in {
+ IngestionJobStatus.SUCCEEDED,
+ IngestionJobStatus.FAILED,
+ IngestionJobStatus.CANCELLED,
+ }
+ for item in items
+ )
+ return cls(
+ batch_id=view.batch.id,
+ status=view.status,
+ total_files=view.batch.total_files,
+ accepted_files=accepted,
+ rejected_files=rejected,
+ completed_files=completed,
+ created_at=view.batch.created_at,
+ items=items,
+ )
+
+
class SearchRequest(BaseModel):
"""Validated public search input; normalization remains in the core service."""
diff --git a/docs/api/API_CONTRACT.md b/docs/api/API_CONTRACT.md
index 9a273ee..2ba44ce 100644
--- a/docs/api/API_CONTRACT.md
+++ b/docs/api/API_CONTRACT.md
@@ -11,11 +11,13 @@ Use OpenAPI as executable source of truth once implementation starts. Provider-s
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
版本化为 `/api/v1/*`。
-`DEMO-015` 已将单文件上传切换为持久化异步 Job;批量提交仍留给 `DEMO-016`。
+`DEMO-015` 已将单文件上传切换为持久化异步 Job;`DEMO-016` 已增加批量提交和聚合查询。
| Method | Path | 行为 |
|---|---|---|
| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
+| `POST` | `/api/v1/ingestion-batches` | multipart `files` 多文件接收;返回批次与逐文件状态 |
+| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新 Job 状态 |
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
@@ -55,7 +57,7 @@ Search 响应:
```
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
-文档或 Job 不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
+文档、Job 或批次不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
厂商响应。
@@ -74,7 +76,7 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
-## 异步入库契约(DEMO-015 已实现;DEMO-016 批量部分待实现)
+## 异步入库契约(DEMO-015~016 已实现)
| Method | Path | 行为 |
|---|---|---|
@@ -101,17 +103,29 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
}
```
-DEMO-016 的批量响应示意:
+批量字段名为 `files`,成功返回 `202`。响应示意:
```json
{
"batch_id": "d21f8012-4c4e-4bb4-89af-e11c0f9f470e",
+ "status": "processing",
+ "total_files": 2,
+ "accepted_files": 1,
+ "rejected_files": 1,
+ "completed_files": 1,
+ "created_at": "2026-09-02T12:00:00Z",
"items": [
{
+ "item_index": 0,
"filename": "退款政策.md",
+ "acceptance_status": "accepted",
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
- "status": "queued"
+ "job_status": "queued",
+ "stage": null,
+ "attempt": 0,
+ "max_attempts": 3,
+ "error_message": null
}
]
}
@@ -119,8 +133,11 @@ DEMO-016 的批量响应示意:
Job 公开状态为 `queued | running | retrying | succeeded | failed | cancelled`,并通过 `stage`
表示 `parsing | cleaning | chunking | embedding | indexing`。错误字段只返回有界、安全摘要。
-批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚其他文件。文件数量、单文件大小和
-请求总大小均由服务端配置限制。
+批次状态为 `processing | completed | partial_failure | failed`;逐文件接收状态为
+`pending | accepted | rejected`。批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚
+其他文件。文件数量、单文件大小和请求总大小分别由 `BATCH_MAX_FILES`、`MAX_UPLOAD_BYTES`
+和 `BATCH_MAX_TOTAL_BYTES` 限制。数量超限返回 `413`;类型、大小或内容不合规作为该文件的
+`rejected` 安全结果返回,其他合规文件继续接收。
自动重试是服务端 Job 行为,不要求客户端再次上传,也不新增公开手动重试接口。每次重试前
清理派生数据但保留原始文件和稳定的 Document ID;达到次数上限或清理失败后进入 `failed`。
diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.md
index db41147..a510d66 100644
--- a/docs/architecture/ARCHITECTURE.md
+++ b/docs/architecture/ARCHITECTURE.md
@@ -179,6 +179,10 @@ Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Fail
API 只负责校验并持久化不可变源文件、Document 和 Ingestion Job,然后返回 `202`。批次是
提交与聚合查询边界,不是跨文件数据库事务;每个文件独立排队、执行和失败。
+DEMO-016 以 canonical `ingestion_batches` / `ingestion_batch_items` 保存输入顺序和接收引用;
+处理阶段、Attempt 与最终结果继续以 `ingestion_jobs` / `ingestion_attempts` 为事实源,批次表
+不复制 Worker 状态。
+
```text
Batch Request
├─ Source A + Document A + Job A ─→ Worker
diff --git a/docs/demo/DEMO_SCOPE.md b/docs/demo/DEMO_SCOPE.md
index fca286f..d467284 100644
--- a/docs/demo/DEMO_SCOPE.md
+++ b/docs/demo/DEMO_SCOPE.md
@@ -340,7 +340,7 @@ API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重
### DEMO-016 批量异步上传
-在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
+**已完成。** 在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
Admin Console 状态展示。批次内文件相互隔离,不包含目录上传、压缩包解包或断点续传。
---
@@ -397,8 +397,8 @@ DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
-任务拆分;`DEMO-012`、`DEMO-015` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
-后续严格按 `DEMO-016`、`DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
+任务拆分;`DEMO-012`、`DEMO-015~016` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
+后续严格按 `DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
可选项,不因本状态记录自动进入范围。
可选:
diff --git a/docs/development/ADMIN_CONSOLE.md b/docs/development/ADMIN_CONSOLE.md
index 4de4b3e..fb27b9a 100644
--- a/docs/development/ADMIN_CONSOLE.md
+++ b/docs/development/ADMIN_CONSOLE.md
@@ -16,7 +16,7 @@ make run
| 页面 | 地址 | 对应业务接口 |
|---|---|---|
-| 文档上传 | `/admin/upload` | `POST /api/v1/documents` |
+| 文档上传 | `/admin/upload` | `POST/GET /api/v1/ingestion-batches*` |
| 文档列表与详情 | `/admin/documents` | `GET/DELETE /api/v1/documents*` |
| Search Lab | `/admin/search` | `POST /api/v1/search` |
@@ -34,9 +34,9 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
## Demo 操作闭环
-1. 在上传页选择一个 PDF、DOCX、TXT 或 Markdown 文件并提交;
-2. 接口返回 `202` 后,当前页面显示已接收;在文档列表刷新查看 `Processing/Ready/Failed`;
-3. 在文档列表查看状态和安全元数据;
+1. 在上传页选择一个或多个 PDF、DOCX、TXT 或 Markdown 文件并提交;
+2. 接口返回 `202` 后,页面自动轮询批次,逐文件显示接收状态、处理阶段、尝试次数和安全错误;
+3. 批次进入 `Completed`、`Partial failure` 或 `Failed` 后停止轮询,再在文档列表查看元数据;
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
Rerank Score;
5. 如需清理 Demo 数据,在文档列表确认后直接删除。
@@ -46,7 +46,8 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
-- 当前 Console 尚不轮询 Job 详情或展示 Attempt;该状态 UI 与批量上传一起由 DEMO-016 实现;
+- Console 只轮询当前批次的聚合接口,不读取 Provider/队列细节,也不提供人工重试按钮;
+- 批量上传不包含目录上传、压缩包解包、断点续传或跨批次去重;
- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
或完整 V1 Admin Console 导航;
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。
diff --git a/docs/development/ASYNC_INGESTION_SETUP.md b/docs/development/ASYNC_INGESTION_SETUP.md
index 20c5bba..ca81bf8 100644
--- a/docs/development/ASYNC_INGESTION_SETUP.md
+++ b/docs/development/ASYNC_INGESTION_SETUP.md
@@ -2,11 +2,12 @@
## 范围
-DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker。OceanBase 中的
+DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker,DEMO-016 在其上增加逐文件
+隔离的批量接收和状态聚合。OceanBase 中的
`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
不携带文件内容、Provider 凭据或业务状态。
-本任务不提供批量上传和批次 UI;这些能力属于 DEMO-016。
+批次只聚合接收与 Job 状态,不改变每个文件的独立执行和重试边界。
## 运行时配置
@@ -19,6 +20,8 @@ INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
+BATCH_MAX_FILES=20
+BATCH_MAX_TOTAL_BYTES=209715200
```
Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
@@ -72,6 +75,18 @@ Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;
`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
+批量提交与查询:
+
+```text
+POST /api/v1/ingestion-batches # multipart 字段 files,可重复
+GET /api/v1/ingestion-batches/{batch_id}
+```
+
+批次创建后逐文件返回 `accepted` 或 `rejected`;已接收文件继续通过上述 Job 状态异步推进。
+批次聚合状态为 `processing | completed | partial_failure | failed`。请求文件数超过
+`BATCH_MAX_FILES` 时整体返回 `413`;单文件或共享总字节预算超限只拒绝当前文件,不删除此前
+已接收文件。默认总预算 200 MiB,且配置不得小于单文件 `MAX_UPLOAD_BYTES`。
+
## 故障检查
1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;
diff --git a/docs/development/REST_API_SETUP.md b/docs/development/REST_API_SETUP.md
index 6860054..aaf8704 100644
--- a/docs/development/REST_API_SETUP.md
+++ b/docs/development/REST_API_SETUP.md
@@ -3,10 +3,10 @@
## 范围
DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
-Job。API 层只负责输入校验、
+Job,DEMO-016 增加批量接收与聚合查询。API 层只负责输入校验、
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
-当前不包含 ACL/RBAC、软删除、批量上传或 Hybrid Search。
+当前不包含 ACL/RBAC、软删除、目录/压缩包上传、断点续传或 Hybrid Search。
## 启动前提
@@ -40,6 +40,16 @@ curl -F 'file=@./policy.md;type=text/markdown' \
curl http://127.0.0.1:8000/api/v1/jobs/
```
+批量上传并查询聚合进度:
+
+```bash
+curl -F 'files=@./policy.md;type=text/markdown' \
+ -F 'files=@./manual.pdf;type=application/pdf' \
+ http://127.0.0.1:8000/api/v1/ingestion-batches
+
+curl http://127.0.0.1:8000/api/v1/ingestion-batches/
+```
+
检索:
```bash
diff --git a/docs/plans/ASYNC_INGESTION_MCP_PLAN.md b/docs/plans/ASYNC_INGESTION_MCP_PLAN.md
index e67d2e2..89d5d5e 100644
--- a/docs/plans/ASYNC_INGESTION_MCP_PLAN.md
+++ b/docs/plans/ASYNC_INGESTION_MCP_PLAN.md
@@ -9,13 +9,13 @@
| 能力 | 当前实现 | 结论 |
|---|---|---|
| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
-| 批量上传 | API 与 Admin Console 都只支持一个文件 | 未实现 |
-| 失败补偿 | 上传元数据失败会删除源对象;解析元数据失败会删除本次解析产物;索引失败会删除当前 Document/Profile 的可能残留向量 | 已有局部补偿,不是自动重试 |
+| 批量上传 | 批次 API、逐文件 Job 状态和 Admin Console 自动轮询 | DEMO-016 已实现 |
+| 失败补偿 | 阶段内补偿 + 重试前清理全部派生数据,保留不可变源和执行历史 | DEMO-015 已实现 |
| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
-| MCP | 已有 MCP 业务契约和可复用 Search Service/REST API,没有可运行 MCP Gateway 或 WorkBuddy 验证 | 未实现 |
+| MCP | 独立 Gateway、认证边界、`search_knowledge` 与 WorkBuddy 协议级验收 | DEMO-012 已实现 |
-PLAN-001 的上述缺口已由 DEMO-015 补齐;批量提交与状态 UI 仍由 DEMO-016 实现。
+PLAN-001 的异步执行缺口已由 DEMO-015 补齐,批量提交与状态 UI 已由 DEMO-016 补齐。
## 2. Task 归并结论
@@ -51,7 +51,7 @@ PLAN-001 的上述缺口已由 DEMO-015 补齐;批量提交与状态 UI 仍由
该 Task 只完成公共知识库入库所需的 Job/Retry 与队列基线,不扩展到完整审计系统、
Compilation、OCR、跨区域高可用或通用企业级 Job 平台。
-### DEMO-016:批量异步上传与状态展示(新增)
+### DEMO-016:批量异步上传与状态展示(已完成)
建立在 DEMO-015 之上,不与 Job 基础合并:
@@ -79,8 +79,8 @@ PLAN-002(已完成)
推荐严格按以下顺序执行并逐项闭环:
1. `PLAN-002`(已完成)
-2. `DEMO-015`
-3. `DEMO-016`
+2. `DEMO-015`(已完成)
+3. `DEMO-016`(已完成)
4. `DEPLOY-001`
5. `DEPLOY-002`
6. `DEPLOY-003`
diff --git a/docs/plans/PROGRESS.md b/docs/plans/PROGRESS.md
index e1b0021..50ee46c 100644
--- a/docs/plans/PROGRESS.md
+++ b/docs/plans/PROGRESS.md
@@ -1,12 +1,12 @@
# 项目进度
-当前阶段:异步单文件入库闭环完成,准备实施批量上传
+当前阶段:批量异步上传闭环完成,准备交付生产镜像
-已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-015、ARCH-001~ARCH-006、PLAN-001~PLAN-002
+已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002
进行中:无
阻塞:无
-下一任务:`DEMO-016`。随后严格执行 `DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
+下一任务:`DEPLOY-001`。随后严格执行 `DEPLOY-002 → DEPLOY-003`。
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
@@ -354,3 +354,26 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
- 下一任务:`DEMO-016`。
+
+### DEMO-016(已完成)
+
+- 新增 canonical `ingestion_batches` / `ingestion_batch_items` 及 forward-only migration,保存
+ 输入顺序、逐文件接收结果与 Document/Job 引用;异步阶段和最终结果仍以 Job 为事实源,
+ 批次不是跨文件事务。
+- 新增 `POST /api/v1/ingestion-batches` 与 `GET /api/v1/ingestion-batches/{batch_id}`;批量
+ 请求字段为 `files`,成功返回 `202`、聚合状态和逐文件接收/Job 状态。类型、大小或内容错误
+ 只拒绝当前文件,错误映射为固定安全摘要,其他文件继续接收。
+- 新增 `BATCH_MAX_FILES` 与 `BATCH_MAX_TOTAL_BYTES`,复用 `MAX_UPLOAD_BYTES` 约束单文件;
+ 上传流共享总字节预算,不把整批文件读入内存。数量超限在创建批次前返回 `413`。
+- Admin Console 支持多选提交,显示当前批次的接收数量、完成数量、逐文件阶段、Attempt 与安全
+ 错误,并每 1.5 秒自动轮询;进入 `completed`、`partial_failure` 或 `failed` 后停止轮询。
+ DOM 数据继续只通过文本节点渲染。
+- 集成测试覆盖顺序稳定、部分拒绝不回滚、数量/单文件/批次总大小、全成功/部分失败聚合、
+ 未知批次、OpenAPI 和前端契约;迁移 drift 增加两张新表。Ruff Format/Lint、Mypy、
+ JavaScript 语法、依赖锁、Alembic Head/离线升级均通过;Unit + Integration `183 passed`,
+ RAG Evaluation `5 passed`。
+- 浏览器验收确认桌面布局和 390px 窄屏布局正常、无根页面横向溢出、无控制台警告或错误;
+ 本地页面不注入云端凭据,批次结果行为由确定性 API 集成测试验证。
+- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续
+ 复用 DEMO-015,Accepted ADR 未变更。
+- 下一任务:`DEPLOY-001`。
diff --git a/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md b/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md
index e76ddfc..cde092b 100644
--- a/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md
+++ b/docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md
@@ -82,7 +82,8 @@ Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务
### DEMO-016
-基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console 状态展示。
+**已完成。** 基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console
+状态展示;下一交付门槛为 DEPLOY-001 的生产镜像。
### DEPLOY-001
diff --git a/docs/plans/TASKS.md b/docs/plans/TASKS.md
index a096eed..ed2a828 100644
--- a/docs/plans/TASKS.md
+++ b/docs/plans/TASKS.md
@@ -29,7 +29,7 @@ validation. Provider credentials and internal endpoints are runtime inputs and n
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
-- [ ] DEMO-016 Batch async upload API + per-file Job status + Admin Console
+- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
- [ ] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
- [ ] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance
diff --git a/docs/plans/V1_EXECUTION_PLAN.md b/docs/plans/V1_EXECUTION_PLAN.md
index f4a33d6..20b26ee 100644
--- a/docs/plans/V1_EXECUTION_PLAN.md
+++ b/docs/plans/V1_EXECUTION_PLAN.md
@@ -17,7 +17,7 @@
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
-3. `DEMO-016`:批量异步上传、批次/Job 查询和 Admin Console 状态展示;
+3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
4. `DEPLOY-001`:API/Admin、MCP Gateway、Async Worker 生产镜像;
5. `DEPLOY-002`:Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。
diff --git a/docs/product/PRD.md b/docs/product/PRD.md
index ef561be..576db92 100644
--- a/docs/product/PRD.md
+++ b/docs/product/PRD.md
@@ -465,6 +465,9 @@ Document processing 必须异步执行。
源文件与 Document/Job 成功持久化后返回 `202 Accepted`。批量上传中的每个文件拥有独立的
Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批次内其他已接收文件。
+当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、
+断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。
+
**REQ-INGEST-004**
展示处理阶段:
diff --git a/migrations/README.md b/migrations/README.md
index 8d5b044..8188767 100644
--- a/migrations/README.md
+++ b/migrations/README.md
@@ -21,3 +21,7 @@ single-active generated unique index.
DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
the latter preserves numbered history and uses a generated nullable active marker plus a unique index
to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.
+
+DEMO-016 adds `ingestion_batches` and `ingestion_batch_items`. They retain input order and per-file
+acceptance references while Job rows remain the processing state source of truth; a batch is an aggregate,
+not a cross-file transaction.
diff --git a/migrations/versions/20260902_0008_create_ingestion_batches.py b/migrations/versions/20260902_0008_create_ingestion_batches.py
new file mode 100644
index 0000000..ba22b8f
--- /dev/null
+++ b/migrations/versions/20260902_0008_create_ingestion_batches.py
@@ -0,0 +1,109 @@
+"""Create ingestion batch aggregation metadata.
+
+Revision ID: 20260902_0008
+Revises: 20260902_0007
+Create Date: 2026-09-02
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "20260902_0008"
+down_revision: str | None = "20260902_0007"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ """Persist ordered per-file acceptance without making a batch transactional."""
+ op.create_table(
+ "ingestion_batches",
+ sa.Column("id", sa.Uuid(), nullable=False),
+ sa.Column("total_files", sa.Integer(), nullable=False),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("CURRENT_TIMESTAMP"),
+ nullable=False,
+ ),
+ sa.CheckConstraint("total_files > 0", name="total_files_positive"),
+ sa.PrimaryKeyConstraint("id", name="pk_ingestion_batches"),
+ )
+ op.create_table(
+ "ingestion_batch_items",
+ sa.Column("id", sa.Uuid(), nullable=False),
+ sa.Column("batch_id", sa.Uuid(), nullable=False),
+ sa.Column("item_index", sa.Integer(), nullable=False),
+ sa.Column("filename", sa.String(length=255), nullable=False),
+ sa.Column(
+ "acceptance_status",
+ sa.Enum(
+ "pending",
+ "accepted",
+ "rejected",
+ name="batch_item_acceptance_status",
+ native_enum=False,
+ create_constraint=False,
+ ),
+ nullable=False,
+ ),
+ sa.Column("document_id", sa.Uuid(), nullable=True),
+ sa.Column("job_id", sa.Uuid(), nullable=True),
+ sa.Column("error_message", sa.Text(), nullable=True),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("CURRENT_TIMESTAMP"),
+ nullable=False,
+ ),
+ sa.Column(
+ "updated_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("CURRENT_TIMESTAMP"),
+ nullable=False,
+ ),
+ sa.CheckConstraint("item_index >= 0", name="item_index_non_negative"),
+ sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
+ sa.CheckConstraint(
+ "acceptance_status IN ('pending', 'accepted', 'rejected')",
+ name="acceptance_status_valid",
+ ),
+ sa.ForeignKeyConstraint(
+ ["batch_id"],
+ ["ingestion_batches.id"],
+ name="fk_ingestion_batch_items_batch_id_ingestion_batches",
+ ondelete="CASCADE",
+ ),
+ sa.ForeignKeyConstraint(
+ ["document_id"],
+ ["documents.id"],
+ name="fk_ingestion_batch_items_document_id_documents",
+ ondelete="SET NULL",
+ ),
+ sa.ForeignKeyConstraint(
+ ["job_id"],
+ ["ingestion_jobs.id"],
+ name="fk_ingestion_batch_items_job_id_ingestion_jobs",
+ ondelete="SET NULL",
+ ),
+ sa.PrimaryKeyConstraint("id", name="pk_ingestion_batch_items"),
+ )
+ op.create_index(
+ "uq_ingestion_batch_items_order",
+ "ingestion_batch_items",
+ ["batch_id", "item_index"],
+ unique=True,
+ )
+ op.create_index(
+ "uq_ingestion_batch_items_job_id",
+ "ingestion_batch_items",
+ ["job_id"],
+ unique=True,
+ )
+
+
+def downgrade() -> None:
+ """Reject downgrade because project migrations are forward-only."""
+ raise NotImplementedError("Database migrations are forward-only")
diff --git a/packages/core/src/ekp_core/config.py b/packages/core/src/ekp_core/config.py
index ba27cff..b8034b4 100644
--- a/packages/core/src/ekp_core/config.py
+++ b/packages/core/src/ekp_core/config.py
@@ -39,6 +39,8 @@ class Settings(BaseSettings):
ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
+ batch_max_files: int = Field(default=20, ge=1, le=100)
+ batch_max_total_bytes: int = Field(default=200 * 1024 * 1024, gt=0)
object_storage_provider: Literal["local", "bos"] = "bos"
object_storage_path: Path = Path("storage")
bos_endpoint: str | None = None
@@ -108,6 +110,8 @@ class Settings(BaseSettings):
)
if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
+ if self.batch_max_total_bytes < self.max_upload_bytes:
+ raise ValueError("batch_max_total_bytes must be at least max_upload_bytes")
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
raise ValueError("mcp_path must start with '/' and must not end with '/'")
if not self.mcp_allowed_hosts:
diff --git a/packages/core/src/ekp_core/ingestion_batches.py b/packages/core/src/ekp_core/ingestion_batches.py
new file mode 100644
index 0000000..4e4da62
--- /dev/null
+++ b/packages/core/src/ekp_core/ingestion_batches.py
@@ -0,0 +1,270 @@
+"""Batch submission and aggregate status over independent ingestion Jobs."""
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Protocol
+from uuid import UUID
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session, sessionmaker
+
+from ekp_core.document_upload import (
+ DocumentTooLargeError,
+ DocumentUploadError,
+ EmptyDocumentError,
+ MediaTypeMismatchError,
+ UnsupportedDocumentTypeError,
+)
+from ekp_core.ingestion_jobs import AsyncIngestionSubmissionService
+from ekp_domain import (
+ BatchItemAcceptanceStatus,
+ IngestionBatch,
+ IngestionBatchItem,
+ IngestionJob,
+ IngestionJobStatus,
+)
+from ekp_object_storage import BinarySource
+
+SAFE_UNSUPPORTED_ERROR = "unsupported document type or media type"
+SAFE_FILE_SIZE_ERROR = "document exceeds the configured upload limit"
+SAFE_BATCH_SIZE_ERROR = "batch exceeds the configured total upload limit"
+SAFE_EMPTY_ERROR = "document must contain content"
+SAFE_ACCEPTANCE_ERROR = "document could not be accepted"
+
+
+class BatchNotFoundError(LookupError):
+ """Raised when a requested ingestion batch does not exist."""
+
+
+class BatchFileCountLimitError(ValueError):
+ """Raised before persistence when the multipart file count is invalid."""
+
+
+class BatchAggregateStatus(StrEnum):
+ """Computed batch state derived from item acceptance and durable Jobs."""
+
+ PROCESSING = "processing"
+ COMPLETED = "completed"
+ PARTIAL_FAILURE = "partial_failure"
+ FAILED = "failed"
+
+
+class BatchTotalSizeLimitError(ValueError):
+ """Raised while streaming the file that crosses the shared batch budget."""
+
+
+class _ReadableSource(Protocol):
+ def read(self, size: int = -1, /) -> bytes:
+ """Read bytes from an uploaded file."""
+
+
+@dataclass(frozen=True, slots=True)
+class BatchSource:
+ """Provider-neutral input for one ordered multipart file."""
+
+ filename: str
+ media_type: str | None
+ source: _ReadableSource
+
+
+@dataclass(frozen=True, slots=True)
+class BatchItemView:
+ """Detached batch item plus its current optional Job state."""
+
+ item: IngestionBatchItem
+ job: IngestionJob | None
+
+
+@dataclass(frozen=True, slots=True)
+class BatchView:
+ """Detached aggregate used by REST without exposing persistence sessions."""
+
+ batch: IngestionBatch
+ items: tuple[BatchItemView, ...]
+
+ @property
+ def status(self) -> BatchAggregateStatus:
+ """Aggregate terminal results without copying Job state into batch rows."""
+ if any(
+ view.item.acceptance_status == BatchItemAcceptanceStatus.PENDING
+ or (
+ view.item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED
+ and (
+ view.job is None
+ or view.job.status
+ in {
+ IngestionJobStatus.QUEUED,
+ IngestionJobStatus.RUNNING,
+ IngestionJobStatus.RETRYING,
+ }
+ )
+ )
+ for view in self.items
+ ):
+ return BatchAggregateStatus.PROCESSING
+ succeeded = sum(
+ view.job is not None and view.job.status == IngestionJobStatus.SUCCEEDED
+ for view in self.items
+ )
+ failed = len(self.items) - succeeded
+ if failed == 0:
+ return BatchAggregateStatus.COMPLETED
+ if succeeded == 0:
+ return BatchAggregateStatus.FAILED
+ return BatchAggregateStatus.PARTIAL_FAILURE
+
+
+class _BatchByteBudget:
+ """Count bytes across independently streamed files without buffering them."""
+
+ def __init__(self, limit_bytes: int) -> None:
+ self._limit_bytes = limit_bytes
+ self._consumed_bytes = 0
+
+ def wrap(self, source: _ReadableSource) -> BinarySource:
+ return _BatchLimitedSource(source, self)
+
+ def consume(self, size_bytes: int) -> None:
+ self._consumed_bytes += size_bytes
+ if self._consumed_bytes > self._limit_bytes:
+ raise BatchTotalSizeLimitError(SAFE_BATCH_SIZE_ERROR)
+
+
+class _BatchLimitedSource:
+ """Apply one shared request budget through the ObjectStorage stream contract."""
+
+ def __init__(self, source: _ReadableSource, budget: _BatchByteBudget) -> None:
+ self._source = source
+ self._budget = budget
+
+ def read(self, size: int = -1, /) -> bytes:
+ chunk = self._source.read(size)
+ self._budget.consume(len(chunk))
+ return chunk
+
+
+class IngestionBatchService:
+ """Accept files independently and persist ordered aggregate status."""
+
+ def __init__(
+ self,
+ session_factory: sessionmaker[Session],
+ submissions: AsyncIngestionSubmissionService,
+ *,
+ max_files: int,
+ max_total_bytes: int,
+ ) -> None:
+ if max_files <= 0 or max_total_bytes <= 0:
+ raise ValueError("batch limits must be positive")
+ self._session_factory = session_factory
+ self._submissions = submissions
+ self._max_files = max_files
+ self._max_total_bytes = max_total_bytes
+
+ def submit(self, sources: tuple[BatchSource, ...]) -> BatchView:
+ """Create a batch and independently accept or reject every ordered file."""
+ if not sources or len(sources) > self._max_files:
+ raise BatchFileCountLimitError(
+ f"batch must contain between 1 and {self._max_files} files"
+ )
+ batch_id, item_ids = self._create_pending_batch(sources)
+ budget = _BatchByteBudget(self._max_total_bytes)
+ for item_id, source in zip(item_ids, sources, strict=True):
+ try:
+ submission = self._submissions.submit(
+ source.filename,
+ source.media_type,
+ budget.wrap(source.source),
+ )
+ except Exception as error:
+ self._record_rejected(item_id, _safe_acceptance_error(error))
+ continue
+ self._record_accepted(
+ item_id,
+ document_id=submission.document.id,
+ job_id=submission.job.id,
+ )
+ return self.get(batch_id)
+
+ def get(self, batch_id: UUID) -> BatchView:
+ """Return ordered items and fresh Job state for polling clients."""
+ with self._session_factory() as session:
+ batch = session.get(IngestionBatch, batch_id)
+ if batch is None:
+ raise BatchNotFoundError(str(batch_id))
+ items = tuple(
+ session.scalars(
+ select(IngestionBatchItem)
+ .where(IngestionBatchItem.batch_id == batch_id)
+ .order_by(IngestionBatchItem.item_index)
+ ).all()
+ )
+ views = tuple(
+ BatchItemView(
+ item=item,
+ job=session.get(IngestionJob, item.job_id) if item.job_id is not None else None,
+ )
+ for item in items
+ )
+ return BatchView(batch=batch, items=views)
+
+ def _create_pending_batch(
+ self, sources: tuple[BatchSource, ...]
+ ) -> tuple[UUID, tuple[UUID, ...]]:
+ """Persist every input position before starting external storage writes."""
+ with self._session_factory() as session:
+ batch = IngestionBatch(total_files=len(sources))
+ session.add(batch)
+ session.flush()
+ items = tuple(
+ IngestionBatchItem(
+ batch_id=batch.id,
+ item_index=index,
+ filename=_display_filename(source.filename),
+ )
+ for index, source in enumerate(sources)
+ )
+ session.add_all(items)
+ session.commit()
+ return batch.id, tuple(item.id for item in items)
+
+ def _record_accepted(self, item_id: UUID, *, document_id: UUID, job_id: UUID) -> None:
+ with self._session_factory() as session:
+ item = session.get(IngestionBatchItem, item_id)
+ if item is None:
+ raise BatchNotFoundError(str(item_id))
+ item.acceptance_status = BatchItemAcceptanceStatus.ACCEPTED
+ item.document_id = document_id
+ item.job_id = job_id
+ item.error_message = None
+ session.commit()
+
+ def _record_rejected(self, item_id: UUID, error_message: str) -> None:
+ with self._session_factory() as session:
+ item = session.get(IngestionBatchItem, item_id)
+ if item is None:
+ raise BatchNotFoundError(str(item_id))
+ item.acceptance_status = BatchItemAcceptanceStatus.REJECTED
+ item.error_message = error_message
+ session.commit()
+
+
+def _display_filename(filename: str) -> str:
+ """Store a bounded safe label even when the upload filename is invalid."""
+ display = filename.replace("\x00", "�").strip()
+ return display[:255] or ""
+
+
+def _safe_acceptance_error(error: Exception) -> str:
+ """Map per-file failures without persisting exception text or credentials."""
+ if isinstance(error, (UnsupportedDocumentTypeError, MediaTypeMismatchError)):
+ return SAFE_UNSUPPORTED_ERROR
+ if isinstance(error, DocumentTooLargeError):
+ return SAFE_FILE_SIZE_ERROR
+ if isinstance(error, BatchTotalSizeLimitError):
+ return SAFE_BATCH_SIZE_ERROR
+ if isinstance(error, EmptyDocumentError):
+ return SAFE_EMPTY_ERROR
+ if isinstance(error, DocumentUploadError):
+ return SAFE_ACCEPTANCE_ERROR
+ return SAFE_ACCEPTANCE_ERROR
diff --git a/packages/domain/src/ekp_domain/__init__.py b/packages/domain/src/ekp_domain/__init__.py
index 054ea88..563fc39 100644
--- a/packages/domain/src/ekp_domain/__init__.py
+++ b/packages/domain/src/ekp_domain/__init__.py
@@ -8,6 +8,11 @@ from ekp_domain.index_profile import (
IndexProfileStatus,
StoredIndexProfile,
)
+from ekp_domain.ingestion_batch import (
+ BatchItemAcceptanceStatus,
+ IngestionBatch,
+ IngestionBatchItem,
+)
from ekp_domain.ingestion_job import (
IngestionAttempt,
IngestionAttemptStatus,
@@ -19,6 +24,7 @@ from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocum
__all__ = [
"Base",
+ "BatchItemAcceptanceStatus",
"Chunk",
"ChunkIndexState",
"Document",
@@ -30,6 +36,8 @@ __all__ = [
"IngestionJob",
"IngestionJobStatus",
"IngestionStage",
+ "IngestionBatch",
+ "IngestionBatchItem",
"ParsedBlock",
"ParsedBlockType",
"ParsedDocument",
diff --git a/packages/domain/src/ekp_domain/ingestion_batch.py b/packages/domain/src/ekp_domain/ingestion_batch.py
new file mode 100644
index 0000000..1b5795c
--- /dev/null
+++ b/packages/domain/src/ekp_domain/ingestion_batch.py
@@ -0,0 +1,96 @@
+"""Canonical batch submission metadata for per-file ingestion tracking."""
+
+from datetime import datetime
+from enum import StrEnum
+from uuid import UUID, uuid4
+
+from sqlalchemy import (
+ CheckConstraint,
+ DateTime,
+ ForeignKey,
+ Index,
+ Integer,
+ String,
+ Text,
+ Uuid,
+ func,
+)
+from sqlalchemy import Enum as SqlEnum
+from sqlalchemy.orm import Mapped, mapped_column
+
+from ekp_domain.document import Base
+
+
+class BatchItemAcceptanceStatus(StrEnum):
+ """Source acceptance state, independent from its asynchronous Job state."""
+
+ PENDING = "pending"
+ ACCEPTED = "accepted"
+ REJECTED = "rejected"
+
+
+class IngestionBatch(Base):
+ """Stable aggregation identifier for one bounded multipart submission."""
+
+ __tablename__ = "ingestion_batches"
+ __table_args__ = (CheckConstraint("total_files > 0", name="total_files_positive"),)
+
+ id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
+ total_files: Mapped[int] = mapped_column(Integer, nullable=False)
+ created_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False, server_default=func.now()
+ )
+
+
+class IngestionBatchItem(Base):
+ """One ordered input and its optional accepted Document/Job references."""
+
+ __tablename__ = "ingestion_batch_items"
+ __table_args__ = (
+ CheckConstraint("item_index >= 0", name="item_index_non_negative"),
+ CheckConstraint("length(filename) > 0", name="filename_not_empty"),
+ CheckConstraint(
+ "acceptance_status IN ('pending', 'accepted', 'rejected')",
+ name="acceptance_status_valid",
+ ),
+ Index("uq_ingestion_batch_items_order", "batch_id", "item_index", unique=True),
+ Index("uq_ingestion_batch_items_job_id", "job_id", unique=True),
+ )
+
+ id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
+ batch_id: Mapped[UUID] = mapped_column(
+ Uuid(as_uuid=True),
+ ForeignKey("ingestion_batches.id", ondelete="CASCADE"),
+ nullable=False,
+ )
+ item_index: Mapped[int] = mapped_column(Integer, nullable=False)
+ filename: Mapped[str] = mapped_column(String(255), nullable=False)
+ acceptance_status: Mapped[BatchItemAcceptanceStatus] = mapped_column(
+ SqlEnum(
+ BatchItemAcceptanceStatus,
+ name="batch_item_acceptance_status",
+ native_enum=False,
+ create_constraint=False,
+ validate_strings=True,
+ values_callable=lambda values: [item.value for item in values],
+ ),
+ nullable=False,
+ default=BatchItemAcceptanceStatus.PENDING,
+ )
+ document_id: Mapped[UUID | None] = mapped_column(
+ Uuid(as_uuid=True),
+ ForeignKey("documents.id", ondelete="SET NULL"),
+ nullable=True,
+ )
+ job_id: Mapped[UUID | None] = mapped_column(
+ Uuid(as_uuid=True),
+ ForeignKey("ingestion_jobs.id", ondelete="SET NULL"),
+ nullable=True,
+ )
+ error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+ created_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False, server_default=func.now()
+ )
+ updated_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
+ )
diff --git a/tests/integration/test_admin_console.py b/tests/integration/test_admin_console.py
index 077271f..e92e9e9 100644
--- a/tests/integration/test_admin_console.py
+++ b/tests/integration/test_admin_console.py
@@ -43,7 +43,9 @@ def test_admin_console_contains_only_demo_pages_and_versioned_api_calls(
assert all(label not in html for label in ("权限管理", "任务管理", "Compiled Knowledge"))
assert 'const API_BASE = "/api/v1"' in script
assert "innerHTML" not in script
- assert all(endpoint in script for endpoint in ('"/documents"', '"/search"'))
+ assert all(endpoint in script for endpoint in ('"/ingestion-batches"', '"/search"'))
+ assert "multiple" in html
+ assert "BATCH_POLL_INTERVAL_MS" in script
assert all(field in script for field in ("Page", "Section", "Dense Score", "Rerank Score"))
assert "--brand: #FF3C00" in stylesheet
assert "prefers-reduced-motion" in stylesheet
diff --git a/tests/integration/test_document_migration.py b/tests/integration/test_document_migration.py
index 0a32b24..d02c09e 100644
--- a/tests/integration/test_document_migration.py
+++ b/tests/integration/test_document_migration.py
@@ -30,6 +30,8 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
"index_profiles",
"ingestion_jobs",
"ingestion_attempts",
+ "ingestion_batches",
+ "ingestion_batch_items",
}.issubset(inspector.get_table_names())
assert "unrelated_provider_table" in inspector.get_table_names()
assert {column["name"] for column in inspector.get_columns("documents")} == {
@@ -101,3 +103,20 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
"started_at",
"finished_at",
}
+ assert {column["name"] for column in inspector.get_columns("ingestion_batches")} == {
+ "id",
+ "total_files",
+ "created_at",
+ }
+ assert {column["name"] for column in inspector.get_columns("ingestion_batch_items")} == {
+ "id",
+ "batch_id",
+ "item_index",
+ "filename",
+ "acceptance_status",
+ "document_id",
+ "job_id",
+ "error_message",
+ "created_at",
+ "updated_at",
+ }
diff --git a/tests/integration/test_document_search_api.py b/tests/integration/test_document_search_api.py
index d859361..0c83df8 100644
--- a/tests/integration/test_document_search_api.py
+++ b/tests/integration/test_document_search_api.py
@@ -19,6 +19,7 @@ from ekp_core.document_indexing import DocumentIndexingPipelineService, Document
from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
+from ekp_core.ingestion_batches import IngestionBatchService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
@@ -26,7 +27,7 @@ from ekp_core.ingestion_jobs import (
IngestionJobService,
)
from ekp_core.search import SearchService
-from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile
+from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile, IngestionBatch
from ekp_embedding import FakeEmbeddingProvider
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
@@ -57,6 +58,10 @@ class RecordingDispatcher:
def _build_api_services(
tmp_path: Path,
+ *,
+ max_upload_bytes: int = 1024 * 1024,
+ batch_max_files: int = 20,
+ batch_max_total_bytes: int = 2 * 1024 * 1024,
) -> tuple[
ApiServices,
sessionmaker[Session],
@@ -84,7 +89,11 @@ def _build_api_services(
embedding_model=embedding.model_name,
vector_dimension=embedding.dimensions,
)
- upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
+ upload = DocumentUploadService(
+ session_factory,
+ storage,
+ max_upload_bytes=max_upload_bytes,
+ )
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
@@ -102,6 +111,12 @@ def _build_api_services(
jobs = IngestionJobService(session_factory, max_attempts=3)
dispatcher = RecordingDispatcher()
submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher)
+ batches = IngestionBatchService(
+ session_factory,
+ submissions,
+ max_files=batch_max_files,
+ max_total_bytes=batch_max_total_bytes,
+ )
runner = IngestionJobRunner(
session_factory,
parsing,
@@ -134,6 +149,7 @@ def _build_api_services(
ApiServices(
ingestion=ingestion,
submissions=submissions,
+ batches=batches,
jobs=jobs,
documents=documents,
search=search,
@@ -238,6 +254,8 @@ def test_rest_validation_not_found_and_openapi_contract(
assert set(openapi["paths"]) >= {
"/api/v1/documents",
"/api/v1/documents/{document_id}",
+ "/api/v1/ingestion-batches",
+ "/api/v1/ingestion-batches/{batch_id}",
"/api/v1/jobs/{job_id}",
"/api/v1/search",
}
@@ -261,3 +279,141 @@ def test_rest_validation_not_found_and_openapi_contract(
assert boolean_top_k.status_code == 422
assert missing_get.status_code == 404
assert missing_delete.status_code == 404
+
+
+def test_batch_upload_isolates_files_and_reports_terminal_status(
+ tmp_path: Path,
+ isolated_settings_environment: None,
+) -> None:
+ """A rejected file must not roll back valid siblings in the same batch."""
+ services, _, _, _, runner = _build_api_services(tmp_path)
+ app = create_app(Settings(app_env="test"), api_services=services)
+
+ with TestClient(app) as client:
+ accepted = client.post(
+ "/api/v1/ingestion-batches",
+ files=[
+ ("files", ("alpha.md", b"# Alpha\n\nFirst policy.", "text/markdown")),
+ ("files", ("blocked.csv", b"secret=must-not-persist", "text/csv")),
+ ("files", ("beta.txt", b"Second policy.", "text/plain")),
+ ],
+ )
+
+ assert accepted.status_code == 202
+ body = accepted.json()
+ assert body["status"] == "processing"
+ assert body["total_files"] == 3
+ assert body["accepted_files"] == 2
+ assert body["rejected_files"] == 1
+ assert body["completed_files"] == 1
+ assert [item["filename"] for item in body["items"]] == [
+ "alpha.md",
+ "blocked.csv",
+ "beta.txt",
+ ]
+ assert [item["acceptance_status"] for item in body["items"]] == [
+ "accepted",
+ "rejected",
+ "accepted",
+ ]
+ assert body["items"][1]["error_message"] == ("unsupported document type or media type")
+ assert "must-not-persist" not in str(body)
+
+ for item in body["items"]:
+ if item["job_id"] is not None:
+ runner.run(UUID(item["job_id"]))
+
+ terminal = client.get(f"/api/v1/ingestion-batches/{body['batch_id']}")
+
+ assert terminal.status_code == 200
+ terminal_body = terminal.json()
+ assert terminal_body["status"] == "partial_failure"
+ assert terminal_body["completed_files"] == 3
+ assert [item["job_status"] for item in terminal_body["items"]] == [
+ "succeeded",
+ None,
+ "succeeded",
+ ]
+
+
+def test_batch_upload_enforces_count_and_shared_byte_limits(
+ tmp_path: Path,
+ isolated_settings_environment: None,
+) -> None:
+ """Request limits are bounded while already accepted siblings remain durable."""
+ services, session_factory, _, _, _ = _build_api_services(
+ tmp_path,
+ max_upload_bytes=10,
+ batch_max_files=2,
+ batch_max_total_bytes=15,
+ )
+ app = create_app(Settings(app_env="test"), api_services=services)
+
+ with TestClient(app) as client:
+ too_many = client.post(
+ "/api/v1/ingestion-batches",
+ files=[
+ ("files", ("one.txt", b"1", "text/plain")),
+ ("files", ("two.txt", b"2", "text/plain")),
+ ("files", ("three.txt", b"3", "text/plain")),
+ ],
+ )
+ bounded = client.post(
+ "/api/v1/ingestion-batches",
+ files=[
+ ("files", ("first.txt", b"1234567890", "text/plain")),
+ ("files", ("second.txt", b"abcdefghij", "text/plain")),
+ ],
+ )
+ oversized = client.post(
+ "/api/v1/ingestion-batches",
+ files=[("files", ("large.txt", b"12345678901", "text/plain"))],
+ )
+
+ assert too_many.status_code == 413
+ assert too_many.json()["detail"] == "batch file count exceeds the configured limit"
+ assert bounded.status_code == 202
+ body = bounded.json()
+ assert body["accepted_files"] == 1
+ assert body["rejected_files"] == 1
+ assert body["items"][1]["error_message"] == ("batch exceeds the configured total upload limit")
+ assert oversized.status_code == 202
+ assert oversized.json()["status"] == "failed"
+ assert oversized.json()["accepted_files"] == 0
+ assert oversized.json()["items"][0]["error_message"] == (
+ "document exceeds the configured upload limit"
+ )
+
+ # A count-rejected request creates no batch; the two accepted requests do.
+ with session_factory() as session:
+ assert len(session.scalars(select(IngestionBatch)).all()) == 2
+
+
+def test_batch_upload_completed_and_missing_contract(
+ tmp_path: Path,
+ isolated_settings_environment: None,
+) -> None:
+ """All-success batches become completed and unknown IDs stay explicit."""
+ services, _, _, _, runner = _build_api_services(tmp_path)
+ app = create_app(Settings(app_env="test"), api_services=services)
+ missing_id = uuid4()
+
+ with TestClient(app) as client:
+ accepted = client.post(
+ "/api/v1/ingestion-batches",
+ files=[
+ ("files", ("one.md", b"# One\n\nContent.", "text/markdown")),
+ ("files", ("two.txt", b"More content.", "text/plain")),
+ ],
+ ).json()
+ for item in accepted["items"]:
+ runner.run(UUID(item["job_id"]))
+
+ completed = client.get(f"/api/v1/ingestion-batches/{accepted['batch_id']}")
+ missing = client.get(f"/api/v1/ingestion-batches/{missing_id}")
+
+ assert completed.status_code == 200
+ assert completed.json()["status"] == "completed"
+ assert completed.json()["completed_files"] == 2
+ assert missing.status_code == 404
+ assert missing.json()["detail"] == "ingestion batch not found"
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index faec677..6e66ed7 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -25,6 +25,8 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
assert settings.ingestion_retry_backoff_seconds == 2
assert settings.ingestion_retry_backoff_max_seconds == 60
assert settings.ingestion_worker_lease_seconds == 300
+ assert settings.batch_max_files == 20
+ assert settings.batch_max_total_bytes == 200 * 1024 * 1024
assert settings.object_storage_provider == "bos"
assert settings.object_storage_path == Path("storage")
assert settings.bos_endpoint is None
@@ -157,3 +159,10 @@ def test_settings_require_worker_lease_longer_than_provider_timeout(
) -> None:
with pytest.raises(ValidationError, match="worker_lease"):
Settings(ingestion_worker_lease_seconds=30, embedding_timeout_seconds=30)
+
+
+def test_settings_require_batch_limit_to_fit_one_file(
+ isolated_settings_environment: None,
+) -> None:
+ with pytest.raises(ValidationError, match="batch_max_total_bytes"):
+ Settings(max_upload_bytes=100, batch_max_total_bytes=99)