merge: complete DEMO-016 batch async upload

This commit is contained in:
d-robotics 2026-09-02 20:43:26 +08:00
commit a1eae16fb1
30 changed files with 1098 additions and 59 deletions

View File

@ -18,6 +18,8 @@ INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
BATCH_MAX_FILES=20
BATCH_MAX_TOTAL_BYTES=209715200
OBJECT_STORAGE_PROVIDER=bos
OBJECT_STORAGE_PATH=storage

View File

@ -133,7 +133,8 @@ source results are rehydrated from canonical relational chunks and retain both
Dense and reranker scores; stale vector content fails closed. See
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
changes upload to asynchronous acceptance and adds Job status. Upload, list, detail, direct delete,
changes upload to asynchronous acceptance and adds Job status. DEMO-016 adds bounded multi-file
submission, batch/per-file status, and automatic Admin Console polling. Upload, list, detail, direct delete,
and reranked source search are documented in
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
outside the public response contract.

View File

@ -1,12 +1,33 @@
// DEMO-010 stays API First: this client owns presentation state only and sends
// every document/search operation through the versioned public REST contract.
// The Admin Console owns presentation state only. Batch acceptance, Job state,
// and retrieval remain behind the versioned public REST contract.
const API_BASE = "/api/v1";
const BATCH_POLL_INTERVAL_MS = 1500;
const terminalBatchStatuses = new Set(["completed", "partial_failure", "failed"]);
let batchPollTimer;
let activeBatchId;
const statusLabels = {
uploaded: "Uploaded",
processing: "Processing",
ready: "Ready",
failed: "Failed",
pending: "Pending",
accepted: "Accepted",
rejected: "Rejected",
queued: "Queued",
running: "Running",
retrying: "Retrying",
succeeded: "Succeeded",
completed: "Completed",
partial_failure: "Partial failure",
};
const stageLabels = {
parsing: "解析",
cleaning: "清理",
chunking: "切分",
embedding: "向量化",
indexing: "索引",
};
const pageTitles = {
@ -93,23 +114,37 @@ function initializeUpload() {
const button = document.querySelector("#upload-button");
const updateFilename = () => {
selected.textContent = input.files[0]?.name ?? "尚未选择文件";
const files = Array.from(input.files);
if (!files.length) {
selected.textContent = "尚未选择文件";
} else if (files.length === 1) {
selected.textContent = files[0].name;
} else {
selected.textContent = `已选择 ${files.length} 个文件`;
}
};
input.addEventListener("change", updateFilename);
form.addEventListener("submit", async (event) => {
event.preventDefault();
if (!input.files.length) return;
window.clearTimeout(batchPollTimer);
activeBatchId = undefined;
setButtonBusy(button, true, "提交中…");
try {
const formData = new FormData();
formData.append("file", input.files[0]);
const documentData = await requestJson("/documents", {
Array.from(input.files).forEach((file) => formData.append("files", file));
const batch = await requestJson("/ingestion-batches", {
method: "POST",
body: formData,
});
renderUploadResult(documentData);
showNotice("文档已接收并进入后台处理队列。", "success");
activeBatchId = batch.batch_id;
renderBatchResult(batch);
scheduleBatchPoll(batch);
showNotice(
`批次已接收:${batch.accepted_files} 个进入后台队列,${batch.rejected_files} 个被拒绝。`,
batch.rejected_files ? "error" : "success",
);
form.reset();
updateFilename();
} catch (error) {
@ -120,18 +155,62 @@ function initializeUpload() {
});
}
function renderUploadResult(documentData) {
function scheduleBatchPoll(batch) {
window.clearTimeout(batchPollTimer);
if (batch.batch_id !== activeBatchId) return;
if (terminalBatchStatuses.has(batch.status)) return;
batchPollTimer = window.setTimeout(async () => {
try {
const latest = await requestJson(`/ingestion-batches/${batch.batch_id}`);
renderBatchResult(latest);
scheduleBatchPoll(latest);
} catch (error) {
showNotice(`批次状态刷新失败:${error.message}`, "error");
scheduleBatchPoll(batch);
}
}, BATCH_POLL_INTERVAL_MS);
}
function renderBatchResult(batch) {
const card = document.querySelector("#upload-result");
document.querySelector("#upload-filename").textContent = documentData.filename;
document.querySelector("#upload-type").textContent = documentData.media_type;
document.querySelector("#upload-size").textContent = formatSize(documentData.size_bytes);
document.querySelector("#upload-time").textContent = formatDate(documentData.created_at);
document.querySelector("#upload-status").replaceWith(statusPill(documentData.status));
document.querySelector("#upload-batch-title").textContent =
`批次 ${batch.batch_id.slice(0, 8)} · ${formatDate(batch.created_at)}`;
document.querySelector("#upload-total").textContent = `${batch.total_files} 个`;
document.querySelector("#upload-completed").textContent =
`${batch.completed_files} / ${batch.total_files}`;
document.querySelector("#upload-accepted").textContent =
`${batch.accepted_files} / ${batch.rejected_files}`;
document.querySelector("#upload-status").replaceWith(statusPill(batch.status));
const status = card.querySelector(".status-pill");
status.id = "upload-status";
const items = document.querySelector("#batch-items");
items.replaceChildren(...batch.items.map(batchItem));
card.hidden = false;
}
function batchItem(item) {
const row = document.createElement("article");
row.className = "batch-item";
const order = document.createElement("span");
order.className = "batch-item-order";
order.textContent = String(item.item_index + 1).padStart(2, "0");
const body = document.createElement("div");
body.className = "batch-item-body";
const name = document.createElement("strong");
name.textContent = item.filename;
const detail = document.createElement("span");
const stage = item.stage ? ` · ${stageLabels[item.stage] ?? item.stage}` : "";
const attempts = item.max_attempts == null ? "" : ` · 尝试 ${item.attempt}/${item.max_attempts}`;
detail.textContent = item.error_message ?? `后台任务${stage}${attempts}`;
body.append(name, detail);
const state = item.job_status ?? item.acceptance_status;
row.append(order, body, statusPill(state));
return row;
}
async function loadDocuments() {
const refresh = document.querySelector("#refresh-documents");
setButtonBusy(refresh, true, "加载中…");
@ -329,3 +408,4 @@ initializeUpload();
initializeSearch();
document.querySelector("#refresh-documents").addEventListener("click", loadDocuments);
if (page === "documents") loadDocuments();
window.addEventListener("pagehide", () => window.clearTimeout(batchPollTimer));

View File

@ -533,6 +533,50 @@ a {
border-block: 1px solid var(--line);
}
.batch-items {
display: grid;
gap: 8px;
margin: -8px 0 24px;
}
.batch-item {
display: grid;
grid-template-columns: 34px minmax(0, 1fr) auto;
gap: 12px;
align-items: center;
min-height: 64px;
padding: 11px 13px;
border: 1px solid var(--line);
border-radius: 3px;
background: #fafaf8;
}
.batch-item-order {
color: var(--brand);
font-family: var(--data-font);
font-size: 10px;
font-weight: 850;
}
.batch-item-body {
display: grid;
min-width: 0;
gap: 5px;
}
.batch-item-body strong {
overflow: hidden;
font-size: 12px;
text-overflow: ellipsis;
white-space: nowrap;
}
.batch-item-body span {
overflow-wrap: anywhere;
color: var(--muted);
font-size: 10px;
}
dt {
color: var(--muted);
font-family: var(--data-font);
@ -655,6 +699,15 @@ tbody tr:hover {
.status-processing { background: #fff0ce; color: var(--warning); }
.status-failed { background: #fbe6e3; color: var(--danger); }
.status-uploaded { background: var(--brand-soft); color: var(--brand-deep); }
.status-pending,
.status-queued,
.status-running,
.status-retrying,
.status-accepted { background: #fff0ce; color: var(--warning); }
.status-succeeded,
.status-completed { background: #e5f3eb; color: var(--success); }
.status-rejected,
.status-partial_failure { background: #fbe6e3; color: var(--danger); }
.empty-state {
min-height: 240px;
@ -958,6 +1011,8 @@ select:focus {
.search-controls { grid-template-columns: minmax(0, 1fr) 90px; }
.search-controls textarea { grid-column: 1 / -1; }
.metadata-grid { grid-template-columns: 1fr; }
.batch-item { grid-template-columns: 28px minmax(0, 1fr); }
.batch-item .status-pill { grid-column: 2; }
}
@media (max-width: 540px) {

View File

@ -67,7 +67,7 @@
<div>
<p class="section-label">01 / INGESTION</p>
<h2>让知识进入检索链路</h2>
<p>上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将在后台完成解析、切分与索引。</p>
<p>批量提交 PDF、DOCX、TXT 或 Markdown 文件,系统将逐个接收并在后台独立完成解析、切分与索引。</p>
</div>
</div>
@ -75,14 +75,15 @@
<label id="drop-zone" class="drop-zone" for="file-input">
<input
id="file-input"
name="file"
name="files"
type="file"
multiple
accept=".pdf,.docx,.txt,.md,application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document,text/plain,text/markdown"
required
/>
<span class="upload-icon" aria-hidden="true">↑</span>
<strong>选择知识文件</strong>
<span>支持 PDF、DOCX、TXT、Markdown · 单次一个文件</span>
<strong>选择一批知识文件</strong>
<span>支持 PDF、DOCX、TXT、Markdown · 每个文件独立处理</span>
<span id="selected-file" class="selected-file">尚未选择文件</span>
</label>
<button id="upload-button" class="primary-button" type="submit">上传并处理</button>
@ -90,15 +91,16 @@
<article id="upload-result" class="result-card" hidden>
<div>
<p class="section-label">LATEST SIGNAL</p>
<h3 id="upload-filename"></h3>
<p class="section-label">LATEST BATCH</p>
<h3 id="upload-batch-title"></h3>
</div>
<span id="upload-status" class="status-pill"></span>
<dl class="metadata-grid">
<div><dt>文件类型</dt><dd id="upload-type"></dd></div>
<div><dt>文件大小</dt><dd id="upload-size"></dd></div>
<div><dt>上传时间</dt><dd id="upload-time"></dd></div>
<div><dt>总文件</dt><dd id="upload-total"></dd></div>
<div><dt>已完成</dt><dd id="upload-completed"></dd></div>
<div><dt>接收 / 拒绝</dt><dd id="upload-accepted"></dd></div>
</dl>
<div id="batch-items" class="batch-items" aria-live="polite"></div>
<a class="text-link" href="/admin/search">前往 Search Lab 检索 <span>→</span></a>
</article>
</section>

View File

@ -18,6 +18,7 @@ from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.index_rebuild import resolve_active_index_profile
from ekp_core.ingestion_batches import IngestionBatchService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
@ -50,6 +51,7 @@ class ApiServices:
ingestion: DocumentIndexingPipelineService
submissions: AsyncIngestionSubmissionService
batches: IngestionBatchService
jobs: IngestionJobService
documents: DocumentManagementService
search: SearchService
@ -310,6 +312,12 @@ def create_api_services(settings: Settings) -> ApiServices:
components.jobs,
components.dispatcher,
)
batches = IngestionBatchService(
components.session_factory,
submissions,
max_files=settings.batch_max_files,
max_total_bytes=settings.batch_max_total_bytes,
)
dense_retrieval = create_dense_retrieval_service(
settings,
components.embedding_provider,
@ -332,6 +340,7 @@ def create_api_services(settings: Settings) -> ApiServices:
return ApiServices(
ingestion=ingestion,
submissions=submissions,
batches=batches,
jobs=components.jobs,
documents=documents,
search=search,

View File

@ -9,6 +9,7 @@ from ekp_api.providers import ApiServices, ProviderConfigurationError, create_ap
from ekp_api.schemas import (
DocumentResponse,
IngestionAcceptedResponse,
IngestionBatchResponse,
IngestionJobResponse,
SearchRequest,
SearchResponse,
@ -25,6 +26,7 @@ from ekp_core.document_upload import (
MediaTypeMismatchError,
UnsupportedDocumentTypeError,
)
from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundError, BatchSource
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
router = APIRouter()
@ -98,6 +100,61 @@ def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
return IngestionJobResponse.from_job(job)
@router.post(
"/ingestion-batches",
response_model=IngestionBatchResponse,
status_code=status.HTTP_202_ACCEPTED,
tags=["jobs"],
)
def upload_ingestion_batch(
services: Services,
files: Annotated[
list[UploadFile],
File(description="One or more PDF, DOCX, TXT, or Markdown files"),
],
) -> IngestionBatchResponse:
"""Accept a bounded batch while isolating every file's result."""
try:
view = services.batches.submit(
tuple(
BatchSource(
filename=file.filename or "",
media_type=file.content_type,
source=file.file,
)
for file in files
)
)
except BatchFileCountLimitError as error:
raise HTTPException(
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
detail="batch file count exceeds the configured limit",
) from error
except Exception as error:
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="batch could not be accepted",
) from error
return IngestionBatchResponse.from_view(view)
@router.get(
"/ingestion-batches/{batch_id}",
response_model=IngestionBatchResponse,
tags=["jobs"],
)
def get_ingestion_batch(batch_id: UUID, services: Services) -> IngestionBatchResponse:
"""Return aggregate and per-file status for Admin Console polling."""
try:
view = services.batches.get(batch_id)
except BatchNotFoundError as error:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="ingestion batch not found",
) from error
return IngestionBatchResponse.from_view(view)
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
def list_documents(services: Services) -> list[DocumentResponse]:
"""List canonical document metadata for the Demo admin page."""

View File

@ -5,9 +5,17 @@ from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
from ekp_core.ingestion_batches import BatchAggregateStatus, BatchItemView, BatchView
from ekp_core.ingestion_jobs import IngestionSubmission
from ekp_core.search import SearchResult
from ekp_domain import Document, DocumentStatus, IngestionJob, IngestionJobStatus, IngestionStage
from ekp_domain import (
BatchItemAcceptanceStatus,
Document,
DocumentStatus,
IngestionJob,
IngestionJobStatus,
IngestionStage,
)
class DocumentResponse(BaseModel):
@ -88,6 +96,80 @@ class IngestionJobResponse(BaseModel):
)
class IngestionBatchItemResponse(BaseModel):
"""Per-file acceptance and latest asynchronous processing state."""
item_index: int
filename: str
acceptance_status: BatchItemAcceptanceStatus
document_id: UUID | None
job_id: UUID | None
job_status: IngestionJobStatus | None
stage: IngestionStage | None
attempt: int
max_attempts: int | None
error_message: str | None
@classmethod
def from_view(cls, view: BatchItemView) -> "IngestionBatchItemResponse":
job = view.job
return cls(
item_index=view.item.item_index,
filename=view.item.filename,
acceptance_status=view.item.acceptance_status,
document_id=view.item.document_id,
job_id=view.item.job_id,
job_status=job.status if job is not None else None,
stage=job.stage if job is not None else None,
attempt=job.attempt_count if job is not None else 0,
max_attempts=job.max_attempts if job is not None else None,
error_message=(job.error_message if job is not None else view.item.error_message),
)
class IngestionBatchResponse(BaseModel):
"""Batch aggregate with ordered independently processed files."""
batch_id: UUID
status: BatchAggregateStatus
total_files: int
accepted_files: int
rejected_files: int
completed_files: int
created_at: datetime
items: list[IngestionBatchItemResponse]
@classmethod
def from_view(cls, view: BatchView) -> "IngestionBatchResponse":
items = [IngestionBatchItemResponse.from_view(item) for item in view.items]
accepted = sum(
item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED for item in items
)
rejected = sum(
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED for item in items
)
completed = sum(
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED
or item.job_status
in {
IngestionJobStatus.SUCCEEDED,
IngestionJobStatus.FAILED,
IngestionJobStatus.CANCELLED,
}
for item in items
)
return cls(
batch_id=view.batch.id,
status=view.status,
total_files=view.batch.total_files,
accepted_files=accepted,
rejected_files=rejected,
completed_files=completed,
created_at=view.batch.created_at,
items=items,
)
class SearchRequest(BaseModel):
"""Validated public search input; normalization remains in the core service."""

View File

@ -11,11 +11,13 @@ Use OpenAPI as executable source of truth once implementation starts. Provider-s
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
版本化为 `/api/v1/*`。
`DEMO-015` 已将单文件上传切换为持久化异步 Job;批量提交仍留给 `DEMO-016`。
`DEMO-015` 已将单文件上传切换为持久化异步 Job;`DEMO-016` 已增加批量提交和聚合查询。
| Method | Path | 行为 |
|---|---|---|
| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
| `POST` | `/api/v1/ingestion-batches` | multipart `files` 多文件接收;返回批次与逐文件状态 |
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新 Job 状态 |
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
@ -55,7 +57,7 @@ Search 响应:
```
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
文档或 Job 不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
文档、Job 或批次不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
厂商响应。
@ -74,7 +76,7 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
## 异步入库契约(DEMO-015 已实现;DEMO-016 批量部分待实现)
## 异步入库契约(DEMO-015~016 已实现)
| Method | Path | 行为 |
|---|---|---|
@ -101,17 +103,29 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
}
```
DEMO-016 的批量响应示意:
批量字段名为 `files`,成功返回 `202`。响应示意:
```json
{
"batch_id": "d21f8012-4c4e-4bb4-89af-e11c0f9f470e",
"status": "processing",
"total_files": 2,
"accepted_files": 1,
"rejected_files": 1,
"completed_files": 1,
"created_at": "2026-09-02T12:00:00Z",
"items": [
{
"item_index": 0,
"filename": "退款政策.md",
"acceptance_status": "accepted",
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
"status": "queued"
"job_status": "queued",
"stage": null,
"attempt": 0,
"max_attempts": 3,
"error_message": null
}
]
}
@ -119,8 +133,11 @@ DEMO-016 的批量响应示意:
Job 公开状态为 `queued | running | retrying | succeeded | failed | cancelled`,并通过 `stage`
表示 `parsing | cleaning | chunking | embedding | indexing`。错误字段只返回有界、安全摘要。
批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚其他文件。文件数量、单文件大小和
请求总大小均由服务端配置限制。
批次状态为 `processing | completed | partial_failure | failed`;逐文件接收状态为
`pending | accepted | rejected`。批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚
其他文件。文件数量、单文件大小和请求总大小分别由 `BATCH_MAX_FILES`、`MAX_UPLOAD_BYTES`
和 `BATCH_MAX_TOTAL_BYTES` 限制。数量超限返回 `413`;类型、大小或内容不合规作为该文件的
`rejected` 安全结果返回,其他合规文件继续接收。
自动重试是服务端 Job 行为,不要求客户端再次上传,也不新增公开手动重试接口。每次重试前
清理派生数据但保留原始文件和稳定的 Document ID;达到次数上限或清理失败后进入 `failed`。

View File

@ -179,6 +179,10 @@ Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Fail
API 只负责校验并持久化不可变源文件、Document 和 Ingestion Job,然后返回 `202`。批次是
提交与聚合查询边界,不是跨文件数据库事务;每个文件独立排队、执行和失败。
DEMO-016 以 canonical `ingestion_batches` / `ingestion_batch_items` 保存输入顺序和接收引用;
处理阶段、Attempt 与最终结果继续以 `ingestion_jobs` / `ingestion_attempts` 为事实源,批次表
不复制 Worker 状态。
```text
Batch Request
├─ Source A + Document A + Job A ─→ Worker

View File

@ -340,7 +340,7 @@ API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重
### DEMO-016 批量异步上传
在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
**已完成。** 在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
Admin Console 状态展示。批次内文件相互隔离,不包含目录上传、压缩包解包或断点续传。
---
@ -397,8 +397,8 @@ DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
任务拆分;`DEMO-012`、`DEMO-015` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
后续严格按 `DEMO-016`、`DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
任务拆分;`DEMO-012`、`DEMO-015~016` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
后续严格按 `DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
可选项,不因本状态记录自动进入范围。
可选:

View File

@ -16,7 +16,7 @@ make run
| 页面 | 地址 | 对应业务接口 |
|---|---|---|
| 文档上传 | `/admin/upload` | `POST /api/v1/documents` |
| 文档上传 | `/admin/upload` | `POST/GET /api/v1/ingestion-batches*` |
| 文档列表与详情 | `/admin/documents` | `GET/DELETE /api/v1/documents*` |
| Search Lab | `/admin/search` | `POST /api/v1/search` |
@ -34,9 +34,9 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
## Demo 操作闭环
1. 在上传页选择一个 PDF、DOCX、TXT 或 Markdown 文件并提交;
2. 接口返回 `202` 后,当前页面显示已接收;在文档列表刷新查看 `Processing/Ready/Failed`;
3. 在文档列表查看状态和安全元数据;
1. 在上传页选择一个或多个 PDF、DOCX、TXT 或 Markdown 文件并提交;
2. 接口返回 `202` 后,页面自动轮询批次,逐文件显示接收状态、处理阶段、尝试次数和安全错误;
3. 批次进入 `Completed`、`Partial failure` 或 `Failed` 后停止轮询,再在文档列表查看元数据;
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
Rerank Score;
5. 如需清理 Demo 数据,在文档列表确认后直接删除。
@ -46,7 +46,8 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
- 当前 Console 尚不轮询 Job 详情或展示 Attempt;该状态 UI 与批量上传一起由 DEMO-016 实现;
- Console 只轮询当前批次的聚合接口,不读取 Provider/队列细节,也不提供人工重试按钮;
- 批量上传不包含目录上传、压缩包解包、断点续传或跨批次去重;
- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
或完整 V1 Admin Console 导航;
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。

View File

@ -2,11 +2,12 @@
## 范围
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker。OceanBase 中的
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker,DEMO-016 在其上增加逐文件
隔离的批量接收和状态聚合。OceanBase 中的
`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
不携带文件内容、Provider 凭据或业务状态。
本任务不提供批量上传和批次 UI;这些能力属于 DEMO-016。
批次只聚合接收与 Job 状态,不改变每个文件的独立执行和重试边界。
## 运行时配置
@ -19,6 +20,8 @@ INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
BATCH_MAX_FILES=20
BATCH_MAX_TOTAL_BYTES=209715200
```
Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
@ -72,6 +75,18 @@ Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;
`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
批量提交与查询:
```text
POST /api/v1/ingestion-batches # multipart 字段 files,可重复
GET /api/v1/ingestion-batches/{batch_id}
```
批次创建后逐文件返回 `accepted` 或 `rejected`;已接收文件继续通过上述 Job 状态异步推进。
批次聚合状态为 `processing | completed | partial_failure | failed`。请求文件数超过
`BATCH_MAX_FILES` 时整体返回 `413`;单文件或共享总字节预算超限只拒绝当前文件,不删除此前
已接收文件。默认总预算 200 MiB,且配置不得小于单文件 `MAX_UPLOAD_BYTES`。
## 故障检查
1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;

View File

@ -3,10 +3,10 @@
## 范围
DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
Job。API 层只负责输入校验、
Job,DEMO-016 增加批量接收与聚合查询。API 层只负责输入校验、
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
当前不包含 ACL/RBAC、软删除、批量上传或 Hybrid Search。
当前不包含 ACL/RBAC、软删除、目录/压缩包上传、断点续传或 Hybrid Search。
## 启动前提
@ -40,6 +40,16 @@ curl -F 'file=@./policy.md;type=text/markdown' \
curl http://127.0.0.1:8000/api/v1/jobs/<job-id>
```
批量上传并查询聚合进度:
```bash
curl -F 'files=@./policy.md;type=text/markdown' \
-F 'files=@./manual.pdf;type=application/pdf' \
http://127.0.0.1:8000/api/v1/ingestion-batches
curl http://127.0.0.1:8000/api/v1/ingestion-batches/<batch-id>
```
检索:
```bash

View File

@ -9,13 +9,13 @@
| 能力 | 当前实现 | 结论 |
|---|---|---|
| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
| 批量上传 | API 与 Admin Console 都只支持一个文件 | 未实现 |
| 失败补偿 | 上传元数据失败会删除源对象;解析元数据失败会删除本次解析产物;索引失败会删除当前 Document/Profile 的可能残留向量 | 已有局部补偿,不是自动重试 |
| 批量上传 | 批次 API、逐文件 Job 状态和 Admin Console 自动轮询 | DEMO-016 已实现 |
| 失败补偿 | 阶段内补偿 + 重试前清理全部派生数据,保留不可变源和执行历史 | DEMO-015 已实现 |
| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
| MCP | 已有 MCP 业务契约和可复用 Search Service/REST API,没有可运行 MCP Gateway 或 WorkBuddy 验证 | 未实现 |
| MCP | 独立 Gateway、认证边界、`search_knowledge` 与 WorkBuddy 协议级验收 | DEMO-012 已实现 |
PLAN-001 的上述缺口已由 DEMO-015 补齐;批量提交与状态 UI 仍由 DEMO-016 实现。
PLAN-001 的异步执行缺口已由 DEMO-015 补齐,批量提交与状态 UI 已由 DEMO-016 补齐。
## 2. Task 归并结论
@ -51,7 +51,7 @@ PLAN-001 的上述缺口已由 DEMO-015 补齐;批量提交与状态 UI 仍由
该 Task 只完成公共知识库入库所需的 Job/Retry 与队列基线,不扩展到完整审计系统、
Compilation、OCR、跨区域高可用或通用企业级 Job 平台。
### DEMO-016:批量异步上传与状态展示(新增)
### DEMO-016:批量异步上传与状态展示(已完成)
建立在 DEMO-015 之上,不与 Job 基础合并:
@ -79,8 +79,8 @@ PLAN-002(已完成)
推荐严格按以下顺序执行并逐项闭环:
1. `PLAN-002`(已完成)
2. `DEMO-015`
3. `DEMO-016`
2. `DEMO-015`(已完成)
3. `DEMO-016`(已完成)
4. `DEPLOY-001`
5. `DEPLOY-002`
6. `DEPLOY-003`

View File

@ -1,12 +1,12 @@
# 项目进度
当前阶段:异步单文件入库闭环完成,准备实施批量上传
当前阶段:批量异步上传闭环完成,准备交付生产镜像
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-015、ARCH-001~ARCH-006、PLAN-001~PLAN-002
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002
进行中:无
阻塞:无
下一任务:`DEMO-016`。随后严格执行 `DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
下一任务:`DEPLOY-001`。随后严格执行 `DEPLOY-002 → DEPLOY-003`。
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
@ -354,3 +354,26 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
- 下一任务:`DEMO-016`。
### DEMO-016(已完成)
- 新增 canonical `ingestion_batches` / `ingestion_batch_items` 及 forward-only migration,保存
输入顺序、逐文件接收结果与 Document/Job 引用;异步阶段和最终结果仍以 Job 为事实源,
批次不是跨文件事务。
- 新增 `POST /api/v1/ingestion-batches` 与 `GET /api/v1/ingestion-batches/{batch_id}`;批量
请求字段为 `files`,成功返回 `202`、聚合状态和逐文件接收/Job 状态。类型、大小或内容错误
只拒绝当前文件,错误映射为固定安全摘要,其他文件继续接收。
- 新增 `BATCH_MAX_FILES` 与 `BATCH_MAX_TOTAL_BYTES`,复用 `MAX_UPLOAD_BYTES` 约束单文件;
上传流共享总字节预算,不把整批文件读入内存。数量超限在创建批次前返回 `413`。
- Admin Console 支持多选提交,显示当前批次的接收数量、完成数量、逐文件阶段、Attempt 与安全
错误,并每 1.5 秒自动轮询;进入 `completed`、`partial_failure` 或 `failed` 后停止轮询。
DOM 数据继续只通过文本节点渲染。
- 集成测试覆盖顺序稳定、部分拒绝不回滚、数量/单文件/批次总大小、全成功/部分失败聚合、
未知批次、OpenAPI 和前端契约;迁移 drift 增加两张新表。Ruff Format/Lint、Mypy、
JavaScript 语法、依赖锁、Alembic Head/离线升级均通过;Unit + Integration `183 passed`,
RAG Evaluation `5 passed`。
- 浏览器验收确认桌面布局和 390px 窄屏布局正常、无根页面横向溢出、无控制台警告或错误;
本地页面不注入云端凭据,批次结果行为由确定性 API 集成测试验证。
- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续
复用 DEMO-015,Accepted ADR 未变更。
- 下一任务:`DEPLOY-001`。

View File

@ -82,7 +82,8 @@ Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务
### DEMO-016
基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console 状态展示。
**已完成。** 基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console
状态展示;下一交付门槛为 DEPLOY-001 的生产镜像。
### DEPLOY-001

View File

@ -29,7 +29,7 @@ validation. Provider credentials and internal endpoints are runtime inputs and n
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
- [ ] DEMO-016 Batch async upload API + per-file Job status + Admin Console
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
- [ ] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
- [ ] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance

View File

@ -17,7 +17,7 @@
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
3. `DEMO-016`:批量异步上传、批次/Job 查询和 Admin Console 状态展示;
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
4. `DEPLOY-001`:API/Admin、MCP Gateway、Async Worker 生产镜像;
5. `DEPLOY-002`:Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。

View File

@ -465,6 +465,9 @@ Document processing 必须异步执行。
源文件与 Document/Job 成功持久化后返回 `202 Accepted`。批量上传中的每个文件拥有独立的
Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批次内其他已接收文件。
当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、
断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。
**REQ-INGEST-004**
展示处理阶段:

View File

@ -21,3 +21,7 @@ single-active generated unique index.
DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
the latter preserves numbered history and uses a generated nullable active marker plus a unique index
to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.
DEMO-016 adds `ingestion_batches` and `ingestion_batch_items`. They retain input order and per-file
acceptance references while Job rows remain the processing state source of truth; a batch is an aggregate,
not a cross-file transaction.

View File

@ -0,0 +1,109 @@
"""Create ingestion batch aggregation metadata.
Revision ID: 20260902_0008
Revises: 20260902_0007
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0008"
down_revision: str | None = "20260902_0007"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Persist ordered per-file acceptance without making a batch transactional."""
op.create_table(
"ingestion_batches",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("total_files", sa.Integer(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.CheckConstraint("total_files > 0", name="total_files_positive"),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batches"),
)
op.create_table(
"ingestion_batch_items",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("batch_id", sa.Uuid(), nullable=False),
sa.Column("item_index", sa.Integer(), nullable=False),
sa.Column("filename", sa.String(length=255), nullable=False),
sa.Column(
"acceptance_status",
sa.Enum(
"pending",
"accepted",
"rejected",
name="batch_item_acceptance_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column("document_id", sa.Uuid(), nullable=True),
sa.Column("job_id", sa.Uuid(), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.CheckConstraint("item_index >= 0", name="item_index_non_negative"),
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
sa.CheckConstraint(
"acceptance_status IN ('pending', 'accepted', 'rejected')",
name="acceptance_status_valid",
),
sa.ForeignKeyConstraint(
["batch_id"],
["ingestion_batches.id"],
name="fk_ingestion_batch_items_batch_id_ingestion_batches",
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["document_id"],
["documents.id"],
name="fk_ingestion_batch_items_document_id_documents",
ondelete="SET NULL",
),
sa.ForeignKeyConstraint(
["job_id"],
["ingestion_jobs.id"],
name="fk_ingestion_batch_items_job_id_ingestion_jobs",
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batch_items"),
)
op.create_index(
"uq_ingestion_batch_items_order",
"ingestion_batch_items",
["batch_id", "item_index"],
unique=True,
)
op.create_index(
"uq_ingestion_batch_items_job_id",
"ingestion_batch_items",
["job_id"],
unique=True,
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -39,6 +39,8 @@ class Settings(BaseSettings):
ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
batch_max_files: int = Field(default=20, ge=1, le=100)
batch_max_total_bytes: int = Field(default=200 * 1024 * 1024, gt=0)
object_storage_provider: Literal["local", "bos"] = "bos"
object_storage_path: Path = Path("storage")
bos_endpoint: str | None = None
@ -108,6 +110,8 @@ class Settings(BaseSettings):
)
if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
if self.batch_max_total_bytes < self.max_upload_bytes:
raise ValueError("batch_max_total_bytes must be at least max_upload_bytes")
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
raise ValueError("mcp_path must start with '/' and must not end with '/'")
if not self.mcp_allowed_hosts:

View File

@ -0,0 +1,270 @@
"""Batch submission and aggregate status over independent ingestion Jobs."""
from dataclasses import dataclass
from enum import StrEnum
from typing import Protocol
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_upload import (
DocumentTooLargeError,
DocumentUploadError,
EmptyDocumentError,
MediaTypeMismatchError,
UnsupportedDocumentTypeError,
)
from ekp_core.ingestion_jobs import AsyncIngestionSubmissionService
from ekp_domain import (
BatchItemAcceptanceStatus,
IngestionBatch,
IngestionBatchItem,
IngestionJob,
IngestionJobStatus,
)
from ekp_object_storage import BinarySource
SAFE_UNSUPPORTED_ERROR = "unsupported document type or media type"
SAFE_FILE_SIZE_ERROR = "document exceeds the configured upload limit"
SAFE_BATCH_SIZE_ERROR = "batch exceeds the configured total upload limit"
SAFE_EMPTY_ERROR = "document must contain content"
SAFE_ACCEPTANCE_ERROR = "document could not be accepted"
class BatchNotFoundError(LookupError):
"""Raised when a requested ingestion batch does not exist."""
class BatchFileCountLimitError(ValueError):
"""Raised before persistence when the multipart file count is invalid."""
class BatchAggregateStatus(StrEnum):
"""Computed batch state derived from item acceptance and durable Jobs."""
PROCESSING = "processing"
COMPLETED = "completed"
PARTIAL_FAILURE = "partial_failure"
FAILED = "failed"
class BatchTotalSizeLimitError(ValueError):
"""Raised while streaming the file that crosses the shared batch budget."""
class _ReadableSource(Protocol):
def read(self, size: int = -1, /) -> bytes:
"""Read bytes from an uploaded file."""
@dataclass(frozen=True, slots=True)
class BatchSource:
"""Provider-neutral input for one ordered multipart file."""
filename: str
media_type: str | None
source: _ReadableSource
@dataclass(frozen=True, slots=True)
class BatchItemView:
"""Detached batch item plus its current optional Job state."""
item: IngestionBatchItem
job: IngestionJob | None
@dataclass(frozen=True, slots=True)
class BatchView:
"""Detached aggregate used by REST without exposing persistence sessions."""
batch: IngestionBatch
items: tuple[BatchItemView, ...]
@property
def status(self) -> BatchAggregateStatus:
"""Aggregate terminal results without copying Job state into batch rows."""
if any(
view.item.acceptance_status == BatchItemAcceptanceStatus.PENDING
or (
view.item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED
and (
view.job is None
or view.job.status
in {
IngestionJobStatus.QUEUED,
IngestionJobStatus.RUNNING,
IngestionJobStatus.RETRYING,
}
)
)
for view in self.items
):
return BatchAggregateStatus.PROCESSING
succeeded = sum(
view.job is not None and view.job.status == IngestionJobStatus.SUCCEEDED
for view in self.items
)
failed = len(self.items) - succeeded
if failed == 0:
return BatchAggregateStatus.COMPLETED
if succeeded == 0:
return BatchAggregateStatus.FAILED
return BatchAggregateStatus.PARTIAL_FAILURE
class _BatchByteBudget:
"""Count bytes across independently streamed files without buffering them."""
def __init__(self, limit_bytes: int) -> None:
self._limit_bytes = limit_bytes
self._consumed_bytes = 0
def wrap(self, source: _ReadableSource) -> BinarySource:
return _BatchLimitedSource(source, self)
def consume(self, size_bytes: int) -> None:
self._consumed_bytes += size_bytes
if self._consumed_bytes > self._limit_bytes:
raise BatchTotalSizeLimitError(SAFE_BATCH_SIZE_ERROR)
class _BatchLimitedSource:
"""Apply one shared request budget through the ObjectStorage stream contract."""
def __init__(self, source: _ReadableSource, budget: _BatchByteBudget) -> None:
self._source = source
self._budget = budget
def read(self, size: int = -1, /) -> bytes:
chunk = self._source.read(size)
self._budget.consume(len(chunk))
return chunk
class IngestionBatchService:
"""Accept files independently and persist ordered aggregate status."""
def __init__(
self,
session_factory: sessionmaker[Session],
submissions: AsyncIngestionSubmissionService,
*,
max_files: int,
max_total_bytes: int,
) -> None:
if max_files <= 0 or max_total_bytes <= 0:
raise ValueError("batch limits must be positive")
self._session_factory = session_factory
self._submissions = submissions
self._max_files = max_files
self._max_total_bytes = max_total_bytes
def submit(self, sources: tuple[BatchSource, ...]) -> BatchView:
"""Create a batch and independently accept or reject every ordered file."""
if not sources or len(sources) > self._max_files:
raise BatchFileCountLimitError(
f"batch must contain between 1 and {self._max_files} files"
)
batch_id, item_ids = self._create_pending_batch(sources)
budget = _BatchByteBudget(self._max_total_bytes)
for item_id, source in zip(item_ids, sources, strict=True):
try:
submission = self._submissions.submit(
source.filename,
source.media_type,
budget.wrap(source.source),
)
except Exception as error:
self._record_rejected(item_id, _safe_acceptance_error(error))
continue
self._record_accepted(
item_id,
document_id=submission.document.id,
job_id=submission.job.id,
)
return self.get(batch_id)
def get(self, batch_id: UUID) -> BatchView:
"""Return ordered items and fresh Job state for polling clients."""
with self._session_factory() as session:
batch = session.get(IngestionBatch, batch_id)
if batch is None:
raise BatchNotFoundError(str(batch_id))
items = tuple(
session.scalars(
select(IngestionBatchItem)
.where(IngestionBatchItem.batch_id == batch_id)
.order_by(IngestionBatchItem.item_index)
).all()
)
views = tuple(
BatchItemView(
item=item,
job=session.get(IngestionJob, item.job_id) if item.job_id is not None else None,
)
for item in items
)
return BatchView(batch=batch, items=views)
def _create_pending_batch(
self, sources: tuple[BatchSource, ...]
) -> tuple[UUID, tuple[UUID, ...]]:
"""Persist every input position before starting external storage writes."""
with self._session_factory() as session:
batch = IngestionBatch(total_files=len(sources))
session.add(batch)
session.flush()
items = tuple(
IngestionBatchItem(
batch_id=batch.id,
item_index=index,
filename=_display_filename(source.filename),
)
for index, source in enumerate(sources)
)
session.add_all(items)
session.commit()
return batch.id, tuple(item.id for item in items)
def _record_accepted(self, item_id: UUID, *, document_id: UUID, job_id: UUID) -> None:
with self._session_factory() as session:
item = session.get(IngestionBatchItem, item_id)
if item is None:
raise BatchNotFoundError(str(item_id))
item.acceptance_status = BatchItemAcceptanceStatus.ACCEPTED
item.document_id = document_id
item.job_id = job_id
item.error_message = None
session.commit()
def _record_rejected(self, item_id: UUID, error_message: str) -> None:
with self._session_factory() as session:
item = session.get(IngestionBatchItem, item_id)
if item is None:
raise BatchNotFoundError(str(item_id))
item.acceptance_status = BatchItemAcceptanceStatus.REJECTED
item.error_message = error_message
session.commit()
def _display_filename(filename: str) -> str:
"""Store a bounded safe label even when the upload filename is invalid."""
display = filename.replace("\x00", "<EFBFBD>").strip()
return display[:255] or "<unnamed>"
def _safe_acceptance_error(error: Exception) -> str:
"""Map per-file failures without persisting exception text or credentials."""
if isinstance(error, (UnsupportedDocumentTypeError, MediaTypeMismatchError)):
return SAFE_UNSUPPORTED_ERROR
if isinstance(error, DocumentTooLargeError):
return SAFE_FILE_SIZE_ERROR
if isinstance(error, BatchTotalSizeLimitError):
return SAFE_BATCH_SIZE_ERROR
if isinstance(error, EmptyDocumentError):
return SAFE_EMPTY_ERROR
if isinstance(error, DocumentUploadError):
return SAFE_ACCEPTANCE_ERROR
return SAFE_ACCEPTANCE_ERROR

View File

@ -8,6 +8,11 @@ from ekp_domain.index_profile import (
IndexProfileStatus,
StoredIndexProfile,
)
from ekp_domain.ingestion_batch import (
BatchItemAcceptanceStatus,
IngestionBatch,
IngestionBatchItem,
)
from ekp_domain.ingestion_job import (
IngestionAttempt,
IngestionAttemptStatus,
@ -19,6 +24,7 @@ from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocum
__all__ = [
"Base",
"BatchItemAcceptanceStatus",
"Chunk",
"ChunkIndexState",
"Document",
@ -30,6 +36,8 @@ __all__ = [
"IngestionJob",
"IngestionJobStatus",
"IngestionStage",
"IngestionBatch",
"IngestionBatchItem",
"ParsedBlock",
"ParsedBlockType",
"ParsedDocument",

View File

@ -0,0 +1,96 @@
"""Canonical batch submission metadata for per-file ingestion tracking."""
from datetime import datetime
from enum import StrEnum
from uuid import UUID, uuid4
from sqlalchemy import (
CheckConstraint,
DateTime,
ForeignKey,
Index,
Integer,
String,
Text,
Uuid,
func,
)
from sqlalchemy import Enum as SqlEnum
from sqlalchemy.orm import Mapped, mapped_column
from ekp_domain.document import Base
class BatchItemAcceptanceStatus(StrEnum):
"""Source acceptance state, independent from its asynchronous Job state."""
PENDING = "pending"
ACCEPTED = "accepted"
REJECTED = "rejected"
class IngestionBatch(Base):
"""Stable aggregation identifier for one bounded multipart submission."""
__tablename__ = "ingestion_batches"
__table_args__ = (CheckConstraint("total_files > 0", name="total_files_positive"),)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
total_files: Mapped[int] = mapped_column(Integer, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class IngestionBatchItem(Base):
"""One ordered input and its optional accepted Document/Job references."""
__tablename__ = "ingestion_batch_items"
__table_args__ = (
CheckConstraint("item_index >= 0", name="item_index_non_negative"),
CheckConstraint("length(filename) > 0", name="filename_not_empty"),
CheckConstraint(
"acceptance_status IN ('pending', 'accepted', 'rejected')",
name="acceptance_status_valid",
),
Index("uq_ingestion_batch_items_order", "batch_id", "item_index", unique=True),
Index("uq_ingestion_batch_items_job_id", "job_id", unique=True),
)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
batch_id: Mapped[UUID] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("ingestion_batches.id", ondelete="CASCADE"),
nullable=False,
)
item_index: Mapped[int] = mapped_column(Integer, nullable=False)
filename: Mapped[str] = mapped_column(String(255), nullable=False)
acceptance_status: Mapped[BatchItemAcceptanceStatus] = mapped_column(
SqlEnum(
BatchItemAcceptanceStatus,
name="batch_item_acceptance_status",
native_enum=False,
create_constraint=False,
validate_strings=True,
values_callable=lambda values: [item.value for item in values],
),
nullable=False,
default=BatchItemAcceptanceStatus.PENDING,
)
document_id: Mapped[UUID | None] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("documents.id", ondelete="SET NULL"),
nullable=True,
)
job_id: Mapped[UUID | None] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("ingestion_jobs.id", ondelete="SET NULL"),
nullable=True,
)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
)

View File

@ -43,7 +43,9 @@ def test_admin_console_contains_only_demo_pages_and_versioned_api_calls(
assert all(label not in html for label in ("权限管理", "任务管理", "Compiled Knowledge"))
assert 'const API_BASE = "/api/v1"' in script
assert "innerHTML" not in script
assert all(endpoint in script for endpoint in ('"/documents"', '"/search"'))
assert all(endpoint in script for endpoint in ('"/ingestion-batches"', '"/search"'))
assert "multiple" in html
assert "BATCH_POLL_INTERVAL_MS" in script
assert all(field in script for field in ("Page", "Section", "Dense Score", "Rerank Score"))
assert "--brand: #FF3C00" in stylesheet
assert "prefers-reduced-motion" in stylesheet

View File

@ -30,6 +30,8 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
"index_profiles",
"ingestion_jobs",
"ingestion_attempts",
"ingestion_batches",
"ingestion_batch_items",
}.issubset(inspector.get_table_names())
assert "unrelated_provider_table" in inspector.get_table_names()
assert {column["name"] for column in inspector.get_columns("documents")} == {
@ -101,3 +103,20 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
"started_at",
"finished_at",
}
assert {column["name"] for column in inspector.get_columns("ingestion_batches")} == {
"id",
"total_files",
"created_at",
}
assert {column["name"] for column in inspector.get_columns("ingestion_batch_items")} == {
"id",
"batch_id",
"item_index",
"filename",
"acceptance_status",
"document_id",
"job_id",
"error_message",
"created_at",
"updated_at",
}

View File

@ -19,6 +19,7 @@ from ekp_core.document_indexing import DocumentIndexingPipelineService, Document
from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.ingestion_batches import IngestionBatchService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
@ -26,7 +27,7 @@ from ekp_core.ingestion_jobs import (
IngestionJobService,
)
from ekp_core.search import SearchService
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile, IngestionBatch
from ekp_embedding import FakeEmbeddingProvider
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
@ -57,6 +58,10 @@ class RecordingDispatcher:
def _build_api_services(
tmp_path: Path,
*,
max_upload_bytes: int = 1024 * 1024,
batch_max_files: int = 20,
batch_max_total_bytes: int = 2 * 1024 * 1024,
) -> tuple[
ApiServices,
sessionmaker[Session],
@ -84,7 +89,11 @@ def _build_api_services(
embedding_model=embedding.model_name,
vector_dimension=embedding.dimensions,
)
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
upload = DocumentUploadService(
session_factory,
storage,
max_upload_bytes=max_upload_bytes,
)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
@ -102,6 +111,12 @@ def _build_api_services(
jobs = IngestionJobService(session_factory, max_attempts=3)
dispatcher = RecordingDispatcher()
submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher)
batches = IngestionBatchService(
session_factory,
submissions,
max_files=batch_max_files,
max_total_bytes=batch_max_total_bytes,
)
runner = IngestionJobRunner(
session_factory,
parsing,
@ -134,6 +149,7 @@ def _build_api_services(
ApiServices(
ingestion=ingestion,
submissions=submissions,
batches=batches,
jobs=jobs,
documents=documents,
search=search,
@ -238,6 +254,8 @@ def test_rest_validation_not_found_and_openapi_contract(
assert set(openapi["paths"]) >= {
"/api/v1/documents",
"/api/v1/documents/{document_id}",
"/api/v1/ingestion-batches",
"/api/v1/ingestion-batches/{batch_id}",
"/api/v1/jobs/{job_id}",
"/api/v1/search",
}
@ -261,3 +279,141 @@ def test_rest_validation_not_found_and_openapi_contract(
assert boolean_top_k.status_code == 422
assert missing_get.status_code == 404
assert missing_delete.status_code == 404
def test_batch_upload_isolates_files_and_reports_terminal_status(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
"""A rejected file must not roll back valid siblings in the same batch."""
services, _, _, _, runner = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
accepted = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("alpha.md", b"# Alpha\n\nFirst policy.", "text/markdown")),
("files", ("blocked.csv", b"secret=must-not-persist", "text/csv")),
("files", ("beta.txt", b"Second policy.", "text/plain")),
],
)
assert accepted.status_code == 202
body = accepted.json()
assert body["status"] == "processing"
assert body["total_files"] == 3
assert body["accepted_files"] == 2
assert body["rejected_files"] == 1
assert body["completed_files"] == 1
assert [item["filename"] for item in body["items"]] == [
"alpha.md",
"blocked.csv",
"beta.txt",
]
assert [item["acceptance_status"] for item in body["items"]] == [
"accepted",
"rejected",
"accepted",
]
assert body["items"][1]["error_message"] == ("unsupported document type or media type")
assert "must-not-persist" not in str(body)
for item in body["items"]:
if item["job_id"] is not None:
runner.run(UUID(item["job_id"]))
terminal = client.get(f"/api/v1/ingestion-batches/{body['batch_id']}")
assert terminal.status_code == 200
terminal_body = terminal.json()
assert terminal_body["status"] == "partial_failure"
assert terminal_body["completed_files"] == 3
assert [item["job_status"] for item in terminal_body["items"]] == [
"succeeded",
None,
"succeeded",
]
def test_batch_upload_enforces_count_and_shared_byte_limits(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
"""Request limits are bounded while already accepted siblings remain durable."""
services, session_factory, _, _, _ = _build_api_services(
tmp_path,
max_upload_bytes=10,
batch_max_files=2,
batch_max_total_bytes=15,
)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
too_many = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("one.txt", b"1", "text/plain")),
("files", ("two.txt", b"2", "text/plain")),
("files", ("three.txt", b"3", "text/plain")),
],
)
bounded = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("first.txt", b"1234567890", "text/plain")),
("files", ("second.txt", b"abcdefghij", "text/plain")),
],
)
oversized = client.post(
"/api/v1/ingestion-batches",
files=[("files", ("large.txt", b"12345678901", "text/plain"))],
)
assert too_many.status_code == 413
assert too_many.json()["detail"] == "batch file count exceeds the configured limit"
assert bounded.status_code == 202
body = bounded.json()
assert body["accepted_files"] == 1
assert body["rejected_files"] == 1
assert body["items"][1]["error_message"] == ("batch exceeds the configured total upload limit")
assert oversized.status_code == 202
assert oversized.json()["status"] == "failed"
assert oversized.json()["accepted_files"] == 0
assert oversized.json()["items"][0]["error_message"] == (
"document exceeds the configured upload limit"
)
# A count-rejected request creates no batch; the two accepted requests do.
with session_factory() as session:
assert len(session.scalars(select(IngestionBatch)).all()) == 2
def test_batch_upload_completed_and_missing_contract(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
"""All-success batches become completed and unknown IDs stay explicit."""
services, _, _, _, runner = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
missing_id = uuid4()
with TestClient(app) as client:
accepted = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("one.md", b"# One\n\nContent.", "text/markdown")),
("files", ("two.txt", b"More content.", "text/plain")),
],
).json()
for item in accepted["items"]:
runner.run(UUID(item["job_id"]))
completed = client.get(f"/api/v1/ingestion-batches/{accepted['batch_id']}")
missing = client.get(f"/api/v1/ingestion-batches/{missing_id}")
assert completed.status_code == 200
assert completed.json()["status"] == "completed"
assert completed.json()["completed_files"] == 2
assert missing.status_code == 404
assert missing.json()["detail"] == "ingestion batch not found"

View File

@ -25,6 +25,8 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
assert settings.ingestion_retry_backoff_seconds == 2
assert settings.ingestion_retry_backoff_max_seconds == 60
assert settings.ingestion_worker_lease_seconds == 300
assert settings.batch_max_files == 20
assert settings.batch_max_total_bytes == 200 * 1024 * 1024
assert settings.object_storage_provider == "bos"
assert settings.object_storage_path == Path("storage")
assert settings.bos_endpoint is None
@ -157,3 +159,10 @@ def test_settings_require_worker_lease_longer_than_provider_timeout(
) -> None:
with pytest.raises(ValidationError, match="worker_lease"):
Settings(ingestion_worker_lease_seconds=30, embedding_timeout_seconds=30)
def test_settings_require_batch_limit_to_fit_one_file(
isolated_settings_environment: None,
) -> None:
with pytest.raises(ValidationError, match="batch_max_total_bytes"):
Settings(max_upload_bytes=100, batch_max_total_bytes=99)