feat: add batch async ingestion
This commit is contained in:
parent
a8dfbe0cfa
commit
0fad825eb0
@ -18,6 +18,8 @@ INGESTION_RETRY_BACKOFF_SECONDS=2
|
||||
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
|
||||
INGESTION_WORKER_LEASE_SECONDS=300
|
||||
INGESTION_RECOVERY_INTERVAL_SECONDS=30
|
||||
BATCH_MAX_FILES=20
|
||||
BATCH_MAX_TOTAL_BYTES=209715200
|
||||
|
||||
OBJECT_STORAGE_PROVIDER=bos
|
||||
OBJECT_STORAGE_PATH=storage
|
||||
|
||||
@ -133,7 +133,8 @@ source results are rehydrated from canonical relational chunks and retain both
|
||||
Dense and reranker scores; stale vector content fails closed. See
|
||||
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
|
||||
DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
|
||||
changes upload to asynchronous acceptance and adds Job status. Upload, list, detail, direct delete,
|
||||
changes upload to asynchronous acceptance and adds Job status. DEMO-016 adds bounded multi-file
|
||||
submission, batch/per-file status, and automatic Admin Console polling. Upload, list, detail, direct delete,
|
||||
and reranked source search are documented in
|
||||
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
|
||||
outside the public response contract.
|
||||
|
||||
@ -1,12 +1,33 @@
|
||||
// DEMO-010 stays API First: this client owns presentation state only and sends
|
||||
// every document/search operation through the versioned public REST contract.
|
||||
// The Admin Console owns presentation state only. Batch acceptance, Job state,
|
||||
// and retrieval remain behind the versioned public REST contract.
|
||||
const API_BASE = "/api/v1";
|
||||
const BATCH_POLL_INTERVAL_MS = 1500;
|
||||
const terminalBatchStatuses = new Set(["completed", "partial_failure", "failed"]);
|
||||
let batchPollTimer;
|
||||
let activeBatchId;
|
||||
|
||||
const statusLabels = {
|
||||
uploaded: "Uploaded",
|
||||
processing: "Processing",
|
||||
ready: "Ready",
|
||||
failed: "Failed",
|
||||
pending: "Pending",
|
||||
accepted: "Accepted",
|
||||
rejected: "Rejected",
|
||||
queued: "Queued",
|
||||
running: "Running",
|
||||
retrying: "Retrying",
|
||||
succeeded: "Succeeded",
|
||||
completed: "Completed",
|
||||
partial_failure: "Partial failure",
|
||||
};
|
||||
|
||||
const stageLabels = {
|
||||
parsing: "解析",
|
||||
cleaning: "清理",
|
||||
chunking: "切分",
|
||||
embedding: "向量化",
|
||||
indexing: "索引",
|
||||
};
|
||||
|
||||
const pageTitles = {
|
||||
@ -93,23 +114,37 @@ function initializeUpload() {
|
||||
const button = document.querySelector("#upload-button");
|
||||
|
||||
const updateFilename = () => {
|
||||
selected.textContent = input.files[0]?.name ?? "尚未选择文件";
|
||||
const files = Array.from(input.files);
|
||||
if (!files.length) {
|
||||
selected.textContent = "尚未选择文件";
|
||||
} else if (files.length === 1) {
|
||||
selected.textContent = files[0].name;
|
||||
} else {
|
||||
selected.textContent = `已选择 ${files.length} 个文件`;
|
||||
}
|
||||
};
|
||||
input.addEventListener("change", updateFilename);
|
||||
|
||||
form.addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
if (!input.files.length) return;
|
||||
window.clearTimeout(batchPollTimer);
|
||||
activeBatchId = undefined;
|
||||
setButtonBusy(button, true, "提交中…");
|
||||
try {
|
||||
const formData = new FormData();
|
||||
formData.append("file", input.files[0]);
|
||||
const documentData = await requestJson("/documents", {
|
||||
Array.from(input.files).forEach((file) => formData.append("files", file));
|
||||
const batch = await requestJson("/ingestion-batches", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
renderUploadResult(documentData);
|
||||
showNotice("文档已接收并进入后台处理队列。", "success");
|
||||
activeBatchId = batch.batch_id;
|
||||
renderBatchResult(batch);
|
||||
scheduleBatchPoll(batch);
|
||||
showNotice(
|
||||
`批次已接收:${batch.accepted_files} 个进入后台队列,${batch.rejected_files} 个被拒绝。`,
|
||||
batch.rejected_files ? "error" : "success",
|
||||
);
|
||||
form.reset();
|
||||
updateFilename();
|
||||
} catch (error) {
|
||||
@ -120,18 +155,62 @@ function initializeUpload() {
|
||||
});
|
||||
}
|
||||
|
||||
function renderUploadResult(documentData) {
|
||||
function scheduleBatchPoll(batch) {
|
||||
window.clearTimeout(batchPollTimer);
|
||||
if (batch.batch_id !== activeBatchId) return;
|
||||
if (terminalBatchStatuses.has(batch.status)) return;
|
||||
batchPollTimer = window.setTimeout(async () => {
|
||||
try {
|
||||
const latest = await requestJson(`/ingestion-batches/${batch.batch_id}`);
|
||||
renderBatchResult(latest);
|
||||
scheduleBatchPoll(latest);
|
||||
} catch (error) {
|
||||
showNotice(`批次状态刷新失败:${error.message}`, "error");
|
||||
scheduleBatchPoll(batch);
|
||||
}
|
||||
}, BATCH_POLL_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function renderBatchResult(batch) {
|
||||
const card = document.querySelector("#upload-result");
|
||||
document.querySelector("#upload-filename").textContent = documentData.filename;
|
||||
document.querySelector("#upload-type").textContent = documentData.media_type;
|
||||
document.querySelector("#upload-size").textContent = formatSize(documentData.size_bytes);
|
||||
document.querySelector("#upload-time").textContent = formatDate(documentData.created_at);
|
||||
document.querySelector("#upload-status").replaceWith(statusPill(documentData.status));
|
||||
document.querySelector("#upload-batch-title").textContent =
|
||||
`批次 ${batch.batch_id.slice(0, 8)} · ${formatDate(batch.created_at)}`;
|
||||
document.querySelector("#upload-total").textContent = `${batch.total_files} 个`;
|
||||
document.querySelector("#upload-completed").textContent =
|
||||
`${batch.completed_files} / ${batch.total_files}`;
|
||||
document.querySelector("#upload-accepted").textContent =
|
||||
`${batch.accepted_files} / ${batch.rejected_files}`;
|
||||
document.querySelector("#upload-status").replaceWith(statusPill(batch.status));
|
||||
const status = card.querySelector(".status-pill");
|
||||
status.id = "upload-status";
|
||||
const items = document.querySelector("#batch-items");
|
||||
items.replaceChildren(...batch.items.map(batchItem));
|
||||
card.hidden = false;
|
||||
}
|
||||
|
||||
function batchItem(item) {
|
||||
const row = document.createElement("article");
|
||||
row.className = "batch-item";
|
||||
|
||||
const order = document.createElement("span");
|
||||
order.className = "batch-item-order";
|
||||
order.textContent = String(item.item_index + 1).padStart(2, "0");
|
||||
|
||||
const body = document.createElement("div");
|
||||
body.className = "batch-item-body";
|
||||
const name = document.createElement("strong");
|
||||
name.textContent = item.filename;
|
||||
const detail = document.createElement("span");
|
||||
const stage = item.stage ? ` · ${stageLabels[item.stage] ?? item.stage}` : "";
|
||||
const attempts = item.max_attempts == null ? "" : ` · 尝试 ${item.attempt}/${item.max_attempts}`;
|
||||
detail.textContent = item.error_message ?? `后台任务${stage}${attempts}`;
|
||||
body.append(name, detail);
|
||||
|
||||
const state = item.job_status ?? item.acceptance_status;
|
||||
row.append(order, body, statusPill(state));
|
||||
return row;
|
||||
}
|
||||
|
||||
async function loadDocuments() {
|
||||
const refresh = document.querySelector("#refresh-documents");
|
||||
setButtonBusy(refresh, true, "加载中…");
|
||||
@ -329,3 +408,4 @@ initializeUpload();
|
||||
initializeSearch();
|
||||
document.querySelector("#refresh-documents").addEventListener("click", loadDocuments);
|
||||
if (page === "documents") loadDocuments();
|
||||
window.addEventListener("pagehide", () => window.clearTimeout(batchPollTimer));
|
||||
|
||||
@ -533,6 +533,50 @@ a {
|
||||
border-block: 1px solid var(--line);
|
||||
}
|
||||
|
||||
.batch-items {
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
margin: -8px 0 24px;
|
||||
}
|
||||
|
||||
.batch-item {
|
||||
display: grid;
|
||||
grid-template-columns: 34px minmax(0, 1fr) auto;
|
||||
gap: 12px;
|
||||
align-items: center;
|
||||
min-height: 64px;
|
||||
padding: 11px 13px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 3px;
|
||||
background: #fafaf8;
|
||||
}
|
||||
|
||||
.batch-item-order {
|
||||
color: var(--brand);
|
||||
font-family: var(--data-font);
|
||||
font-size: 10px;
|
||||
font-weight: 850;
|
||||
}
|
||||
|
||||
.batch-item-body {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
.batch-item-body strong {
|
||||
overflow: hidden;
|
||||
font-size: 12px;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.batch-item-body span {
|
||||
overflow-wrap: anywhere;
|
||||
color: var(--muted);
|
||||
font-size: 10px;
|
||||
}
|
||||
|
||||
dt {
|
||||
color: var(--muted);
|
||||
font-family: var(--data-font);
|
||||
@ -655,6 +699,15 @@ tbody tr:hover {
|
||||
.status-processing { background: #fff0ce; color: var(--warning); }
|
||||
.status-failed { background: #fbe6e3; color: var(--danger); }
|
||||
.status-uploaded { background: var(--brand-soft); color: var(--brand-deep); }
|
||||
.status-pending,
|
||||
.status-queued,
|
||||
.status-running,
|
||||
.status-retrying,
|
||||
.status-accepted { background: #fff0ce; color: var(--warning); }
|
||||
.status-succeeded,
|
||||
.status-completed { background: #e5f3eb; color: var(--success); }
|
||||
.status-rejected,
|
||||
.status-partial_failure { background: #fbe6e3; color: var(--danger); }
|
||||
|
||||
.empty-state {
|
||||
min-height: 240px;
|
||||
@ -958,6 +1011,8 @@ select:focus {
|
||||
.search-controls { grid-template-columns: minmax(0, 1fr) 90px; }
|
||||
.search-controls textarea { grid-column: 1 / -1; }
|
||||
.metadata-grid { grid-template-columns: 1fr; }
|
||||
.batch-item { grid-template-columns: 28px minmax(0, 1fr); }
|
||||
.batch-item .status-pill { grid-column: 2; }
|
||||
}
|
||||
|
||||
@media (max-width: 540px) {
|
||||
|
||||
@ -67,7 +67,7 @@
|
||||
<div>
|
||||
<p class="section-label">01 / INGESTION</p>
|
||||
<h2>让知识进入检索链路</h2>
|
||||
<p>上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将在后台完成解析、切分与索引。</p>
|
||||
<p>批量提交 PDF、DOCX、TXT 或 Markdown 文件,系统将逐个接收并在后台独立完成解析、切分与索引。</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -75,14 +75,15 @@
|
||||
<label id="drop-zone" class="drop-zone" for="file-input">
|
||||
<input
|
||||
id="file-input"
|
||||
name="file"
|
||||
name="files"
|
||||
type="file"
|
||||
multiple
|
||||
accept=".pdf,.docx,.txt,.md,application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document,text/plain,text/markdown"
|
||||
required
|
||||
/>
|
||||
<span class="upload-icon" aria-hidden="true">↑</span>
|
||||
<strong>选择知识文件</strong>
|
||||
<span>支持 PDF、DOCX、TXT、Markdown · 单次一个文件</span>
|
||||
<strong>选择一批知识文件</strong>
|
||||
<span>支持 PDF、DOCX、TXT、Markdown · 每个文件独立处理</span>
|
||||
<span id="selected-file" class="selected-file">尚未选择文件</span>
|
||||
</label>
|
||||
<button id="upload-button" class="primary-button" type="submit">上传并处理</button>
|
||||
@ -90,15 +91,16 @@
|
||||
|
||||
<article id="upload-result" class="result-card" hidden>
|
||||
<div>
|
||||
<p class="section-label">LATEST SIGNAL</p>
|
||||
<h3 id="upload-filename"></h3>
|
||||
<p class="section-label">LATEST BATCH</p>
|
||||
<h3 id="upload-batch-title"></h3>
|
||||
</div>
|
||||
<span id="upload-status" class="status-pill"></span>
|
||||
<dl class="metadata-grid">
|
||||
<div><dt>文件类型</dt><dd id="upload-type"></dd></div>
|
||||
<div><dt>文件大小</dt><dd id="upload-size"></dd></div>
|
||||
<div><dt>上传时间</dt><dd id="upload-time"></dd></div>
|
||||
<div><dt>总文件</dt><dd id="upload-total"></dd></div>
|
||||
<div><dt>已完成</dt><dd id="upload-completed"></dd></div>
|
||||
<div><dt>接收 / 拒绝</dt><dd id="upload-accepted"></dd></div>
|
||||
</dl>
|
||||
<div id="batch-items" class="batch-items" aria-live="polite"></div>
|
||||
<a class="text-link" href="/admin/search">前往 Search Lab 检索 <span>→</span></a>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
@ -18,6 +18,7 @@ from ekp_core.document_management import DocumentManagementService
|
||||
from ekp_core.document_parsing import DocumentParsingService
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.index_rebuild import resolve_active_index_profile
|
||||
from ekp_core.ingestion_batches import IngestionBatchService
|
||||
from ekp_core.ingestion_jobs import (
|
||||
AsyncIngestionSubmissionService,
|
||||
DerivedDocumentCleanupService,
|
||||
@ -50,6 +51,7 @@ class ApiServices:
|
||||
|
||||
ingestion: DocumentIndexingPipelineService
|
||||
submissions: AsyncIngestionSubmissionService
|
||||
batches: IngestionBatchService
|
||||
jobs: IngestionJobService
|
||||
documents: DocumentManagementService
|
||||
search: SearchService
|
||||
@ -310,6 +312,12 @@ def create_api_services(settings: Settings) -> ApiServices:
|
||||
components.jobs,
|
||||
components.dispatcher,
|
||||
)
|
||||
batches = IngestionBatchService(
|
||||
components.session_factory,
|
||||
submissions,
|
||||
max_files=settings.batch_max_files,
|
||||
max_total_bytes=settings.batch_max_total_bytes,
|
||||
)
|
||||
dense_retrieval = create_dense_retrieval_service(
|
||||
settings,
|
||||
components.embedding_provider,
|
||||
@ -332,6 +340,7 @@ def create_api_services(settings: Settings) -> ApiServices:
|
||||
return ApiServices(
|
||||
ingestion=ingestion,
|
||||
submissions=submissions,
|
||||
batches=batches,
|
||||
jobs=components.jobs,
|
||||
documents=documents,
|
||||
search=search,
|
||||
|
||||
@ -9,6 +9,7 @@ from ekp_api.providers import ApiServices, ProviderConfigurationError, create_ap
|
||||
from ekp_api.schemas import (
|
||||
DocumentResponse,
|
||||
IngestionAcceptedResponse,
|
||||
IngestionBatchResponse,
|
||||
IngestionJobResponse,
|
||||
SearchRequest,
|
||||
SearchResponse,
|
||||
@ -25,6 +26,7 @@ from ekp_core.document_upload import (
|
||||
MediaTypeMismatchError,
|
||||
UnsupportedDocumentTypeError,
|
||||
)
|
||||
from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundError, BatchSource
|
||||
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
|
||||
|
||||
router = APIRouter()
|
||||
@ -98,6 +100,61 @@ def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
|
||||
return IngestionJobResponse.from_job(job)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/ingestion-batches",
|
||||
response_model=IngestionBatchResponse,
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
tags=["jobs"],
|
||||
)
|
||||
def upload_ingestion_batch(
|
||||
services: Services,
|
||||
files: Annotated[
|
||||
list[UploadFile],
|
||||
File(description="One or more PDF, DOCX, TXT, or Markdown files"),
|
||||
],
|
||||
) -> IngestionBatchResponse:
|
||||
"""Accept a bounded batch while isolating every file's result."""
|
||||
try:
|
||||
view = services.batches.submit(
|
||||
tuple(
|
||||
BatchSource(
|
||||
filename=file.filename or "",
|
||||
media_type=file.content_type,
|
||||
source=file.file,
|
||||
)
|
||||
for file in files
|
||||
)
|
||||
)
|
||||
except BatchFileCountLimitError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
|
||||
detail="batch file count exceeds the configured limit",
|
||||
) from error
|
||||
except Exception as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="batch could not be accepted",
|
||||
) from error
|
||||
return IngestionBatchResponse.from_view(view)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/ingestion-batches/{batch_id}",
|
||||
response_model=IngestionBatchResponse,
|
||||
tags=["jobs"],
|
||||
)
|
||||
def get_ingestion_batch(batch_id: UUID, services: Services) -> IngestionBatchResponse:
|
||||
"""Return aggregate and per-file status for Admin Console polling."""
|
||||
try:
|
||||
view = services.batches.get(batch_id)
|
||||
except BatchNotFoundError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="ingestion batch not found",
|
||||
) from error
|
||||
return IngestionBatchResponse.from_view(view)
|
||||
|
||||
|
||||
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
|
||||
def list_documents(services: Services) -> list[DocumentResponse]:
|
||||
"""List canonical document metadata for the Demo admin page."""
|
||||
|
||||
@ -5,9 +5,17 @@ from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from ekp_core.ingestion_batches import BatchAggregateStatus, BatchItemView, BatchView
|
||||
from ekp_core.ingestion_jobs import IngestionSubmission
|
||||
from ekp_core.search import SearchResult
|
||||
from ekp_domain import Document, DocumentStatus, IngestionJob, IngestionJobStatus, IngestionStage
|
||||
from ekp_domain import (
|
||||
BatchItemAcceptanceStatus,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
IngestionJob,
|
||||
IngestionJobStatus,
|
||||
IngestionStage,
|
||||
)
|
||||
|
||||
|
||||
class DocumentResponse(BaseModel):
|
||||
@ -88,6 +96,80 @@ class IngestionJobResponse(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
class IngestionBatchItemResponse(BaseModel):
|
||||
"""Per-file acceptance and latest asynchronous processing state."""
|
||||
|
||||
item_index: int
|
||||
filename: str
|
||||
acceptance_status: BatchItemAcceptanceStatus
|
||||
document_id: UUID | None
|
||||
job_id: UUID | None
|
||||
job_status: IngestionJobStatus | None
|
||||
stage: IngestionStage | None
|
||||
attempt: int
|
||||
max_attempts: int | None
|
||||
error_message: str | None
|
||||
|
||||
@classmethod
|
||||
def from_view(cls, view: BatchItemView) -> "IngestionBatchItemResponse":
|
||||
job = view.job
|
||||
return cls(
|
||||
item_index=view.item.item_index,
|
||||
filename=view.item.filename,
|
||||
acceptance_status=view.item.acceptance_status,
|
||||
document_id=view.item.document_id,
|
||||
job_id=view.item.job_id,
|
||||
job_status=job.status if job is not None else None,
|
||||
stage=job.stage if job is not None else None,
|
||||
attempt=job.attempt_count if job is not None else 0,
|
||||
max_attempts=job.max_attempts if job is not None else None,
|
||||
error_message=(job.error_message if job is not None else view.item.error_message),
|
||||
)
|
||||
|
||||
|
||||
class IngestionBatchResponse(BaseModel):
|
||||
"""Batch aggregate with ordered independently processed files."""
|
||||
|
||||
batch_id: UUID
|
||||
status: BatchAggregateStatus
|
||||
total_files: int
|
||||
accepted_files: int
|
||||
rejected_files: int
|
||||
completed_files: int
|
||||
created_at: datetime
|
||||
items: list[IngestionBatchItemResponse]
|
||||
|
||||
@classmethod
|
||||
def from_view(cls, view: BatchView) -> "IngestionBatchResponse":
|
||||
items = [IngestionBatchItemResponse.from_view(item) for item in view.items]
|
||||
accepted = sum(
|
||||
item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED for item in items
|
||||
)
|
||||
rejected = sum(
|
||||
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED for item in items
|
||||
)
|
||||
completed = sum(
|
||||
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED
|
||||
or item.job_status
|
||||
in {
|
||||
IngestionJobStatus.SUCCEEDED,
|
||||
IngestionJobStatus.FAILED,
|
||||
IngestionJobStatus.CANCELLED,
|
||||
}
|
||||
for item in items
|
||||
)
|
||||
return cls(
|
||||
batch_id=view.batch.id,
|
||||
status=view.status,
|
||||
total_files=view.batch.total_files,
|
||||
accepted_files=accepted,
|
||||
rejected_files=rejected,
|
||||
completed_files=completed,
|
||||
created_at=view.batch.created_at,
|
||||
items=items,
|
||||
)
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
"""Validated public search input; normalization remains in the core service."""
|
||||
|
||||
|
||||
@ -11,11 +11,13 @@ Use OpenAPI as executable source of truth once implementation starts. Provider-s
|
||||
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
|
||||
版本化为 `/api/v1/*`。
|
||||
|
||||
`DEMO-015` 已将单文件上传切换为持久化异步 Job;批量提交仍留给 `DEMO-016`。
|
||||
`DEMO-015` 已将单文件上传切换为持久化异步 Job;`DEMO-016` 已增加批量提交和聚合查询。
|
||||
|
||||
| Method | Path | 行为 |
|
||||
|---|---|---|
|
||||
| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
|
||||
| `POST` | `/api/v1/ingestion-batches` | multipart `files` 多文件接收;返回批次与逐文件状态 |
|
||||
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新 Job 状态 |
|
||||
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
|
||||
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
|
||||
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
|
||||
@ -55,7 +57,7 @@ Search 响应:
|
||||
```
|
||||
|
||||
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
|
||||
文档或 Job 不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
|
||||
文档、Job 或批次不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
|
||||
`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
|
||||
厂商响应。
|
||||
|
||||
@ -74,7 +76,7 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
|
||||
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
|
||||
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
|
||||
|
||||
## 异步入库契约(DEMO-015 已实现;DEMO-016 批量部分待实现)
|
||||
## 异步入库契约(DEMO-015~016 已实现)
|
||||
|
||||
| Method | Path | 行为 |
|
||||
|---|---|---|
|
||||
@ -101,17 +103,29 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
|
||||
}
|
||||
```
|
||||
|
||||
DEMO-016 的批量响应示意:
|
||||
批量字段名为 `files`,成功返回 `202`。响应示意:
|
||||
|
||||
```json
|
||||
{
|
||||
"batch_id": "d21f8012-4c4e-4bb4-89af-e11c0f9f470e",
|
||||
"status": "processing",
|
||||
"total_files": 2,
|
||||
"accepted_files": 1,
|
||||
"rejected_files": 1,
|
||||
"completed_files": 1,
|
||||
"created_at": "2026-09-02T12:00:00Z",
|
||||
"items": [
|
||||
{
|
||||
"item_index": 0,
|
||||
"filename": "退款政策.md",
|
||||
"acceptance_status": "accepted",
|
||||
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
|
||||
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
|
||||
"status": "queued"
|
||||
"job_status": "queued",
|
||||
"stage": null,
|
||||
"attempt": 0,
|
||||
"max_attempts": 3,
|
||||
"error_message": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -119,8 +133,11 @@ DEMO-016 的批量响应示意:
|
||||
|
||||
Job 公开状态为 `queued | running | retrying | succeeded | failed | cancelled`,并通过 `stage`
|
||||
表示 `parsing | cleaning | chunking | embedding | indexing`。错误字段只返回有界、安全摘要。
|
||||
批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚其他文件。文件数量、单文件大小和
|
||||
请求总大小均由服务端配置限制。
|
||||
批次状态为 `processing | completed | partial_failure | failed`;逐文件接收状态为
|
||||
`pending | accepted | rejected`。批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚
|
||||
其他文件。文件数量、单文件大小和请求总大小分别由 `BATCH_MAX_FILES`、`MAX_UPLOAD_BYTES`
|
||||
和 `BATCH_MAX_TOTAL_BYTES` 限制。数量超限返回 `413`;类型、大小或内容不合规作为该文件的
|
||||
`rejected` 安全结果返回,其他合规文件继续接收。
|
||||
|
||||
自动重试是服务端 Job 行为,不要求客户端再次上传,也不新增公开手动重试接口。每次重试前
|
||||
清理派生数据但保留原始文件和稳定的 Document ID;达到次数上限或清理失败后进入 `failed`。
|
||||
|
||||
@ -179,6 +179,10 @@ Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Fail
|
||||
API 只负责校验并持久化不可变源文件、Document 和 Ingestion Job,然后返回 `202`。批次是
|
||||
提交与聚合查询边界,不是跨文件数据库事务;每个文件独立排队、执行和失败。
|
||||
|
||||
DEMO-016 以 canonical `ingestion_batches` / `ingestion_batch_items` 保存输入顺序和接收引用;
|
||||
处理阶段、Attempt 与最终结果继续以 `ingestion_jobs` / `ingestion_attempts` 为事实源,批次表
|
||||
不复制 Worker 状态。
|
||||
|
||||
```text
|
||||
Batch Request
|
||||
├─ Source A + Document A + Job A ─→ Worker
|
||||
|
||||
@ -340,7 +340,7 @@ API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重
|
||||
|
||||
### DEMO-016 批量异步上传
|
||||
|
||||
在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
|
||||
**已完成。** 在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
|
||||
Admin Console 状态展示。批次内文件相互隔离,不包含目录上传、压缩包解包或断点续传。
|
||||
|
||||
---
|
||||
@ -397,8 +397,8 @@ DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
|
||||
|
||||
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
|
||||
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
|
||||
任务拆分;`DEMO-012`、`DEMO-015` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
|
||||
后续严格按 `DEMO-016`、`DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
|
||||
任务拆分;`DEMO-012`、`DEMO-015~016` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
|
||||
后续严格按 `DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
|
||||
可选项,不因本状态记录自动进入范围。
|
||||
|
||||
可选:
|
||||
|
||||
@ -16,7 +16,7 @@ make run
|
||||
|
||||
| 页面 | 地址 | 对应业务接口 |
|
||||
|---|---|---|
|
||||
| 文档上传 | `/admin/upload` | `POST /api/v1/documents` |
|
||||
| 文档上传 | `/admin/upload` | `POST/GET /api/v1/ingestion-batches*` |
|
||||
| 文档列表与详情 | `/admin/documents` | `GET/DELETE /api/v1/documents*` |
|
||||
| Search Lab | `/admin/search` | `POST /api/v1/search` |
|
||||
|
||||
@ -34,9 +34,9 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
|
||||
|
||||
## Demo 操作闭环
|
||||
|
||||
1. 在上传页选择一个 PDF、DOCX、TXT 或 Markdown 文件并提交;
|
||||
2. 接口返回 `202` 后,当前页面显示已接收;在文档列表刷新查看 `Processing/Ready/Failed`;
|
||||
3. 在文档列表查看状态和安全元数据;
|
||||
1. 在上传页选择一个或多个 PDF、DOCX、TXT 或 Markdown 文件并提交;
|
||||
2. 接口返回 `202` 后,页面自动轮询批次,逐文件显示接收状态、处理阶段、尝试次数和安全错误;
|
||||
3. 批次进入 `Completed`、`Partial failure` 或 `Failed` 后停止轮询,再在文档列表查看元数据;
|
||||
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
|
||||
Rerank Score;
|
||||
5. 如需清理 Demo 数据,在文档列表确认后直接删除。
|
||||
@ -46,7 +46,8 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
|
||||
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
|
||||
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
|
||||
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
|
||||
- 当前 Console 尚不轮询 Job 详情或展示 Attempt;该状态 UI 与批量上传一起由 DEMO-016 实现;
|
||||
- Console 只轮询当前批次的聚合接口,不读取 Provider/队列细节,也不提供人工重试按钮;
|
||||
- 批量上传不包含目录上传、压缩包解包、断点续传或跨批次去重;
|
||||
- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
|
||||
或完整 V1 Admin Console 导航;
|
||||
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。
|
||||
|
||||
@ -2,11 +2,12 @@
|
||||
|
||||
## 范围
|
||||
|
||||
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker。OceanBase 中的
|
||||
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker,DEMO-016 在其上增加逐文件
|
||||
隔离的批量接收和状态聚合。OceanBase 中的
|
||||
`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
|
||||
不携带文件内容、Provider 凭据或业务状态。
|
||||
|
||||
本任务不提供批量上传和批次 UI;这些能力属于 DEMO-016。
|
||||
批次只聚合接收与 Job 状态,不改变每个文件的独立执行和重试边界。
|
||||
|
||||
## 运行时配置
|
||||
|
||||
@ -19,6 +20,8 @@ INGESTION_RETRY_BACKOFF_SECONDS=2
|
||||
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
|
||||
INGESTION_WORKER_LEASE_SECONDS=300
|
||||
INGESTION_RECOVERY_INTERVAL_SECONDS=30
|
||||
BATCH_MAX_FILES=20
|
||||
BATCH_MAX_TOTAL_BYTES=209715200
|
||||
```
|
||||
|
||||
Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
|
||||
@ -72,6 +75,18 @@ Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;
|
||||
`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
|
||||
返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
|
||||
|
||||
批量提交与查询:
|
||||
|
||||
```text
|
||||
POST /api/v1/ingestion-batches # multipart 字段 files,可重复
|
||||
GET /api/v1/ingestion-batches/{batch_id}
|
||||
```
|
||||
|
||||
批次创建后逐文件返回 `accepted` 或 `rejected`;已接收文件继续通过上述 Job 状态异步推进。
|
||||
批次聚合状态为 `processing | completed | partial_failure | failed`。请求文件数超过
|
||||
`BATCH_MAX_FILES` 时整体返回 `413`;单文件或共享总字节预算超限只拒绝当前文件,不删除此前
|
||||
已接收文件。默认总预算 200 MiB,且配置不得小于单文件 `MAX_UPLOAD_BYTES`。
|
||||
|
||||
## 故障检查
|
||||
|
||||
1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;
|
||||
|
||||
@ -3,10 +3,10 @@
|
||||
## 范围
|
||||
|
||||
DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
|
||||
Job。API 层只负责输入校验、
|
||||
Job,DEMO-016 增加批量接收与聚合查询。API 层只负责输入校验、
|
||||
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
|
||||
|
||||
当前不包含 ACL/RBAC、软删除、批量上传或 Hybrid Search。
|
||||
当前不包含 ACL/RBAC、软删除、目录/压缩包上传、断点续传或 Hybrid Search。
|
||||
|
||||
## 启动前提
|
||||
|
||||
@ -40,6 +40,16 @@ curl -F 'file=@./policy.md;type=text/markdown' \
|
||||
curl http://127.0.0.1:8000/api/v1/jobs/<job-id>
|
||||
```
|
||||
|
||||
批量上传并查询聚合进度:
|
||||
|
||||
```bash
|
||||
curl -F 'files=@./policy.md;type=text/markdown' \
|
||||
-F 'files=@./manual.pdf;type=application/pdf' \
|
||||
http://127.0.0.1:8000/api/v1/ingestion-batches
|
||||
|
||||
curl http://127.0.0.1:8000/api/v1/ingestion-batches/<batch-id>
|
||||
```
|
||||
|
||||
检索:
|
||||
|
||||
```bash
|
||||
|
||||
@ -9,13 +9,13 @@
|
||||
| 能力 | 当前实现 | 结论 |
|
||||
|---|---|---|
|
||||
| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
|
||||
| 批量上传 | API 与 Admin Console 都只支持一个文件 | 未实现 |
|
||||
| 失败补偿 | 上传元数据失败会删除源对象;解析元数据失败会删除本次解析产物;索引失败会删除当前 Document/Profile 的可能残留向量 | 已有局部补偿,不是自动重试 |
|
||||
| 批量上传 | 批次 API、逐文件 Job 状态和 Admin Console 自动轮询 | DEMO-016 已实现 |
|
||||
| 失败补偿 | 阶段内补偿 + 重试前清理全部派生数据,保留不可变源和执行历史 | DEMO-015 已实现 |
|
||||
| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
|
||||
| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
|
||||
| MCP | 已有 MCP 业务契约和可复用 Search Service/REST API,没有可运行 MCP Gateway 或 WorkBuddy 验证 | 未实现 |
|
||||
| MCP | 独立 Gateway、认证边界、`search_knowledge` 与 WorkBuddy 协议级验收 | DEMO-012 已实现 |
|
||||
|
||||
PLAN-001 的上述缺口已由 DEMO-015 补齐;批量提交与状态 UI 仍由 DEMO-016 实现。
|
||||
PLAN-001 的异步执行缺口已由 DEMO-015 补齐,批量提交与状态 UI 已由 DEMO-016 补齐。
|
||||
|
||||
## 2. Task 归并结论
|
||||
|
||||
@ -51,7 +51,7 @@ PLAN-001 的上述缺口已由 DEMO-015 补齐;批量提交与状态 UI 仍由
|
||||
该 Task 只完成公共知识库入库所需的 Job/Retry 与队列基线,不扩展到完整审计系统、
|
||||
Compilation、OCR、跨区域高可用或通用企业级 Job 平台。
|
||||
|
||||
### DEMO-016:批量异步上传与状态展示(新增)
|
||||
### DEMO-016:批量异步上传与状态展示(已完成)
|
||||
|
||||
建立在 DEMO-015 之上,不与 Job 基础合并:
|
||||
|
||||
@ -79,8 +79,8 @@ PLAN-002(已完成)
|
||||
推荐严格按以下顺序执行并逐项闭环:
|
||||
|
||||
1. `PLAN-002`(已完成)
|
||||
2. `DEMO-015`
|
||||
3. `DEMO-016`
|
||||
2. `DEMO-015`(已完成)
|
||||
3. `DEMO-016`(已完成)
|
||||
4. `DEPLOY-001`
|
||||
5. `DEPLOY-002`
|
||||
6. `DEPLOY-003`
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
# 项目进度
|
||||
|
||||
当前阶段:异步单文件入库闭环完成,准备实施批量上传
|
||||
当前阶段:批量异步上传闭环完成,准备交付生产镜像
|
||||
|
||||
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-015、ARCH-001~ARCH-006、PLAN-001~PLAN-002
|
||||
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002
|
||||
进行中:无
|
||||
阻塞:无
|
||||
|
||||
下一任务:`DEMO-016`。随后严格执行 `DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
|
||||
下一任务:`DEPLOY-001`。随后严格执行 `DEPLOY-002 → DEPLOY-003`。
|
||||
|
||||
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
|
||||
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
|
||||
@ -354,3 +354,26 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
|
||||
真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
|
||||
重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
|
||||
- 下一任务:`DEMO-016`。
|
||||
|
||||
### DEMO-016(已完成)
|
||||
|
||||
- 新增 canonical `ingestion_batches` / `ingestion_batch_items` 及 forward-only migration,保存
|
||||
输入顺序、逐文件接收结果与 Document/Job 引用;异步阶段和最终结果仍以 Job 为事实源,
|
||||
批次不是跨文件事务。
|
||||
- 新增 `POST /api/v1/ingestion-batches` 与 `GET /api/v1/ingestion-batches/{batch_id}`;批量
|
||||
请求字段为 `files`,成功返回 `202`、聚合状态和逐文件接收/Job 状态。类型、大小或内容错误
|
||||
只拒绝当前文件,错误映射为固定安全摘要,其他文件继续接收。
|
||||
- 新增 `BATCH_MAX_FILES` 与 `BATCH_MAX_TOTAL_BYTES`,复用 `MAX_UPLOAD_BYTES` 约束单文件;
|
||||
上传流共享总字节预算,不把整批文件读入内存。数量超限在创建批次前返回 `413`。
|
||||
- Admin Console 支持多选提交,显示当前批次的接收数量、完成数量、逐文件阶段、Attempt 与安全
|
||||
错误,并每 1.5 秒自动轮询;进入 `completed`、`partial_failure` 或 `failed` 后停止轮询。
|
||||
DOM 数据继续只通过文本节点渲染。
|
||||
- 集成测试覆盖顺序稳定、部分拒绝不回滚、数量/单文件/批次总大小、全成功/部分失败聚合、
|
||||
未知批次、OpenAPI 和前端契约;迁移 drift 增加两张新表。Ruff Format/Lint、Mypy、
|
||||
JavaScript 语法、依赖锁、Alembic Head/离线升级均通过;Unit + Integration `183 passed`,
|
||||
RAG Evaluation `5 passed`。
|
||||
- 浏览器验收确认桌面布局和 390px 窄屏布局正常、无根页面横向溢出、无控制台警告或错误;
|
||||
本地页面不注入云端凭据,批次结果行为由确定性 API 集成测试验证。
|
||||
- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续
|
||||
复用 DEMO-015,Accepted ADR 未变更。
|
||||
- 下一任务:`DEPLOY-001`。
|
||||
|
||||
@ -82,7 +82,8 @@ Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务
|
||||
|
||||
### DEMO-016
|
||||
|
||||
基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console 状态展示。
|
||||
**已完成。** 基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console
|
||||
状态展示;下一交付门槛为 DEPLOY-001 的生产镜像。
|
||||
|
||||
### DEPLOY-001
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ validation. Provider credentials and internal endpoints are runtime inputs and n
|
||||
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
|
||||
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
|
||||
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
|
||||
- [ ] DEMO-016 Batch async upload API + per-file Job status + Admin Console
|
||||
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
|
||||
- [ ] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
|
||||
- [ ] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
|
||||
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
|
||||
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
|
||||
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
|
||||
3. `DEMO-016`:批量异步上传、批次/Job 查询和 Admin Console 状态展示;
|
||||
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
|
||||
4. `DEPLOY-001`:API/Admin、MCP Gateway、Async Worker 生产镜像;
|
||||
5. `DEPLOY-002`:Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
|
||||
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。
|
||||
|
||||
@ -465,6 +465,9 @@ Document processing 必须异步执行。
|
||||
源文件与 Document/Job 成功持久化后返回 `202 Accepted`。批量上传中的每个文件拥有独立的
|
||||
Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批次内其他已接收文件。
|
||||
|
||||
当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、
|
||||
断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。
|
||||
|
||||
**REQ-INGEST-004**
|
||||
|
||||
展示处理阶段:
|
||||
|
||||
@ -21,3 +21,7 @@ single-active generated unique index.
|
||||
DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
|
||||
the latter preserves numbered history and uses a generated nullable active marker plus a unique index
|
||||
to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.
|
||||
|
||||
DEMO-016 adds `ingestion_batches` and `ingestion_batch_items`. They retain input order and per-file
|
||||
acceptance references while Job rows remain the processing state source of truth; a batch is an aggregate,
|
||||
not a cross-file transaction.
|
||||
|
||||
109
migrations/versions/20260902_0008_create_ingestion_batches.py
Normal file
109
migrations/versions/20260902_0008_create_ingestion_batches.py
Normal file
@ -0,0 +1,109 @@
|
||||
"""Create ingestion batch aggregation metadata.
|
||||
|
||||
Revision ID: 20260902_0008
|
||||
Revises: 20260902_0007
|
||||
Create Date: 2026-09-02
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260902_0008"
|
||||
down_revision: str | None = "20260902_0007"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Persist ordered per-file acceptance without making a batch transactional."""
|
||||
op.create_table(
|
||||
"ingestion_batches",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("total_files", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint("total_files > 0", name="total_files_positive"),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batches"),
|
||||
)
|
||||
op.create_table(
|
||||
"ingestion_batch_items",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("batch_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("item_index", sa.Integer(), nullable=False),
|
||||
sa.Column("filename", sa.String(length=255), nullable=False),
|
||||
sa.Column(
|
||||
"acceptance_status",
|
||||
sa.Enum(
|
||||
"pending",
|
||||
"accepted",
|
||||
"rejected",
|
||||
name="batch_item_acceptance_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("document_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("job_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint("item_index >= 0", name="item_index_non_negative"),
|
||||
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
|
||||
sa.CheckConstraint(
|
||||
"acceptance_status IN ('pending', 'accepted', 'rejected')",
|
||||
name="acceptance_status_valid",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["batch_id"],
|
||||
["ingestion_batches.id"],
|
||||
name="fk_ingestion_batch_items_batch_id_ingestion_batches",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["document_id"],
|
||||
["documents.id"],
|
||||
name="fk_ingestion_batch_items_document_id_documents",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["job_id"],
|
||||
["ingestion_jobs.id"],
|
||||
name="fk_ingestion_batch_items_job_id_ingestion_jobs",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batch_items"),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_ingestion_batch_items_order",
|
||||
"ingestion_batch_items",
|
||||
["batch_id", "item_index"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_index(
|
||||
"uq_ingestion_batch_items_job_id",
|
||||
"ingestion_batch_items",
|
||||
["job_id"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -39,6 +39,8 @@ class Settings(BaseSettings):
|
||||
ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
|
||||
ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
|
||||
ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
|
||||
batch_max_files: int = Field(default=20, ge=1, le=100)
|
||||
batch_max_total_bytes: int = Field(default=200 * 1024 * 1024, gt=0)
|
||||
object_storage_provider: Literal["local", "bos"] = "bos"
|
||||
object_storage_path: Path = Path("storage")
|
||||
bos_endpoint: str | None = None
|
||||
@ -108,6 +110,8 @@ class Settings(BaseSettings):
|
||||
)
|
||||
if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
|
||||
raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
|
||||
if self.batch_max_total_bytes < self.max_upload_bytes:
|
||||
raise ValueError("batch_max_total_bytes must be at least max_upload_bytes")
|
||||
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
|
||||
raise ValueError("mcp_path must start with '/' and must not end with '/'")
|
||||
if not self.mcp_allowed_hosts:
|
||||
|
||||
270
packages/core/src/ekp_core/ingestion_batches.py
Normal file
270
packages/core/src/ekp_core/ingestion_batches.py
Normal file
@ -0,0 +1,270 @@
|
||||
"""Batch submission and aggregate status over independent ingestion Jobs."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from typing import Protocol
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.document_upload import (
|
||||
DocumentTooLargeError,
|
||||
DocumentUploadError,
|
||||
EmptyDocumentError,
|
||||
MediaTypeMismatchError,
|
||||
UnsupportedDocumentTypeError,
|
||||
)
|
||||
from ekp_core.ingestion_jobs import AsyncIngestionSubmissionService
|
||||
from ekp_domain import (
|
||||
BatchItemAcceptanceStatus,
|
||||
IngestionBatch,
|
||||
IngestionBatchItem,
|
||||
IngestionJob,
|
||||
IngestionJobStatus,
|
||||
)
|
||||
from ekp_object_storage import BinarySource
|
||||
|
||||
SAFE_UNSUPPORTED_ERROR = "unsupported document type or media type"
|
||||
SAFE_FILE_SIZE_ERROR = "document exceeds the configured upload limit"
|
||||
SAFE_BATCH_SIZE_ERROR = "batch exceeds the configured total upload limit"
|
||||
SAFE_EMPTY_ERROR = "document must contain content"
|
||||
SAFE_ACCEPTANCE_ERROR = "document could not be accepted"
|
||||
|
||||
|
||||
class BatchNotFoundError(LookupError):
|
||||
"""Raised when a requested ingestion batch does not exist."""
|
||||
|
||||
|
||||
class BatchFileCountLimitError(ValueError):
|
||||
"""Raised before persistence when the multipart file count is invalid."""
|
||||
|
||||
|
||||
class BatchAggregateStatus(StrEnum):
|
||||
"""Computed batch state derived from item acceptance and durable Jobs."""
|
||||
|
||||
PROCESSING = "processing"
|
||||
COMPLETED = "completed"
|
||||
PARTIAL_FAILURE = "partial_failure"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class BatchTotalSizeLimitError(ValueError):
|
||||
"""Raised while streaming the file that crosses the shared batch budget."""
|
||||
|
||||
|
||||
class _ReadableSource(Protocol):
|
||||
def read(self, size: int = -1, /) -> bytes:
|
||||
"""Read bytes from an uploaded file."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchSource:
|
||||
"""Provider-neutral input for one ordered multipart file."""
|
||||
|
||||
filename: str
|
||||
media_type: str | None
|
||||
source: _ReadableSource
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchItemView:
|
||||
"""Detached batch item plus its current optional Job state."""
|
||||
|
||||
item: IngestionBatchItem
|
||||
job: IngestionJob | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchView:
|
||||
"""Detached aggregate used by REST without exposing persistence sessions."""
|
||||
|
||||
batch: IngestionBatch
|
||||
items: tuple[BatchItemView, ...]
|
||||
|
||||
@property
|
||||
def status(self) -> BatchAggregateStatus:
|
||||
"""Aggregate terminal results without copying Job state into batch rows."""
|
||||
if any(
|
||||
view.item.acceptance_status == BatchItemAcceptanceStatus.PENDING
|
||||
or (
|
||||
view.item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED
|
||||
and (
|
||||
view.job is None
|
||||
or view.job.status
|
||||
in {
|
||||
IngestionJobStatus.QUEUED,
|
||||
IngestionJobStatus.RUNNING,
|
||||
IngestionJobStatus.RETRYING,
|
||||
}
|
||||
)
|
||||
)
|
||||
for view in self.items
|
||||
):
|
||||
return BatchAggregateStatus.PROCESSING
|
||||
succeeded = sum(
|
||||
view.job is not None and view.job.status == IngestionJobStatus.SUCCEEDED
|
||||
for view in self.items
|
||||
)
|
||||
failed = len(self.items) - succeeded
|
||||
if failed == 0:
|
||||
return BatchAggregateStatus.COMPLETED
|
||||
if succeeded == 0:
|
||||
return BatchAggregateStatus.FAILED
|
||||
return BatchAggregateStatus.PARTIAL_FAILURE
|
||||
|
||||
|
||||
class _BatchByteBudget:
|
||||
"""Count bytes across independently streamed files without buffering them."""
|
||||
|
||||
def __init__(self, limit_bytes: int) -> None:
|
||||
self._limit_bytes = limit_bytes
|
||||
self._consumed_bytes = 0
|
||||
|
||||
def wrap(self, source: _ReadableSource) -> BinarySource:
|
||||
return _BatchLimitedSource(source, self)
|
||||
|
||||
def consume(self, size_bytes: int) -> None:
|
||||
self._consumed_bytes += size_bytes
|
||||
if self._consumed_bytes > self._limit_bytes:
|
||||
raise BatchTotalSizeLimitError(SAFE_BATCH_SIZE_ERROR)
|
||||
|
||||
|
||||
class _BatchLimitedSource:
|
||||
"""Apply one shared request budget through the ObjectStorage stream contract."""
|
||||
|
||||
def __init__(self, source: _ReadableSource, budget: _BatchByteBudget) -> None:
|
||||
self._source = source
|
||||
self._budget = budget
|
||||
|
||||
def read(self, size: int = -1, /) -> bytes:
|
||||
chunk = self._source.read(size)
|
||||
self._budget.consume(len(chunk))
|
||||
return chunk
|
||||
|
||||
|
||||
class IngestionBatchService:
|
||||
"""Accept files independently and persist ordered aggregate status."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker[Session],
|
||||
submissions: AsyncIngestionSubmissionService,
|
||||
*,
|
||||
max_files: int,
|
||||
max_total_bytes: int,
|
||||
) -> None:
|
||||
if max_files <= 0 or max_total_bytes <= 0:
|
||||
raise ValueError("batch limits must be positive")
|
||||
self._session_factory = session_factory
|
||||
self._submissions = submissions
|
||||
self._max_files = max_files
|
||||
self._max_total_bytes = max_total_bytes
|
||||
|
||||
def submit(self, sources: tuple[BatchSource, ...]) -> BatchView:
|
||||
"""Create a batch and independently accept or reject every ordered file."""
|
||||
if not sources or len(sources) > self._max_files:
|
||||
raise BatchFileCountLimitError(
|
||||
f"batch must contain between 1 and {self._max_files} files"
|
||||
)
|
||||
batch_id, item_ids = self._create_pending_batch(sources)
|
||||
budget = _BatchByteBudget(self._max_total_bytes)
|
||||
for item_id, source in zip(item_ids, sources, strict=True):
|
||||
try:
|
||||
submission = self._submissions.submit(
|
||||
source.filename,
|
||||
source.media_type,
|
||||
budget.wrap(source.source),
|
||||
)
|
||||
except Exception as error:
|
||||
self._record_rejected(item_id, _safe_acceptance_error(error))
|
||||
continue
|
||||
self._record_accepted(
|
||||
item_id,
|
||||
document_id=submission.document.id,
|
||||
job_id=submission.job.id,
|
||||
)
|
||||
return self.get(batch_id)
|
||||
|
||||
def get(self, batch_id: UUID) -> BatchView:
|
||||
"""Return ordered items and fresh Job state for polling clients."""
|
||||
with self._session_factory() as session:
|
||||
batch = session.get(IngestionBatch, batch_id)
|
||||
if batch is None:
|
||||
raise BatchNotFoundError(str(batch_id))
|
||||
items = tuple(
|
||||
session.scalars(
|
||||
select(IngestionBatchItem)
|
||||
.where(IngestionBatchItem.batch_id == batch_id)
|
||||
.order_by(IngestionBatchItem.item_index)
|
||||
).all()
|
||||
)
|
||||
views = tuple(
|
||||
BatchItemView(
|
||||
item=item,
|
||||
job=session.get(IngestionJob, item.job_id) if item.job_id is not None else None,
|
||||
)
|
||||
for item in items
|
||||
)
|
||||
return BatchView(batch=batch, items=views)
|
||||
|
||||
def _create_pending_batch(
|
||||
self, sources: tuple[BatchSource, ...]
|
||||
) -> tuple[UUID, tuple[UUID, ...]]:
|
||||
"""Persist every input position before starting external storage writes."""
|
||||
with self._session_factory() as session:
|
||||
batch = IngestionBatch(total_files=len(sources))
|
||||
session.add(batch)
|
||||
session.flush()
|
||||
items = tuple(
|
||||
IngestionBatchItem(
|
||||
batch_id=batch.id,
|
||||
item_index=index,
|
||||
filename=_display_filename(source.filename),
|
||||
)
|
||||
for index, source in enumerate(sources)
|
||||
)
|
||||
session.add_all(items)
|
||||
session.commit()
|
||||
return batch.id, tuple(item.id for item in items)
|
||||
|
||||
def _record_accepted(self, item_id: UUID, *, document_id: UUID, job_id: UUID) -> None:
|
||||
with self._session_factory() as session:
|
||||
item = session.get(IngestionBatchItem, item_id)
|
||||
if item is None:
|
||||
raise BatchNotFoundError(str(item_id))
|
||||
item.acceptance_status = BatchItemAcceptanceStatus.ACCEPTED
|
||||
item.document_id = document_id
|
||||
item.job_id = job_id
|
||||
item.error_message = None
|
||||
session.commit()
|
||||
|
||||
def _record_rejected(self, item_id: UUID, error_message: str) -> None:
|
||||
with self._session_factory() as session:
|
||||
item = session.get(IngestionBatchItem, item_id)
|
||||
if item is None:
|
||||
raise BatchNotFoundError(str(item_id))
|
||||
item.acceptance_status = BatchItemAcceptanceStatus.REJECTED
|
||||
item.error_message = error_message
|
||||
session.commit()
|
||||
|
||||
|
||||
def _display_filename(filename: str) -> str:
|
||||
"""Store a bounded safe label even when the upload filename is invalid."""
|
||||
display = filename.replace("\x00", "<EFBFBD>").strip()
|
||||
return display[:255] or "<unnamed>"
|
||||
|
||||
|
||||
def _safe_acceptance_error(error: Exception) -> str:
|
||||
"""Map per-file failures without persisting exception text or credentials."""
|
||||
if isinstance(error, (UnsupportedDocumentTypeError, MediaTypeMismatchError)):
|
||||
return SAFE_UNSUPPORTED_ERROR
|
||||
if isinstance(error, DocumentTooLargeError):
|
||||
return SAFE_FILE_SIZE_ERROR
|
||||
if isinstance(error, BatchTotalSizeLimitError):
|
||||
return SAFE_BATCH_SIZE_ERROR
|
||||
if isinstance(error, EmptyDocumentError):
|
||||
return SAFE_EMPTY_ERROR
|
||||
if isinstance(error, DocumentUploadError):
|
||||
return SAFE_ACCEPTANCE_ERROR
|
||||
return SAFE_ACCEPTANCE_ERROR
|
||||
@ -8,6 +8,11 @@ from ekp_domain.index_profile import (
|
||||
IndexProfileStatus,
|
||||
StoredIndexProfile,
|
||||
)
|
||||
from ekp_domain.ingestion_batch import (
|
||||
BatchItemAcceptanceStatus,
|
||||
IngestionBatch,
|
||||
IngestionBatchItem,
|
||||
)
|
||||
from ekp_domain.ingestion_job import (
|
||||
IngestionAttempt,
|
||||
IngestionAttemptStatus,
|
||||
@ -19,6 +24,7 @@ from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocum
|
||||
|
||||
__all__ = [
|
||||
"Base",
|
||||
"BatchItemAcceptanceStatus",
|
||||
"Chunk",
|
||||
"ChunkIndexState",
|
||||
"Document",
|
||||
@ -30,6 +36,8 @@ __all__ = [
|
||||
"IngestionJob",
|
||||
"IngestionJobStatus",
|
||||
"IngestionStage",
|
||||
"IngestionBatch",
|
||||
"IngestionBatchItem",
|
||||
"ParsedBlock",
|
||||
"ParsedBlockType",
|
||||
"ParsedDocument",
|
||||
|
||||
96
packages/domain/src/ekp_domain/ingestion_batch.py
Normal file
96
packages/domain/src/ekp_domain/ingestion_batch.py
Normal file
@ -0,0 +1,96 @@
|
||||
"""Canonical batch submission metadata for per-file ingestion tracking."""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
Uuid,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy import Enum as SqlEnum
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ekp_domain.document import Base
|
||||
|
||||
|
||||
class BatchItemAcceptanceStatus(StrEnum):
|
||||
"""Source acceptance state, independent from its asynchronous Job state."""
|
||||
|
||||
PENDING = "pending"
|
||||
ACCEPTED = "accepted"
|
||||
REJECTED = "rejected"
|
||||
|
||||
|
||||
class IngestionBatch(Base):
|
||||
"""Stable aggregation identifier for one bounded multipart submission."""
|
||||
|
||||
__tablename__ = "ingestion_batches"
|
||||
__table_args__ = (CheckConstraint("total_files > 0", name="total_files_positive"),)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
|
||||
total_files: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class IngestionBatchItem(Base):
|
||||
"""One ordered input and its optional accepted Document/Job references."""
|
||||
|
||||
__tablename__ = "ingestion_batch_items"
|
||||
__table_args__ = (
|
||||
CheckConstraint("item_index >= 0", name="item_index_non_negative"),
|
||||
CheckConstraint("length(filename) > 0", name="filename_not_empty"),
|
||||
CheckConstraint(
|
||||
"acceptance_status IN ('pending', 'accepted', 'rejected')",
|
||||
name="acceptance_status_valid",
|
||||
),
|
||||
Index("uq_ingestion_batch_items_order", "batch_id", "item_index", unique=True),
|
||||
Index("uq_ingestion_batch_items_job_id", "job_id", unique=True),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
|
||||
batch_id: Mapped[UUID] = mapped_column(
|
||||
Uuid(as_uuid=True),
|
||||
ForeignKey("ingestion_batches.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
item_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
filename: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
acceptance_status: Mapped[BatchItemAcceptanceStatus] = mapped_column(
|
||||
SqlEnum(
|
||||
BatchItemAcceptanceStatus,
|
||||
name="batch_item_acceptance_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
validate_strings=True,
|
||||
values_callable=lambda values: [item.value for item in values],
|
||||
),
|
||||
nullable=False,
|
||||
default=BatchItemAcceptanceStatus.PENDING,
|
||||
)
|
||||
document_id: Mapped[UUID | None] = mapped_column(
|
||||
Uuid(as_uuid=True),
|
||||
ForeignKey("documents.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
job_id: Mapped[UUID | None] = mapped_column(
|
||||
Uuid(as_uuid=True),
|
||||
ForeignKey("ingestion_jobs.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
@ -43,7 +43,9 @@ def test_admin_console_contains_only_demo_pages_and_versioned_api_calls(
|
||||
assert all(label not in html for label in ("权限管理", "任务管理", "Compiled Knowledge"))
|
||||
assert 'const API_BASE = "/api/v1"' in script
|
||||
assert "innerHTML" not in script
|
||||
assert all(endpoint in script for endpoint in ('"/documents"', '"/search"'))
|
||||
assert all(endpoint in script for endpoint in ('"/ingestion-batches"', '"/search"'))
|
||||
assert "multiple" in html
|
||||
assert "BATCH_POLL_INTERVAL_MS" in script
|
||||
assert all(field in script for field in ("Page", "Section", "Dense Score", "Rerank Score"))
|
||||
assert "--brand: #FF3C00" in stylesheet
|
||||
assert "prefers-reduced-motion" in stylesheet
|
||||
|
||||
@ -30,6 +30,8 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
|
||||
"index_profiles",
|
||||
"ingestion_jobs",
|
||||
"ingestion_attempts",
|
||||
"ingestion_batches",
|
||||
"ingestion_batch_items",
|
||||
}.issubset(inspector.get_table_names())
|
||||
assert "unrelated_provider_table" in inspector.get_table_names()
|
||||
assert {column["name"] for column in inspector.get_columns("documents")} == {
|
||||
@ -101,3 +103,20 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
|
||||
"started_at",
|
||||
"finished_at",
|
||||
}
|
||||
assert {column["name"] for column in inspector.get_columns("ingestion_batches")} == {
|
||||
"id",
|
||||
"total_files",
|
||||
"created_at",
|
||||
}
|
||||
assert {column["name"] for column in inspector.get_columns("ingestion_batch_items")} == {
|
||||
"id",
|
||||
"batch_id",
|
||||
"item_index",
|
||||
"filename",
|
||||
"acceptance_status",
|
||||
"document_id",
|
||||
"job_id",
|
||||
"error_message",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
}
|
||||
|
||||
@ -19,6 +19,7 @@ from ekp_core.document_indexing import DocumentIndexingPipelineService, Document
|
||||
from ekp_core.document_management import DocumentManagementService
|
||||
from ekp_core.document_parsing import DocumentParsingService
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.ingestion_batches import IngestionBatchService
|
||||
from ekp_core.ingestion_jobs import (
|
||||
AsyncIngestionSubmissionService,
|
||||
DerivedDocumentCleanupService,
|
||||
@ -26,7 +27,7 @@ from ekp_core.ingestion_jobs import (
|
||||
IngestionJobService,
|
||||
)
|
||||
from ekp_core.search import SearchService
|
||||
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile
|
||||
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile, IngestionBatch
|
||||
from ekp_embedding import FakeEmbeddingProvider
|
||||
from ekp_object_storage import LocalObjectStorage
|
||||
from ekp_parser import ParserRouter
|
||||
@ -57,6 +58,10 @@ class RecordingDispatcher:
|
||||
|
||||
def _build_api_services(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
max_upload_bytes: int = 1024 * 1024,
|
||||
batch_max_files: int = 20,
|
||||
batch_max_total_bytes: int = 2 * 1024 * 1024,
|
||||
) -> tuple[
|
||||
ApiServices,
|
||||
sessionmaker[Session],
|
||||
@ -84,7 +89,11 @@ def _build_api_services(
|
||||
embedding_model=embedding.model_name,
|
||||
vector_dimension=embedding.dimensions,
|
||||
)
|
||||
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
|
||||
upload = DocumentUploadService(
|
||||
session_factory,
|
||||
storage,
|
||||
max_upload_bytes=max_upload_bytes,
|
||||
)
|
||||
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
|
||||
chunking = DocumentChunkingService(
|
||||
session_factory,
|
||||
@ -102,6 +111,12 @@ def _build_api_services(
|
||||
jobs = IngestionJobService(session_factory, max_attempts=3)
|
||||
dispatcher = RecordingDispatcher()
|
||||
submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher)
|
||||
batches = IngestionBatchService(
|
||||
session_factory,
|
||||
submissions,
|
||||
max_files=batch_max_files,
|
||||
max_total_bytes=batch_max_total_bytes,
|
||||
)
|
||||
runner = IngestionJobRunner(
|
||||
session_factory,
|
||||
parsing,
|
||||
@ -134,6 +149,7 @@ def _build_api_services(
|
||||
ApiServices(
|
||||
ingestion=ingestion,
|
||||
submissions=submissions,
|
||||
batches=batches,
|
||||
jobs=jobs,
|
||||
documents=documents,
|
||||
search=search,
|
||||
@ -238,6 +254,8 @@ def test_rest_validation_not_found_and_openapi_contract(
|
||||
assert set(openapi["paths"]) >= {
|
||||
"/api/v1/documents",
|
||||
"/api/v1/documents/{document_id}",
|
||||
"/api/v1/ingestion-batches",
|
||||
"/api/v1/ingestion-batches/{batch_id}",
|
||||
"/api/v1/jobs/{job_id}",
|
||||
"/api/v1/search",
|
||||
}
|
||||
@ -261,3 +279,141 @@ def test_rest_validation_not_found_and_openapi_contract(
|
||||
assert boolean_top_k.status_code == 422
|
||||
assert missing_get.status_code == 404
|
||||
assert missing_delete.status_code == 404
|
||||
|
||||
|
||||
def test_batch_upload_isolates_files_and_reports_terminal_status(
|
||||
tmp_path: Path,
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
"""A rejected file must not roll back valid siblings in the same batch."""
|
||||
services, _, _, _, runner = _build_api_services(tmp_path)
|
||||
app = create_app(Settings(app_env="test"), api_services=services)
|
||||
|
||||
with TestClient(app) as client:
|
||||
accepted = client.post(
|
||||
"/api/v1/ingestion-batches",
|
||||
files=[
|
||||
("files", ("alpha.md", b"# Alpha\n\nFirst policy.", "text/markdown")),
|
||||
("files", ("blocked.csv", b"secret=must-not-persist", "text/csv")),
|
||||
("files", ("beta.txt", b"Second policy.", "text/plain")),
|
||||
],
|
||||
)
|
||||
|
||||
assert accepted.status_code == 202
|
||||
body = accepted.json()
|
||||
assert body["status"] == "processing"
|
||||
assert body["total_files"] == 3
|
||||
assert body["accepted_files"] == 2
|
||||
assert body["rejected_files"] == 1
|
||||
assert body["completed_files"] == 1
|
||||
assert [item["filename"] for item in body["items"]] == [
|
||||
"alpha.md",
|
||||
"blocked.csv",
|
||||
"beta.txt",
|
||||
]
|
||||
assert [item["acceptance_status"] for item in body["items"]] == [
|
||||
"accepted",
|
||||
"rejected",
|
||||
"accepted",
|
||||
]
|
||||
assert body["items"][1]["error_message"] == ("unsupported document type or media type")
|
||||
assert "must-not-persist" not in str(body)
|
||||
|
||||
for item in body["items"]:
|
||||
if item["job_id"] is not None:
|
||||
runner.run(UUID(item["job_id"]))
|
||||
|
||||
terminal = client.get(f"/api/v1/ingestion-batches/{body['batch_id']}")
|
||||
|
||||
assert terminal.status_code == 200
|
||||
terminal_body = terminal.json()
|
||||
assert terminal_body["status"] == "partial_failure"
|
||||
assert terminal_body["completed_files"] == 3
|
||||
assert [item["job_status"] for item in terminal_body["items"]] == [
|
||||
"succeeded",
|
||||
None,
|
||||
"succeeded",
|
||||
]
|
||||
|
||||
|
||||
def test_batch_upload_enforces_count_and_shared_byte_limits(
|
||||
tmp_path: Path,
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
"""Request limits are bounded while already accepted siblings remain durable."""
|
||||
services, session_factory, _, _, _ = _build_api_services(
|
||||
tmp_path,
|
||||
max_upload_bytes=10,
|
||||
batch_max_files=2,
|
||||
batch_max_total_bytes=15,
|
||||
)
|
||||
app = create_app(Settings(app_env="test"), api_services=services)
|
||||
|
||||
with TestClient(app) as client:
|
||||
too_many = client.post(
|
||||
"/api/v1/ingestion-batches",
|
||||
files=[
|
||||
("files", ("one.txt", b"1", "text/plain")),
|
||||
("files", ("two.txt", b"2", "text/plain")),
|
||||
("files", ("three.txt", b"3", "text/plain")),
|
||||
],
|
||||
)
|
||||
bounded = client.post(
|
||||
"/api/v1/ingestion-batches",
|
||||
files=[
|
||||
("files", ("first.txt", b"1234567890", "text/plain")),
|
||||
("files", ("second.txt", b"abcdefghij", "text/plain")),
|
||||
],
|
||||
)
|
||||
oversized = client.post(
|
||||
"/api/v1/ingestion-batches",
|
||||
files=[("files", ("large.txt", b"12345678901", "text/plain"))],
|
||||
)
|
||||
|
||||
assert too_many.status_code == 413
|
||||
assert too_many.json()["detail"] == "batch file count exceeds the configured limit"
|
||||
assert bounded.status_code == 202
|
||||
body = bounded.json()
|
||||
assert body["accepted_files"] == 1
|
||||
assert body["rejected_files"] == 1
|
||||
assert body["items"][1]["error_message"] == ("batch exceeds the configured total upload limit")
|
||||
assert oversized.status_code == 202
|
||||
assert oversized.json()["status"] == "failed"
|
||||
assert oversized.json()["accepted_files"] == 0
|
||||
assert oversized.json()["items"][0]["error_message"] == (
|
||||
"document exceeds the configured upload limit"
|
||||
)
|
||||
|
||||
# A count-rejected request creates no batch; the two accepted requests do.
|
||||
with session_factory() as session:
|
||||
assert len(session.scalars(select(IngestionBatch)).all()) == 2
|
||||
|
||||
|
||||
def test_batch_upload_completed_and_missing_contract(
|
||||
tmp_path: Path,
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
"""All-success batches become completed and unknown IDs stay explicit."""
|
||||
services, _, _, _, runner = _build_api_services(tmp_path)
|
||||
app = create_app(Settings(app_env="test"), api_services=services)
|
||||
missing_id = uuid4()
|
||||
|
||||
with TestClient(app) as client:
|
||||
accepted = client.post(
|
||||
"/api/v1/ingestion-batches",
|
||||
files=[
|
||||
("files", ("one.md", b"# One\n\nContent.", "text/markdown")),
|
||||
("files", ("two.txt", b"More content.", "text/plain")),
|
||||
],
|
||||
).json()
|
||||
for item in accepted["items"]:
|
||||
runner.run(UUID(item["job_id"]))
|
||||
|
||||
completed = client.get(f"/api/v1/ingestion-batches/{accepted['batch_id']}")
|
||||
missing = client.get(f"/api/v1/ingestion-batches/{missing_id}")
|
||||
|
||||
assert completed.status_code == 200
|
||||
assert completed.json()["status"] == "completed"
|
||||
assert completed.json()["completed_files"] == 2
|
||||
assert missing.status_code == 404
|
||||
assert missing.json()["detail"] == "ingestion batch not found"
|
||||
|
||||
@ -25,6 +25,8 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
|
||||
assert settings.ingestion_retry_backoff_seconds == 2
|
||||
assert settings.ingestion_retry_backoff_max_seconds == 60
|
||||
assert settings.ingestion_worker_lease_seconds == 300
|
||||
assert settings.batch_max_files == 20
|
||||
assert settings.batch_max_total_bytes == 200 * 1024 * 1024
|
||||
assert settings.object_storage_provider == "bos"
|
||||
assert settings.object_storage_path == Path("storage")
|
||||
assert settings.bos_endpoint is None
|
||||
@ -157,3 +159,10 @@ def test_settings_require_worker_lease_longer_than_provider_timeout(
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match="worker_lease"):
|
||||
Settings(ingestion_worker_lease_seconds=30, embedding_timeout_seconds=30)
|
||||
|
||||
|
||||
def test_settings_require_batch_limit_to_fit_one_file(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match="batch_max_total_bytes"):
|
||||
Settings(max_upload_bytes=100, batch_max_total_bytes=99)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user