Compare commits

..

9 Commits

Author SHA1 Message Date
d-robotics
03dd5ffbfd merge: complete DEPLOY-002 Kubernetes Helm delivery 2026-09-03 13:07:19 +08:00
d-robotics
5100d98e3c deploy: add Kubernetes Helm delivery 2026-09-03 12:21:54 +08:00
d-robotics
d727d0a118 merge: complete DEPLOY-001 production images 2026-09-03 11:01:58 +08:00
d-robotics
72dc248bbc build: add production process images 2026-09-02 21:16:33 +08:00
d-robotics
a1eae16fb1 merge: complete DEMO-016 batch async upload 2026-09-02 20:43:26 +08:00
d-robotics
0fad825eb0 feat: add batch async ingestion 2026-09-02 20:42:52 +08:00
d-robotics
a8dfbe0cfa merge: DEMO-015 durable async ingestion 2026-09-02 20:19:28 +08:00
d-robotics
7938c0b9cf feat: add durable async ingestion jobs 2026-09-02 20:18:58 +08:00
d-robotics
4d73465888 merge: PLAN-002 public cluster delivery plan 2026-09-02 19:43:15 +08:00
80 changed files with 5745 additions and 153 deletions

21
.dockerignore Normal file
View File

@ -0,0 +1,21 @@
# Keep credentials, local state, caches, and developer artifacts out of every image context.
.env
.env.*
!.env.example
.git
.github
.idea
.vscode
.DS_Store
.mypy_cache
.pytest_cache
.ruff_cache
.venv
__pycache__
*.py[cod]
*.sqlite*
dist
storage
tests
docs
docker-compose.dev.yml

View File

@ -4,6 +4,8 @@ LOG_LEVEL=INFO
API_HOST=0.0.0.0
API_PORT=8000
API_PREFIX=/api/v1
# Optional outside source checkouts; production image fixes this to /app/apps/admin-web.
# ADMIN_WEB_ROOT=/absolute/path/to/apps/admin-web
DATABASE_URL=mysql+pymysql://ekp:change-me@localhost:2881/ekp
DATABASE_POOL_SIZE=10
@ -11,8 +13,15 @@ DATABASE_MAX_OVERFLOW=10
DATABASE_POOL_RECYCLE_SECONDS=600
DATABASE_POOL_TIMEOUT_SECONDS=30
DATABASE_CONNECT_TIMEOUT_SECONDS=30
REDIS_URL=redis://localhost:6379/0
RABBITMQ_URL=amqp://guest:guest@localhost:5672//
# DEMO-015 queue. Inject the real credential at runtime; never commit its value.
RABBITMQ_URL=
INGESTION_MAX_ATTEMPTS=3
INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
BATCH_MAX_FILES=20
BATCH_MAX_TOTAL_BYTES=209715200
OBJECT_STORAGE_PROVIDER=bos
OBJECT_STORAGE_PATH=storage

View File

@ -1,4 +1,4 @@
.PHONY: help install run run-mcp format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
export UV_CACHE_DIR
@ -8,6 +8,13 @@ help:
@echo " install Install development dependencies"
@echo " run Start the development API"
@echo " run-mcp Start the authenticated MCP gateway"
@echo " run-worker Start the ingestion worker and recovery scheduler"
@echo " images Build production images for the current builder"
@echo " images-print Render the multi-architecture image build plan"
@echo " images-push Build and push amd64/arm64 production images"
@echo " deploy Install or upgrade Kubernetes with VALUES=<file>"
@echo " deploy-lint Validate Helm values and templates"
@echo " deploy-render Render non-secret Kubernetes manifests"
@echo " format Format code"
@echo " format-check Check formatting without modifying files"
@echo " lint Run linters"
@ -27,6 +34,22 @@ run:
uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}"
run-mcp:
uv run ekp-mcp
run-worker:
uv run ekp-worker
images:
docker buildx bake -f deploy/docker/docker-bake.hcl
images-print:
docker buildx bake -f deploy/docker/docker-bake.hcl --print
images-push:
docker buildx bake -f deploy/docker/docker-bake.hcl --push
deploy:
./scripts/deploy.sh "$(VALUES)"
deploy-lint:
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
helm lint deploy/helm/knowledge-platform --values "$(VALUES)"
deploy-render:
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
helm template "$${RELEASE:-knowledge-platform}" deploy/helm/knowledge-platform --namespace "$${NAMESPACE:-knowledge-platform}" --values "$(VALUES)"
format:
uv run ruff format .
format-check:

View File

@ -53,6 +53,7 @@ Install dependencies and run the API:
```bash
cp .env.example .env
make install
make run-worker
make run
```
@ -75,6 +76,11 @@ The Demo Admin Console is available at `http://localhost:8000/admin` and contain
only document upload, document management, and Search Lab pages. See
`docs/development/ADMIN_CONSOLE.md` for the operation and scope boundaries.
DEPLOY-001 provides locked, non-root `api`, `mcp`, and `worker` production image targets with an
amd64/arm64 Buildx Bake plan. DEPLOY-002 adds the Kubernetes/Helm Chart, runtime Secret references,
serialized Migration, Ingress/TLS and the `make deploy VALUES=<file>` entrypoint. See
`docs/development/CONTAINER_IMAGES.md` and `docs/development/KUBERNETES_DEPLOYMENT.md`.
Run the development checks with:
```bash
@ -100,8 +106,11 @@ Chunk metadata, including deterministic IDs, filename, page range, section,
index, and strategy version, is stored atomically in canonical relational tables. The lightweight
token counter is replaceable so DEMO-006 can inject model-specific behavior.
Documents remain `processing` until embedding/indexing completes; pipeline
failures become `failed` without deleting the immutable original. DEMO-009
exposes this pipeline through the versioned upload endpoint.
failures become `failed` without deleting the immutable original. DEMO-015 now
accepts a source plus durable Job with HTTP `202`, while a Celery/RabbitMQ worker
runs the pipeline. Bounded transient retries clean every derived artifact before
starting the next Attempt; broker redelivery and expired worker leases converge on
OceanBase Job state. See `docs/development/ASYNC_INGESTION_SETUP.md`.
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a
validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase.
@ -128,8 +137,9 @@ adapter with the `bce-reranker-base` baseline. The final Top20-to-Top5 pipeline
source results are rehydrated from canonical relational chunks and retain both
Dense and reranker scores; stale vector content fails closed. See
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
DEMO-009 exposes the completed synchronous pipeline through versioned document
and search REST endpoints under `/api/v1`. Upload, list, detail, direct delete,
DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
changes upload to asynchronous acceptance and adds Job status. DEMO-016 adds bounded multi-file
submission, batch/per-file status, and automatic Admin Console polling. Upload, list, detail, direct delete,
and reranked source search are documented in
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
outside the public response contract.

View File

@ -1,12 +1,33 @@
// DEMO-010 stays API First: this client owns presentation state only and sends
// every document/search operation through the versioned public REST contract.
// The Admin Console owns presentation state only. Batch acceptance, Job state,
// and retrieval remain behind the versioned public REST contract.
const API_BASE = "/api/v1";
const BATCH_POLL_INTERVAL_MS = 1500;
const terminalBatchStatuses = new Set(["completed", "partial_failure", "failed"]);
let batchPollTimer;
let activeBatchId;
const statusLabels = {
uploaded: "Uploaded",
processing: "Processing",
ready: "Ready",
failed: "Failed",
pending: "Pending",
accepted: "Accepted",
rejected: "Rejected",
queued: "Queued",
running: "Running",
retrying: "Retrying",
succeeded: "Succeeded",
completed: "Completed",
partial_failure: "Partial failure",
};
const stageLabels = {
parsing: "解析",
cleaning: "清理",
chunking: "切分",
embedding: "向量化",
indexing: "索引",
};
const pageTitles = {
@ -93,23 +114,37 @@ function initializeUpload() {
const button = document.querySelector("#upload-button");
const updateFilename = () => {
selected.textContent = input.files[0]?.name ?? "尚未选择文件";
const files = Array.from(input.files);
if (!files.length) {
selected.textContent = "尚未选择文件";
} else if (files.length === 1) {
selected.textContent = files[0].name;
} else {
selected.textContent = `已选择 ${files.length} 个文件`;
}
};
input.addEventListener("change", updateFilename);
form.addEventListener("submit", async (event) => {
event.preventDefault();
if (!input.files.length) return;
setButtonBusy(button, true, "处理中…");
window.clearTimeout(batchPollTimer);
activeBatchId = undefined;
setButtonBusy(button, true, "提交中…");
try {
const formData = new FormData();
formData.append("file", input.files[0]);
const documentData = await requestJson("/documents", {
Array.from(input.files).forEach((file) => formData.append("files", file));
const batch = await requestJson("/ingestion-batches", {
method: "POST",
body: formData,
});
renderUploadResult(documentData);
showNotice("文档已上传并完成处理。可以前往 Search Lab 检索。", "success");
activeBatchId = batch.batch_id;
renderBatchResult(batch);
scheduleBatchPoll(batch);
showNotice(
`批次已接收:${batch.accepted_files} 个进入后台队列,${batch.rejected_files} 个被拒绝。`,
batch.rejected_files ? "error" : "success",
);
form.reset();
updateFilename();
} catch (error) {
@ -120,18 +155,62 @@ function initializeUpload() {
});
}
function renderUploadResult(documentData) {
function scheduleBatchPoll(batch) {
window.clearTimeout(batchPollTimer);
if (batch.batch_id !== activeBatchId) return;
if (terminalBatchStatuses.has(batch.status)) return;
batchPollTimer = window.setTimeout(async () => {
try {
const latest = await requestJson(`/ingestion-batches/${batch.batch_id}`);
renderBatchResult(latest);
scheduleBatchPoll(latest);
} catch (error) {
showNotice(`批次状态刷新失败:${error.message}`, "error");
scheduleBatchPoll(batch);
}
}, BATCH_POLL_INTERVAL_MS);
}
function renderBatchResult(batch) {
const card = document.querySelector("#upload-result");
document.querySelector("#upload-filename").textContent = documentData.filename;
document.querySelector("#upload-type").textContent = documentData.media_type;
document.querySelector("#upload-size").textContent = formatSize(documentData.size_bytes);
document.querySelector("#upload-time").textContent = formatDate(documentData.created_at);
document.querySelector("#upload-status").replaceWith(statusPill(documentData.status));
document.querySelector("#upload-batch-title").textContent =
`批次 ${batch.batch_id.slice(0, 8)} · ${formatDate(batch.created_at)}`;
document.querySelector("#upload-total").textContent = `${batch.total_files} 个`;
document.querySelector("#upload-completed").textContent =
`${batch.completed_files} / ${batch.total_files}`;
document.querySelector("#upload-accepted").textContent =
`${batch.accepted_files} / ${batch.rejected_files}`;
document.querySelector("#upload-status").replaceWith(statusPill(batch.status));
const status = card.querySelector(".status-pill");
status.id = "upload-status";
const items = document.querySelector("#batch-items");
items.replaceChildren(...batch.items.map(batchItem));
card.hidden = false;
}
function batchItem(item) {
const row = document.createElement("article");
row.className = "batch-item";
const order = document.createElement("span");
order.className = "batch-item-order";
order.textContent = String(item.item_index + 1).padStart(2, "0");
const body = document.createElement("div");
body.className = "batch-item-body";
const name = document.createElement("strong");
name.textContent = item.filename;
const detail = document.createElement("span");
const stage = item.stage ? ` · ${stageLabels[item.stage] ?? item.stage}` : "";
const attempts = item.max_attempts == null ? "" : ` · 尝试 ${item.attempt}/${item.max_attempts}`;
detail.textContent = item.error_message ?? `后台任务${stage}${attempts}`;
body.append(name, detail);
const state = item.job_status ?? item.acceptance_status;
row.append(order, body, statusPill(state));
return row;
}
async function loadDocuments() {
const refresh = document.querySelector("#refresh-documents");
setButtonBusy(refresh, true, "加载中…");
@ -329,3 +408,4 @@ initializeUpload();
initializeSearch();
document.querySelector("#refresh-documents").addEventListener("click", loadDocuments);
if (page === "documents") loadDocuments();
window.addEventListener("pagehide", () => window.clearTimeout(batchPollTimer));

View File

@ -533,6 +533,50 @@ a {
border-block: 1px solid var(--line);
}
.batch-items {
display: grid;
gap: 8px;
margin: -8px 0 24px;
}
.batch-item {
display: grid;
grid-template-columns: 34px minmax(0, 1fr) auto;
gap: 12px;
align-items: center;
min-height: 64px;
padding: 11px 13px;
border: 1px solid var(--line);
border-radius: 3px;
background: #fafaf8;
}
.batch-item-order {
color: var(--brand);
font-family: var(--data-font);
font-size: 10px;
font-weight: 850;
}
.batch-item-body {
display: grid;
min-width: 0;
gap: 5px;
}
.batch-item-body strong {
overflow: hidden;
font-size: 12px;
text-overflow: ellipsis;
white-space: nowrap;
}
.batch-item-body span {
overflow-wrap: anywhere;
color: var(--muted);
font-size: 10px;
}
dt {
color: var(--muted);
font-family: var(--data-font);
@ -655,6 +699,15 @@ tbody tr:hover {
.status-processing { background: #fff0ce; color: var(--warning); }
.status-failed { background: #fbe6e3; color: var(--danger); }
.status-uploaded { background: var(--brand-soft); color: var(--brand-deep); }
.status-pending,
.status-queued,
.status-running,
.status-retrying,
.status-accepted { background: #fff0ce; color: var(--warning); }
.status-succeeded,
.status-completed { background: #e5f3eb; color: var(--success); }
.status-rejected,
.status-partial_failure { background: #fbe6e3; color: var(--danger); }
.empty-state {
min-height: 240px;
@ -958,6 +1011,8 @@ select:focus {
.search-controls { grid-template-columns: minmax(0, 1fr) 90px; }
.search-controls textarea { grid-column: 1 / -1; }
.metadata-grid { grid-template-columns: 1fr; }
.batch-item { grid-template-columns: 28px minmax(0, 1fr); }
.batch-item .status-pill { grid-column: 2; }
}
@media (max-width: 540px) {

View File

@ -67,7 +67,7 @@
<div>
<p class="section-label">01 / INGESTION</p>
<h2>让知识进入检索链路</h2>
<p>上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将同步完成解析、切分与索引。</p>
<p>批量提交 PDF、DOCX、TXT 或 Markdown 文件,系统将逐个接收并在后台独立完成解析、切分与索引。</p>
</div>
</div>
@ -75,14 +75,15 @@
<label id="drop-zone" class="drop-zone" for="file-input">
<input
id="file-input"
name="file"
name="files"
type="file"
multiple
accept=".pdf,.docx,.txt,.md,application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document,text/plain,text/markdown"
required
/>
<span class="upload-icon" aria-hidden="true">↑</span>
<strong>选择知识文件</strong>
<span>支持 PDF、DOCX、TXT、Markdown · 单次一个文件</span>
<strong>选择一批知识文件</strong>
<span>支持 PDF、DOCX、TXT、Markdown · 每个文件独立处理</span>
<span id="selected-file" class="selected-file">尚未选择文件</span>
</label>
<button id="upload-button" class="primary-button" type="submit">上传并处理</button>
@ -90,15 +91,16 @@
<article id="upload-result" class="result-card" hidden>
<div>
<p class="section-label">LATEST SIGNAL</p>
<h3 id="upload-filename"></h3>
<p class="section-label">LATEST BATCH</p>
<h3 id="upload-batch-title"></h3>
</div>
<span id="upload-status" class="status-pill"></span>
<dl class="metadata-grid">
<div><dt>文件类型</dt><dd id="upload-type"></dd></div>
<div><dt>文件大小</dt><dd id="upload-size"></dd></div>
<div><dt>上传时间</dt><dd id="upload-time"></dd></div>
<div><dt>总文件</dt><dd id="upload-total"></dd></div>
<div><dt>已完成</dt><dd id="upload-completed"></dd></div>
<div><dt>接收 / 拒绝</dt><dd id="upload-accepted"></dd></div>
</dl>
<div id="batch-items" class="batch-items" aria-live="polite"></div>
<a class="text-link" href="/admin/search">前往 Search Lab 检索 <span>→</span></a>
</article>
</section>

View File

@ -0,0 +1,55 @@
"""Celery/RabbitMQ adapter behind the queue-neutral ingestion dispatcher."""
from uuid import UUID
from celery import Celery # type: ignore[import-untyped]
from ekp_core.config import Settings
INGESTION_TASK_NAME = "ekp.ingestion.run"
RECOVERY_TASK_NAME = "ekp.ingestion.recover"
def create_celery_app(settings: Settings) -> Celery:
"""Create a secret-safe broker client without opening a connection eagerly."""
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
raise ValueError("RABBITMQ_URL is required for asynchronous ingestion")
app = Celery(
"enterprise-knowledge-platform",
broker=settings.rabbitmq_url.get_secret_value(),
)
app.conf.update(
task_ignore_result=True,
task_acks_late=True,
task_reject_on_worker_lost=True,
worker_prefetch_multiplier=1,
broker_connection_retry_on_startup=True,
timezone="UTC",
enable_utc=True,
task_routes={
INGESTION_TASK_NAME: {"queue": "ingestion"},
RECOVERY_TASK_NAME: {"queue": "ingestion"},
},
beat_schedule={
"recover-durable-ingestion-jobs": {
"task": RECOVERY_TASK_NAME,
"schedule": float(settings.ingestion_recovery_interval_seconds),
}
},
)
return app
class CeleryIngestionDispatcher:
"""Publish only a durable Job id; source bytes and state stay canonical."""
def __init__(self, app: Celery) -> None:
self._app = app
def enqueue(self, job_id: UUID) -> None:
"""Deliver one idempotent execution hint to the ingestion queue."""
self._app.send_task(
INGESTION_TASK_NAME,
args=[str(job_id)],
queue="ingestion",
)

View File

@ -37,12 +37,13 @@ def create_app(
application.state.settings = resolved_settings
application.state.api_services = api_services
application.include_router(router, prefix=resolved_settings.api_prefix)
admin_web_root = (resolved_settings.admin_web_root or ADMIN_WEB_ROOT).resolve()
# The Demo console is a deliberately thin, same-origin client. It contains
# no business logic and calls only the versioned public REST API.
application.mount(
"/admin/assets",
StaticFiles(directory=ADMIN_WEB_ROOT / "assets"),
StaticFiles(directory=admin_web_root / "assets"),
name="admin-assets",
)
@ -52,7 +53,7 @@ def create_app(
@application.get("/admin/search", include_in_schema=False)
async def admin_console() -> FileResponse:
"""Serve the three-page Demo Admin Console shell."""
return FileResponse(ADMIN_WEB_ROOT / "index.html")
return FileResponse(admin_web_root / "index.html")
@application.get("/healthz", response_model=HealthResponse, tags=["system"])
async def health() -> HealthResponse:

View File

@ -4,6 +4,7 @@ from dataclasses import dataclass
from sqlalchemy.orm import Session, sessionmaker
from ekp_api.celery_queue import CeleryIngestionDispatcher, create_celery_app
from ekp_core.chunking import StructureAwareChunker
from ekp_core.config import Settings
from ekp_core.database import create_database_engine, create_session_factory
@ -17,6 +18,13 @@ from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.index_rebuild import resolve_active_index_profile
from ekp_core.ingestion_batches import IngestionBatchService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
)
from ekp_core.search import SearchService
from ekp_domain import IndexProfile
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
@ -42,10 +50,39 @@ class ApiServices:
"""Business services injected into REST routes without provider leakage."""
ingestion: DocumentIndexingPipelineService
submissions: AsyncIngestionSubmissionService
batches: IngestionBatchService
jobs: IngestionJobService
documents: DocumentManagementService
search: SearchService
@dataclass(frozen=True, slots=True)
class WorkerServices:
"""Provider graph required by Celery without REST search-only services."""
runner: IngestionJobRunner
jobs: IngestionJobService
dispatcher: CeleryIngestionDispatcher
@dataclass(frozen=True, slots=True)
class _IngestionComponents:
"""Shared composition result used by API and independent worker processes."""
session_factory: sessionmaker[Session]
storage: ObjectStorage
vector_store: VectorStore
embedding_provider: EmbeddingProvider
index_profile: IndexProfile
upload: DocumentUploadService
parsing: DocumentParsingService
chunking: DocumentChunkingService
indexing: DocumentIndexingService
jobs: IngestionJobService
dispatcher: CeleryIngestionDispatcher
def create_index_profile(
settings: Settings,
session_factory: sessionmaker[Session] | None = None,
@ -161,8 +198,31 @@ def create_search_service(
)
def create_api_services(settings: Settings) -> ApiServices:
"""Compose the synchronous Demo runtime once for all business endpoints."""
def create_runtime_search_service(settings: Settings) -> SearchService:
"""Compose only read-path providers for the independent MCP Gateway."""
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
index_profile = create_index_profile(settings, session_factory)
embedding_provider = create_embedding_provider(settings)
vector_store = create_vector_store(settings)
dense_retrieval = create_dense_retrieval_service(
settings,
embedding_provider,
vector_store,
index_profile,
)
return create_search_service(
settings,
session_factory,
dense_retrieval,
create_reranker_provider(settings),
)
def _create_ingestion_components(settings: Settings) -> _IngestionComponents:
"""Compose provider-neutral ingestion stages and the Celery delivery adapter."""
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
raise ProviderConfigurationError("RABBITMQ_URL is required")
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
storage = create_object_storage(settings)
@ -185,35 +245,103 @@ def create_api_services(settings: Settings) -> ApiServices:
max_characters=settings.chunk_max_characters,
),
)
document_pipeline = DocumentPipelineService(
upload_service,
parsing_service,
chunking_service,
)
indexing_service = DocumentIndexingService(
session_factory,
embedding_provider,
vector_store,
index_profile,
)
ingestion = DocumentIndexingPipelineService(document_pipeline, indexing_service)
jobs = IngestionJobService(session_factory, settings.ingestion_max_attempts)
dispatcher = CeleryIngestionDispatcher(create_celery_app(settings))
return _IngestionComponents(
session_factory=session_factory,
storage=storage,
vector_store=vector_store,
embedding_provider=embedding_provider,
index_profile=index_profile,
upload=upload_service,
parsing=parsing_service,
chunking=chunking_service,
indexing=indexing_service,
jobs=jobs,
dispatcher=dispatcher,
)
def _create_runner(settings: Settings, components: _IngestionComponents) -> IngestionJobRunner:
"""Compose retry cleanup and execution policy from non-sensitive settings."""
cleanup = DerivedDocumentCleanupService(
components.session_factory,
components.storage,
components.vector_store,
components.index_profile.profile_id,
)
return IngestionJobRunner(
components.session_factory,
components.parsing,
components.chunking,
components.indexing,
cleanup,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
retry_backoff_max_seconds=settings.ingestion_retry_backoff_max_seconds,
lease_seconds=settings.ingestion_worker_lease_seconds,
)
def create_worker_services(settings: Settings) -> WorkerServices:
"""Build the independent worker runtime from the same application boundaries."""
components = _create_ingestion_components(settings)
return WorkerServices(
runner=_create_runner(settings, components),
jobs=components.jobs,
dispatcher=components.dispatcher,
)
def create_api_services(settings: Settings) -> ApiServices:
"""Compose REST services while ingestion execution remains in workers."""
components = _create_ingestion_components(settings)
document_pipeline = DocumentPipelineService(
components.upload,
components.parsing,
components.chunking,
)
ingestion = DocumentIndexingPipelineService(document_pipeline, components.indexing)
submissions = AsyncIngestionSubmissionService(
components.upload,
components.jobs,
components.dispatcher,
)
batches = IngestionBatchService(
components.session_factory,
submissions,
max_files=settings.batch_max_files,
max_total_bytes=settings.batch_max_total_bytes,
)
dense_retrieval = create_dense_retrieval_service(
settings,
embedding_provider,
vector_store,
index_profile,
components.embedding_provider,
components.vector_store,
components.index_profile,
)
reranker_provider = create_reranker_provider(settings)
search = create_search_service(
settings,
session_factory,
components.session_factory,
dense_retrieval,
reranker_provider,
)
documents = DocumentManagementService(
session_factory,
storage,
vector_store,
active_index_profile_id=index_profile.profile_id,
components.session_factory,
components.storage,
components.vector_store,
active_index_profile_id=components.index_profile.profile_id,
)
return ApiServices(
ingestion=ingestion,
submissions=submissions,
batches=batches,
jobs=components.jobs,
documents=documents,
search=search,
)
return ApiServices(ingestion=ingestion, documents=documents, search=search)

View File

@ -6,10 +6,18 @@ from uuid import UUID
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services
from ekp_api.schemas import DocumentResponse, SearchRequest, SearchResponse
from ekp_api.schemas import (
DocumentResponse,
IngestionAcceptedResponse,
IngestionBatchResponse,
IngestionJobResponse,
SearchRequest,
SearchResponse,
)
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
from ekp_core.document_management import (
DocumentArtifactCleanupError,
DocumentIngestionActiveError,
DocumentNotFoundError,
)
from ekp_core.document_upload import (
@ -18,7 +26,8 @@ from ekp_core.document_upload import (
MediaTypeMismatchError,
UnsupportedDocumentTypeError,
)
from ekp_parser import DocumentParseError
from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundError, BatchSource
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
router = APIRouter()
@ -44,17 +53,17 @@ Services = Annotated[ApiServices, Depends(get_api_services)]
@router.post(
"/documents",
response_model=DocumentResponse,
status_code=status.HTTP_201_CREATED,
response_model=IngestionAcceptedResponse,
status_code=status.HTTP_202_ACCEPTED,
tags=["documents"],
)
def upload_document(
services: Services,
file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")],
) -> DocumentResponse:
"""Upload one file and synchronously run the complete Demo ingestion pipeline."""
) -> IngestionAcceptedResponse:
"""Persist one immutable source and durable Job, then return immediately."""
try:
result = services.ingestion.ingest(file.filename or "", file.content_type, file.file)
submission = services.submissions.submit(file.filename or "", file.content_type, file.file)
except DocumentTooLargeError as error:
raise HTTPException(
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
@ -65,21 +74,85 @@ def upload_document(
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
detail="unsupported document type or media type",
) from error
except (DocumentUploadError, DocumentParseError) as error:
except DocumentUploadError as error:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail="document could not be accepted or parsed",
detail="document could not be accepted",
) from error
except Exception as error:
# Provider and persistence exceptions are intentionally not reflected to clients.
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="document processing failed",
detail="document acceptance failed",
) from error
# Reload after every pipeline stage so server-generated timestamps and the
# final persisted status, not a detached intermediate ORM object, are returned.
document = services.documents.get_document(result.ingestion.document.id)
return DocumentResponse.from_document(document)
return IngestionAcceptedResponse.from_submission(submission)
@router.get("/jobs/{job_id}", response_model=IngestionJobResponse, tags=["jobs"])
def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
"""Return durable stage, attempt and safe failure state for one Job."""
try:
job = services.jobs.get(job_id)
except IngestionJobNotFoundError as error:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="job not found"
) from error
return IngestionJobResponse.from_job(job)
@router.post(
"/ingestion-batches",
response_model=IngestionBatchResponse,
status_code=status.HTTP_202_ACCEPTED,
tags=["jobs"],
)
def upload_ingestion_batch(
services: Services,
files: Annotated[
list[UploadFile],
File(description="One or more PDF, DOCX, TXT, or Markdown files"),
],
) -> IngestionBatchResponse:
"""Accept a bounded batch while isolating every file's result."""
try:
view = services.batches.submit(
tuple(
BatchSource(
filename=file.filename or "",
media_type=file.content_type,
source=file.file,
)
for file in files
)
)
except BatchFileCountLimitError as error:
raise HTTPException(
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
detail="batch file count exceeds the configured limit",
) from error
except Exception as error:
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="batch could not be accepted",
) from error
return IngestionBatchResponse.from_view(view)
@router.get(
"/ingestion-batches/{batch_id}",
response_model=IngestionBatchResponse,
tags=["jobs"],
)
def get_ingestion_batch(batch_id: UUID, services: Services) -> IngestionBatchResponse:
"""Return aggregate and per-file status for Admin Console polling."""
try:
view = services.batches.get(batch_id)
except BatchNotFoundError as error:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="ingestion batch not found",
) from error
return IngestionBatchResponse.from_view(view)
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
@ -115,6 +188,11 @@ def delete_document(document_id: UUID, services: Services) -> Response:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
) from error
except DocumentIngestionActiveError as error:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="document ingestion is still active",
) from error
except DocumentArtifactCleanupError as error:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,

View File

@ -5,8 +5,17 @@ from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
from ekp_core.ingestion_batches import BatchAggregateStatus, BatchItemView, BatchView
from ekp_core.ingestion_jobs import IngestionSubmission
from ekp_core.search import SearchResult
from ekp_domain import Document, DocumentStatus
from ekp_domain import (
BatchItemAcceptanceStatus,
Document,
DocumentStatus,
IngestionJob,
IngestionJobStatus,
IngestionStage,
)
class DocumentResponse(BaseModel):
@ -29,6 +38,138 @@ class DocumentResponse(BaseModel):
return cls.model_validate(document)
class IngestionAcceptedResponse(DocumentResponse):
"""Single-file acceptance with the durable Job needed for status polling."""
job_id: UUID
job_status: IngestionJobStatus
max_attempts: int
@classmethod
def from_submission(cls, submission: IngestionSubmission) -> "IngestionAcceptedResponse":
"""Flatten identifiers while retaining the existing document fields."""
document = submission.document
job = submission.job
return cls(
id=document.id,
filename=document.filename,
media_type=document.media_type,
size_bytes=document.size_bytes,
status=document.status,
error_message=document.error_message,
created_at=document.created_at,
updated_at=document.updated_at,
job_id=job.id,
job_status=job.status,
max_attempts=job.max_attempts,
)
class IngestionJobResponse(BaseModel):
"""Public queue-independent Job status without provider or broker details."""
id: UUID
document_id: UUID
status: IngestionJobStatus
stage: IngestionStage | None
attempt: int
max_attempts: int
error_message: str | None
created_at: datetime
updated_at: datetime
completed_at: datetime | None
@classmethod
def from_job(cls, job: IngestionJob) -> "IngestionJobResponse":
"""Serialize only client-actionable state and safe diagnostics."""
return cls(
id=job.id,
document_id=job.document_id,
status=job.status,
stage=job.stage,
attempt=job.attempt_count,
max_attempts=job.max_attempts,
error_message=job.error_message,
created_at=job.created_at,
updated_at=job.updated_at,
completed_at=job.completed_at,
)
class IngestionBatchItemResponse(BaseModel):
"""Per-file acceptance and latest asynchronous processing state."""
item_index: int
filename: str
acceptance_status: BatchItemAcceptanceStatus
document_id: UUID | None
job_id: UUID | None
job_status: IngestionJobStatus | None
stage: IngestionStage | None
attempt: int
max_attempts: int | None
error_message: str | None
@classmethod
def from_view(cls, view: BatchItemView) -> "IngestionBatchItemResponse":
job = view.job
return cls(
item_index=view.item.item_index,
filename=view.item.filename,
acceptance_status=view.item.acceptance_status,
document_id=view.item.document_id,
job_id=view.item.job_id,
job_status=job.status if job is not None else None,
stage=job.stage if job is not None else None,
attempt=job.attempt_count if job is not None else 0,
max_attempts=job.max_attempts if job is not None else None,
error_message=(job.error_message if job is not None else view.item.error_message),
)
class IngestionBatchResponse(BaseModel):
"""Batch aggregate with ordered independently processed files."""
batch_id: UUID
status: BatchAggregateStatus
total_files: int
accepted_files: int
rejected_files: int
completed_files: int
created_at: datetime
items: list[IngestionBatchItemResponse]
@classmethod
def from_view(cls, view: BatchView) -> "IngestionBatchResponse":
items = [IngestionBatchItemResponse.from_view(item) for item in view.items]
accepted = sum(
item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED for item in items
)
rejected = sum(
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED for item in items
)
completed = sum(
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED
or item.job_status
in {
IngestionJobStatus.SUCCEEDED,
IngestionJobStatus.FAILED,
IngestionJobStatus.CANCELLED,
}
for item in items
)
return cls(
batch_id=view.batch.id,
status=view.status,
total_files=view.batch.total_files,
accepted_files=accepted,
rejected_files=rejected,
completed_files=completed,
created_at=view.batch.created_at,
items=items,
)
class SearchRequest(BaseModel):
"""Validated public search input; normalization remains in the core service."""

View File

@ -0,0 +1,51 @@
"""Celery worker entry point for durable ingestion Jobs."""
from functools import lru_cache
from typing import Any
from uuid import UUID
from ekp_api.celery_queue import (
INGESTION_TASK_NAME,
RECOVERY_TASK_NAME,
create_celery_app,
)
from ekp_api.providers import WorkerServices, create_worker_services
from ekp_core.config import get_settings
from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs
_settings = get_settings()
celery_app = create_celery_app(_settings)
@lru_cache
def _worker_services() -> WorkerServices:
"""Build one provider/service graph per worker process."""
return create_worker_services(_settings)
@celery_app.task( # type: ignore[untyped-decorator]
bind=True,
name=INGESTION_TASK_NAME,
max_retries=None,
)
def run_ingestion(self: Any, job_id: str) -> None:
"""Execute one delivery and ask Celery to redeliver only when required.
Celery delivery retries are unbounded because valid-lease deferrals are not
business Attempts; the durable runner independently enforces max_attempts.
"""
result = _worker_services().runner.run(UUID(job_id))
if result.action == JobRunAction.RETRY:
raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None)
@celery_app.task(name=RECOVERY_TASK_NAME) # type: ignore[untyped-decorator]
def recover_ingestion_jobs() -> int:
"""Redeliver due durable Jobs after API/broker or worker interruptions."""
services = _worker_services()
return dispatch_pending_jobs(services.jobs, services.dispatcher)
def main() -> None:
"""Run one worker with the embedded Demo recovery scheduler."""
celery_app.worker_main(["worker", "--loglevel=INFO", "--queues=ingestion", "--beat"])

View File

@ -12,7 +12,7 @@ from mcp.server.transport_security import TransportSecuritySettings
from pydantic import Field
from starlette.applications import Starlette
from ekp_api.providers import ApiServices, create_api_services
from ekp_api.providers import ApiServices, create_runtime_search_service
from ekp_core.config import Settings, get_settings
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
from ekp_mcp.auth import StaticServiceTokenVerifier, current_mcp_identity
@ -29,8 +29,10 @@ def _create_search_application(
settings: Settings,
api_services: ApiServices | None,
) -> McpSearchApplication:
services = api_services or create_api_services(settings)
return McpSearchApplication(services.search, settings.mcp_required_scope)
searcher = (
api_services.search if api_services is not None else create_runtime_search_service(settings)
)
return McpSearchApplication(searcher, settings.mcp_required_scope)
def create_mcp_server(

61
deploy/docker/Dockerfile Normal file
View File

@ -0,0 +1,61 @@
# syntax=docker/dockerfile:1.7
# One locked runtime produces three independently deployable process images.
ARG PYTHON_IMAGE=python:3.12.14-slim-bookworm@sha256:782412e85d0f0984994c290652577d4018aff08145c85b262bb63dc0c7522254
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1
FROM ${UV_IMAGE} AS uv-bin
FROM ${PYTHON_IMAGE} AS runtime
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
VIRTUAL_ENV=/app/.venv \
PATH=/app/.venv/bin:${PATH} \
HOME=/tmp/ekp \
XDG_CACHE_HOME=/tmp/ekp/cache \
APP_ENV=production \
API_HOST=0.0.0.0 \
MCP_HOST=0.0.0.0 \
ADMIN_WEB_ROOT=/app/apps/admin-web
WORKDIR /app
COPY --from=uv-bin /uv /uvx /usr/local/bin/
COPY pyproject.toml uv.lock README.md alembic.ini ./
COPY apps ./apps
COPY packages ./packages
COPY migrations ./migrations
# --frozen prevents lock changes; --no-dev and --no-editable keep the runtime
# independent from source-tree import hooks while retaining migration sources.
RUN uv sync --frozen --no-dev --no-editable \
&& test -f /app/apps/admin-web/index.html \
&& groupadd --gid 10001 ekp \
&& useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin ekp \
&& mkdir -p /tmp/ekp/cache \
&& chown -R 10001:10001 /tmp/ekp \
&& rm -rf /root/.cache/uv
USER 10001:10001
FROM runtime AS api
EXPOSE 8000
# This endpoint is intentionally credential-free and suitable for container liveness.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=2)"]
CMD ["uvicorn", "ekp_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
FROM runtime AS mcp
EXPOSE 8766
# MCP requests require authentication, so the image-level check only verifies PID 1.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import os; os.kill(1, 0)"]
CMD ["ekp-mcp"]
FROM runtime AS worker
# Worker health is process-level here; durable Job recovery is validated by the service tests.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import os; os.kill(1, 0)"]
CMD ["ekp-worker"]

View File

@ -0,0 +1,36 @@
# Reproducible target names consumed by local and CI buildx invocations.
variable "IMAGE_REGISTRY" {
default = "knowledge-platform"
}
variable "IMAGE_TAG" {
default = "dev"
}
group "default" {
targets = ["api", "mcp", "worker"]
}
target "common" {
context = "."
dockerfile = "deploy/docker/Dockerfile"
platforms = ["linux/amd64", "linux/arm64"]
}
target "api" {
inherits = ["common"]
target = "api"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-api:${IMAGE_TAG}"]
}
target "mcp" {
inherits = ["common"]
target = "mcp"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-mcp:${IMAGE_TAG}"]
}
target "worker" {
inherits = ["common"]
target = "worker"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-worker:${IMAGE_TAG}"]
}

View File

@ -0,0 +1,50 @@
# Copy this file outside the repository for one environment. It contains only
# non-sensitive settings and Kubernetes Secret references; never add values for
# DATABASE_URL, RABBITMQ_URL, provider keys, passwords or MCP_AUTH_TOKEN here.
existingSecret: knowledge-platform-runtime
images:
api:
repository: registry.example.internal/knowledge/enterprise-knowledge-api
tag: "0.1.0"
digest: ""
mcp:
repository: registry.example.internal/knowledge/enterprise-knowledge-mcp
tag: "0.1.0"
digest: ""
worker:
repository: registry.example.internal/knowledge/enterprise-knowledge-worker
tag: "0.1.0"
digest: ""
ingress:
enabled: true
className: nginx
api:
host: knowledge.example.internal
tlsSecretName: knowledge-platform-api-tls
mcp:
host: mcp.example.internal
tlsSecretName: knowledge-platform-mcp-tls
config:
objectStorage:
endpoint: https://s3.example.internal
bucket: knowledge-platform
vectorStore:
uri: oceanbase.example.internal:2881
user: knowledge-platform
database: knowledge_platform
mcp:
publicUrl: https://mcp.example.internal/mcp
issuerUrl: https://mcp.example.internal
allowedHosts:
- mcp.example.internal
allowedOrigins:
- https://mcp.example.internal
# External RabbitMQ is the production default. If this local dependency is
# explicitly enabled for a demo cluster, create its Secret before deployment.
rabbitmq:
enabled: false
existingSecret: knowledge-platform-rabbitmq

View File

@ -0,0 +1,15 @@
apiVersion: v2
name: knowledge-platform
description: Enterprise Knowledge Platform public-knowledge deployment
type: application
version: 0.1.0
appVersion: "0.1.0"
# This local dependency is disabled by default. Production deployments should
# use an externally managed RabbitMQ service with an independently owned data
# lifecycle; the subchart exists only for explicit development/demo use.
dependencies:
- name: rabbitmq
version: 0.1.0
repository: file://charts/rabbitmq
condition: rabbitmq.enabled

View File

@ -0,0 +1,6 @@
apiVersion: v2
name: rabbitmq
description: Minimal single-node RabbitMQ dependency for local demo clusters only
type: application
version: 0.1.0
appVersion: "4.1.4"

View File

@ -0,0 +1,11 @@
{{- define "rabbitmq.fullname" -}}
{{- printf "%s-rabbitmq" .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "rabbitmq.image" -}}
{{- if .Values.image.digest -}}
{{- printf "%s@%s" .Values.image.repository .Values.image.digest -}}
{{- else -}}
{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}}
{{- end -}}
{{- end -}}

View File

@ -0,0 +1,16 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "rabbitmq.fullname" . }}
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
clusterIP: None
ports:
- name: amqp
port: {{ .Values.service.port }}
targetPort: amqp
selector:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}

View File

@ -0,0 +1,8 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "rabbitmq.fullname" . }}
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
automountServiceAccountToken: false

View File

@ -0,0 +1,109 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ include "rabbitmq.fullname" . }}
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
serviceName: {{ include "rabbitmq.fullname" . }}
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
serviceAccountName: {{ include "rabbitmq.fullname" . }}
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 999
runAsGroup: 999
fsGroup: 999
seccompProfile:
type: RuntimeDefault
containers:
- name: rabbitmq
image: {{ include "rabbitmq.image" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
ports:
- name: amqp
containerPort: 5672
env:
- name: RABBITMQ_DEFAULT_USER
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: {{ .Values.secretKeys.username }}
- name: RABBITMQ_DEFAULT_PASS
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: {{ .Values.secretKeys.password }}
- name: RABBITMQ_ERLANG_COOKIE
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: {{ .Values.secretKeys.erlangCookie }}
startupProbe:
exec:
command: ["rabbitmq-diagnostics", "-q", "ping"]
periodSeconds: 5
failureThreshold: 24
readinessProbe:
exec:
command: ["rabbitmq-diagnostics", "-q", "check_running"]
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
livenessProbe:
exec:
command: ["rabbitmq-diagnostics", "-q", "ping"]
periodSeconds: 20
timeoutSeconds: 5
failureThreshold: 3
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: data
mountPath: /var/lib/rabbitmq
- name: config
mountPath: /etc/rabbitmq/conf.d
- name: log
mountPath: /var/log/rabbitmq
- name: tmp
mountPath: /tmp
volumes:
- name: config
emptyDir: {}
- name: log
emptyDir: {}
- name: tmp
emptyDir: {}
{{- if not .Values.persistence.enabled }}
- name: data
emptyDir: {}
{{- end }}
{{- if .Values.persistence.enabled }}
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [{{ .Values.persistence.accessMode | quote }}]
{{- if .Values.persistence.storageClass }}
storageClassName: {{ .Values.persistence.storageClass | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.persistence.size }}
{{- end }}

View File

@ -0,0 +1,69 @@
{
"$schema": "https://json-schema.org/draft-07/schema#",
"type": "object",
"additionalProperties": false,
"required": ["enabled", "existingSecret", "secretKeys", "image", "service", "persistence", "resources"],
"properties": {
"global": {"type": "object"},
"enabled": {"type": "boolean"},
"existingSecret": {"type": "string", "minLength": 1},
"secretKeys": {
"type": "object",
"additionalProperties": false,
"required": ["username", "password", "erlangCookie"],
"properties": {
"username": {"type": "string", "minLength": 1},
"password": {"type": "string", "minLength": 1},
"erlangCookie": {"type": "string", "minLength": 1}
}
},
"image": {
"type": "object",
"additionalProperties": false,
"required": ["repository", "tag", "digest", "pullPolicy"],
"properties": {
"repository": {"type": "string", "minLength": 1},
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
}
},
"service": {
"type": "object",
"additionalProperties": false,
"required": ["port"],
"properties": {"port": {"type": "integer", "minimum": 1, "maximum": 65535}}
},
"persistence": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "storageClass", "accessMode", "size"],
"properties": {
"enabled": {"type": "boolean"},
"storageClass": {"type": "string"},
"accessMode": {"enum": ["ReadWriteOnce", "ReadWriteMany"]},
"size": {"type": "string", "minLength": 2}
}
},
"resources": {
"type": "object",
"additionalProperties": false,
"required": ["requests", "limits"],
"properties": {
"requests": {"$ref": "#/definitions/resourcePair"},
"limits": {"$ref": "#/definitions/resourcePair"}
}
}
},
"definitions": {
"resourcePair": {
"type": "object",
"additionalProperties": false,
"required": ["cpu", "memory"],
"properties": {
"cpu": {"type": ["string", "number"]},
"memory": {"type": "string", "minLength": 2}
}
}
}
}

View File

@ -0,0 +1,25 @@
enabled: false
existingSecret: knowledge-platform-rabbitmq
secretKeys:
username: username
password: password
erlangCookie: erlang-cookie
image:
repository: rabbitmq
tag: 4.1.4-alpine
digest: sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2
pullPolicy: IfNotPresent
service:
port: 5672
persistence:
enabled: true
storageClass: ""
accessMode: ReadWriteOnce
size: 8Gi
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi

View File

@ -0,0 +1,12 @@
Enterprise Knowledge Platform {{ .Chart.AppVersion }} is managed by Helm release {{ .Release.Name }}.
Runtime credentials are read only from Secret {{ .Values.existingSecret }}.
{{- if .Values.ingress.enabled }}
Admin/API: https://{{ .Values.ingress.api.host }}{{ .Values.ingress.api.path }}
MCP: https://{{ .Values.ingress.mcp.host }}{{ .Values.ingress.mcp.path }}
{{- else }}
Ingress is disabled. Enable it in the environment values file before remote use.
{{- end }}
Inspect rollout state without printing environment variables:
kubectl -n {{ .Release.Namespace }} get deploy,job,pod

View File

@ -0,0 +1,106 @@
{{/* Stable resource name helpers keep every workload under one release identity. */}}
{{- define "knowledge-platform.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "knowledge-platform.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := include "knowledge-platform.name" . -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- define "knowledge-platform.labels" -}}
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | quote }}
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end -}}
{{- define "knowledge-platform.selectorLabels" -}}
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end -}}
{{- define "knowledge-platform.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{- default (include "knowledge-platform.fullname" .) .Values.serviceAccount.name -}}
{{- else -}}
{{- required "serviceAccount.name is required when serviceAccount.create=false" .Values.serviceAccount.name -}}
{{- end -}}
{{- end -}}
{{/* A supplied digest takes precedence over the immutable release tag. */}}
{{- define "knowledge-platform.image" -}}
{{- if .digest -}}
{{- printf "%s@%s" .repository .digest -}}
{{- else -}}
{{- printf "%s:%s" .repository .tag -}}
{{- end -}}
{{- end -}}
{{- define "knowledge-platform.configMapName" -}}
{{- printf "%s-runtime" (include "knowledge-platform.fullname" .) -}}
{{- end -}}
{{/* Cross-field rules complement values.schema.json's structural checks. */}}
{{- define "knowledge-platform.validateValues" -}}
{{- if lt (float64 .Values.config.ingestion.retryBackoffMaxSeconds) (float64 .Values.config.ingestion.retryBackoffSeconds) -}}
{{- fail "config.ingestion.retryBackoffMaxSeconds must be >= retryBackoffSeconds" -}}
{{- end -}}
{{- if lt (float64 .Values.config.ingestion.batchMaxTotalBytes) (float64 .Values.config.ingestion.maxUploadBytes) -}}
{{- fail "config.ingestion.batchMaxTotalBytes must be >= maxUploadBytes" -}}
{{- end -}}
{{- if ge (float64 .Values.config.chunking.overlapTokens) (float64 .Values.config.chunking.maxTokens) -}}
{{- fail "config.chunking.overlapTokens must be < maxTokens" -}}
{{- end -}}
{{- if gt (float64 .Values.config.retrieval.searchTopK) (float64 .Values.config.retrieval.denseTopN) -}}
{{- fail "config.retrieval.searchTopK must be <= denseTopN" -}}
{{- end -}}
{{- end -}}
{{- define "knowledge-platform.podSecurityContext" -}}
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
{{- end -}}
{{- define "knowledge-platform.containerSecurityContext" -}}
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
capabilities:
drop:
- ALL
{{- end -}}
{{- define "knowledge-platform.envFrom" -}}
- configMapRef:
name: {{ include "knowledge-platform.configMapName" . }}
- secretRef:
name: {{ .Values.existingSecret }}
{{- end -}}
{{- define "knowledge-platform.tmpVolume" -}}
- name: tmp
emptyDir:
sizeLimit: 1Gi
{{- end -}}
{{- define "knowledge-platform.tmpMount" -}}
- name: tmp
mountPath: /tmp
{{- end -}}

View File

@ -0,0 +1,75 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-api
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
replicas: {{ .Values.api.replicaCount }}
minReadySeconds: 5
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: api
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: api
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: api
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
ports:
- name: http
containerPort: 8000
protocol: TCP
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
httpGet:
path: /healthz
port: http
{{- toYaml .Values.api.probes.startup | nindent 12 }}
readinessProbe:
httpGet:
path: /healthz
port: http
{{- toYaml .Values.api.probes.readiness | nindent 12 }}
livenessProbe:
httpGet:
path: /healthz
port: http
{{- toYaml .Values.api.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.api.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}

View File

@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "knowledge-platform.fullname" . }}-api
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
type: {{ .Values.api.service.type }}
ports:
- name: http
port: {{ .Values.api.service.port }}
targetPort: http
protocol: TCP
selector:
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: api

View File

@ -0,0 +1,66 @@
{{- include "knowledge-platform.validateValues" . }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "knowledge-platform.configMapName" . }}
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
data:
# Only the explicitly enumerated non-sensitive settings are rendered here.
APP_ENV: "production"
APP_NAME: {{ .Values.config.appName | quote }}
LOG_LEVEL: {{ .Values.config.logLevel | quote }}
API_HOST: "0.0.0.0"
API_PORT: "8000"
API_PREFIX: "/api/v1"
DATABASE_POOL_SIZE: {{ .Values.config.database.poolSize | quote }}
DATABASE_MAX_OVERFLOW: {{ .Values.config.database.maxOverflow | quote }}
DATABASE_POOL_RECYCLE_SECONDS: {{ .Values.config.database.poolRecycleSeconds | quote }}
DATABASE_POOL_TIMEOUT_SECONDS: {{ .Values.config.database.poolTimeoutSeconds | quote }}
DATABASE_CONNECT_TIMEOUT_SECONDS: {{ .Values.config.database.connectTimeoutSeconds | quote }}
INGESTION_MAX_ATTEMPTS: {{ .Values.config.ingestion.maxAttempts | quote }}
INGESTION_RETRY_BACKOFF_SECONDS: {{ .Values.config.ingestion.retryBackoffSeconds | quote }}
INGESTION_RETRY_BACKOFF_MAX_SECONDS: {{ .Values.config.ingestion.retryBackoffMaxSeconds | quote }}
INGESTION_WORKER_LEASE_SECONDS: {{ .Values.config.ingestion.workerLeaseSeconds | quote }}
INGESTION_RECOVERY_INTERVAL_SECONDS: {{ .Values.config.ingestion.recoveryIntervalSeconds | quote }}
BATCH_MAX_FILES: {{ .Values.config.ingestion.batchMaxFiles | quote }}
BATCH_MAX_TOTAL_BYTES: {{ printf "%.0f" .Values.config.ingestion.batchMaxTotalBytes | quote }}
MAX_UPLOAD_BYTES: {{ printf "%.0f" .Values.config.ingestion.maxUploadBytes | quote }}
OBJECT_STORAGE_PROVIDER: {{ .Values.config.objectStorage.provider | quote }}
BOS_ENDPOINT: {{ .Values.config.objectStorage.endpoint | quote }}
BOS_BUCKET: {{ .Values.config.objectStorage.bucket | quote }}
BOS_KEY_PREFIX: {{ .Values.config.objectStorage.keyPrefix | quote }}
BOS_TIMEOUT_SECONDS: {{ .Values.config.objectStorage.timeoutSeconds | quote }}
CHUNK_MAX_TOKENS: {{ .Values.config.chunking.maxTokens | quote }}
CHUNK_OVERLAP_TOKENS: {{ .Values.config.chunking.overlapTokens | quote }}
CHUNK_MAX_CHARACTERS: {{ .Values.config.chunking.maxCharacters | quote }}
EMBEDDING_PROVIDER: {{ .Values.config.embedding.provider | quote }}
EMBEDDING_MODEL: {{ .Values.config.embedding.model | quote }}
EMBEDDING_ENDPOINT: {{ .Values.config.embedding.endpoint | quote }}
EMBEDDING_TIMEOUT_SECONDS: {{ .Values.config.embedding.timeoutSeconds | quote }}
VECTOR_STORE_PROVIDER: {{ .Values.config.vectorStore.provider | quote }}
OCEANBASE_VECTOR_URI: {{ .Values.config.vectorStore.uri | quote }}
OCEANBASE_VECTOR_USER: {{ .Values.config.vectorStore.user | quote }}
OCEANBASE_VECTOR_DATABASE: {{ .Values.config.vectorStore.database | quote }}
OCEANBASE_VECTOR_TABLE: {{ .Values.config.vectorStore.table | quote }}
OCEANBASE_VECTOR_INDEX: {{ .Values.config.vectorStore.index | quote }}
OCEANBASE_VECTOR_BATCH_SIZE: {{ .Values.config.vectorStore.batchSize | quote }}
OCEANBASE_VECTOR_TIMEOUT_SECONDS: {{ .Values.config.vectorStore.timeoutSeconds | quote }}
INDEX_PROFILE_ID: {{ .Values.config.retrieval.indexProfileId | quote }}
DENSE_TOP_N: {{ .Values.config.retrieval.denseTopN | quote }}
SEARCH_TOP_K: {{ .Values.config.retrieval.searchTopK | quote }}
RERANKER_PROVIDER: {{ .Values.config.reranker.provider | quote }}
RERANKER_MODEL: {{ .Values.config.reranker.model | quote }}
RERANKER_ENDPOINT: {{ .Values.config.reranker.endpoint | quote }}
RERANKER_TIMEOUT_SECONDS: {{ .Values.config.reranker.timeoutSeconds | quote }}
MCP_HOST: "0.0.0.0"
MCP_PORT: "8766"
MCP_PATH: "/mcp"
MCP_PUBLIC_URL: {{ .Values.config.mcp.publicUrl | quote }}
MCP_ISSUER_URL: {{ .Values.config.mcp.issuerUrl | quote }}
MCP_CLIENT_ID: {{ .Values.config.mcp.clientId | quote }}
MCP_SERVICE_IDENTITY: {{ .Values.config.mcp.serviceIdentity | quote }}
MCP_REQUIRED_SCOPE: {{ .Values.config.mcp.requiredScope | quote }}
MCP_ALLOWED_HOSTS: {{ .Values.config.mcp.allowedHosts | toJson | quote }}
MCP_ALLOWED_ORIGINS: {{ .Values.config.mcp.allowedOrigins | toJson | quote }}
MCP_MAX_REQUEST_BODY_BYTES: {{ printf "%.0f" .Values.config.mcp.maxRequestBodyBytes | quote }}

View File

@ -0,0 +1,42 @@
{{- if .Values.ingress.enabled }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "knowledge-platform.fullname" . }}
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
ingressClassName: {{ .Values.ingress.className }}
tls:
- hosts:
- {{ .Values.ingress.api.host | quote }}
secretName: {{ .Values.ingress.api.tlsSecretName }}
- hosts:
- {{ .Values.ingress.mcp.host | quote }}
secretName: {{ .Values.ingress.mcp.tlsSecretName }}
rules:
- host: {{ .Values.ingress.api.host | quote }}
http:
paths:
- path: {{ .Values.ingress.api.path }}
pathType: Prefix
backend:
service:
name: {{ include "knowledge-platform.fullname" . }}-api
port:
number: {{ .Values.api.service.port }}
- host: {{ .Values.ingress.mcp.host | quote }}
http:
paths:
- path: {{ .Values.ingress.mcp.path }}
pathType: Prefix
backend:
service:
name: {{ include "knowledge-platform.fullname" . }}-mcp
port:
number: {{ .Values.mcp.service.port }}
{{- end }}

View File

@ -0,0 +1,72 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-mcp
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: mcp
spec:
replicas: {{ .Values.mcp.replicaCount }}
minReadySeconds: 5
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: mcp
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: mcp
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: mcp
image: {{ include "knowledge-platform.image" .Values.images.mcp | quote }}
imagePullPolicy: {{ .Values.images.mcp.pullPolicy }}
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
ports:
- name: mcp
containerPort: 8766
protocol: TCP
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
tcpSocket:
port: mcp
{{- toYaml .Values.mcp.probes.startup | nindent 12 }}
readinessProbe:
tcpSocket:
port: mcp
{{- toYaml .Values.mcp.probes.readiness | nindent 12 }}
livenessProbe:
tcpSocket:
port: mcp
{{- toYaml .Values.mcp.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.mcp.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}

View File

@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "knowledge-platform.fullname" . }}-mcp
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: mcp
spec:
type: {{ .Values.mcp.service.type }}
ports:
- name: mcp
port: {{ .Values.mcp.service.port }}
targetPort: mcp
protocol: TCP
selector:
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: mcp

View File

@ -0,0 +1,59 @@
{{- if .Values.migration.enabled }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "knowledge-platform.fullname" . }}-migration
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: migration
annotations:
# Helm waits for pre-install/pre-upgrade hooks. A failed migration blocks
# the application rollout, and the previous Job remains until the next run.
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-weight: "-10"
helm.sh/hook-delete-policy: before-hook-creation
spec:
backoffLimit: {{ .Values.migration.backoffLimit }}
activeDeadlineSeconds: {{ .Values.migration.activeDeadlineSeconds }}
ttlSecondsAfterFinished: {{ .Values.migration.ttlSecondsAfterFinished }}
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: migration
spec:
restartPolicy: Never
# Pre-install hooks run before the release ServiceAccount exists. The
# namespace default account is used with token mounting disabled; the
# migration needs no Kubernetes API permissions.
serviceAccountName: default
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: migration
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
command: ["alembic"]
args: ["upgrade", "head"]
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
env:
- name: APP_ENV
value: production
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: DATABASE_URL
resources:
{{- toYaml .Values.migration.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
{{- end }}

View File

@ -0,0 +1,71 @@
{{- if .Values.scheduler.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-scheduler
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: scheduler
spec:
# A single Celery Beat owns periodic durable-Job recovery. Recreate prevents
# two schedulers from overlapping during upgrades.
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: scheduler
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: scheduler
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: scheduler
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
workingDir: /tmp
command: ["celery"]
args:
- --app=ekp_api.worker:celery_app
- beat
- --loglevel=INFO
- --schedule=/tmp/celerybeat-schedule
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.scheduler.probes.startup | nindent 12 }}
livenessProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.scheduler.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.scheduler.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
{{- end }}

View File

@ -0,0 +1,13 @@
{{- if .Values.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "knowledge-platform.serviceAccountName" . }}
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: false
{{- end }}

View File

@ -0,0 +1,17 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "knowledge-platform.fullname" . }}-preflight-references
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
annotations:
# The Make preflight renders this test hook to discover only resource
# names. It never renders or reads Secret data.
helm.sh/hook: test
data:
runtimeSecret: {{ .Values.existingSecret | quote }}
ingressEnabled: {{ .Values.ingress.enabled | quote }}
apiTlsSecret: {{ .Values.ingress.api.tlsSecretName | quote }}
mcpTlsSecret: {{ .Values.ingress.mcp.tlsSecretName | quote }}
rabbitmqEnabled: {{ .Values.rabbitmq.enabled | quote }}
rabbitmqSecret: {{ .Values.rabbitmq.existingSecret | quote }}

View File

@ -0,0 +1,71 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-worker
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: worker
spec:
replicas: {{ .Values.worker.replicaCount }}
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 1
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: worker
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: worker
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: worker
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
workingDir: /tmp
command: ["celery"]
args:
- --app=ekp_api.worker:celery_app
- worker
- --loglevel=INFO
- --queues=ingestion
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.worker.probes.startup | nindent 12 }}
livenessProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.worker.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.worker.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}

View File

@ -0,0 +1,338 @@
{
"$schema": "https://json-schema.org/draft-07/schema#",
"title": "Enterprise Knowledge Platform Helm values",
"type": "object",
"additionalProperties": false,
"required": [
"existingSecret",
"images",
"serviceAccount",
"api",
"mcp",
"worker",
"scheduler",
"migration",
"ingress",
"config",
"rabbitmq"
],
"properties": {
"nameOverride": {"type": "string"},
"fullnameOverride": {"type": "string"},
"existingSecret": {"$ref": "#/definitions/kubernetesName"},
"imagePullSecrets": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["name"],
"properties": {"name": {"$ref": "#/definitions/kubernetesName"}}
}
},
"images": {
"type": "object",
"additionalProperties": false,
"required": ["api", "mcp", "worker"],
"properties": {
"api": {"$ref": "#/definitions/image"},
"mcp": {"$ref": "#/definitions/image"},
"worker": {"$ref": "#/definitions/image"}
}
},
"serviceAccount": {
"type": "object",
"additionalProperties": false,
"required": ["create", "name", "annotations"],
"properties": {
"create": {"type": "boolean"},
"name": {"type": "string"},
"annotations": {"$ref": "#/definitions/stringMap"}
}
},
"podAnnotations": {"$ref": "#/definitions/stringMap"},
"podLabels": {"$ref": "#/definitions/stringMap"},
"api": {"$ref": "#/definitions/httpWorkload"},
"mcp": {"$ref": "#/definitions/httpWorkload"},
"worker": {
"type": "object",
"additionalProperties": false,
"required": ["replicaCount", "terminationGracePeriodSeconds", "resources", "probes"],
"properties": {
"replicaCount": {"type": "integer", "minimum": 1, "maximum": 100},
"terminationGracePeriodSeconds": {"type": "integer", "minimum": 30, "maximum": 3600},
"resources": {"$ref": "#/definitions/resources"},
"probes": {"$ref": "#/definitions/processProbes"}
}
},
"scheduler": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "resources", "probes"],
"properties": {
"enabled": {"type": "boolean"},
"resources": {"$ref": "#/definitions/resources"},
"probes": {"$ref": "#/definitions/processProbes"}
}
},
"migration": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "activeDeadlineSeconds", "backoffLimit", "ttlSecondsAfterFinished", "resources"],
"properties": {
"enabled": {"type": "boolean"},
"activeDeadlineSeconds": {"type": "integer", "minimum": 60, "maximum": 3600},
"backoffLimit": {"type": "integer", "minimum": 0, "maximum": 3},
"ttlSecondsAfterFinished": {"type": "integer", "minimum": 0},
"resources": {"$ref": "#/definitions/resources"}
}
},
"ingress": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "className", "annotations", "api", "mcp"],
"properties": {
"enabled": {"type": "boolean"},
"className": {"type": "string", "minLength": 1},
"annotations": {"$ref": "#/definitions/stringMap"},
"api": {"$ref": "#/definitions/ingressRoute"},
"mcp": {"$ref": "#/definitions/ingressRoute"}
}
},
"config": {
"type": "object",
"additionalProperties": false,
"required": ["appName", "logLevel", "database", "ingestion", "objectStorage", "chunking", "embedding", "vectorStore", "retrieval", "reranker", "mcp"],
"properties": {
"appName": {"type": "string", "minLength": 1},
"logLevel": {"enum": ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]},
"database": {"$ref": "#/definitions/databaseConfig"},
"ingestion": {"$ref": "#/definitions/ingestionConfig"},
"objectStorage": {"$ref": "#/definitions/objectStorageConfig"},
"chunking": {"$ref": "#/definitions/chunkingConfig"},
"embedding": {"$ref": "#/definitions/embeddingConfig"},
"vectorStore": {"$ref": "#/definitions/vectorStoreConfig"},
"retrieval": {"$ref": "#/definitions/retrievalConfig"},
"reranker": {"$ref": "#/definitions/rerankerConfig"},
"mcp": {"$ref": "#/definitions/mcpConfig"}
}
},
"rabbitmq": {
"type": "object",
"required": ["enabled", "existingSecret"],
"properties": {
"enabled": {"type": "boolean"},
"existingSecret": {"$ref": "#/definitions/kubernetesName"}
}
}
},
"definitions": {
"kubernetesName": {
"type": "string",
"minLength": 1,
"maxLength": 253,
"pattern": "^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$"
},
"stringMap": {
"type": "object",
"additionalProperties": {"type": "string"}
},
"image": {
"type": "object",
"additionalProperties": false,
"required": ["repository", "tag", "digest", "pullPolicy"],
"properties": {
"repository": {"type": "string", "minLength": 1},
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
}
},
"probeTiming": {
"type": "object",
"additionalProperties": false,
"required": ["failureThreshold", "periodSeconds"],
"properties": {
"failureThreshold": {"type": "integer", "minimum": 1},
"periodSeconds": {"type": "integer", "minimum": 1},
"timeoutSeconds": {"type": "integer", "minimum": 1}
}
},
"httpProbes": {
"type": "object",
"additionalProperties": false,
"required": ["startup", "readiness", "liveness"],
"properties": {
"startup": {"$ref": "#/definitions/probeTiming"},
"readiness": {"$ref": "#/definitions/probeTiming"},
"liveness": {"$ref": "#/definitions/probeTiming"}
}
},
"processProbes": {
"type": "object",
"additionalProperties": false,
"required": ["startup", "liveness"],
"properties": {
"startup": {"$ref": "#/definitions/probeTiming"},
"liveness": {"$ref": "#/definitions/probeTiming"}
}
},
"resources": {
"type": "object",
"additionalProperties": false,
"required": ["requests", "limits"],
"properties": {
"requests": {"$ref": "#/definitions/resourcePair"},
"limits": {"$ref": "#/definitions/resourcePair"}
}
},
"resourcePair": {
"type": "object",
"additionalProperties": false,
"required": ["cpu", "memory"],
"properties": {
"cpu": {"type": ["string", "number"]},
"memory": {"type": "string", "minLength": 2}
}
},
"httpWorkload": {
"type": "object",
"additionalProperties": false,
"required": ["replicaCount", "service", "resources", "probes"],
"properties": {
"replicaCount": {"type": "integer", "minimum": 2, "maximum": 100},
"service": {
"type": "object",
"additionalProperties": false,
"required": ["type", "port"],
"properties": {
"type": {"enum": ["ClusterIP"]},
"port": {"type": "integer", "minimum": 1, "maximum": 65535}
}
},
"resources": {"$ref": "#/definitions/resources"},
"probes": {"$ref": "#/definitions/httpProbes"}
}
},
"ingressRoute": {
"type": "object",
"additionalProperties": false,
"required": ["host", "path", "tlsSecretName"],
"properties": {
"host": {"type": "string", "minLength": 1},
"path": {"type": "string", "pattern": "^/"},
"tlsSecretName": {"$ref": "#/definitions/kubernetesName"}
}
},
"databaseConfig": {
"type": "object",
"additionalProperties": false,
"required": ["poolSize", "maxOverflow", "poolRecycleSeconds", "poolTimeoutSeconds", "connectTimeoutSeconds"],
"properties": {
"poolSize": {"type": "integer", "minimum": 1, "maximum": 100},
"maxOverflow": {"type": "integer", "minimum": 0, "maximum": 100},
"poolRecycleSeconds": {"type": "integer", "minimum": 1},
"poolTimeoutSeconds": {"type": "number", "exclusiveMinimum": 0},
"connectTimeoutSeconds": {"type": "integer", "minimum": 1}
}
},
"ingestionConfig": {
"type": "object",
"additionalProperties": false,
"required": ["maxAttempts", "retryBackoffSeconds", "retryBackoffMaxSeconds", "workerLeaseSeconds", "recoveryIntervalSeconds", "batchMaxFiles", "batchMaxTotalBytes", "maxUploadBytes"],
"properties": {
"maxAttempts": {"type": "integer", "minimum": 1, "maximum": 10},
"retryBackoffSeconds": {"type": "integer", "minimum": 1},
"retryBackoffMaxSeconds": {"type": "integer", "minimum": 1},
"workerLeaseSeconds": {"type": "integer", "minimum": 30},
"recoveryIntervalSeconds": {"type": "integer", "minimum": 5},
"batchMaxFiles": {"type": "integer", "minimum": 1, "maximum": 100},
"batchMaxTotalBytes": {"type": "integer", "minimum": 1},
"maxUploadBytes": {"type": "integer", "minimum": 1}
}
},
"objectStorageConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "endpoint", "bucket", "keyPrefix", "timeoutSeconds"],
"properties": {
"provider": {"const": "bos"},
"endpoint": {"type": "string", "minLength": 1},
"bucket": {"type": "string", "minLength": 1},
"keyPrefix": {"type": "string", "minLength": 1},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"chunkingConfig": {
"type": "object",
"additionalProperties": false,
"required": ["maxTokens", "overlapTokens", "maxCharacters"],
"properties": {
"maxTokens": {"type": "integer", "minimum": 1},
"overlapTokens": {"type": "integer", "minimum": 0},
"maxCharacters": {"type": "integer", "minimum": 4}
}
},
"embeddingConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
"properties": {
"provider": {"const": "qianfan"},
"model": {"const": "bge-large-zh"},
"endpoint": {"type": "string", "minLength": 1},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"vectorStoreConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "uri", "user", "database", "table", "index", "batchSize", "timeoutSeconds"],
"properties": {
"provider": {"const": "oceanbase"},
"uri": {"type": "string", "minLength": 1},
"user": {"type": "string", "minLength": 1},
"database": {"type": "string", "minLength": 1},
"table": {"type": "string", "minLength": 1},
"index": {"type": "string", "minLength": 1},
"batchSize": {"type": "integer", "minimum": 1, "maximum": 1000},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"retrievalConfig": {
"type": "object",
"additionalProperties": false,
"required": ["indexProfileId", "denseTopN", "searchTopK"],
"properties": {
"indexProfileId": {"type": "string", "minLength": 1},
"denseTopN": {"type": "integer", "minimum": 1, "maximum": 64},
"searchTopK": {"type": "integer", "minimum": 1, "maximum": 64}
}
},
"rerankerConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
"properties": {
"provider": {"const": "qianfan"},
"model": {"const": "bce-reranker-base"},
"endpoint": {"type": "string", "minLength": 1},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"mcpConfig": {
"type": "object",
"additionalProperties": false,
"required": ["publicUrl", "issuerUrl", "clientId", "serviceIdentity", "requiredScope", "allowedHosts", "allowedOrigins", "maxRequestBodyBytes"],
"properties": {
"publicUrl": {"type": "string", "pattern": "^https://"},
"issuerUrl": {"type": "string", "pattern": "^https://"},
"clientId": {"type": "string", "minLength": 1},
"serviceIdentity": {"type": "string", "minLength": 1},
"requiredScope": {"const": "knowledge:search"},
"allowedHosts": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},
"allowedOrigins": {"type": "array", "minItems": 1, "items": {"type": "string", "pattern": "^https://"}},
"maxRequestBodyBytes": {"type": "integer", "minimum": 1024, "maximum": 4194304}
}
}
}
}

View File

@ -0,0 +1,218 @@
# DEPLOY-002 safe defaults. Real endpoints, immutable image coordinates and
# Secret names are supplied in an environment-specific values file.
nameOverride: ""
fullnameOverride: ""
existingSecret: knowledge-platform-runtime
imagePullSecrets: []
images:
api:
repository: knowledge-platform/enterprise-knowledge-api
tag: "0.1.0"
digest: ""
pullPolicy: IfNotPresent
mcp:
repository: knowledge-platform/enterprise-knowledge-mcp
tag: "0.1.0"
digest: ""
pullPolicy: IfNotPresent
worker:
repository: knowledge-platform/enterprise-knowledge-worker
tag: "0.1.0"
digest: ""
pullPolicy: IfNotPresent
serviceAccount:
create: true
name: ""
annotations: {}
podAnnotations: {}
podLabels: {}
api:
replicaCount: 2
service:
type: ClusterIP
port: 80
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
readiness:
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
mcp:
replicaCount: 2
service:
type: ClusterIP
port: 80
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
readiness:
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
worker:
replicaCount: 2
terminationGracePeriodSeconds: 60
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "2"
memory: 2Gi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
# Celery Beat is separated from scalable workers so only one recovery
# scheduler publishes due durable Jobs at a time.
scheduler:
enabled: true
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
migration:
enabled: true
activeDeadlineSeconds: 600
backoffLimit: 0
ttlSecondsAfterFinished: 86400
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
ingress:
enabled: false
className: nginx
annotations: {}
api:
host: knowledge.example.invalid
path: /
tlsSecretName: knowledge-platform-api-tls
mcp:
host: mcp.example.invalid
path: /mcp
tlsSecretName: knowledge-platform-mcp-tls
# Only non-sensitive runtime settings belong here. The templates enumerate
# keys explicitly so credentials cannot be added to this ConfigMap by mistake.
config:
appName: enterprise-knowledge-platform
logLevel: INFO
database:
poolSize: 10
maxOverflow: 10
poolRecycleSeconds: 600
poolTimeoutSeconds: 30
connectTimeoutSeconds: 30
ingestion:
maxAttempts: 3
retryBackoffSeconds: 2
retryBackoffMaxSeconds: 60
workerLeaseSeconds: 300
recoveryIntervalSeconds: 30
batchMaxFiles: 20
batchMaxTotalBytes: 209715200
maxUploadBytes: 52428800
objectStorage:
provider: bos
endpoint: https://s3.example.invalid
bucket: knowledge-platform
keyPrefix: knowledge-platform
timeoutSeconds: 30
chunking:
maxTokens: 450
overlapTokens: 75
maxCharacters: 2000
embedding:
provider: qianfan
model: bge-large-zh
endpoint: https://qianfan.baidubce.com/v2/embeddings
timeoutSeconds: 30
vectorStore:
provider: oceanbase
uri: oceanbase.example.invalid:2881
user: knowledge-platform
database: knowledge_platform
table: knowledge_chunks_v1
index: vidx_knowledge_chunks_embedding
batchSize: 100
timeoutSeconds: 30
retrieval:
indexProfileId: demo-bge-large-zh-structure-v1-oceanbase-v1
denseTopN: 20
searchTopK: 5
reranker:
provider: qianfan
model: bce-reranker-base
endpoint: https://qianfan.baidubce.com/v2/rerank
timeoutSeconds: 30
mcp:
publicUrl: https://mcp.example.invalid/mcp
issuerUrl: https://mcp.example.invalid
clientId: workbuddy-enterprise
serviceIdentity: workbuddy-public-knowledge
requiredScope: knowledge:search
allowedHosts:
- mcp.example.invalid
allowedOrigins:
- https://mcp.example.invalid
maxRequestBodyBytes: 1048576
# Optional local dependency. Keep disabled for production and provide
# RABBITMQ_URL through existingSecret for an externally managed broker.
rabbitmq:
enabled: false
existingSecret: knowledge-platform-rabbitmq

View File

@ -11,19 +11,22 @@ Use OpenAPI as executable source of truth once implementation starts. Provider-s
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
版本化为 `/api/v1/*`。
下表描述当前已实现的同步 Demo 契约;它将在 `DEMO-015` 完成时由后文的异步契约替代。
`DEMO-015` 已将单文件上传切换为持久化异步 Job;`DEMO-016` 已增加批量提交和聚合查询。
| Method | Path | 行为 |
|---|---|---|
| `POST` | `/api/v1/documents` | multipart 单文件上传并同步执行完整入库流水线 |
| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
| `POST` | `/api/v1/ingestion-batches` | multipart `files` 多文件接收;返回批次与逐文件状态 |
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新 Job 状态 |
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
| `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 |
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `201` 和 Document;
Demo 同步流水线完成后状态为 `ready`。Document 响应不暴露 `storage_key`、Provider 配置或
索引实现。
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `202`、Document 字段及
`job_id`、`job_status=queued`、`max_attempts`;响应不等待 Parse、Clean、Chunk、Embedding 或
Index。Document 响应不暴露 `storage_key`、Provider 配置或索引实现。
Search 请求:
@ -54,8 +57,9 @@ Search 响应:
```
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
文档不存在返回 `404`,运行时 Provider 未配置返回 `503`。上游 Provider 或持久化异常只返回
经过收敛的 `5xx` 信息,不回显凭证、请求正文或厂商响应。
文档、Job 或批次不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
厂商响应。
## Demo Admin 客户端路由
@ -72,26 +76,56 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
## Planned async ingestion contract(DEMO-015/016,尚未实现)
## 异步入库契约(DEMO-015~016 已实现)
| Method | Path | 行为 |
|---|---|---|
| `POST` | `/api/v1/documents` | 单文件接收;持久化 Source、Document、Job 后返回 `202` |
| `POST` | `/api/v1/documents` | 已实现:单文件接收并返回 Document/Job 标识 |
| `POST` | `/api/v1/ingestion-batches` | multipart 多文件接收,返回批次及逐文件 Document/Job 标识 |
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新状态 |
| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
| `GET` | `/api/v1/jobs/{job_id}` | 已实现:返回持久化 Job 状态 |
单文件与批量上传都不等待解析、Chunk、Embedding 或索引完成。批量响应示意:
单文件响应示意:
```json
{
"id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
"filename": "退款政策.md",
"media_type": "text/markdown",
"size_bytes": 1024,
"status": "uploaded",
"error_message": null,
"created_at": "2026-09-02T12:00:00Z",
"updated_at": "2026-09-02T12:00:00Z",
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
"job_status": "queued",
"max_attempts": 3
}
```
批量字段名为 `files`,成功返回 `202`。响应示意:
```json
{
"batch_id": "d21f8012-4c4e-4bb4-89af-e11c0f9f470e",
"status": "processing",
"total_files": 2,
"accepted_files": 1,
"rejected_files": 1,
"completed_files": 1,
"created_at": "2026-09-02T12:00:00Z",
"items": [
{
"item_index": 0,
"filename": "退款政策.md",
"acceptance_status": "accepted",
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
"status": "queued"
"job_status": "queued",
"stage": null,
"attempt": 0,
"max_attempts": 3,
"error_message": null
}
]
}
@ -99,8 +133,11 @@ Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏
Job 公开状态为 `queued | running | retrying | succeeded | failed | cancelled`,并通过 `stage`
表示 `parsing | cleaning | chunking | embedding | indexing`。错误字段只返回有界、安全摘要。
批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚其他文件。文件数量、单文件大小和
请求总大小均由服务端配置限制。
批次状态为 `processing | completed | partial_failure | failed`;逐文件接收状态为
`pending | accepted | rejected`。批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚
其他文件。文件数量、单文件大小和请求总大小分别由 `BATCH_MAX_FILES`、`MAX_UPLOAD_BYTES`
和 `BATCH_MAX_TOTAL_BYTES` 限制。数量超限返回 `413`;类型、大小或内容不合规作为该文件的
`rejected` 安全结果返回,其他合规文件继续接收。
自动重试是服务端 Job 行为,不要求客户端再次上传,也不新增公开手动重试接口。每次重试前
清理派生数据但保留原始文件和稳定的 Document ID;达到次数上限或清理失败后进入 `failed`。

View File

@ -179,6 +179,10 @@ Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Fail
API 只负责校验并持久化不可变源文件、Document 和 Ingestion Job,然后返回 `202`。批次是
提交与聚合查询边界,不是跨文件数据库事务;每个文件独立排队、执行和失败。
DEMO-016 以 canonical `ingestion_batches` / `ingestion_batch_items` 保存输入顺序和接收引用;
处理阶段、Attempt 与最终结果继续以 `ingestion_jobs` / `ingestion_attempts` 为事实源,批次表
不复制 Worker 状态。
```text
Batch Request
├─ Source A + Document A + Job A ─→ Worker
@ -207,6 +211,9 @@ Celery task 只负责恢复应用上下文并调用 Ingestion Application Servic
清理顺序和业务幂等仍由应用层持有。Broker 消息不是 Job 的 source of truth,Worker 重启或消息
重复投递必须以 OceanBase 中的持久化 Job/Attempt 状态收敛。
DEPLOY-001 将 API/Admin、MCP Gateway 与 Async Worker 构建为三个共享锁定 Python 运行时的
非 root 镜像 target;进程边界与上述模块边界一致,不新增服务或把 Provider 配置烘焙进镜像。
---
### 4.3 文档结构与 Chunk
@ -756,6 +763,11 @@ readiness/liveness、资源限制、非 root 与只读根文件系统、多副
本地开发资产,不能作为集群模板。详细依赖和验收见
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
DEPLOY-002 的运行拓扑将 Celery Worker 与 Celery Beat 恢复调度器拆成独立 Pod:Worker 可横向
扩缩,Beat 固定单副本并使用 `Recreate`,避免每个 Worker 都发布周期恢复消息。二者仍调用同一
应用服务与持久化 Job/Attempt 状态,不形成新业务服务,也不改变 ADR-002。Migration 使用 Helm
`pre-install,pre-upgrade` Hook;失败会在业务 Deployment 更新前终止原子升级。
---
## 17. V1 验收重点

View File

@ -333,14 +333,14 @@ Search Service 或 `/api/v1/search`,用于验证标准 MCP、identity 边界
### DEMO-015 异步入库 Job 与自动重试清理
将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
**已完成。** 将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
删除旧解析产物、Chunk、索引状态和向量数据,但保留不可变原始文件、Document 和 Attempt
历史。清理失败时终止,不在未知脏状态继续执行。通过执行器抽象接入 Celery/RabbitMQ,使
API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重复消息。
### DEMO-016 批量异步上传
在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
**已完成。** 在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
Admin Console 状态展示。批次内文件相互隔离,不包含目录上传、压缩包解包或断点续传。
---
@ -397,8 +397,8 @@ DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
任务拆分;`DEMO-012` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,后续严格按
`DEMO-015`、`DEMO-016`、`DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
任务拆分;`DEMO-012`、`DEMO-015~016` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
后续严格按 `DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
可选项,不因本状态记录自动进入范围。
可选:
@ -547,7 +547,7 @@ docs/demo/DEMO_SCOPE.md
完整 V1 PRD
只允许实现本文件明确列出的 Demo、Architecture、Plan 与 Deploy Task,并且一次只执行一个
Task。当前严格顺序为 DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
Task。当前严格顺序为 DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
当前公共知识库阶段的业务目标保持为:

View File

@ -5,9 +5,10 @@ DEMO-010 在 FastAPI 服务中提供一个轻量、同源的管理客户端。
## 启动与访问
完成运行时配置后启动 API:
完成运行时配置后启动 Worker 与 API:
```bash
make run-worker
make run
```
@ -15,7 +16,7 @@ make run
| 页面 | 地址 | 对应业务接口 |
|---|---|---|
| 文档上传 | `/admin/upload` | `POST /api/v1/documents` |
| 文档上传 | `/admin/upload` | `POST/GET /api/v1/ingestion-batches*` |
| 文档列表与详情 | `/admin/documents` | `GET/DELETE /api/v1/documents*` |
| Search Lab | `/admin/search` | `POST /api/v1/search` |
@ -33,9 +34,9 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
## Demo 操作闭环
1. 在上传页选择一个 PDF、DOCX、TXT 或 Markdown 文件并提交;
2. 等待同步流水线返回 `Ready`,失败时页面显示经过 API 收敛的错误;
3. 在文档列表查看状态和安全元数据;
1. 在上传页选择一个或多个 PDF、DOCX、TXT 或 Markdown 文件并提交;
2. 接口返回 `202` 后,页面自动轮询批次,逐文件显示接收状态、处理阶段、尝试次数和安全错误;
3. 批次进入 `Completed`、`Partial failure` 或 `Failed` 后停止轮询,再在文档列表查看元数据;
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
Rerank Score;
5. 如需清理 Demo 数据,在文档列表确认后直接删除。
@ -45,8 +46,10 @@ DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
- 本任务不包含 ACL/RBAC、审计、异步任务、Document Version、Compiled Knowledge、MCP、
Hybrid/BM25 或完整 V1 Admin Console 导航;
- Console 只轮询当前批次的聚合接口,不读取 Provider/队列细节,也不提供人工重试按钮;
- 批量上传不包含目录上传、压缩包解包、断点续传或跨批次去重;
- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
或完整 V1 Admin Console 导航;
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。
## 验证

View File

@ -0,0 +1,97 @@
# 异步入库与失败恢复说明
## 范围
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker,DEMO-016 在其上增加逐文件
隔离的批量接收和状态聚合。OceanBase 中的
`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
不携带文件内容、Provider 凭据或业务状态。
批次只聚合接收与 Job 状态,不改变每个文件的独立执行和重试边界。
## 运行时配置
以下值必须通过环境变量或部署 Secret 注入,不得提交真实值:
```text
RABBITMQ_URL=amqp://<user>:<password>@<host>:5672/<vhost>
INGESTION_MAX_ATTEMPTS=3
INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
BATCH_MAX_FILES=20
BATCH_MAX_TOTAL_BYTES=209715200
```
Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
RabbitMQ 用户应只具备指定 vhost/queue 的最小权限。
## 启动顺序
```bash
uv run alembic upgrade head
make run-worker
make run
```
`make run-worker` 在当前 Demo 中同时启动 Worker 和恢复调度器。恢复调度器周期扫描:
- 尚未成功投递的 `queued` Job;
- 已到重试时间的 `retrying` Job;
- Worker 丢失后租约已过期的 `running` Job。
任务投递使用 late acknowledgement、worker-lost reject 和单任务 prefetch。重复消息会先锁定
Job 行;有效租约内不会创建第二个 Attempt。租约过期后,旧 Attempt 以安全错误关闭,再按重试
规则恢复。
## 重试与清理
确定性的内容错误(无法解析、清洗后为空、无 Chunk、派生数据身份不一致)直接进入 `failed`。
其余运行时/Provider 错误按指数退避重试,默认最多三次总 Attempt。
第二次及后续 Attempt 创建前严格执行:
```text
OceanBase Vector rows
→ ChunkIndexState
→ Chunk
→ parsed JSON object
→ reset Document processing state
```
不可变源文件、Document、Job 和已结束 Attempt 始终保留。任一清理步骤失败时不创建新 Attempt,
Job 直接以 `retry cleanup failed` 终止,禁止在未知脏状态继续执行。
## API 状态
`POST /api/v1/documents` 成功返回 `202` 和 `job_id`。查询:
```text
GET /api/v1/jobs/{job_id}
```
Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;阶段为
`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
批量提交与查询:
```text
POST /api/v1/ingestion-batches # multipart 字段 files,可重复
GET /api/v1/ingestion-batches/{batch_id}
```
批次创建后逐文件返回 `accepted` 或 `rejected`;已接收文件继续通过上述 Job 状态异步推进。
批次聚合状态为 `processing | completed | partial_failure | failed`。请求文件数超过
`BATCH_MAX_FILES` 时整体返回 `413`;单文件或共享总字节预算超限只拒绝当前文件,不删除此前
已接收文件。默认总预算 200 MiB,且配置不得小于单文件 `MAX_UPLOAD_BYTES`。
## 故障检查
1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;
2. Job 长期为 `running`:检查 Worker,租约到期后恢复调度器会重新投递;
3. Job 为 `retrying`:等待 `next_run_at`,客户端不应重新上传同一文件;
4. Job 为 `failed` 且错误为 `retry cleanup failed`:停止人工重试,先核对向量、关系数据与 BOS
派生对象的一致性;
5. 不得通过日志打印 `RABBITMQ_URL`、数据库 URL、BOS/模型凭据或原始 Provider 响应。

View File

@ -0,0 +1,91 @@
# 生产容器镜像说明
## 范围
DEPLOY-001 使用同一个锁定运行时构建三个独立镜像目标:
| Target | 默认镜像名 | 入口 | 端口 | 镜像健康检查 |
|---|---|---|---|---|
| `api` | `enterprise-knowledge-api` | Uvicorn + API/Admin | `8000` | `GET /healthz` |
| `mcp` | `enterprise-knowledge-mcp` | `ekp-mcp` | `8766` | PID 1 存活 |
| `worker` | `enterprise-knowledge-worker` | `ekp-worker` | 无 | PID 1 存活 |
镜像没有内置数据库迁移、Helm、Ingress 或 Secret 对象;DEPLOY-002 已在独立 Chart 中提供这些
编排能力。API 镜像包含 Alembic 和 migrations,由一次性 Migration Job 覆盖命令后复用。详见
[`KUBERNETES_DEPLOYMENT.md`](KUBERNETES_DEPLOYMENT.md)。
## 可重复构建基线
- Python 基础镜像固定为 `3.12.14-slim-bookworm`,uv 固定为 `0.12.5`,两者同时锁定 manifest
digest;
- Python 依赖只使用提交的 `uv.lock`,以 `uv sync --frozen --no-dev --no-editable` 安装;
- 三个目标共享相同依赖层,不复制不同版本的业务运行时;
- 进程均以固定 UID/GID `10001:10001` 运行,不具备 root 权限;
- `.dockerignore` 排除 `.env`、Git、虚拟环境、缓存、本地存储、测试和开发产物;
- Bake 计划同时声明 `linux/amd64` 与 `linux/arm64`,不下载或打包平台专属模型。
基础镜像 tag/digest 与应用依赖均已固定;镜像发布系统仍应保存本项目镜像的构建 digest,并让
Helm 按 digest 部署。本项目镜像 digest 由实际 Registry 发布产生,不把某个环境的 Registry
地址写入仓库。
## 构建命令
先确认 Docker Buildx builder 支持两个目标平台:
```bash
make images-print
```
构建全部目标:
```bash
IMAGE_REGISTRY=registry.example.internal/knowledge \
IMAGE_TAG=<immutable-release-tag> \
make images
```
只有发布流水线才执行推送:
```bash
IMAGE_REGISTRY=registry.example.internal/knowledge \
IMAGE_TAG=<immutable-release-tag> \
make images-push
```
不要把 Registry 密码放入变量或命令行;构建节点应提前通过其凭据助手登录。禁止使用
`latest` 作为集群部署版本。
## 运行时配置
镜像只内置以下非敏感进程默认值:`APP_ENV=production`、容器监听地址、静态页面绝对路径和
临时缓存目录。OceanBase、BOS、RabbitMQ、Embedding、Reranker 与 MCP Token 必须在容器启动
时由 Secret 注入。镜像构建参数、Layer、标签、健康检查和启动命令都不得包含这些值。
API 的 `/healthz` 只证明进程存活,不读取云端凭据或探测依赖。Kubernetes readiness、资源限制、
只读根文件系统所需临时卷及依赖就绪策略由 DEPLOY-002 Chart 配置。MCP endpoint 需要认证,Worker
没有 HTTP server,因此这两个镜像的 Docker 健康检查只验证主进程;业务可恢复性仍以持久化
Job/Attempt 和 DEPLOY-003 故障验收为准。
## 本地单目标检查
需要加载到本地 Docker 时,可直接构建当前平台的一个 target:
```bash
docker build --target api -f deploy/docker/Dockerfile -t ekp-api:test .
docker build --target mcp -f deploy/docker/Dockerfile -t ekp-mcp:test .
docker build --target worker -f deploy/docker/Dockerfile -t ekp-worker:test .
```
启动时使用 `--env-file` 或编排系统的 Secret 引用,不在命令中展开敏感值。API/Admin smoke test
至少检查 `/healthz`、`/admin/upload` 和 `/api/v1/openapi.json`;MCP 与 Worker 的真实依赖启动
验收在具备 RabbitMQ 和目标 Provider 的受控环境执行。
## DEPLOY-001 构建验收
2026-09-02 已使用 Buildx 分别构建并加载三个 `linux/arm64` target 和三个 `linux/amd64`
target。API/Admin 以非 root 用户完成健康、静态页面和 OpenAPI smoke test;MCP/Worker 完成非
root 安装入口检查。构建历史与容器文件系统检查未发现 `.env`、Git、测试文件或敏感配置。
这次验收只生成本地 `deploy001-*` 测试 tag,没有登录或推送 Registry,因此不产生可供集群
引用的项目镜像 digest。DEPLOY-002 必须使用操作者指定的不可变 Registry tag/digest,不得把
本地测试 tag 当作发布版本。

View File

@ -0,0 +1,143 @@
# Kubernetes / Helm 部署说明
## 1. 交付范围
DEPLOY-002 提供单一 Helm Chart,将以下运行角色部署到已有 Kubernetes 集群:
- API/Admin:默认两个副本,提供 `/healthz`、`/admin/*` 和 `/api/v1/*`;
- MCP Gateway:默认两个副本,提供经 Bearer Token 保护的 `/mcp`;
- Async Worker:默认两个副本,只消费 `ingestion` 队列;
- Recovery Scheduler:固定单副本,以 `Recreate` 策略运行 Celery Beat;
- Migration Job:每次安装/升级前串行执行 `alembic upgrade head`;
- 可选 RabbitMQ:只供开发/演示集群显式启用,生产默认使用外部 Broker。
OceanBase MySQL、BOS、Embedding/Reranker 服务继续使用集群外已有资源。Chart 不创建数据库、
BOS Bucket、域名、证书或云账号,也没有扩大公共知识库阶段的业务范围。
## 2. 前置条件
操作者需要提前准备:
1. 可用的 Kubernetes Namespace 和对应 `kubectl` 上下文;
2. Helm 3/4 与 Kubernetes 客户端;
3. 已推送到目标 Registry 的 API、MCP、Worker 不可变 tag 或 digest;
4. Namespace 内的运行时 Secret、TLS Secret,以及私有 Registry 所需的 ImagePullSecret;
5. 集群到 OceanBase、BOS、RabbitMQ 和模型服务的网络连通性。
Chart 默认引用 `knowledge-platform-runtime`,但不会创建它。该 Secret 至少包含以下键:
```text
DATABASE_URL
RABBITMQ_URL
BOS_ACCESS_KEY_ID
BOS_SECRET_ACCESS_KEY
BAIDU_QIANFAN_API_KEY
OCEANBASE_VECTOR_PASSWORD
MCP_AUTH_TOKEN
```
`RERANKER_API_KEY` 可单独提供;未提供时应用按既有契约复用
`BAIDU_QIANFAN_API_KEY`。`MCP_AUTH_TOKEN` 至少 32 个字符,只具备
`knowledge:search` 服务范围。不要使用 `kubectl --from-literal`、Shell 参数或 Helm
`--set` 传递以上值;应由受控 Secret 管理系统或运维流程预先写入 Namespace。
## 3. 环境 Values
复制 [`values.example.yaml`](../../deploy/helm/environments/values.example.yaml) 到仓库外的受控
路径,填写以下非敏感信息:
- 三个镜像的 Registry 路径与不可变 tag/digest;
- `existingSecret` 和 `imagePullSecrets` 的资源名称;
- OceanBase/BOS 的非敏感 Endpoint、数据库标识和 Bucket;
- Admin/API 与 MCP 的 Ingress Host、TLS Secret 名称;
- 与外部地址一致的 MCP Public URL、Issuer、Allowed Host/Origin;
- 副本、资源、超时和批量上传限制。
`values.schema.json` 拒绝未知顶层字段、`latest` tag、无效 digest、单副本 API/MCP、错误类型和
空的关键引用;模板还会校验 Chunk、检索和入库限制之间的大小关系。Values 只允许非敏感配置,
模板逐项生成 ConfigMap,不提供任意 `extraEnv` 入口,避免把凭据误写进渲染产物。
## 4. 安装与幂等升级
先做离线校验和清单检查:
```bash
make deploy-lint VALUES=/secure/path/environment-values.yaml
make deploy-render VALUES=/secure/path/environment-values.yaml
```
安装或升级:
```bash
NAMESPACE=knowledge-platform \
RELEASE=knowledge-platform \
make deploy VALUES=/secure/path/environment-values.yaml
```
`make deploy` 只接收 Values 文件路径。脚本会先验证 Chart、Namespace、运行时 Secret 及必需
键名、可选 RabbitMQ Secret 键名,以及启用 Ingress 时的两个 TLS Secret 引用,然后执行带失败
回滚和等待语义的 `helm upgrade --install`;脚本对 Helm 3 使用 `--atomic`,对 Helm 4 使用等价
的 `--rollback-on-failure`。它只验证 Secret 资源和键是否存在,不读取或输出 Secret 值。
Migration 是 `pre-install,pre-upgrade` Hook,固定名称配合
`before-hook-creation` 保证同一 Release 内不会并发执行。Helm 等待 Migration 成功后才更新业务
工作负载;失败时 Helm 保留上一版本。完成的 Job 最长保留一天供日志追查,并在下次升级前删除。
安装命令可以重复执行;数据库迁移必须保持 forward-only 与幂等。
## 5. Ingress、探针与 Pod 安全
生产环境启用 Ingress 后,Admin/API 与 MCP 使用独立稳定 HTTPS Host,也可由同一 Ingress
Controller 托管。证书只通过预建 TLS Secret 引用,不进入 Values。
- API 的 startup/readiness/liveness 调用无凭据 `/healthz`;
- MCP 的三类探针使用 TCP 端口,不发送或记录 Token;
- Worker/Scheduler 使用 PID 1 存活探针,业务恢复正确性由持久化 Job/Attempt 保证;
- 全部应用容器固定 UID/GID `10001`、禁止提权、丢弃 Linux capabilities、使用
`RuntimeDefault` seccomp 和只读根文件系统;
- `/tmp` 使用有大小上限的 `emptyDir`,ServiceAccount 不挂载 API Token;
- API/MCP 滚动更新使用 `maxUnavailable: 0`,未 Ready Pod 不进入 Service Endpoint。
`/healthz` 当前只表示 API 进程存活,不探测所有外部 Provider;真实依赖故障、滚动重启和
Worker 恢复行为由 DEPLOY-003 在目标集群验收。
## 6. 可选 Demo RabbitMQ
生产默认 `rabbitmq.enabled=false`,`RABBITMQ_URL` 指向外部受管 Broker。仅开发/演示时可开启
本地子 Chart;它使用单节点 StatefulSet 和 PVC,并要求预建 Secret 包含:
```text
username
password
erlang-cookie
```
同时必须让运行时 Secret 中的 `RABBITMQ_URL` 指向该 Service。子 Chart 使用固定
`rabbitmq:4.1.4-alpine` tag 与 multi-architecture manifest digest;它不具备生产 RabbitMQ 的
高可用、备份或升级保证,禁止作为生产默认值。卸载 Release 时 StatefulSet PVC 默认保留,
删除前必须由数据责任人确认。
## 7. 回滚、卸载与 Secret 轮换
查看历史并回滚时不要输出 Pod 环境变量:
```bash
helm -n knowledge-platform history knowledge-platform
helm -n knowledge-platform rollback knowledge-platform <revision> --wait
```
更新外部 Secret 后,Secret 名称不变;用户和 WorkBuddy 配置无需重新注入。当前应通过受控
滚动重启让 Pod 读取新值,目标集群的无中断验证属于 DEPLOY-003。
卸载只删除本 Release 管理的 Kubernetes 对象:
```bash
helm -n knowledge-platform uninstall knowledge-platform
```
外部 OceanBase、BOS、RabbitMQ、TLS/运行时 Secret 和可选 RabbitMQ PVC 不由卸载命令删除。
## 8. DEPLOY-002 验收边界
本任务完成 Chart lint、双路径渲染、严格 Kubernetes Schema、Values 失败关闭、Secret 泄漏扫描
以及本地集群两次安装/升级验证。目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实
Provider 和远程 WorkBuddy 验收统一属于后续 `DEPLOY-003`,不得提前宣称完成。

View File

@ -2,19 +2,21 @@
## 范围
DEMO-009 将已经完成的同步入库和检索服务公开为版本化 REST API。API 层只负责输入校验、
DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
Job,DEMO-016 增加批量接收与聚合查询。API 层只负责输入校验、
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
本任务不包含 Admin Console、ACL/RBAC、异步任务、软删除或 Hybrid Search。
当前不包含 ACL/RBAC、软删除、目录/压缩包上传、断点续传或 Hybrid Search。
## 启动前提
1. 按 `docs/development/BOS_SETUP.md` 配置 BOS;
2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding;
3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
4. 执行 `uv run alembic upgrade head`;
5. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
6. 执行 `make run`。
4. 按 `docs/development/ASYNC_INGESTION_SETUP.md` 配置 RabbitMQ 并启动 Worker;
5. 执行 `uv run alembic upgrade head`;
6. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
7. 执行 `make run`。
`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时
延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。
@ -25,13 +27,29 @@ Swagger UI:`/api/v1/docs`
## 示例
上传并同步入库:
上传并异步排队:
```bash
curl -F 'file=@./policy.md;type=text/markdown' \
http://127.0.0.1:8000/api/v1/documents
```
使用响应中的 `job_id` 查询进度:
```bash
curl http://127.0.0.1:8000/api/v1/jobs/<job-id>
```
批量上传并查询聚合进度:
```bash
curl -F 'files=@./policy.md;type=text/markdown' \
-F 'files=@./manual.pdf;type=application/pdf' \
http://127.0.0.1:8000/api/v1/ingestion-batches
curl http://127.0.0.1:8000/api/v1/ingestion-batches/<batch-id>
```
检索:
```bash
@ -40,8 +58,8 @@ curl -H 'Content-Type: application/json' \
http://127.0.0.1:8000/api/v1/search
```
Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase Upsert 完成。大文
件或云端调用较慢时,请求耗时会相应增加;异步任务属于 Demo 后置能力。
上传只等待不可变源文件、Document 与 Job 持久化,成功返回 `202`。Worker 后台执行 Parse、
Clean、Chunk、Embedding 和 OceanBase Upsert;客户端通过 Job 状态判断何时可检索。
## 删除一致性
@ -51,6 +69,8 @@ Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase
2. 删除 OceanBase canonical 关系表中的 ChunkIndexState、Chunk 和 Document;
3. 删除解析 JSON 与不可变原始文件。
`queued`、`running` 或 `retrying` Job 对应的 Document 返回 `409`,避免删除与 Worker 并发。
向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口
返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1
应通过可重试清理任务和审计记录强化。

View File

@ -11,6 +11,11 @@ Provide deterministic fakes:
External integrations use separate integration suites/markers.
Async ingestion tests run with an in-process recording dispatcher and deterministic providers. They
must cover persisted Job/Attempt history, safe error summaries, bounded retry, cleanup-before-retry,
active-lease duplicate delivery, expired-lease recovery, immutable-source preservation and terminal
cleanup failure without requiring a live RabbitMQ broker.
E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval.
Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation.

View File

@ -8,14 +8,14 @@
| 能力 | 当前实现 | 结论 |
|---|---|---|
| 单文件上传 | `POST /api/v1/documents` 接收一个 `file`,并在请求线程内同步完成解析、Chunk、Embedding 和索引 | 已实现,但不满足 V1 异步要求 |
| 批量上传 | API 与 Admin Console 都只支持一个文件 | 未实现 |
| 失败补偿 | 上传元数据失败会删除源对象;解析元数据失败会删除本次解析产物;索引失败会删除当前 Document/Profile 的可能残留向量 | 已有局部补偿,不是自动重试 |
| 自动重试 | 没有持久化 Job、Attempt、调度器、退避策略或自动重新执行入口 | 未实现 |
| 重试前清理 | 影子索引重建能清理目标 Profile 的部分写入;正常上传流水线没有统一清理解析产物、Chunk、索引状态和向量的能力 | 未实现 |
| MCP | 已有 MCP 业务契约和可复用 Search Service/REST API,没有可运行 MCP Gateway 或 WorkBuddy 验证 | 未实现 |
| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
| 批量上传 | 批次 API、逐文件 Job 状态和 Admin Console 自动轮询 | DEMO-016 已实现 |
| 失败补偿 | 阶段内补偿 + 重试前清理全部派生数据,保留不可变源和执行历史 | DEMO-015 已实现 |
| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
| MCP | 独立 Gateway、认证边界、`search_knowledge` 与 WorkBuddy 协议级验收 | DEMO-012 已实现 |
因此,现有失败补偿只能降低一次执行留下半成品的概率,不能满足“失败后自动重新执行,并在重新执行前清理旧数据”。
PLAN-001 的异步执行缺口已由 DEMO-015 补齐,批量提交与状态 UI 已由 DEMO-016 补齐。
## 2. Task 归并结论
@ -51,7 +51,7 @@
该 Task 只完成公共知识库入库所需的 Job/Retry 与队列基线,不扩展到完整审计系统、
Compilation、OCR、跨区域高可用或通用企业级 Job 平台。
### DEMO-016:批量异步上传与状态展示(新增)
### DEMO-016:批量异步上传与状态展示(已完成)
建立在 DEMO-015 之上,不与 Job 基础合并:
@ -79,8 +79,8 @@ PLAN-002(已完成)
推荐严格按以下顺序执行并逐项闭环:
1. `PLAN-002`(已完成)
2. `DEMO-015`
3. `DEMO-016`
2. `DEMO-015`(已完成)
3. `DEMO-016`(已完成)
4. `DEPLOY-001`
5. `DEPLOY-002`
6. `DEPLOY-003`

View File

@ -1,12 +1,12 @@
# 项目进度
当前阶段:公共知识库集群交付规划完成,准备实施异步入库
当前阶段:生产容器镜像闭环完成,准备实施 Kubernetes/Helm 一键部署
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014、ARCH-001~ARCH-006、PLAN-001~PLAN-002
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001
进行中:无
阻塞:无
下一任务:`DEMO-015`。随后严格执行 `DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
下一任务:`DEPLOY-002`,完成并合并后执行 `DEPLOY-003`。
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
@ -332,3 +332,95 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
- 验证结果:Markdown 相对链接、敏感值模式和 Git diff 检查通过;Ruff Format/Lint、Mypy、
Alembic Head/离线升级通过;Unit + Integration `165 passed`,RAG Evaluation `5 passed`。
- 下一任务:`DEMO-015`。
### DEMO-015(已完成)
- 新增 OceanBase canonical `ingestion_jobs` / `ingestion_attempts` 与 forward-only migration;
Document 唯一 Job、Attempt 编号上限和单一运行中 Attempt 均由数据库约束保护。
- `POST /api/v1/documents` 改为持久化不可变 Source、Document、Job 后返回 `202`,新增
`GET /api/v1/jobs/{job_id}`;活动 Job 的 Document 删除返回 `409`,避免与 Worker 竞态。
- 新增队列无关的 Dispatcher 与 Job Runner,首个适配器使用 Celery/RabbitMQ;消息只包含 Job
ID,启用 late ack、worker-lost reject、单任务 prefetch 与周期性持久化 Job 恢复扫描。
- 瞬时错误按配置执行指数退避和最多三次总 Attempt;Worker 租约有效时重复投递不创建新
Attempt,租约过期后关闭旧 Attempt 并恢复。错误记录为固定安全摘要,不保存 Provider 正文。
- 第二次及后续 Attempt 前依次清理向量、ChunkIndexState、Chunk 与 parsed JSON,再从不可变
源文件重建;清理失败直接终止且不创建新 Attempt。Job 创建失败会补偿 Document 与源对象。
- Admin Console 仅将上传提示调整为“已接收并进入队列”;批量选择、Job 轮询和逐文件状态 UI
没有提前实现,继续属于 `DEMO-016`。
- 验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、依赖锁、Alembic Head/离线 MySQL SQL
与 SQLite schema drift 均通过;Unit + Integration `179 passed`,RAG Evaluation `5 passed`;
Worker 入口确认注册 `ekp.ingestion.run` 与 `ekp.ingestion.recover`。
- 环境限制:当前工作站没有 Docker/RabbitMQ,也没有持久化注入目标 OceanBase URL,因此未做
真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
- 下一任务:`DEMO-016`。
### DEMO-016(已完成)
- 新增 canonical `ingestion_batches` / `ingestion_batch_items` 及 forward-only migration,保存
输入顺序、逐文件接收结果与 Document/Job 引用;异步阶段和最终结果仍以 Job 为事实源,
批次不是跨文件事务。
- 新增 `POST /api/v1/ingestion-batches` 与 `GET /api/v1/ingestion-batches/{batch_id}`;批量
请求字段为 `files`,成功返回 `202`、聚合状态和逐文件接收/Job 状态。类型、大小或内容错误
只拒绝当前文件,错误映射为固定安全摘要,其他文件继续接收。
- 新增 `BATCH_MAX_FILES` 与 `BATCH_MAX_TOTAL_BYTES`,复用 `MAX_UPLOAD_BYTES` 约束单文件;
上传流共享总字节预算,不把整批文件读入内存。数量超限在创建批次前返回 `413`。
- Admin Console 支持多选提交,显示当前批次的接收数量、完成数量、逐文件阶段、Attempt 与安全
错误,并每 1.5 秒自动轮询;进入 `completed`、`partial_failure` 或 `failed` 后停止轮询。
DOM 数据继续只通过文本节点渲染。
- 集成测试覆盖顺序稳定、部分拒绝不回滚、数量/单文件/批次总大小、全成功/部分失败聚合、
未知批次、OpenAPI 和前端契约;迁移 drift 增加两张新表。Ruff Format/Lint、Mypy、
JavaScript 语法、依赖锁、Alembic Head/离线升级均通过;Unit + Integration `183 passed`,
RAG Evaluation `5 passed`。
- 浏览器验收确认桌面布局和 390px 窄屏布局正常、无根页面横向溢出、无控制台警告或错误;
本地页面不注入云端凭据,批次结果行为由确定性 API 集成测试验证。
- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续
复用 DEMO-015,Accepted ADR 未变更。
- 下一任务:`DEPLOY-001`。
### DEPLOY-001(已完成)
- 新增共享生产 Dockerfile 的 `api`、`mcp`、`worker` 三个 target,使用精确 Python/uv 版本、
frozen production dependencies、固定非 root UID/GID、exec-form 启动命令和逐 target 健康检查。
- 新增 amd64/arm64 Buildx Bake 计划、镜像构建/渲染/推送 Make 入口和 `.dockerignore`;构建上下文
排除凭据、本地状态、缓存、测试与开发产物,镜像不包含环境 Endpoint 或 Secret。
- Admin 静态目录新增类型化运行时覆盖,使非 editable 安装后的 API 镜像仍能提供同源页面;
源码运行继续使用既有默认目录。
- 新增不依赖 Docker daemon 的镜像契约测试与中文运行文档;基础 Python/uv 镜像同时固定精确
tag 与 manifest digest,Registry 地址和本项目发布 digest 继续由部署环境提供。
- 使用 Docker Buildx `0.36.1` + Colima 分别实际构建并加载全部三个 `linux/arm64` 和三个
`linux/amd64` target;最终 digest 固定版本再次构建命中相同业务 layer。没有执行 Registry
push,也没有在构建参数、history 或镜像文件系统中发现凭据。
- API 镜像以 `10001:10001` 启动并进入 healthy,`/healthz`、`/admin/upload`、OpenAPI 及
DEMO-016 批量路径 smoke test 通过;MCP/Worker 镜像以同一非 root 用户完成安装入口导入,
默认 CMD 与 HEALTHCHECK 元数据核对通过。真实 Provider/Broker 启动仍属于 DEPLOY-003。
- Ruff Format/Lint、Mypy、Docker Bake 渲染、JavaScript/依赖锁/Alembic 与完整测试门禁通过;
Unit + Integration `187 passed`,RAG Evaluation `5 passed`。下一任务:`DEPLOY-002`。
### DEPLOY-002(已完成)
- 新增 `deploy/helm/knowledge-platform` Helm Chart,部署 API/Admin、MCP Gateway、Async Worker、
单实例 Recovery Scheduler、串行 Migration Job、Service、Ingress/TLS 和可选 Demo RabbitMQ;
API/MCP 默认各两个副本,Worker 可独立扩缩,Beat 使用 `Recreate` 避免重复调度。
- 新增严格 `values.schema.json` 与仓库外环境 Values 示例,拒绝未知顶层字段、`latest` tag、
非法 digest、API/MCP 单副本和错误类型;跨字段模板校验继续约束批量大小、重试、Chunk 和
Top-K 关系。应用和可选 RabbitMQ 镜像均支持 digest,Demo RabbitMQ 已固定 manifest digest。
- 敏感配置只通过预建 `existingSecret` 注入;ConfigMap 仅渲染显式白名单内的非敏感字段,
没有任意 `extraEnv` 或 Secret 对象。`scripts/deploy.sh` 在部署前校验 Values、Namespace、运行时
Secret/必需键、可选 RabbitMQ Secret/键和启用 Ingress 时的 TLS Secret 引用,不读取 Secret
值;Helm 3/4 分别使用等价的原子失败回滚参数。
- Migration 使用 `pre-install,pre-upgrade` Hook、固定名称和 `before-hook-creation`,安装/升级在
Migration 成功前不更新业务 Pod。应用容器固定非 root UID/GID、只读根文件系统、丢弃全部
capabilities、禁用 ServiceAccount Token,并为临时目录、RabbitMQ 日志和 Beat 状态提供
`emptyDir`;API/MCP/Worker/Scheduler 均配置启动和存活/就绪探针及资源 requests/limits。
- 本机使用 K3s `v1.35.0+k3s1`、Helm `4.2.4` 和 DEPLOY-001 ARM64 镜像完成真实安装;Migration
成功,API/MCP 为 `2/2`,Worker、Scheduler 与可选 RabbitMQ 全部 Ready。首次安装暴露的
RabbitMQ 只读日志目录和 Beat 调度文件权限问题已修复,随后相同 Values 连续升级至 Revision
3~5 均成功;API `/healthz` 和 Admin 上传页 smoke test 通过。
- 默认、环境示例和 RabbitMQ 三种 Helm lint 均通过;含 Ingress/TLS/RabbitMQ 的 14 个最终
资源通过 kubeconform strict 校验。仓库、Helm manifest、Pod 日志三路 Secret 值扫描均为 `0`;
Ruff Format/Lint、Mypy、Alembic、Bash 语法与完整门禁通过,Unit + Integration `195 passed`,
RAG Evaluation `5 passed`。
- 目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实外部依赖监测与远程 WorkBuddy
仍属于 `DEPLOY-003`;本任务没有把这些项提前标记完成,也没有改变 Accepted ADR 或公共知识库
范围。下一任务:`DEPLOY-003`。

View File

@ -82,18 +82,21 @@ Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务
### DEMO-016
基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console 状态展示。
**已完成。** 基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console
状态展示;下一交付门槛为 DEPLOY-001 的生产镜像。
### DEPLOY-001
交付 API/Admin、MCP Gateway、Async Worker 的生产容器镜像与启动约定;镜像使用固定依赖、
**已完成。** 交付 API/Admin、MCP Gateway、Async Worker 的生产容器镜像与启动约定;镜像使用固定依赖、
非 root 用户、健康检查和多架构可构建基线,不内置凭据。
### DEPLOY-002
交付 Kubernetes/Helm Chart、环境 Values Schema、`existingSecret`、Migration Job、Ingress/TLS、
Pod 安全、资源限制、探针和 `make deploy` 入口。现有 `docker-compose.dev.yml` 仍仅作为旧的
本地开发文件,因其 PostgreSQL/MinIO 假设与 ADR-012 不一致,不得作为集群交付模板。
**已完成。** 交付 Kubernetes/Helm Chart、严格环境 Values Schema、`existingSecret`、串行
Migration Hook、Ingress/TLS、Pod 安全、资源限制、探针和 `make deploy` 入口;Worker 与单实例
恢复调度器已分离。现有 `docker-compose.dev.yml` 仍仅作为旧的本地开发文件,因其
PostgreSQL/MinIO 假设与 ADR-012 不一致,不得作为集群交付模板。下一交付门槛为 DEPLOY-003
的目标集群可用性与远程验收。
### DEPLOY-003

View File

@ -28,10 +28,10 @@ validation. Provider credentials and internal endpoints are runtime inputs and n
- [x] PLAN-001 MCP 与异步批量入库现状审计、任务拆分和契约基线
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
- [ ] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
- [ ] DEMO-016 Batch async upload API + per-file Job status + Admin Console
- [ ] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
- [ ] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
- [x] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
- [x] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance
Dependencies and acceptance boundaries are defined in

View File

@ -16,10 +16,10 @@
“单一公共知识可见域、企业受控网络、持续可用和一键集群部署”推进,不提前实现用户/权限隔离:
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
2. `DEMO-015`:持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
3. `DEMO-016`:批量异步上传、批次/Job 查询和 Admin Console 状态展示;
4. `DEPLOY-001`:API/Admin、MCP Gateway、Async Worker 生产镜像;
5. `DEPLOY-002`:Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
4. `DEPLOY-001`(已完成):API/Admin、MCP Gateway、Async Worker 生产镜像;
5. `DEPLOY-002`(已完成):Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。
严格依赖为:

View File

@ -465,6 +465,16 @@ Document processing 必须异步执行。
源文件与 Document/Job 成功持久化后返回 `202 Accepted`。批量上传中的每个文件拥有独立的
Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批次内其他已接收文件。
当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、
断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。
公共知识库集群交付使用分别可扩缩的 API/Admin、MCP Gateway 和 Async Worker 生产镜像;
镜像只包含锁定代码与非敏感默认值,环境凭据必须在部署时注入。
集群编排由 Helm Chart 统一管理:API/MCP 默认多副本,Worker 与单实例恢复调度器分离;敏感值
只引用预建 Kubernetes Secret,数据库 Migration 在安装/升级前串行完成。Ingress/TLS、探针、
资源限制和 Pod 安全默认值属于部署契约,不改变 REST/MCP 业务接口。
**REQ-INGEST-004**
展示处理阶段:

View File

@ -17,3 +17,11 @@ defaults, and primary keys while excluding those three falsely reported object
types. Full constraint comparison still runs against SQLite in integration
tests; target-cluster checks separately verify HNSW/PRE-FILTER and the
single-active generated unique index.
DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
the latter preserves numbered history and uses a generated nullable active marker plus a unique index
to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.
DEMO-016 adds `ingestion_batches` and `ingestion_batch_items`. They retain input order and per-file
acceptance references while Job rows remain the processing state source of truth; a batch is an aggregate,
not a cross-file transaction.

View File

@ -0,0 +1,174 @@
"""Create durable ingestion jobs and attempt history.
Revision ID: 20260902_0007
Revises: 20260902_0006
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0007"
down_revision: str | None = "20260902_0006"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Persist queue-independent Job state and bounded execution attempts."""
op.create_table(
"ingestion_jobs",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("document_id", sa.Uuid(), nullable=False),
sa.Column(
"status",
sa.Enum(
"queued",
"running",
"retrying",
"succeeded",
"failed",
"cancelled",
name="ingestion_job_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column(
"stage",
sa.Enum(
"parsing",
"cleaning",
"chunking",
"embedding",
"indexing",
name="ingestion_stage",
native_enum=False,
create_constraint=False,
),
nullable=True,
),
sa.Column("attempt_count", sa.Integer(), nullable=False),
sa.Column("max_attempts", sa.Integer(), nullable=False),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
sa.CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
sa.CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
sa.CheckConstraint(
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
name="status_valid",
),
sa.CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
sa.ForeignKeyConstraint(
["document_id"],
["documents.id"],
name="fk_ingestion_jobs_document_id_documents",
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_jobs"),
sa.UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
)
op.create_index(
"ix_ingestion_jobs_status_next_run",
"ingestion_jobs",
["status", "next_run_at"],
unique=False,
)
op.create_table(
"ingestion_attempts",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("job_id", sa.Uuid(), nullable=False),
sa.Column("attempt_number", sa.Integer(), nullable=False),
sa.Column(
"status",
sa.Enum(
"running",
"succeeded",
"failed",
name="ingestion_attempt_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column(
"stage",
sa.Enum(
"parsing",
"cleaning",
"chunking",
"embedding",
"indexing",
name="ingestion_attempt_stage",
native_enum=False,
create_constraint=False,
),
nullable=True,
),
sa.Column("retryable", sa.Boolean(), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column(
"active_marker",
sa.Integer(),
sa.Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
nullable=True,
),
sa.Column(
"started_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
sa.CheckConstraint(
"status IN ('running', 'succeeded', 'failed')",
name="status_valid",
),
sa.CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
sa.ForeignKeyConstraint(
["job_id"],
["ingestion_jobs.id"],
name="fk_ingestion_attempts_job_id_ingestion_jobs",
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_attempts"),
sa.UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
)
op.create_index(
"uq_ingestion_attempts_active",
"ingestion_attempts",
["job_id", "active_marker"],
unique=True,
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -0,0 +1,109 @@
"""Create ingestion batch aggregation metadata.
Revision ID: 20260902_0008
Revises: 20260902_0007
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0008"
down_revision: str | None = "20260902_0007"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Persist ordered per-file acceptance without making a batch transactional."""
op.create_table(
"ingestion_batches",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("total_files", sa.Integer(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.CheckConstraint("total_files > 0", name="total_files_positive"),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batches"),
)
op.create_table(
"ingestion_batch_items",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("batch_id", sa.Uuid(), nullable=False),
sa.Column("item_index", sa.Integer(), nullable=False),
sa.Column("filename", sa.String(length=255), nullable=False),
sa.Column(
"acceptance_status",
sa.Enum(
"pending",
"accepted",
"rejected",
name="batch_item_acceptance_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column("document_id", sa.Uuid(), nullable=True),
sa.Column("job_id", sa.Uuid(), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.CheckConstraint("item_index >= 0", name="item_index_non_negative"),
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
sa.CheckConstraint(
"acceptance_status IN ('pending', 'accepted', 'rejected')",
name="acceptance_status_valid",
),
sa.ForeignKeyConstraint(
["batch_id"],
["ingestion_batches.id"],
name="fk_ingestion_batch_items_batch_id_ingestion_batches",
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["document_id"],
["documents.id"],
name="fk_ingestion_batch_items_document_id_documents",
ondelete="SET NULL",
),
sa.ForeignKeyConstraint(
["job_id"],
["ingestion_jobs.id"],
name="fk_ingestion_batch_items_job_id_ingestion_jobs",
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batch_items"),
)
op.create_index(
"uq_ingestion_batch_items_order",
"ingestion_batch_items",
["batch_id", "item_index"],
unique=True,
)
op.create_index(
"uq_ingestion_batch_items_job_id",
"ingestion_batch_items",
["job_id"],
unique=True,
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -25,6 +25,7 @@ class Settings(BaseSettings):
api_host: str = "0.0.0.0"
api_port: int = Field(default=8000, ge=1, le=65535)
api_prefix: str = "/api/v1"
admin_web_root: Path | None = None
# SQLAlchemy URLs commonly embed a password, so configuration rendering
# must mask this value and unwrap it only at the database driver boundary.
database_url: SecretStr = SecretStr("mysql+pymysql://localhost:2881/ekp")
@ -33,6 +34,14 @@ class Settings(BaseSettings):
database_pool_recycle_seconds: int = Field(default=600, ge=1)
database_pool_timeout_seconds: float = Field(default=30.0, gt=0)
database_connect_timeout_seconds: int = Field(default=30, ge=1)
rabbitmq_url: SecretStr | None = None
ingestion_max_attempts: int = Field(default=3, ge=1, le=10)
ingestion_retry_backoff_seconds: int = Field(default=2, ge=1, le=3600)
ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
batch_max_files: int = Field(default=20, ge=1, le=100)
batch_max_total_bytes: int = Field(default=200 * 1024 * 1024, gt=0)
object_storage_provider: Literal["local", "bos"] = "bos"
object_storage_path: Path = Path("storage")
bos_endpoint: str | None = None
@ -91,6 +100,19 @@ class Settings(BaseSettings):
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
if self.search_top_k > self.dense_top_n:
raise ValueError("search_top_k must not exceed dense_top_n")
if self.ingestion_retry_backoff_max_seconds < self.ingestion_retry_backoff_seconds:
raise ValueError(
"ingestion_retry_backoff_max_seconds must be at least the initial backoff"
)
longest_provider_timeout = max(
self.bos_timeout_seconds,
self.embedding_timeout_seconds,
self.oceanbase_vector_timeout_seconds,
)
if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
if self.batch_max_total_bytes < self.max_upload_bytes:
raise ValueError("batch_max_total_bytes must be at least max_upload_bytes")
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
raise ValueError("mcp_path must start with '/' and must not end with '/'")
if not self.mcp_allowed_hosts:

View File

@ -1,5 +1,6 @@
"""Application services for building and persisting document chunks."""
from collections.abc import Callable
from dataclasses import dataclass
from hashlib import sha256
from uuid import UUID, uuid5
@ -11,7 +12,7 @@ from ekp_core.chunking import ChunkDraft, ChunkingStrategy
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import deserialize_parsed_document
from ekp_domain import Chunk, Document, DocumentStatus, ParsedDocument
from ekp_domain import Chunk, Document, DocumentStatus, IngestionStage, ParsedDocument
from ekp_object_storage import BinarySource, ObjectStorage
MAX_CHUNKING_ERROR_LENGTH = 2000
@ -54,10 +55,16 @@ class DocumentChunkingService:
self._storage = storage
self._strategy = strategy
def process(self, document_id: UUID) -> tuple[Chunk, ...]:
"""Run DEMO-005 while reserving READY for the later index stage."""
def process(
self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> tuple[Chunk, ...]:
"""Build chunks while optionally reporting the durable Job stage."""
filename, parsed_key = self._start_chunking(document_id)
try:
if stage_callback is not None:
stage_callback(IngestionStage.CHUNKING)
with self._storage.open(parsed_key) as source:
parsed = deserialize_parsed_document(source.read())
if parsed.document_id != document_id or parsed.filename != filename:

View File

@ -1,5 +1,6 @@
"""Embedding and vector-index orchestration for canonical document chunks."""
from collections.abc import Callable
from dataclasses import dataclass
from uuid import UUID, uuid5
@ -7,7 +8,14 @@ from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService
from ekp_domain import Chunk, ChunkIndexState, Document, DocumentStatus, IndexProfile
from ekp_domain import (
Chunk,
ChunkIndexState,
Document,
DocumentStatus,
IndexProfile,
IngestionStage,
)
from ekp_embedding import EmbeddingProvider
from ekp_object_storage import BinarySource
from ekp_vector_store import VectorRecord, VectorStore
@ -67,12 +75,16 @@ class DocumentIndexingService:
self._vector_store = vector_store
self._index_profile = index_profile
def process(self, document_id: UUID) -> DocumentIndexingResult:
"""Index all chunks for one profile, compensating any partial provider write."""
def process(
self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> DocumentIndexingResult:
"""Embed/index chunks while optionally reporting durable Job stages."""
chunks = self._start_indexing(document_id)
upsert_attempted = False
try:
vectors = self._embed_chunks(chunks)
vectors = self._embed_chunks(chunks, stage_callback)
records = tuple(
VectorRecord(
index_record_id=uuid5(
@ -93,6 +105,8 @@ class DocumentIndexingService:
)
for chunk, vector in zip(chunks, vectors, strict=True)
)
if stage_callback is not None:
stage_callback(IngestionStage.INDEXING)
upsert_attempted = True
self._vector_store.upsert(records)
self._record_success(document_id, chunks)
@ -152,13 +166,19 @@ class DocumentIndexingService:
session.commit()
return chunks
def _embed_chunks(self, chunks: tuple[Chunk, ...]) -> tuple[tuple[float, ...], ...]:
"""Respect provider batch limits and validate every returned vector."""
def _embed_chunks(
self,
chunks: tuple[Chunk, ...],
stage_callback: Callable[[IngestionStage], None] | None,
) -> tuple[tuple[float, ...], ...]:
"""Embed bounded batches and renew an optional Job lease before each call."""
vectors: list[tuple[float, ...]] = []
batch_size = self._embedding_provider.max_batch_size
if batch_size <= 0:
raise EmbeddingResultError("embedding provider has an invalid batch size")
for start in range(0, len(chunks), batch_size):
if stage_callback is not None:
stage_callback(IngestionStage.EMBEDDING)
batch = chunks[start : start + batch_size]
embedded = self._embedding_provider.embed([chunk.content for chunk in batch])
if len(embedded) != len(batch):

View File

@ -5,7 +5,7 @@ from uuid import UUID
from sqlalchemy import delete, select
from sqlalchemy.orm import Session, sessionmaker
from ekp_domain import Chunk, ChunkIndexState, Document
from ekp_domain import Chunk, ChunkIndexState, Document, IngestionJob, IngestionJobStatus
from ekp_object_storage import ObjectStorage
from ekp_vector_store import VectorStore
@ -18,6 +18,10 @@ class DocumentArtifactCleanupError(RuntimeError):
"""Raised when deleted metadata leaves an external storage artifact behind."""
class DocumentIngestionActiveError(RuntimeError):
"""Raised when direct deletion would race an active asynchronous Job."""
class DocumentManagementService:
"""Read canonical documents and coordinate the Demo's direct deletion.
@ -63,6 +67,22 @@ class DocumentManagementService:
document = session.get(Document, document_id)
if document is None:
raise DocumentNotFoundError(str(document_id))
active_job_id = session.scalar(
select(IngestionJob.id)
.where(
IngestionJob.document_id == document_id,
IngestionJob.status.in_(
(
IngestionJobStatus.QUEUED,
IngestionJobStatus.RUNNING,
IngestionJobStatus.RETRYING,
)
),
)
.limit(1)
)
if active_job_id is not None:
raise DocumentIngestionActiveError(str(document_id))
source_keys = tuple(
key

View File

@ -1,5 +1,6 @@
"""Application services for parsing and persisting derived document content."""
from collections.abc import Callable
from io import BytesIO
from uuid import UUID
@ -8,7 +9,7 @@ from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_cleaner import clean_parsed_document
from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import serialize_parsed_document
from ekp_domain import Document, DocumentStatus, ParsedDocument
from ekp_domain import Document, DocumentStatus, IngestionStage, ParsedDocument
from ekp_object_storage import BinarySource, ObjectStorage
from ekp_parser import DocumentParseError, Parser
@ -48,15 +49,23 @@ class DocumentParsingService:
self._storage = storage
self._parser = parser
def process(self, document_id: UUID) -> ParsedDocument:
"""Run DEMO-004 synchronously and persist success or failure state."""
def process(
self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> ParsedDocument:
"""Parse and clean while optionally reporting durable Job stage boundaries."""
filename, source_key = self._start_processing(document_id)
parsed_key = f"parsed/{document_id}/document.json"
parsed_object_stored = False
try:
if stage_callback is not None:
stage_callback(IngestionStage.PARSING)
with self._storage.open(source_key) as source:
parsed = self._parser.parse(document_id, filename, source)
if stage_callback is not None:
stage_callback(IngestionStage.CLEANING)
cleaned = clean_parsed_document(parsed)
if not cleaned.blocks:
raise EmptyParsedDocumentError(

View File

@ -1,8 +1,9 @@
"""Validated, transactional source-document upload application service."""
from pathlib import Path
from uuid import uuid4
from uuid import UUID, uuid4
from sqlalchemy import delete
from sqlalchemy.orm import Session, sessionmaker
from ekp_domain import Document
@ -132,6 +133,26 @@ class DocumentUploadService:
return document
def rollback_acceptance(self, document_id: UUID) -> None:
"""Compensate a just-uploaded source when durable Job creation fails.
This narrow method is only valid before parsing starts; normal document
deletion remains coordinated by ``DocumentManagementService``.
"""
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
return
storage_key = document.storage_key
session.execute(delete(Document).where(Document.id == document.id))
session.commit()
try:
self._storage.delete(storage_key)
except Exception as cleanup_error:
raise UploadRollbackError(
f"job creation failed and source cleanup failed for {storage_key}"
) from cleanup_error
def _validate_filename(filename: str) -> str:
"""Accept a plain basename that can be traced without path ambiguity."""

View File

@ -0,0 +1,270 @@
"""Batch submission and aggregate status over independent ingestion Jobs."""
from dataclasses import dataclass
from enum import StrEnum
from typing import Protocol
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_upload import (
DocumentTooLargeError,
DocumentUploadError,
EmptyDocumentError,
MediaTypeMismatchError,
UnsupportedDocumentTypeError,
)
from ekp_core.ingestion_jobs import AsyncIngestionSubmissionService
from ekp_domain import (
BatchItemAcceptanceStatus,
IngestionBatch,
IngestionBatchItem,
IngestionJob,
IngestionJobStatus,
)
from ekp_object_storage import BinarySource
SAFE_UNSUPPORTED_ERROR = "unsupported document type or media type"
SAFE_FILE_SIZE_ERROR = "document exceeds the configured upload limit"
SAFE_BATCH_SIZE_ERROR = "batch exceeds the configured total upload limit"
SAFE_EMPTY_ERROR = "document must contain content"
SAFE_ACCEPTANCE_ERROR = "document could not be accepted"
class BatchNotFoundError(LookupError):
"""Raised when a requested ingestion batch does not exist."""
class BatchFileCountLimitError(ValueError):
"""Raised before persistence when the multipart file count is invalid."""
class BatchAggregateStatus(StrEnum):
"""Computed batch state derived from item acceptance and durable Jobs."""
PROCESSING = "processing"
COMPLETED = "completed"
PARTIAL_FAILURE = "partial_failure"
FAILED = "failed"
class BatchTotalSizeLimitError(ValueError):
"""Raised while streaming the file that crosses the shared batch budget."""
class _ReadableSource(Protocol):
def read(self, size: int = -1, /) -> bytes:
"""Read bytes from an uploaded file."""
@dataclass(frozen=True, slots=True)
class BatchSource:
"""Provider-neutral input for one ordered multipart file."""
filename: str
media_type: str | None
source: _ReadableSource
@dataclass(frozen=True, slots=True)
class BatchItemView:
"""Detached batch item plus its current optional Job state."""
item: IngestionBatchItem
job: IngestionJob | None
@dataclass(frozen=True, slots=True)
class BatchView:
"""Detached aggregate used by REST without exposing persistence sessions."""
batch: IngestionBatch
items: tuple[BatchItemView, ...]
@property
def status(self) -> BatchAggregateStatus:
"""Aggregate terminal results without copying Job state into batch rows."""
if any(
view.item.acceptance_status == BatchItemAcceptanceStatus.PENDING
or (
view.item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED
and (
view.job is None
or view.job.status
in {
IngestionJobStatus.QUEUED,
IngestionJobStatus.RUNNING,
IngestionJobStatus.RETRYING,
}
)
)
for view in self.items
):
return BatchAggregateStatus.PROCESSING
succeeded = sum(
view.job is not None and view.job.status == IngestionJobStatus.SUCCEEDED
for view in self.items
)
failed = len(self.items) - succeeded
if failed == 0:
return BatchAggregateStatus.COMPLETED
if succeeded == 0:
return BatchAggregateStatus.FAILED
return BatchAggregateStatus.PARTIAL_FAILURE
class _BatchByteBudget:
"""Count bytes across independently streamed files without buffering them."""
def __init__(self, limit_bytes: int) -> None:
self._limit_bytes = limit_bytes
self._consumed_bytes = 0
def wrap(self, source: _ReadableSource) -> BinarySource:
return _BatchLimitedSource(source, self)
def consume(self, size_bytes: int) -> None:
self._consumed_bytes += size_bytes
if self._consumed_bytes > self._limit_bytes:
raise BatchTotalSizeLimitError(SAFE_BATCH_SIZE_ERROR)
class _BatchLimitedSource:
"""Apply one shared request budget through the ObjectStorage stream contract."""
def __init__(self, source: _ReadableSource, budget: _BatchByteBudget) -> None:
self._source = source
self._budget = budget
def read(self, size: int = -1, /) -> bytes:
chunk = self._source.read(size)
self._budget.consume(len(chunk))
return chunk
class IngestionBatchService:
"""Accept files independently and persist ordered aggregate status."""
def __init__(
self,
session_factory: sessionmaker[Session],
submissions: AsyncIngestionSubmissionService,
*,
max_files: int,
max_total_bytes: int,
) -> None:
if max_files <= 0 or max_total_bytes <= 0:
raise ValueError("batch limits must be positive")
self._session_factory = session_factory
self._submissions = submissions
self._max_files = max_files
self._max_total_bytes = max_total_bytes
def submit(self, sources: tuple[BatchSource, ...]) -> BatchView:
"""Create a batch and independently accept or reject every ordered file."""
if not sources or len(sources) > self._max_files:
raise BatchFileCountLimitError(
f"batch must contain between 1 and {self._max_files} files"
)
batch_id, item_ids = self._create_pending_batch(sources)
budget = _BatchByteBudget(self._max_total_bytes)
for item_id, source in zip(item_ids, sources, strict=True):
try:
submission = self._submissions.submit(
source.filename,
source.media_type,
budget.wrap(source.source),
)
except Exception as error:
self._record_rejected(item_id, _safe_acceptance_error(error))
continue
self._record_accepted(
item_id,
document_id=submission.document.id,
job_id=submission.job.id,
)
return self.get(batch_id)
def get(self, batch_id: UUID) -> BatchView:
"""Return ordered items and fresh Job state for polling clients."""
with self._session_factory() as session:
batch = session.get(IngestionBatch, batch_id)
if batch is None:
raise BatchNotFoundError(str(batch_id))
items = tuple(
session.scalars(
select(IngestionBatchItem)
.where(IngestionBatchItem.batch_id == batch_id)
.order_by(IngestionBatchItem.item_index)
).all()
)
views = tuple(
BatchItemView(
item=item,
job=session.get(IngestionJob, item.job_id) if item.job_id is not None else None,
)
for item in items
)
return BatchView(batch=batch, items=views)
def _create_pending_batch(
self, sources: tuple[BatchSource, ...]
) -> tuple[UUID, tuple[UUID, ...]]:
"""Persist every input position before starting external storage writes."""
with self._session_factory() as session:
batch = IngestionBatch(total_files=len(sources))
session.add(batch)
session.flush()
items = tuple(
IngestionBatchItem(
batch_id=batch.id,
item_index=index,
filename=_display_filename(source.filename),
)
for index, source in enumerate(sources)
)
session.add_all(items)
session.commit()
return batch.id, tuple(item.id for item in items)
def _record_accepted(self, item_id: UUID, *, document_id: UUID, job_id: UUID) -> None:
with self._session_factory() as session:
item = session.get(IngestionBatchItem, item_id)
if item is None:
raise BatchNotFoundError(str(item_id))
item.acceptance_status = BatchItemAcceptanceStatus.ACCEPTED
item.document_id = document_id
item.job_id = job_id
item.error_message = None
session.commit()
def _record_rejected(self, item_id: UUID, error_message: str) -> None:
with self._session_factory() as session:
item = session.get(IngestionBatchItem, item_id)
if item is None:
raise BatchNotFoundError(str(item_id))
item.acceptance_status = BatchItemAcceptanceStatus.REJECTED
item.error_message = error_message
session.commit()
def _display_filename(filename: str) -> str:
"""Store a bounded safe label even when the upload filename is invalid."""
display = filename.replace("\x00", "<EFBFBD>").strip()
return display[:255] or "<unnamed>"
def _safe_acceptance_error(error: Exception) -> str:
"""Map per-file failures without persisting exception text or credentials."""
if isinstance(error, (UnsupportedDocumentTypeError, MediaTypeMismatchError)):
return SAFE_UNSUPPORTED_ERROR
if isinstance(error, DocumentTooLargeError):
return SAFE_FILE_SIZE_ERROR
if isinstance(error, BatchTotalSizeLimitError):
return SAFE_BATCH_SIZE_ERROR
if isinstance(error, EmptyDocumentError):
return SAFE_EMPTY_ERROR
if isinstance(error, DocumentUploadError):
return SAFE_ACCEPTANCE_ERROR
return SAFE_ACCEPTANCE_ERROR

View File

@ -0,0 +1,588 @@
"""Durable asynchronous ingestion orchestration and retry cleanup."""
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from enum import StrEnum
from math import ceil
from typing import Protocol
from uuid import UUID
from sqlalchemy import delete, select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_chunking import (
DocumentAlreadyChunkedError,
DocumentChunkingService,
DocumentNotParsedError,
EmptyChunkSetError,
ParsedDocumentMismatchError,
)
from ekp_core.document_chunking import (
DocumentNotFoundError as ChunkingDocumentNotFoundError,
)
from ekp_core.document_indexing import (
DocumentAlreadyIndexedError,
DocumentHasNoChunksError,
DocumentIndexingService,
)
from ekp_core.document_indexing import (
DocumentNotFoundError as IndexingDocumentNotFoundError,
)
from ekp_core.document_parsing import (
DocumentAlreadyParsedError,
DocumentParsingService,
EmptyParsedDocumentError,
)
from ekp_core.document_parsing import (
DocumentNotFoundError as ParsingDocumentNotFoundError,
)
from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import ParsedDocumentArtifactError
from ekp_domain import (
Chunk,
ChunkIndexState,
Document,
DocumentStatus,
IngestionAttempt,
IngestionAttemptStatus,
IngestionJob,
IngestionJobStatus,
IngestionStage,
)
from ekp_object_storage import BinarySource, ObjectStorage
from ekp_parser import DocumentParseError
from ekp_vector_store import VectorStore
SAFE_TRANSIENT_ERROR = "temporary ingestion failure"
SAFE_PERMANENT_ERROR = "document content could not be processed"
SAFE_CLEANUP_ERROR = "retry cleanup failed"
SAFE_LEASE_ERROR = "worker execution lease expired"
class IngestionJobNotFoundError(LookupError):
"""Raised when a requested ingestion Job does not exist."""
class IngestionCleanupError(RuntimeError):
"""Raised when derived data cannot be restored to a known clean state."""
class JobDispatcher(Protocol):
"""Replaceable message-delivery boundary used by submission services."""
def enqueue(self, job_id: UUID) -> None:
"""Request asynchronous execution of one already-persisted Job."""
@dataclass(frozen=True, slots=True)
class IngestionSubmission:
"""Stable identifiers returned after source and Job persistence."""
document: Document
job: IngestionJob
class JobRunAction(StrEnum):
"""Instruction returned to a queue adapter after one delivery."""
COMPLETE = "complete"
RETRY = "retry"
@dataclass(frozen=True, slots=True)
class JobRunResult:
"""Queue-neutral result for acknowledging or rescheduling a delivery."""
action: JobRunAction
status: IngestionJobStatus
retry_delay_seconds: int | None = None
@dataclass(frozen=True, slots=True)
class _Claim:
"""Internal durable claim outcome used before external provider work."""
run: bool
cleanup_required: bool = False
result: JobRunResult | None = None
class IngestionJobService:
"""Create and query Job state independently from the broker implementation."""
def __init__(self, session_factory: sessionmaker[Session], max_attempts: int) -> None:
if max_attempts <= 0:
raise ValueError("max_attempts must be positive")
self._session_factory = session_factory
self._max_attempts = max_attempts
def create(self, document_id: UUID) -> IngestionJob:
"""Create exactly one queued ingestion Job for a canonical Document."""
with self._session_factory() as session:
job = IngestionJob(document_id=document_id, max_attempts=self._max_attempts)
session.add(job)
session.commit()
return job
def get(self, job_id: UUID) -> IngestionJob:
"""Return one detached Job or a stable not-found error."""
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
return job
def list_dispatchable(self, limit: int = 100) -> tuple[UUID, ...]:
"""Find due queued/retrying Jobs and expired running leases for recovery."""
if not 1 <= limit <= 1000:
raise ValueError("dispatch limit must be between 1 and 1000")
now = datetime.now(UTC)
with self._session_factory() as session:
jobs = session.scalars(
select(IngestionJob)
.where(
(IngestionJob.status == IngestionJobStatus.QUEUED)
| (
(IngestionJob.status == IngestionJobStatus.RETRYING)
& ((IngestionJob.next_run_at.is_(None)) | (IngestionJob.next_run_at <= now))
)
| (
(IngestionJob.status == IngestionJobStatus.RUNNING)
& (
(IngestionJob.lease_expires_at.is_(None))
| (IngestionJob.lease_expires_at <= now)
)
)
)
.order_by(IngestionJob.created_at, IngestionJob.id)
.limit(limit)
).all()
return tuple(job.id for job in jobs)
class AsyncIngestionSubmissionService:
"""Persist immutable source and Job before best-effort broker delivery."""
def __init__(
self,
upload_service: DocumentUploadService,
jobs: IngestionJobService,
dispatcher: JobDispatcher,
) -> None:
self._upload_service = upload_service
self._jobs = jobs
self._dispatcher = dispatcher
def submit(
self,
filename: str,
media_type: str | None,
source: BinarySource,
) -> IngestionSubmission:
"""Accept one source and return without executing expensive stages inline.
Broker delivery is deliberately best effort after the canonical commit.
A periodic dispatcher recovers queued Jobs when the broker was temporarily
unavailable, so the request never treats a message as the source of truth.
"""
document = self._upload_service.upload(filename, media_type, source)
try:
job = self._jobs.create(document.id)
except Exception:
self._upload_service.rollback_acceptance(document.id)
raise
try:
self._dispatcher.enqueue(job.id)
except Exception:
# Keeping the Job queued is intentional. The recovery dispatcher can
# safely redeliver it without asking the client to upload again.
pass
return IngestionSubmission(document=document, job=job)
class DerivedDocumentCleanupService:
"""Remove only rebuildable derivatives before a retry begins."""
def __init__(
self,
session_factory: sessionmaker[Session],
storage: ObjectStorage,
vector_store: VectorStore,
active_index_profile_id: str,
) -> None:
self._session_factory = session_factory
self._storage = storage
self._vector_store = vector_store
self._active_index_profile_id = active_index_profile_id
def cleanup(self, document_id: UUID) -> None:
"""Clean vector, relational and parsed artifacts while preserving source/Job."""
try:
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise IngestionCleanupError("document metadata is missing")
parsed_key = document.parsed_storage_key
profile_ids = set(
session.scalars(
select(ChunkIndexState.index_profile_id)
.join(Chunk, Chunk.chunk_id == ChunkIndexState.chunk_id)
.where(Chunk.document_id == document_id)
.distinct()
).all()
)
profile_ids.add(self._active_index_profile_id)
# External vector rows are deleted before their canonical evidence.
for profile_id in sorted(profile_ids):
self._vector_store.delete_document(document_id, profile_id)
with self._session_factory() as session:
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
session.execute(
delete(ChunkIndexState).where(ChunkIndexState.chunk_id.in_(chunk_ids))
)
session.execute(delete(Chunk).where(Chunk.document_id == document_id))
session.commit()
if parsed_key is not None and self._storage.exists(parsed_key):
self._storage.delete(parsed_key)
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise IngestionCleanupError("document metadata is missing")
document.parsed_storage_key = None
document.status = DocumentStatus.UPLOADED
document.error_message = None
session.commit()
except IngestionCleanupError:
raise
except Exception as error:
raise IngestionCleanupError(SAFE_CLEANUP_ERROR) from error
class IngestionJobRunner:
"""Execute one durable attempt and return queue-neutral retry instructions."""
def __init__(
self,
session_factory: sessionmaker[Session],
parsing: DocumentParsingService,
chunking: DocumentChunkingService,
indexing: DocumentIndexingService,
cleanup: DerivedDocumentCleanupService,
*,
retry_backoff_seconds: int,
retry_backoff_max_seconds: int,
lease_seconds: int,
) -> None:
if retry_backoff_seconds <= 0 or retry_backoff_max_seconds <= 0 or lease_seconds <= 0:
raise ValueError("retry and lease durations must be positive")
self._session_factory = session_factory
self._parsing = parsing
self._chunking = chunking
self._indexing = indexing
self._cleanup = cleanup
self._retry_backoff_seconds = retry_backoff_seconds
self._retry_backoff_max_seconds = retry_backoff_max_seconds
self._lease_seconds = lease_seconds
def run(self, job_id: UUID) -> JobRunResult:
"""Claim, optionally clean, execute and durably finalize one delivery."""
claim = self._claim(job_id)
if not claim.run:
assert claim.result is not None
return claim.result
if claim.cleanup_required:
try:
document_id = self._job_document_id(job_id)
self._cleanup.cleanup(document_id)
except Exception:
self._finish_cleanup_failure(job_id)
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.FAILED)
attempt_number, document_id = self._start_attempt(job_id)
def stage_callback(stage: IngestionStage) -> None:
"""Bind this attempt while pipeline stages remain Job-agnostic."""
self._record_stage(job_id, attempt_number, stage)
try:
self._parsing.process(document_id, stage_callback=stage_callback)
self._chunking.process(document_id, stage_callback=stage_callback)
self._indexing.process(document_id, stage_callback=stage_callback)
except Exception as error:
return self._finish_failure(job_id, attempt_number, document_id, error)
self._finish_success(job_id, attempt_number, document_id)
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.SUCCEEDED)
def _claim(self, job_id: UUID) -> _Claim:
"""Serialize deliveries and recover only after an execution lease expires."""
now = datetime.now(UTC)
with self._session_factory() as session:
job = session.scalar(
select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
if job.status in {
IngestionJobStatus.SUCCEEDED,
IngestionJobStatus.FAILED,
IngestionJobStatus.CANCELLED,
}:
return _Claim(
run=False,
result=JobRunResult(JobRunAction.COMPLETE, job.status),
)
if job.status == IngestionJobStatus.RUNNING:
lease_expiry = _as_utc(job.lease_expires_at)
if lease_expiry is not None and lease_expiry > now:
delay = max(1, ceil((lease_expiry - now).total_seconds()))
return _Claim(
run=False,
result=JobRunResult(
JobRunAction.RETRY,
IngestionJobStatus.RUNNING,
delay,
),
)
self._expire_active_attempt(session, job, now)
if job.attempt_count >= job.max_attempts:
job.status = IngestionJobStatus.FAILED
job.lease_expires_at = None
job.completed_at = now
document = session.get(Document, job.document_id)
if document is not None:
document.status = DocumentStatus.FAILED
document.error_message = SAFE_LEASE_ERROR
session.commit()
return _Claim(
run=False,
result=JobRunResult(JobRunAction.COMPLETE, job.status),
)
job.status = IngestionJobStatus.RETRYING
next_run = _as_utc(job.next_run_at)
if (
job.status == IngestionJobStatus.RETRYING
and next_run is not None
and next_run > now
):
delay = max(1, ceil((next_run - now).total_seconds()))
return _Claim(
run=False,
result=JobRunResult(
JobRunAction.RETRY,
IngestionJobStatus.RETRYING,
delay,
),
)
cleanup_required = job.attempt_count > 0
job.status = IngestionJobStatus.RUNNING
job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
job.next_run_at = None
session.commit()
return _Claim(run=True, cleanup_required=cleanup_required)
def _expire_active_attempt(self, session: Session, job: IngestionJob, now: datetime) -> None:
"""Close an abandoned Attempt before a replacement can be created."""
attempt = session.scalar(
select(IngestionAttempt)
.where(
IngestionAttempt.job_id == job.id,
IngestionAttempt.status == IngestionAttemptStatus.RUNNING,
)
.with_for_update()
)
if attempt is not None:
attempt.status = IngestionAttemptStatus.FAILED
attempt.retryable = True
attempt.error_message = SAFE_LEASE_ERROR
attempt.finished_at = now
job.error_message = SAFE_LEASE_ERROR
def _job_document_id(self, job_id: UUID) -> UUID:
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
return job.document_id
def _start_attempt(self, job_id: UUID) -> tuple[int, UUID]:
"""Create the next numbered Attempt only after retry cleanup succeeds."""
now = datetime.now(UTC)
with self._session_factory() as session:
job = session.scalar(
select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
attempt_number = job.attempt_count + 1
job.attempt_count = attempt_number
job.stage = IngestionStage.PARSING
job.error_message = None
job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
session.add(
IngestionAttempt(
job_id=job.id,
attempt_number=attempt_number,
stage=IngestionStage.PARSING,
)
)
session.commit()
return attempt_number, job.document_id
def _record_stage(self, job_id: UUID, attempt_number: int, stage: IngestionStage) -> None:
"""Persist progress and renew the worker lease at every stage boundary."""
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
attempt = session.scalar(
select(IngestionAttempt).where(
IngestionAttempt.job_id == job_id,
IngestionAttempt.attempt_number == attempt_number,
)
)
if job is None or attempt is None:
raise IngestionJobNotFoundError(str(job_id))
job.stage = stage
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=self._lease_seconds)
attempt.stage = stage
session.commit()
def _finish_success(self, job_id: UUID, attempt_number: int, document_id: UUID) -> None:
"""Atomically close Job/Attempt after the document becomes ready."""
now = datetime.now(UTC)
with self._session_factory() as session:
job, attempt = self._load_job_attempt(session, job_id, attempt_number)
document = session.get(Document, document_id)
if document is None:
raise IngestionJobNotFoundError(str(job_id))
attempt.status = IngestionAttemptStatus.SUCCEEDED
attempt.retryable = False
attempt.finished_at = now
job.status = IngestionJobStatus.SUCCEEDED
job.error_message = None
job.lease_expires_at = None
job.completed_at = now
document.status = DocumentStatus.READY
document.error_message = None
session.commit()
def _finish_failure(
self,
job_id: UUID,
attempt_number: int,
document_id: UUID,
error: Exception,
) -> JobRunResult:
"""Classify safely, bound retry count, and schedule exponential backoff."""
retryable = _is_retryable(error)
safe_error = SAFE_TRANSIENT_ERROR if retryable else SAFE_PERMANENT_ERROR
now = datetime.now(UTC)
with self._session_factory() as session:
job, attempt = self._load_job_attempt(session, job_id, attempt_number)
document = session.get(Document, document_id)
attempt.status = IngestionAttemptStatus.FAILED
attempt.retryable = retryable
attempt.error_message = safe_error
attempt.finished_at = now
job.error_message = safe_error
job.lease_expires_at = None
if retryable and job.attempt_count < job.max_attempts:
delay = min(
self._retry_backoff_seconds * (2 ** (job.attempt_count - 1)),
self._retry_backoff_max_seconds,
)
job.status = IngestionJobStatus.RETRYING
job.next_run_at = now + timedelta(seconds=delay)
if document is not None:
document.status = DocumentStatus.PROCESSING
document.error_message = safe_error
session.commit()
return JobRunResult(JobRunAction.RETRY, job.status, delay)
job.status = IngestionJobStatus.FAILED
job.next_run_at = None
job.completed_at = now
if document is not None:
document.status = DocumentStatus.FAILED
document.error_message = safe_error
session.commit()
return JobRunResult(JobRunAction.COMPLETE, job.status)
def _finish_cleanup_failure(self, job_id: UUID) -> None:
"""Fail closed without creating a new Attempt on unknown dirty state."""
now = datetime.now(UTC)
with self._session_factory() as session:
job = session.get(IngestionJob, job_id)
if job is None:
raise IngestionJobNotFoundError(str(job_id))
document = session.get(Document, job.document_id)
job.status = IngestionJobStatus.FAILED
job.error_message = SAFE_CLEANUP_ERROR
job.next_run_at = None
job.lease_expires_at = None
job.completed_at = now
if document is not None:
document.status = DocumentStatus.FAILED
document.error_message = SAFE_CLEANUP_ERROR
session.commit()
@staticmethod
def _load_job_attempt(
session: Session, job_id: UUID, attempt_number: int
) -> tuple[IngestionJob, IngestionAttempt]:
job = session.get(IngestionJob, job_id)
attempt = session.scalar(
select(IngestionAttempt).where(
IngestionAttempt.job_id == job_id,
IngestionAttempt.attempt_number == attempt_number,
)
)
if job is None or attempt is None:
raise IngestionJobNotFoundError(str(job_id))
return job, attempt
def dispatch_pending_jobs(jobs: IngestionJobService, dispatcher: JobDispatcher) -> int:
"""Best-effort redelivery loop used by Celery beat after broker/API failures."""
dispatched = 0
for job_id in jobs.list_dispatchable():
try:
dispatcher.enqueue(job_id)
except Exception:
continue
dispatched += 1
return dispatched
def _is_retryable(error: Exception) -> bool:
"""Treat known content/state violations as permanent and providers as transient."""
permanent_errors = (
DocumentParseError,
EmptyParsedDocumentError,
EmptyChunkSetError,
ParsedDocumentMismatchError,
ParsedDocumentArtifactError,
DocumentAlreadyParsedError,
DocumentAlreadyChunkedError,
DocumentAlreadyIndexedError,
DocumentNotParsedError,
DocumentHasNoChunksError,
ParsingDocumentNotFoundError,
ChunkingDocumentNotFoundError,
IndexingDocumentNotFoundError,
)
return not isinstance(error, permanent_errors)
def _as_utc(value: datetime | None) -> datetime | None:
"""Normalize SQLite's naive DateTime values to the UTC runtime contract."""
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)

View File

@ -8,16 +8,36 @@ from ekp_domain.index_profile import (
IndexProfileStatus,
StoredIndexProfile,
)
from ekp_domain.ingestion_batch import (
BatchItemAcceptanceStatus,
IngestionBatch,
IngestionBatchItem,
)
from ekp_domain.ingestion_job import (
IngestionAttempt,
IngestionAttemptStatus,
IngestionJob,
IngestionJobStatus,
IngestionStage,
)
from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocument
__all__ = [
"Base",
"BatchItemAcceptanceStatus",
"Chunk",
"ChunkIndexState",
"Document",
"DocumentStatus",
"IndexProfile",
"IndexProfileStatus",
"IngestionAttempt",
"IngestionAttemptStatus",
"IngestionJob",
"IngestionJobStatus",
"IngestionStage",
"IngestionBatch",
"IngestionBatchItem",
"ParsedBlock",
"ParsedBlockType",
"ParsedDocument",

View File

@ -0,0 +1,96 @@
"""Canonical batch submission metadata for per-file ingestion tracking."""
from datetime import datetime
from enum import StrEnum
from uuid import UUID, uuid4
from sqlalchemy import (
CheckConstraint,
DateTime,
ForeignKey,
Index,
Integer,
String,
Text,
Uuid,
func,
)
from sqlalchemy import Enum as SqlEnum
from sqlalchemy.orm import Mapped, mapped_column
from ekp_domain.document import Base
class BatchItemAcceptanceStatus(StrEnum):
"""Source acceptance state, independent from its asynchronous Job state."""
PENDING = "pending"
ACCEPTED = "accepted"
REJECTED = "rejected"
class IngestionBatch(Base):
"""Stable aggregation identifier for one bounded multipart submission."""
__tablename__ = "ingestion_batches"
__table_args__ = (CheckConstraint("total_files > 0", name="total_files_positive"),)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
total_files: Mapped[int] = mapped_column(Integer, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class IngestionBatchItem(Base):
"""One ordered input and its optional accepted Document/Job references."""
__tablename__ = "ingestion_batch_items"
__table_args__ = (
CheckConstraint("item_index >= 0", name="item_index_non_negative"),
CheckConstraint("length(filename) > 0", name="filename_not_empty"),
CheckConstraint(
"acceptance_status IN ('pending', 'accepted', 'rejected')",
name="acceptance_status_valid",
),
Index("uq_ingestion_batch_items_order", "batch_id", "item_index", unique=True),
Index("uq_ingestion_batch_items_job_id", "job_id", unique=True),
)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
batch_id: Mapped[UUID] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("ingestion_batches.id", ondelete="CASCADE"),
nullable=False,
)
item_index: Mapped[int] = mapped_column(Integer, nullable=False)
filename: Mapped[str] = mapped_column(String(255), nullable=False)
acceptance_status: Mapped[BatchItemAcceptanceStatus] = mapped_column(
SqlEnum(
BatchItemAcceptanceStatus,
name="batch_item_acceptance_status",
native_enum=False,
create_constraint=False,
validate_strings=True,
values_callable=lambda values: [item.value for item in values],
),
nullable=False,
default=BatchItemAcceptanceStatus.PENDING,
)
document_id: Mapped[UUID | None] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("documents.id", ondelete="SET NULL"),
nullable=True,
)
job_id: Mapped[UUID | None] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("ingestion_jobs.id", ondelete="SET NULL"),
nullable=True,
)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
)

View File

@ -0,0 +1,166 @@
"""Durable ingestion Job and Attempt state for asynchronous processing."""
from datetime import datetime
from enum import StrEnum
from uuid import UUID, uuid4
from sqlalchemy import (
Boolean,
CheckConstraint,
Computed,
DateTime,
ForeignKey,
Index,
Integer,
Text,
UniqueConstraint,
Uuid,
func,
)
from sqlalchemy import Enum as SqlEnum
from sqlalchemy.orm import Mapped, mapped_column
from ekp_domain.document import Base
class IngestionJobStatus(StrEnum):
"""Public lifecycle states for one document ingestion Job."""
QUEUED = "queued"
RUNNING = "running"
RETRYING = "retrying"
SUCCEEDED = "succeeded"
FAILED = "failed"
CANCELLED = "cancelled"
class IngestionStage(StrEnum):
"""Observable stages within one ingestion attempt."""
PARSING = "parsing"
CLEANING = "cleaning"
CHUNKING = "chunking"
EMBEDDING = "embedding"
INDEXING = "indexing"
class IngestionAttemptStatus(StrEnum):
"""Terminal or active state of one bounded execution attempt."""
RUNNING = "running"
SUCCEEDED = "succeeded"
FAILED = "failed"
def _enum_column(enum_type: type[StrEnum], name: str) -> SqlEnum:
"""Build consistent portable string enums for OceanBase and SQLite tests."""
return SqlEnum(
enum_type,
name=name,
native_enum=False,
create_constraint=False,
validate_strings=True,
values_callable=lambda values: [item.value for item in values],
)
class IngestionJob(Base):
"""Canonical orchestration state; broker messages are only delivery hints."""
__tablename__ = "ingestion_jobs"
__table_args__ = (
CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
CheckConstraint(
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
name="status_valid",
),
CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
Index("ix_ingestion_jobs_status_next_run", "status", "next_run_at"),
)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
document_id: Mapped[UUID] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("documents.id", ondelete="CASCADE"),
nullable=False,
)
status: Mapped[IngestionJobStatus] = mapped_column(
_enum_column(IngestionJobStatus, "ingestion_job_status"),
nullable=False,
default=IngestionJobStatus.QUEUED,
)
stage: Mapped[IngestionStage | None] = mapped_column(
_enum_column(IngestionStage, "ingestion_stage"),
nullable=True,
)
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
lease_expires_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
)
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class IngestionAttempt(Base):
"""Immutable-numbered execution history with at most one active row per Job."""
__tablename__ = "ingestion_attempts"
__table_args__ = (
CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
CheckConstraint(
"status IN ('running', 'succeeded', 'failed')",
name="status_valid",
),
CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
# NULL values may repeat, while every running attempt maps to 1. This
# provides portable partial uniqueness on OceanBase MySQL and SQLite.
Index("uq_ingestion_attempts_active", "job_id", "active_marker", unique=True),
)
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
job_id: Mapped[UUID] = mapped_column(
Uuid(as_uuid=True),
ForeignKey("ingestion_jobs.id", ondelete="CASCADE"),
nullable=False,
)
attempt_number: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[IngestionAttemptStatus] = mapped_column(
_enum_column(IngestionAttemptStatus, "ingestion_attempt_status"),
nullable=False,
default=IngestionAttemptStatus.RUNNING,
)
stage: Mapped[IngestionStage | None] = mapped_column(
_enum_column(IngestionStage, "ingestion_attempt_stage"),
nullable=True,
)
retryable: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
active_marker: Mapped[int | None] = mapped_column(
Integer,
Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
nullable=True,
)
started_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

View File

@ -11,6 +11,7 @@ requires-python = ">=3.12,<3.14"
dependencies = [
"alembic>=1.18,<2",
"bce-python-sdk>=0.9.72,<0.10",
"celery>=5.5,<6",
"fastapi>=0.115,<1",
"mcp>=2,<3",
"pymysql>=1.1,<2",
@ -27,6 +28,7 @@ dependencies = [
ekp-index-rebuild = "ekp_api.index_rebuild:main"
ekp-live-acceptance = "ekp_api.live_acceptance:main"
ekp-mcp = "ekp_mcp.main:main"
ekp-worker = "ekp_api.worker:main"
[dependency-groups]
dev = [

130
scripts/deploy.sh Executable file
View File

@ -0,0 +1,130 @@
#!/usr/bin/env bash
set -euo pipefail
# DEPLOY-002 accepts a values file path only. Secret contents remain in
# Kubernetes and are never expanded into this process or Helm arguments.
values_file="${1:-}"
chart_dir="${CHART:-deploy/helm/knowledge-platform}"
release_name="${RELEASE:-knowledge-platform}"
namespace="${NAMESPACE:-knowledge-platform}"
deploy_timeout="${DEPLOY_TIMEOUT:-10m}"
if [[ -z "${values_file}" || ! -f "${values_file}" ]]; then
echo "VALUES must point to an existing environment values file" >&2
exit 2
fi
for tool in helm kubectl; do
if ! command -v "${tool}" >/dev/null 2>&1; then
echo "required deployment tool is missing: ${tool}" >&2
exit 2
fi
done
# Helm 4 renamed --atomic to --rollback-on-failure. Select the equivalent flag
# by major version while preserving compatibility with supported Helm 3.
helm_version="$(helm version --template '{{.Version}}')"
if [[ "${helm_version}" =~ ^v?([0-9]+) ]] && (( BASH_REMATCH[1] >= 4 )); then
rollback_flag=(--rollback-on-failure)
else
rollback_flag=(--atomic)
fi
helm lint "${chart_dir}" --values "${values_file}"
# The test-hook ConfigMap contains resource names only. Its fixed, controlled
# shape lets awk extract names without adding a YAML parser or reading Secrets.
preflight_yaml="$({
helm template "${release_name}" "${chart_dir}" \
--namespace "${namespace}" \
--values "${values_file}" \
--show-only templates/tests/preflight-references.yaml
})"
runtime_secret="$({
printf '%s\n' "${preflight_yaml}" | awk '$1 == "runtimeSecret:" {gsub(/\"/, "", $2); print $2}'
})"
ingress_enabled="$({
printf '%s\n' "${preflight_yaml}" | awk '$1 == "ingressEnabled:" {gsub(/\"/, "", $2); print $2}'
})"
rabbitmq_enabled="$({
printf '%s\n' "${preflight_yaml}" | awk '$1 == "rabbitmqEnabled:" {gsub(/\"/, "", $2); print $2}'
})"
if [[ -z "${runtime_secret}" || -z "${ingress_enabled}" || -z "${rabbitmq_enabled}" ]]; then
echo "failed to resolve Helm Secret references" >&2
exit 2
fi
secret_has_key() {
local secret_name="$1"
local key_name="$2"
local present
present="$({
kubectl -n "${namespace}" get secret "${secret_name}" \
-o go-template="{{with .data}}{{if index . \"${key_name}\"}}present{{end}}{{end}}"
})"
[[ "${present}" == "present" ]]
}
if ! kubectl get namespace "${namespace}" >/dev/null; then
echo "target namespace does not exist: ${namespace}" >&2
exit 2
fi
if ! kubectl -n "${namespace}" get secret "${runtime_secret}" >/dev/null; then
echo "referenced runtime Secret does not exist: ${runtime_secret}" >&2
exit 2
fi
required_runtime_keys=(
DATABASE_URL
RABBITMQ_URL
BOS_ACCESS_KEY_ID
BOS_SECRET_ACCESS_KEY
BAIDU_QIANFAN_API_KEY
OCEANBASE_VECTOR_PASSWORD
MCP_AUTH_TOKEN
)
for key in "${required_runtime_keys[@]}"; do
if ! secret_has_key "${runtime_secret}" "${key}"; then
echo "referenced runtime Secret is missing required key: ${key}" >&2
exit 2
fi
done
if [[ "${ingress_enabled}" == "true" ]]; then
for key in apiTlsSecret mcpTlsSecret; do
tls_secret="$({
printf '%s\n' "${preflight_yaml}" | awk -v field="${key}:" '$1 == field {gsub(/\"/, "", $2); print $2}'
})"
if ! kubectl -n "${namespace}" get secret "${tls_secret}" >/dev/null; then
echo "referenced TLS Secret does not exist: ${tls_secret}" >&2
exit 2
fi
done
fi
if [[ "${rabbitmq_enabled}" == "true" ]]; then
rabbitmq_secret="$({
printf '%s\n' "${preflight_yaml}" | awk '$1 == "rabbitmqSecret:" {gsub(/\"/, "", $2); print $2}'
})"
if ! kubectl -n "${namespace}" get secret "${rabbitmq_secret}" >/dev/null; then
echo "referenced RabbitMQ Secret does not exist: ${rabbitmq_secret}" >&2
exit 2
fi
for key in username password erlang-cookie; do
if ! secret_has_key "${rabbitmq_secret}" "${key}"; then
echo "referenced RabbitMQ Secret is missing required key: ${key}" >&2
exit 2
fi
done
fi
# The version-specific rollback flag keeps the previous release active when a
# pre-upgrade migration or rollout fails. Helm serializes operations for this
# release and waits for hook/workload readiness before returning success.
helm upgrade --install "${release_name}" "${chart_dir}" \
--namespace "${namespace}" \
--values "${values_file}" \
"${rollback_flag[@]}" \
--wait \
--timeout "${deploy_timeout}" \
--history-max 10

View File

@ -43,7 +43,9 @@ def test_admin_console_contains_only_demo_pages_and_versioned_api_calls(
assert all(label not in html for label in ("权限管理", "任务管理", "Compiled Knowledge"))
assert 'const API_BASE = "/api/v1"' in script
assert "innerHTML" not in script
assert all(endpoint in script for endpoint in ('"/documents"', '"/search"'))
assert all(endpoint in script for endpoint in ('"/ingestion-batches"', '"/search"'))
assert "multiple" in html
assert "BATCH_POLL_INTERVAL_MS" in script
assert all(field in script for field in ("Page", "Section", "Dense Score", "Rerank Score"))
assert "--brand: #FF3C00" in stylesheet
assert "prefers-reduced-motion" in stylesheet

View File

@ -0,0 +1,431 @@
"""Durable Job, bounded retry and cleanup-before-retry integration tests."""
from collections.abc import Sequence
from datetime import UTC, datetime, timedelta
from io import BytesIO
from pathlib import Path
from typing import BinaryIO
from uuid import UUID
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.chunking import StructureAwareChunker
from ekp_core.database import create_session_factory
from ekp_core.document_chunking import DocumentChunkingService
from ekp_core.document_indexing import DocumentIndexingService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
JobRunAction,
dispatch_pending_jobs,
)
from ekp_domain import (
Base,
Chunk,
Document,
IndexProfile,
IngestionAttempt,
IngestionAttemptStatus,
IngestionJob,
IngestionJobStatus,
ParsedDocument,
)
from ekp_embedding import EmbeddingProviderError, FakeEmbeddingProvider
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_vector_store import FakeVectorStore
class RecordingDispatcher:
"""Capture delivery requests while preserving the dispatcher contract."""
def __init__(self, *, fail: bool = False) -> None:
self.fail = fail
self.job_ids: list[UUID] = []
def enqueue(self, job_id: UUID) -> None:
if self.fail:
raise RuntimeError("simulated broker outage containing secret=must-not-persist")
self.job_ids.append(job_id)
class FailingJobService(IngestionJobService):
"""Inject metadata failure after immutable source upload."""
def create(self, document_id: UUID) -> IngestionJob:
del document_id
raise RuntimeError("simulated Job persistence failure")
class FailOnceEmbeddingProvider:
"""Inject one transient provider failure after parse/chunk persistence."""
def __init__(self) -> None:
self._delegate = FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
self.calls = 0
@property
def model_name(self) -> str:
return self._delegate.model_name
@property
def dimensions(self) -> int:
return self._delegate.dimensions
@property
def max_batch_size(self) -> int:
return self._delegate.max_batch_size
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
self.calls += 1
if self.calls == 1:
raise EmbeddingProviderError("temporary failure token=must-not-persist")
return self._delegate.embed(texts)
class AlwaysFailEmbeddingProvider(FailOnceEmbeddingProvider):
"""Inject a transient error on every attempt to verify the retry bound."""
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
del texts
self.calls += 1
raise EmbeddingProviderError("temporary failure token=must-not-persist")
class RetryObservingParser:
"""Verify cleanup finished before the second provider execution begins."""
def __init__(
self,
session_factory: sessionmaker[Session],
storage: LocalObjectStorage,
vector_store: FakeVectorStore,
) -> None:
self._delegate = ParserRouter()
self._session_factory = session_factory
self._storage = storage
self._vector_store = vector_store
self.calls = 0
self.retry_started_clean = False
def parse(
self,
document_id: UUID,
filename: str,
source: BinaryIO,
) -> ParsedDocument:
self.calls += 1
if self.calls == 2:
with self._session_factory() as session:
document = session.get(Document, document_id)
chunks = session.scalars(
select(Chunk).where(Chunk.document_id == document_id)
).all()
# The source stream is already open here, proving it survived cleanup.
self.retry_started_clean = (
document is not None
and document.parsed_storage_key is None
and not chunks
and not self._vector_store.records
and self._storage.exists(document.storage_key)
)
return self._delegate.parse(document_id, filename, source)
class CleanupFailingVectorStore(FakeVectorStore):
"""Inject a retry cleanup failure without affecting the first attempt."""
fail_delete = False
def delete_document(self, document_id: UUID, index_profile_id: str) -> None:
if self.fail_delete:
raise RuntimeError("cleanup provider failed secret=must-not-persist")
super().delete_document(document_id, index_profile_id)
def _build_runner(
tmp_path: Path,
*,
embedding: FailOnceEmbeddingProvider | FakeEmbeddingProvider | None = None,
vector_store: FakeVectorStore | None = None,
max_attempts: int = 3,
) -> tuple[
DocumentUploadService,
IngestionJobService,
IngestionJobRunner,
sessionmaker[Session],
LocalObjectStorage,
FakeVectorStore,
RetryObservingParser,
]:
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
selected_store = vector_store or FakeVectorStore()
selected_embedding = embedding or FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
profile = IndexProfile(
profile_id="test-profile",
chunk_strategy_version="structure_v1",
embedding_model=selected_embedding.model_name,
vector_dimension=selected_embedding.dimensions,
)
parser = RetryObservingParser(session_factory, storage, selected_store)
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, parser)
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
)
indexing = DocumentIndexingService(
session_factory,
selected_embedding,
selected_store,
profile,
)
jobs = IngestionJobService(session_factory, max_attempts=max_attempts)
cleanup = DerivedDocumentCleanupService(
session_factory,
storage,
selected_store,
profile.profile_id,
)
runner = IngestionJobRunner(
session_factory,
parsing,
chunking,
indexing,
cleanup,
retry_backoff_seconds=1,
retry_backoff_max_seconds=4,
lease_seconds=30,
)
return upload, jobs, runner, session_factory, storage, selected_store, parser
def _submit(
upload: DocumentUploadService,
jobs: IngestionJobService,
payload: bytes = b"# Policy\n\nRefunds take seven days.",
) -> tuple[UUID, UUID, RecordingDispatcher]:
dispatcher = RecordingDispatcher()
result = AsyncIngestionSubmissionService(upload, jobs, dispatcher).submit(
"policy.md", "text/markdown", BytesIO(payload)
)
return result.document.id, result.job.id, dispatcher
def _make_retry_due(session_factory: sessionmaker[Session], job_id: UUID) -> None:
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
job.next_run_at = datetime.now(UTC) - timedelta(seconds=1)
session.commit()
def test_transient_failure_cleans_derivatives_before_retry(tmp_path: Path) -> None:
embedding = FailOnceEmbeddingProvider()
upload, jobs, runner, session_factory, storage, vector_store, parser = _build_runner(
tmp_path, embedding=embedding
)
document_id, job_id, dispatcher = _submit(upload, jobs)
first = runner.run(job_id)
assert first.action == JobRunAction.RETRY
assert first.retry_delay_seconds == 1
assert dispatcher.job_ids == [job_id]
with session_factory() as session:
stored_job = session.get(IngestionJob, job_id)
assert stored_job is not None
assert stored_job.status == IngestionJobStatus.RETRYING
assert stored_job.attempt_count == 1
assert "must-not-persist" not in (stored_job.error_message or "")
_make_retry_due(session_factory, job_id)
second = runner.run(job_id)
assert second.status == IngestionJobStatus.SUCCEEDED
assert parser.retry_started_clean
with session_factory() as session:
stored_job = session.get(IngestionJob, job_id)
attempts = session.scalars(
select(IngestionAttempt)
.where(IngestionAttempt.job_id == job_id)
.order_by(IngestionAttempt.attempt_number)
).all()
assert stored_job is not None
assert stored_job.attempt_count == 2
assert [attempt.status for attempt in attempts] == [
IngestionAttemptStatus.FAILED,
IngestionAttemptStatus.SUCCEEDED,
]
document = session.get(Document, document_id)
assert document is not None
assert storage.exists(document.storage_key)
assert document.parsed_storage_key is not None
assert storage.exists(document.parsed_storage_key)
assert vector_store.records
def test_permanent_content_failure_does_not_retry(tmp_path: Path) -> None:
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs, payload=b" \n\t")
result = runner.run(job_id)
assert result.action == JobRunAction.COMPLETE
assert result.status == IngestionJobStatus.FAILED
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
assert job.attempt_count == 1
assert job.error_message == "document content could not be processed"
def test_cleanup_failure_is_terminal_and_preserves_source(tmp_path: Path) -> None:
embedding = FailOnceEmbeddingProvider()
vector_store = CleanupFailingVectorStore()
upload, jobs, runner, session_factory, storage, _, _ = _build_runner(
tmp_path,
embedding=embedding,
vector_store=vector_store,
)
document_id, job_id, _ = _submit(upload, jobs)
assert runner.run(job_id).action == JobRunAction.RETRY
_make_retry_due(session_factory, job_id)
vector_store.fail_delete = True
result = runner.run(job_id)
assert result.status == IngestionJobStatus.FAILED
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
assert job.attempt_count == 1 # Cleanup failed before attempt 2 was created.
assert job.error_message == "retry cleanup failed"
document = session.get(Document, document_id)
assert document is not None
assert storage.exists(document.storage_key)
def test_duplicate_delivery_defers_while_active_lease_is_valid(tmp_path: Path) -> None:
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs)
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
job.status = IngestionJobStatus.RUNNING
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=30)
session.commit()
result = runner.run(job_id)
assert result.action == JobRunAction.RETRY
assert result.status == IngestionJobStatus.RUNNING
with session_factory() as session:
assert session.scalars(select(IngestionAttempt)).all() == []
def test_expired_worker_lease_is_recovered_as_a_new_attempt(tmp_path: Path) -> None:
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs)
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
job.status = IngestionJobStatus.RUNNING
job.attempt_count = 1
job.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
session.add(IngestionAttempt(job_id=job_id, attempt_number=1))
session.commit()
result = runner.run(job_id)
assert result.status == IngestionJobStatus.SUCCEEDED
with session_factory() as session:
attempts = session.scalars(
select(IngestionAttempt)
.where(IngestionAttempt.job_id == job_id)
.order_by(IngestionAttempt.attempt_number)
).all()
assert [attempt.status for attempt in attempts] == [
IngestionAttemptStatus.FAILED,
IngestionAttemptStatus.SUCCEEDED,
]
assert attempts[0].error_message == "worker execution lease expired"
def test_transient_failures_stop_at_configured_attempt_bound(tmp_path: Path) -> None:
embedding = AlwaysFailEmbeddingProvider()
upload, jobs, runner, session_factory, _, _, _ = _build_runner(
tmp_path,
embedding=embedding,
max_attempts=2,
)
_, job_id, _ = _submit(upload, jobs)
assert runner.run(job_id).action == JobRunAction.RETRY
_make_retry_due(session_factory, job_id)
result = runner.run(job_id)
assert result.action == JobRunAction.COMPLETE
assert result.status == IngestionJobStatus.FAILED
assert runner.run(job_id).status == IngestionJobStatus.FAILED
with session_factory() as session:
job = session.get(IngestionJob, job_id)
attempts = session.scalars(
select(IngestionAttempt).where(IngestionAttempt.job_id == job_id)
).all()
assert job is not None
assert job.attempt_count == job.max_attempts == 2
assert len(attempts) == 2
def test_recovery_dispatches_persisted_job_after_broker_outage(tmp_path: Path) -> None:
upload, jobs, _, _, _, _, _ = _build_runner(tmp_path)
failing = RecordingDispatcher(fail=True)
submission = AsyncIngestionSubmissionService(upload, jobs, failing).submit(
"policy.md", "text/markdown", BytesIO(b"policy")
)
recovered = RecordingDispatcher()
assert dispatch_pending_jobs(jobs, recovered) == 1
assert recovered.job_ids == [submission.job.id]
def test_job_persistence_failure_compensates_document_and_source(tmp_path: Path) -> None:
upload, _, _, session_factory, _, _, _ = _build_runner(tmp_path)
failing_jobs = FailingJobService(session_factory, max_attempts=3)
with pytest.raises(RuntimeError, match="Job persistence failure"):
AsyncIngestionSubmissionService(upload, failing_jobs, RecordingDispatcher()).submit(
"policy.md", "text/markdown", BytesIO(b"policy")
)
with session_factory() as session:
assert session.scalars(select(Document)).all() == []
assert session.scalars(select(IngestionJob)).all() == []
assert not [path for path in tmp_path.rglob("*") if path.is_file()]
def test_database_rejects_two_active_attempts_for_one_job(tmp_path: Path) -> None:
upload, jobs, _, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs)
with session_factory() as session:
session.add_all(
[
IngestionAttempt(job_id=job_id, attempt_number=1),
IngestionAttempt(job_id=job_id, attempt_number=2),
]
)
with pytest.raises(IntegrityError):
session.commit()
session.rollback()

View File

@ -23,9 +23,16 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
command.check(config)
inspector = inspect(create_engine(database_url))
assert {"documents", "chunks", "chunk_index_states", "index_profiles"}.issubset(
inspector.get_table_names()
)
assert {
"documents",
"chunks",
"chunk_index_states",
"index_profiles",
"ingestion_jobs",
"ingestion_attempts",
"ingestion_batches",
"ingestion_batch_items",
}.issubset(inspector.get_table_names())
assert "unrelated_provider_table" in inspector.get_table_names()
assert {column["name"] for column in inspector.get_columns("documents")} == {
"id",
@ -70,3 +77,46 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
"activated_at",
"active_marker",
}
assert {column["name"] for column in inspector.get_columns("ingestion_jobs")} == {
"id",
"document_id",
"status",
"stage",
"attempt_count",
"max_attempts",
"error_message",
"next_run_at",
"lease_expires_at",
"created_at",
"updated_at",
"completed_at",
}
assert {column["name"] for column in inspector.get_columns("ingestion_attempts")} == {
"id",
"job_id",
"attempt_number",
"status",
"stage",
"retryable",
"error_message",
"active_marker",
"started_at",
"finished_at",
}
assert {column["name"] for column in inspector.get_columns("ingestion_batches")} == {
"id",
"total_files",
"created_at",
}
assert {column["name"] for column in inspector.get_columns("ingestion_batch_items")} == {
"id",
"batch_id",
"item_index",
"filename",
"acceptance_status",
"document_id",
"job_id",
"error_message",
"created_at",
"updated_at",
}

View File

@ -19,8 +19,15 @@ from ekp_core.document_indexing import DocumentIndexingPipelineService, Document
from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.ingestion_batches import IngestionBatchService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
)
from ekp_core.search import SearchService
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile, IngestionBatch
from ekp_embedding import FakeEmbeddingProvider
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
@ -39,13 +46,28 @@ REQUIRED_SOURCE_FIELDS = {
}
class RecordingDispatcher:
"""Capture durable Job deliveries without starting a real broker."""
def __init__(self) -> None:
self.job_ids: list[UUID] = []
def enqueue(self, job_id: UUID) -> None:
self.job_ids.append(job_id)
def _build_api_services(
tmp_path: Path,
*,
max_upload_bytes: int = 1024 * 1024,
batch_max_files: int = 20,
batch_max_total_bytes: int = 2 * 1024 * 1024,
) -> tuple[
ApiServices,
sessionmaker[Session],
LocalObjectStorage,
FakeVectorStore,
IngestionJobRunner,
]:
"""Build the production service graph with offline providers and shared SQLite."""
engine = create_engine(
@ -67,7 +89,11 @@ def _build_api_services(
embedding_model=embedding.model_name,
vector_dimension=embedding.dimensions,
)
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
upload = DocumentUploadService(
session_factory,
storage,
max_upload_bytes=max_upload_bytes,
)
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
chunking = DocumentChunkingService(
session_factory,
@ -82,6 +108,30 @@ def _build_api_services(
profile,
)
ingestion = DocumentIndexingPipelineService(pipeline, indexing)
jobs = IngestionJobService(session_factory, max_attempts=3)
dispatcher = RecordingDispatcher()
submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher)
batches = IngestionBatchService(
session_factory,
submissions,
max_files=batch_max_files,
max_total_bytes=batch_max_total_bytes,
)
runner = IngestionJobRunner(
session_factory,
parsing,
chunking,
indexing,
DerivedDocumentCleanupService(
session_factory,
storage,
vector_store,
profile.profile_id,
),
retry_backoff_seconds=1,
retry_backoff_max_seconds=4,
lease_seconds=30,
)
dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20)
search = SearchService(
session_factory,
@ -96,10 +146,18 @@ def _build_api_services(
active_index_profile_id=profile.profile_id,
)
return (
ApiServices(ingestion=ingestion, documents=documents, search=search),
ApiServices(
ingestion=ingestion,
submissions=submissions,
batches=batches,
jobs=jobs,
documents=documents,
search=search,
),
session_factory,
storage,
vector_store,
runner,
)
@ -107,7 +165,7 @@ def test_document_and_search_rest_flow(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
services, session_factory, storage, vector_store = _build_api_services(tmp_path)
services, session_factory, storage, vector_store, runner = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
@ -121,19 +179,33 @@ def test_document_and_search_rest_flow(
)
},
)
assert upload_response.status_code == 201
assert upload_response.status_code == 202
uploaded = upload_response.json()
document_id = UUID(uploaded["id"])
assert uploaded["status"] == "ready"
job_id = UUID(uploaded["job_id"])
assert uploaded["status"] == "uploaded"
assert uploaded["job_status"] == "queued"
assert uploaded["filename"] == "退款政策.md"
assert "storage_key" not in uploaded
queued = client.get(f"/api/v1/jobs/{job_id}")
assert queued.status_code == 200
assert queued.json()["attempt"] == 0
assert queued.json()["status"] == "queued"
assert client.delete(f"/api/v1/documents/{document_id}").status_code == 409
runner.run(job_id)
list_response = client.get("/api/v1/documents")
detail_response = client.get(f"/api/v1/documents/{document_id}")
assert list_response.status_code == 200
assert [item["id"] for item in list_response.json()] == [str(document_id)]
assert detail_response.status_code == 200
assert detail_response.json() == uploaded
assert detail_response.json()["status"] == "ready"
completed = client.get(f"/api/v1/jobs/{job_id}").json()
assert completed["status"] == "succeeded"
assert completed["stage"] == "indexing"
assert completed["attempt"] == 1
search_response = client.post(
"/api/v1/search",
@ -173,7 +245,7 @@ def test_rest_validation_not_found_and_openapi_contract(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
services, _, _, _ = _build_api_services(tmp_path)
services, _, _, _, _ = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
missing_id = uuid4()
@ -182,6 +254,9 @@ def test_rest_validation_not_found_and_openapi_contract(
assert set(openapi["paths"]) >= {
"/api/v1/documents",
"/api/v1/documents/{document_id}",
"/api/v1/ingestion-batches",
"/api/v1/ingestion-batches/{batch_id}",
"/api/v1/jobs/{job_id}",
"/api/v1/search",
}
assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"}
@ -204,3 +279,141 @@ def test_rest_validation_not_found_and_openapi_contract(
assert boolean_top_k.status_code == 422
assert missing_get.status_code == 404
assert missing_delete.status_code == 404
def test_batch_upload_isolates_files_and_reports_terminal_status(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
"""A rejected file must not roll back valid siblings in the same batch."""
services, _, _, _, runner = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
accepted = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("alpha.md", b"# Alpha\n\nFirst policy.", "text/markdown")),
("files", ("blocked.csv", b"secret=must-not-persist", "text/csv")),
("files", ("beta.txt", b"Second policy.", "text/plain")),
],
)
assert accepted.status_code == 202
body = accepted.json()
assert body["status"] == "processing"
assert body["total_files"] == 3
assert body["accepted_files"] == 2
assert body["rejected_files"] == 1
assert body["completed_files"] == 1
assert [item["filename"] for item in body["items"]] == [
"alpha.md",
"blocked.csv",
"beta.txt",
]
assert [item["acceptance_status"] for item in body["items"]] == [
"accepted",
"rejected",
"accepted",
]
assert body["items"][1]["error_message"] == ("unsupported document type or media type")
assert "must-not-persist" not in str(body)
for item in body["items"]:
if item["job_id"] is not None:
runner.run(UUID(item["job_id"]))
terminal = client.get(f"/api/v1/ingestion-batches/{body['batch_id']}")
assert terminal.status_code == 200
terminal_body = terminal.json()
assert terminal_body["status"] == "partial_failure"
assert terminal_body["completed_files"] == 3
assert [item["job_status"] for item in terminal_body["items"]] == [
"succeeded",
None,
"succeeded",
]
def test_batch_upload_enforces_count_and_shared_byte_limits(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
"""Request limits are bounded while already accepted siblings remain durable."""
services, session_factory, _, _, _ = _build_api_services(
tmp_path,
max_upload_bytes=10,
batch_max_files=2,
batch_max_total_bytes=15,
)
app = create_app(Settings(app_env="test"), api_services=services)
with TestClient(app) as client:
too_many = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("one.txt", b"1", "text/plain")),
("files", ("two.txt", b"2", "text/plain")),
("files", ("three.txt", b"3", "text/plain")),
],
)
bounded = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("first.txt", b"1234567890", "text/plain")),
("files", ("second.txt", b"abcdefghij", "text/plain")),
],
)
oversized = client.post(
"/api/v1/ingestion-batches",
files=[("files", ("large.txt", b"12345678901", "text/plain"))],
)
assert too_many.status_code == 413
assert too_many.json()["detail"] == "batch file count exceeds the configured limit"
assert bounded.status_code == 202
body = bounded.json()
assert body["accepted_files"] == 1
assert body["rejected_files"] == 1
assert body["items"][1]["error_message"] == ("batch exceeds the configured total upload limit")
assert oversized.status_code == 202
assert oversized.json()["status"] == "failed"
assert oversized.json()["accepted_files"] == 0
assert oversized.json()["items"][0]["error_message"] == (
"document exceeds the configured upload limit"
)
# A count-rejected request creates no batch; the two accepted requests do.
with session_factory() as session:
assert len(session.scalars(select(IngestionBatch)).all()) == 2
def test_batch_upload_completed_and_missing_contract(
tmp_path: Path,
isolated_settings_environment: None,
) -> None:
"""All-success batches become completed and unknown IDs stay explicit."""
services, _, _, _, runner = _build_api_services(tmp_path)
app = create_app(Settings(app_env="test"), api_services=services)
missing_id = uuid4()
with TestClient(app) as client:
accepted = client.post(
"/api/v1/ingestion-batches",
files=[
("files", ("one.md", b"# One\n\nContent.", "text/markdown")),
("files", ("two.txt", b"More content.", "text/plain")),
],
).json()
for item in accepted["items"]:
runner.run(UUID(item["job_id"]))
completed = client.get(f"/api/v1/ingestion-batches/{accepted['batch_id']}")
missing = client.get(f"/api/v1/ingestion-batches/{missing_id}")
assert completed.status_code == 200
assert completed.json()["status"] == "completed"
assert completed.json()["completed_files"] == 2
assert missing.status_code == 404
assert missing.json()["detail"] == "ingestion batch not found"

View File

@ -0,0 +1,57 @@
"""Unit tests for the Celery adapter without opening a broker connection."""
from unittest.mock import Mock
from uuid import uuid4
import pytest
from pydantic import SecretStr
from ekp_api.celery_queue import (
INGESTION_TASK_NAME,
RECOVERY_TASK_NAME,
CeleryIngestionDispatcher,
create_celery_app,
)
from ekp_core.config import Settings
def test_celery_app_enables_late_ack_and_recovery_schedule(
isolated_settings_environment: None,
) -> None:
settings = Settings(rabbitmq_url=SecretStr("amqp://user:test@queue.example/ekp"))
app = create_celery_app(settings)
assert app.conf.task_acks_late is True
assert app.conf.task_reject_on_worker_lost is True
assert app.conf.worker_prefetch_multiplier == 1
assert app.conf.task_routes[INGESTION_TASK_NAME]["queue"] == "ingestion"
assert app.conf.task_routes[RECOVERY_TASK_NAME]["queue"] == "ingestion"
assert app.conf.beat_schedule["recover-durable-ingestion-jobs"]["schedule"] == 30.0
def test_celery_dispatcher_sends_only_job_identifier(
monkeypatch: pytest.MonkeyPatch,
isolated_settings_environment: None,
) -> None:
app = create_celery_app(Settings(rabbitmq_url=SecretStr("amqp://user:test@queue/ekp")))
send_task = Mock()
monkeypatch.setattr(app, "send_task", send_task)
job_id = uuid4()
CeleryIngestionDispatcher(app).enqueue(job_id)
send_task.assert_called_once_with(
INGESTION_TASK_NAME,
args=[str(job_id)],
queue="ingestion",
)
def test_celery_app_requires_runtime_broker_secret(
isolated_settings_environment: None,
) -> None:
with pytest.raises(ValueError, match="RABBITMQ_URL"):
create_celery_app(Settings())
with pytest.raises(ValueError, match="RABBITMQ_URL"):
create_celery_app(Settings(rabbitmq_url=SecretStr("")))

View File

@ -14,12 +14,20 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
assert settings.app_env == "development"
assert settings.api_prefix == "/api/v1"
assert settings.api_port == 8000
assert settings.admin_web_root is None
assert settings.database_url.get_secret_value().startswith("mysql+pymysql://")
assert settings.database_pool_size == 10
assert settings.database_max_overflow == 10
assert settings.database_pool_recycle_seconds == 600
assert settings.database_pool_timeout_seconds == 30
assert settings.database_connect_timeout_seconds == 30
assert settings.rabbitmq_url is None
assert settings.ingestion_max_attempts == 3
assert settings.ingestion_retry_backoff_seconds == 2
assert settings.ingestion_retry_backoff_max_seconds == 60
assert settings.ingestion_worker_lease_seconds == 300
assert settings.batch_max_files == 20
assert settings.batch_max_total_bytes == 200 * 1024 * 1024
assert settings.object_storage_provider == "bos"
assert settings.object_storage_path == Path("storage")
assert settings.bos_endpoint is None
@ -90,6 +98,7 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) -
oceanbase_vector_password=SecretStr("oceanbase-secret"),
reranker_api_key=SecretStr("reranker-secret"),
mcp_auth_token=SecretStr("mcp-secret"),
rabbitmq_url=SecretStr("amqp://user:rabbit-secret@queue.example/ekp"),
)
rendered = repr(settings)
@ -100,6 +109,7 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) -
assert "oceanbase-secret" not in rendered
assert "reranker-secret" not in rendered
assert "mcp-secret" not in rendered
assert "rabbit-secret" not in rendered
def test_settings_restrict_sqlite_to_test_environment(
@ -136,3 +146,24 @@ def test_settings_reject_empty_mcp_allowed_hosts(
) -> None:
with pytest.raises(ValidationError, match="mcp_allowed_hosts"):
Settings(mcp_allowed_hosts=[])
def test_settings_reject_retry_backoff_larger_than_cap(
isolated_settings_environment: None,
) -> None:
with pytest.raises(ValidationError, match="initial backoff"):
Settings(ingestion_retry_backoff_seconds=10, ingestion_retry_backoff_max_seconds=5)
def test_settings_require_worker_lease_longer_than_provider_timeout(
isolated_settings_environment: None,
) -> None:
with pytest.raises(ValidationError, match="worker_lease"):
Settings(ingestion_worker_lease_seconds=30, embedding_timeout_seconds=30)
def test_settings_require_batch_limit_to_fit_one_file(
isolated_settings_environment: None,
) -> None:
with pytest.raises(ValidationError, match="batch_max_total_bytes"):
Settings(max_upload_bytes=100, batch_max_total_bytes=99)

View File

@ -0,0 +1,154 @@
"""Static and rendered contracts for the DEPLOY-002 Helm delivery."""
import json
import shutil
import subprocess
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
CHART = ROOT / "deploy" / "helm" / "knowledge-platform"
EXAMPLE_VALUES = ROOT / "deploy" / "helm" / "environments" / "values.example.yaml"
def _helm(*arguments: str, expect_success: bool = True) -> subprocess.CompletedProcess[str]:
"""Run Helm when installed; static checks still run on Python-only hosts."""
executable = shutil.which("helm")
if executable is None:
pytest.skip("Helm is required for rendered chart assertions")
result = subprocess.run(
[executable, *arguments],
cwd=ROOT,
check=False,
capture_output=True,
text=True,
)
if expect_success and result.returncode != 0:
pytest.fail(f"Helm failed: {result.stderr or result.stdout}")
return result
def _render(*extra_arguments: str) -> str:
"""Render deterministic manifests without contacting a cluster."""
result = _helm(
"template",
"knowledge-platform",
str(CHART),
"--namespace",
"knowledge-platform",
"--values",
str(EXAMPLE_VALUES),
*extra_arguments,
)
return result.stdout
def test_chart_has_machine_validated_values_schemas() -> None:
"""Both the application and optional broker expose valid JSON Schemas."""
for schema_path in (
CHART / "values.schema.json",
CHART / "charts" / "rabbitmq" / "values.schema.json",
):
schema = json.loads(schema_path.read_text())
assert schema["$schema"] == "https://json-schema.org/draft-07/schema#"
assert schema["type"] == "object"
def test_default_chart_renders_secret_only_sensitive_configuration() -> None:
"""Credentials are references; the chart never creates or embeds a Secret."""
rendered = _render()
assert "kind: Secret\n" not in rendered
assert "secretRef:\n name: knowledge-platform-runtime" in rendered
assert "key: DATABASE_URL" in rendered
assert "helm.sh/hook: pre-install,pre-upgrade" in rendered
assert 'args: ["upgrade", "head"]' in rendered
assert "value: mysql+pymysql://" not in rendered
assert "change-me" not in rendered
def test_workloads_render_availability_and_restricted_pod_defaults() -> None:
"""API/MCP are multi-replica and every application process is non-root."""
rendered = _render()
assert "name: knowledge-platform-api" in rendered
assert "name: knowledge-platform-mcp" in rendered
assert rendered.count("replicas: 2") >= 3
assert rendered.count("readOnlyRootFilesystem: true") >= 5
assert rendered.count("allowPrivilegeEscalation: false") >= 5
assert rendered.count("automountServiceAccountToken: false") >= 6
assert "maxUnavailable: 0" in rendered
assert "readinessProbe:" in rendered
assert "livenessProbe:" in rendered
assert "startupProbe:" in rendered
def test_scheduler_is_singleton_and_workers_do_not_embed_beat() -> None:
"""Scaled workers cannot accidentally create multiple periodic schedulers."""
rendered = _render()
scheduler = rendered[rendered.index("name: knowledge-platform-scheduler") :]
scheduler = scheduler[: scheduler.index("---", 1)]
worker = rendered[rendered.index("name: knowledge-platform-worker") :]
worker = worker[: worker.index("---", 1)]
assert "replicas: 1" in scheduler
assert "type: Recreate" in scheduler
assert "- beat" in scheduler
assert "- worker" in worker
assert "- beat" not in worker
def test_ingress_and_optional_rabbitmq_are_explicitly_renderable() -> None:
"""TLS routes and the persistent demo broker remain opt-in values."""
rendered = _render("--set", "rabbitmq.enabled=true")
assert "kind: Ingress" in rendered
assert "secretName: knowledge-platform-api-tls" in rendered
assert "secretName: knowledge-platform-mcp-tls" in rendered
assert "kind: StatefulSet" in rendered
assert "volumeClaimTemplates:" in rendered
assert "name: RABBITMQ_DEFAULT_PASS" in rendered
assert "valueFrom:" in rendered
def test_values_schema_rejects_latest_image_tags() -> None:
"""Cluster releases require a non-latest immutable tag or digest."""
result = _helm(
"lint",
str(CHART),
"--set",
"images.api.tag=latest",
expect_success=False,
)
assert result.returncode != 0
assert "/images/api/tag" in (result.stdout + result.stderr)
def test_cross_field_validation_rejects_unsafe_ingestion_limits() -> None:
"""Rendered settings preserve the application's own configuration rules."""
result = _helm(
"template",
"knowledge-platform",
str(CHART),
"--set",
"config.ingestion.batchMaxTotalBytes=1",
expect_success=False,
)
assert result.returncode != 0
assert "batchMaxTotalBytes must be >= maxUploadBytes" in (result.stdout + result.stderr)
def test_deploy_entrypoint_uses_atomic_upgrade_and_never_reads_secret_values() -> None:
"""The one-command wrapper checks references and performs an atomic wait."""
script = (ROOT / "scripts" / "deploy.sh").read_text()
assert "helm upgrade --install" in script
assert "--atomic" in script
assert "--rollback-on-failure" in script
assert "--wait" in script
assert 'kubectl -n "${namespace}" get secret' in script
assert 'awk \'$1 == "runtimeSecret:"' in script
assert "DATABASE_URL}'" not in script

View File

@ -0,0 +1,65 @@
"""Static production-image contract tests that do not require a Docker daemon."""
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
DOCKERFILE = PROJECT_ROOT / "deploy/docker/Dockerfile"
BAKE_FILE = PROJECT_ROOT / "deploy/docker/docker-bake.hcl"
def test_runtime_is_locked_non_root_and_secret_free() -> None:
"""Guard the supply-chain and least-privilege choices in the shared runtime."""
dockerfile = DOCKERFILE.read_text()
assert "python:3.12.14-slim-bookworm" in dockerfile
assert "ghcr.io/astral-sh/uv:0.12.5" in dockerfile
assert dockerfile.count("@sha256:") == 2
assert "uv sync --frozen --no-dev --no-editable" in dockerfile
assert "USER 10001:10001" in dockerfile
assert "COPY . " not in dockerfile
assert all(
sensitive not in dockerfile
for sensitive in ("DATABASE_URL=", "RABBITMQ_URL=", "API_KEY=", "PASSWORD=")
)
def test_three_process_targets_have_healthchecks_and_exec_commands() -> None:
"""Every independently deployed process must have an explicit runtime contract."""
dockerfile = DOCKERFILE.read_text()
assert dockerfile.count("HEALTHCHECK ") == 3
assert "FROM runtime AS api" in dockerfile
assert "FROM runtime AS mcp" in dockerfile
assert "FROM runtime AS worker" in dockerfile
assert 'CMD ["uvicorn", "ekp_api.main:app"' in dockerfile
assert 'CMD ["ekp-mcp"]' in dockerfile
assert 'CMD ["ekp-worker"]' in dockerfile
def test_bake_plan_covers_both_architectures_and_all_images() -> None:
"""The checked-in build plan must stay suitable for amd64 and arm64 clusters."""
bake = BAKE_FILE.read_text()
assert 'platforms = ["linux/amd64", "linux/arm64"]' in bake
assert 'context = "."' in bake
assert 'targets = ["api", "mcp", "worker"]' in bake
assert bake.count('inherits = ["common"]') == 3
assert all(
image in bake
for image in (
"enterprise-knowledge-api",
"enterprise-knowledge-mcp",
"enterprise-knowledge-worker",
)
)
def test_docker_context_excludes_credentials_and_local_state() -> None:
"""Prevent common developer files from entering any build layer."""
ignored = (PROJECT_ROOT / ".dockerignore").read_text().splitlines()
assert ".env" in ignored
assert ".git" in ignored
assert ".venv" in ignored
assert "storage" in ignored
assert "tests" in ignored

158
uv.lock generated
View File

@ -28,6 +28,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/20/89/e62cc37b69ad357cc8ecd6e7367f5245f523d3cbb338a66197212bdf6749/alembic-1.19.1-py3-none-any.whl", hash = "sha256:b39018cb3d9413a19cbd54cf3c02ad33998641f0538eb77413a488a21c3e14be", size = 265946, upload-time = "2026-08-08T16:32:03.153Z" },
]
[[package]]
name = "amqp"
version = "5.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "vine" },
]
sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" },
]
[[package]]
name = "annotated-doc"
version = "0.0.5"
@ -83,6 +95,35 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ef/de/52b763b183e23f4dd73811a2dd2664824ae7fc6ceb1aa8dcd3f1102b1094/bce_python_sdk-0.9.79-py3-none-any.whl", hash = "sha256:71799ac8740505e0759d30873f6f1a478fa8f83aedf425d511f1419a5f30082e", size = 440268, upload-time = "2026-09-01T05:48:50.459Z" },
]
[[package]]
name = "billiard"
version = "4.2.4"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" },
]
[[package]]
name = "celery"
version = "5.6.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "billiard" },
{ name = "click" },
{ name = "click-didyoumean" },
{ name = "click-plugins" },
{ name = "click-repl" },
{ name = "kombu" },
{ name = "python-dateutil" },
{ name = "tzlocal" },
{ name = "vine" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" },
]
[[package]]
name = "cffi"
version = "2.1.1"
@ -129,6 +170,43 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" },
]
[[package]]
name = "click-didyoumean"
version = "0.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
]
sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" },
]
[[package]]
name = "click-plugins"
version = "1.1.1.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" },
]
[[package]]
name = "click-repl"
version = "0.3.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
{ name = "prompt-toolkit" },
]
sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" },
]
[[package]]
name = "colorama"
version = "0.4.6"
@ -249,6 +327,7 @@ source = { editable = "." }
dependencies = [
{ name = "alembic" },
{ name = "bce-python-sdk" },
{ name = "celery" },
{ name = "fastapi" },
{ name = "mcp" },
{ name = "pydantic-settings" },
@ -274,6 +353,7 @@ dev = [
requires-dist = [
{ name = "alembic", specifier = ">=1.18,<2" },
{ name = "bce-python-sdk", specifier = ">=0.9.72,<0.10" },
{ name = "celery", specifier = ">=5.5,<6" },
{ name = "fastapi", specifier = ">=0.115,<1" },
{ name = "mcp", specifier = ">=2,<3" },
{ name = "pydantic-settings", specifier = ">=2.7,<3" },
@ -459,6 +539,21 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" },
]
[[package]]
name = "kombu"
version = "5.6.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "amqp" },
{ name = "packaging" },
{ name = "tzdata" },
{ name = "vine" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" },
]
[[package]]
name = "librt"
version = "0.15.0"
@ -740,6 +835,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
]
[[package]]
name = "prompt-toolkit"
version = "3.0.53"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "wcwidth" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" },
]
[[package]]
name = "pycparser"
version = "3.0"
@ -949,6 +1056,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" },
]
[[package]]
name = "python-dateutil"
version = "2.9.0.post0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "six" },
]
sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
]
[[package]]
name = "python-docx"
version = "1.2.0"
@ -1199,6 +1318,27 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" },
]
[[package]]
name = "tzdata"
version = "2026.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" },
]
[[package]]
name = "tzlocal"
version = "5.4.4"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "tzdata", marker = "sys_platform == 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/81/5b/879b2f932adfa7a053c360d50bc896c977fa6426109185f7c12ebdd0cb9d/tzlocal-5.4.4.tar.gz", hash = "sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4", size = 31170, upload-time = "2026-06-29T08:03:40.026Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9e/a4/017a7a6cbe387d961a688ec31364ae60a5c4e22c96ae9921b79a947c855d/tzlocal-5.4.4-py3-none-any.whl", hash = "sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15", size = 18115, upload-time = "2026-06-29T08:03:38.666Z" },
]
[[package]]
name = "uvicorn"
version = "0.52.4"
@ -1242,6 +1382,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" },
]
[[package]]
name = "vine"
version = "5.1.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" },
]
[[package]]
name = "watchfiles"
version = "1.2.0"
@ -1292,6 +1441,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" },
]
[[package]]
name = "wcwidth"
version = "0.8.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/36/57/ed58088fafdf4c55a0ad6bde846502567645424d7ebf325230b9237f4085/wcwidth-0.8.3.tar.gz", hash = "sha256:d128512515fbf4612e0ff21fd6380399210318b7b54a9af59dff8454cf9730eb", size = 1458450, upload-time = "2026-08-28T18:10:06.875Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/c4/0e/57f6bb3024a597b2e8ec4aee710ffe62ddc95af2e2bb1ee7a7abdc22c68c/wcwidth-0.8.3-py3-none-any.whl", hash = "sha256:d5b73dba6158a595ec9370350e7f2637bcac8d6c5e4fde34f30fcffb6103a5e4", size = 331669, upload-time = "2026-08-28T18:10:04.909Z" },
]
[[package]]
name = "websockets"
version = "17.1"