Compare commits
19 Commits
codex/demo
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
03dd5ffbfd | ||
|
|
5100d98e3c | ||
|
|
d727d0a118 | ||
|
|
72dc248bbc | ||
|
|
a1eae16fb1 | ||
|
|
0fad825eb0 | ||
|
|
a8dfbe0cfa | ||
|
|
7938c0b9cf | ||
|
|
4d73465888 | ||
|
|
f47634e104 | ||
|
|
6f16c971e4 | ||
|
|
123864b80d | ||
|
|
9d53bdbe4b | ||
|
|
77119f4a80 | ||
|
|
6af4ab465d | ||
|
|
e3b5b4d50f | ||
|
|
fe9d85b6da | ||
|
|
f16c738035 | ||
|
|
8fba051da4 |
21
.dockerignore
Normal file
21
.dockerignore
Normal file
@ -0,0 +1,21 @@
|
||||
# Keep credentials, local state, caches, and developer artifacts out of every image context.
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
.git
|
||||
.github
|
||||
.idea
|
||||
.vscode
|
||||
.DS_Store
|
||||
.mypy_cache
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
.venv
|
||||
__pycache__
|
||||
*.py[cod]
|
||||
*.sqlite*
|
||||
dist
|
||||
storage
|
||||
tests
|
||||
docs
|
||||
docker-compose.dev.yml
|
||||
27
.env.example
27
.env.example
@ -4,6 +4,8 @@ LOG_LEVEL=INFO
|
||||
API_HOST=0.0.0.0
|
||||
API_PORT=8000
|
||||
API_PREFIX=/api/v1
|
||||
# Optional outside source checkouts; production image fixes this to /app/apps/admin-web.
|
||||
# ADMIN_WEB_ROOT=/absolute/path/to/apps/admin-web
|
||||
|
||||
DATABASE_URL=mysql+pymysql://ekp:change-me@localhost:2881/ekp
|
||||
DATABASE_POOL_SIZE=10
|
||||
@ -11,8 +13,15 @@ DATABASE_MAX_OVERFLOW=10
|
||||
DATABASE_POOL_RECYCLE_SECONDS=600
|
||||
DATABASE_POOL_TIMEOUT_SECONDS=30
|
||||
DATABASE_CONNECT_TIMEOUT_SECONDS=30
|
||||
REDIS_URL=redis://localhost:6379/0
|
||||
RABBITMQ_URL=amqp://guest:guest@localhost:5672//
|
||||
# DEMO-015 queue. Inject the real credential at runtime; never commit its value.
|
||||
RABBITMQ_URL=
|
||||
INGESTION_MAX_ATTEMPTS=3
|
||||
INGESTION_RETRY_BACKOFF_SECONDS=2
|
||||
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
|
||||
INGESTION_WORKER_LEASE_SECONDS=300
|
||||
INGESTION_RECOVERY_INTERVAL_SECONDS=30
|
||||
BATCH_MAX_FILES=20
|
||||
BATCH_MAX_TOTAL_BYTES=209715200
|
||||
|
||||
OBJECT_STORAGE_PROVIDER=bos
|
||||
OBJECT_STORAGE_PATH=storage
|
||||
@ -52,6 +61,20 @@ RERANKER_API_KEY=
|
||||
RERANKER_TIMEOUT_SECONDS=30
|
||||
SEARCH_TOP_K=5
|
||||
|
||||
# DEMO-012 MCP gateway. Inject MCP_AUTH_TOKEN at runtime; never commit its value.
|
||||
MCP_HOST=127.0.0.1
|
||||
MCP_PORT=8766
|
||||
MCP_PATH=/mcp
|
||||
MCP_PUBLIC_URL=http://127.0.0.1:8766/mcp
|
||||
MCP_ISSUER_URL=http://127.0.0.1:8766
|
||||
MCP_AUTH_TOKEN=
|
||||
MCP_CLIENT_ID=workbuddy-enterprise
|
||||
MCP_SERVICE_IDENTITY=workbuddy-demo
|
||||
MCP_REQUIRED_SCOPE=knowledge:search
|
||||
MCP_ALLOWED_HOSTS='["127.0.0.1:*","localhost:*","[::1]:*","testserver"]'
|
||||
MCP_ALLOWED_ORIGINS='["http://127.0.0.1:*","http://localhost:*","http://[::1]:*"]'
|
||||
MCP_MAX_REQUEST_BODY_BYTES=1048576
|
||||
|
||||
# Controlled ARCH-006 acceptance; keep disabled outside an authorized test run.
|
||||
EKP_RUN_LIVE_TESTS=0
|
||||
EKP_CREDENTIALS_AUTHORIZED=0
|
||||
|
||||
30
Makefile
30
Makefile
@ -1,4 +1,4 @@
|
||||
.PHONY: help install run format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
|
||||
.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
|
||||
|
||||
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
|
||||
export UV_CACHE_DIR
|
||||
@ -7,6 +7,14 @@ help:
|
||||
@echo "Enterprise Knowledge Platform"
|
||||
@echo " install Install development dependencies"
|
||||
@echo " run Start the development API"
|
||||
@echo " run-mcp Start the authenticated MCP gateway"
|
||||
@echo " run-worker Start the ingestion worker and recovery scheduler"
|
||||
@echo " images Build production images for the current builder"
|
||||
@echo " images-print Render the multi-architecture image build plan"
|
||||
@echo " images-push Build and push amd64/arm64 production images"
|
||||
@echo " deploy Install or upgrade Kubernetes with VALUES=<file>"
|
||||
@echo " deploy-lint Validate Helm values and templates"
|
||||
@echo " deploy-render Render non-secret Kubernetes manifests"
|
||||
@echo " format Format code"
|
||||
@echo " format-check Check formatting without modifying files"
|
||||
@echo " lint Run linters"
|
||||
@ -24,6 +32,24 @@ install:
|
||||
uv sync --all-groups
|
||||
run:
|
||||
uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}"
|
||||
run-mcp:
|
||||
uv run ekp-mcp
|
||||
run-worker:
|
||||
uv run ekp-worker
|
||||
images:
|
||||
docker buildx bake -f deploy/docker/docker-bake.hcl
|
||||
images-print:
|
||||
docker buildx bake -f deploy/docker/docker-bake.hcl --print
|
||||
images-push:
|
||||
docker buildx bake -f deploy/docker/docker-bake.hcl --push
|
||||
deploy:
|
||||
./scripts/deploy.sh "$(VALUES)"
|
||||
deploy-lint:
|
||||
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
|
||||
helm lint deploy/helm/knowledge-platform --values "$(VALUES)"
|
||||
deploy-render:
|
||||
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
|
||||
helm template "$${RELEASE:-knowledge-platform}" deploy/helm/knowledge-platform --namespace "$${NAMESPACE:-knowledge-platform}" --values "$(VALUES)"
|
||||
format:
|
||||
uv run ruff format .
|
||||
format-check:
|
||||
@ -31,7 +57,7 @@ format-check:
|
||||
lint:
|
||||
uv run ruff check .
|
||||
typecheck:
|
||||
uv run mypy apps/api/src packages tests
|
||||
uv run mypy apps/api/src apps/mcp-server/src packages tests
|
||||
test:
|
||||
uv run pytest tests/unit tests/integration
|
||||
test-unit:
|
||||
|
||||
35
README.md
35
README.md
@ -53,9 +53,22 @@ Install dependencies and run the API:
|
||||
```bash
|
||||
cp .env.example .env
|
||||
make install
|
||||
make run-worker
|
||||
make run
|
||||
```
|
||||
|
||||
Run the independent authenticated MCP Gateway after injecting a dedicated
|
||||
runtime token of at least 32 characters:
|
||||
|
||||
```bash
|
||||
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
|
||||
make run-mcp
|
||||
```
|
||||
|
||||
The Streamable HTTP endpoint defaults to `http://127.0.0.1:8766/mcp` and exposes
|
||||
only `search_knowledge`. See `docs/development/MCP_SETUP.md` for WorkBuddy configuration,
|
||||
identity boundaries, remote allowlists, and protocol verification.
|
||||
|
||||
The health endpoint is available at `http://localhost:8000/healthz`, and the
|
||||
versioned OpenAPI document is available at
|
||||
`http://localhost:8000/api/v1/openapi.json`.
|
||||
@ -63,6 +76,11 @@ The Demo Admin Console is available at `http://localhost:8000/admin` and contain
|
||||
only document upload, document management, and Search Lab pages. See
|
||||
`docs/development/ADMIN_CONSOLE.md` for the operation and scope boundaries.
|
||||
|
||||
DEPLOY-001 provides locked, non-root `api`, `mcp`, and `worker` production image targets with an
|
||||
amd64/arm64 Buildx Bake plan. DEPLOY-002 adds the Kubernetes/Helm Chart, runtime Secret references,
|
||||
serialized Migration, Ingress/TLS and the `make deploy VALUES=<file>` entrypoint. See
|
||||
`docs/development/CONTAINER_IMAGES.md` and `docs/development/KUBERNETES_DEPLOYMENT.md`.
|
||||
|
||||
Run the development checks with:
|
||||
|
||||
```bash
|
||||
@ -88,8 +106,11 @@ Chunk metadata, including deterministic IDs, filename, page range, section,
|
||||
index, and strategy version, is stored atomically in canonical relational tables. The lightweight
|
||||
token counter is replaceable so DEMO-006 can inject model-specific behavior.
|
||||
Documents remain `processing` until embedding/indexing completes; pipeline
|
||||
failures become `failed` without deleting the immutable original. DEMO-009
|
||||
exposes this pipeline through the versioned upload endpoint.
|
||||
failures become `failed` without deleting the immutable original. DEMO-015 now
|
||||
accepts a source plus durable Job with HTTP `202`, while a Celery/RabbitMQ worker
|
||||
runs the pipeline. Bounded transient retries clean every derived artifact before
|
||||
starting the next Attempt; broker redelivery and expired worker leases converge on
|
||||
OceanBase Job state. See `docs/development/ASYNC_INGESTION_SETUP.md`.
|
||||
|
||||
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a
|
||||
validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase.
|
||||
@ -116,8 +137,14 @@ adapter with the `bce-reranker-base` baseline. The final Top20-to-Top5 pipeline
|
||||
source results are rehydrated from canonical relational chunks and retain both
|
||||
Dense and reranker scores; stale vector content fails closed. See
|
||||
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
|
||||
DEMO-009 exposes the completed synchronous pipeline through versioned document
|
||||
and search REST endpoints under `/api/v1`. Upload, list, detail, direct delete,
|
||||
DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
|
||||
changes upload to asynchronous acceptance and adds Job status. DEMO-016 adds bounded multi-file
|
||||
submission, batch/per-file status, and automatic Admin Console polling. Upload, list, detail, direct delete,
|
||||
and reranked source search are documented in
|
||||
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
|
||||
outside the public response contract.
|
||||
|
||||
DEMO-012 adds the independent authenticated MCP Gateway. It uses the official
|
||||
MCP Python SDK, maps a runtime-only Bearer credential to a least-privilege service
|
||||
identity, and delegates `search_knowledge` to the same `SearchService` used by REST.
|
||||
No provider, storage, database, model, or index-profile details are exposed.
|
||||
|
||||
@ -1,12 +1,33 @@
|
||||
// DEMO-010 stays API First: this client owns presentation state only and sends
|
||||
// every document/search operation through the versioned public REST contract.
|
||||
// The Admin Console owns presentation state only. Batch acceptance, Job state,
|
||||
// and retrieval remain behind the versioned public REST contract.
|
||||
const API_BASE = "/api/v1";
|
||||
const BATCH_POLL_INTERVAL_MS = 1500;
|
||||
const terminalBatchStatuses = new Set(["completed", "partial_failure", "failed"]);
|
||||
let batchPollTimer;
|
||||
let activeBatchId;
|
||||
|
||||
const statusLabels = {
|
||||
uploaded: "Uploaded",
|
||||
processing: "Processing",
|
||||
ready: "Ready",
|
||||
failed: "Failed",
|
||||
pending: "Pending",
|
||||
accepted: "Accepted",
|
||||
rejected: "Rejected",
|
||||
queued: "Queued",
|
||||
running: "Running",
|
||||
retrying: "Retrying",
|
||||
succeeded: "Succeeded",
|
||||
completed: "Completed",
|
||||
partial_failure: "Partial failure",
|
||||
};
|
||||
|
||||
const stageLabels = {
|
||||
parsing: "解析",
|
||||
cleaning: "清理",
|
||||
chunking: "切分",
|
||||
embedding: "向量化",
|
||||
indexing: "索引",
|
||||
};
|
||||
|
||||
const pageTitles = {
|
||||
@ -93,23 +114,37 @@ function initializeUpload() {
|
||||
const button = document.querySelector("#upload-button");
|
||||
|
||||
const updateFilename = () => {
|
||||
selected.textContent = input.files[0]?.name ?? "尚未选择文件";
|
||||
const files = Array.from(input.files);
|
||||
if (!files.length) {
|
||||
selected.textContent = "尚未选择文件";
|
||||
} else if (files.length === 1) {
|
||||
selected.textContent = files[0].name;
|
||||
} else {
|
||||
selected.textContent = `已选择 ${files.length} 个文件`;
|
||||
}
|
||||
};
|
||||
input.addEventListener("change", updateFilename);
|
||||
|
||||
form.addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
if (!input.files.length) return;
|
||||
setButtonBusy(button, true, "处理中…");
|
||||
window.clearTimeout(batchPollTimer);
|
||||
activeBatchId = undefined;
|
||||
setButtonBusy(button, true, "提交中…");
|
||||
try {
|
||||
const formData = new FormData();
|
||||
formData.append("file", input.files[0]);
|
||||
const documentData = await requestJson("/documents", {
|
||||
Array.from(input.files).forEach((file) => formData.append("files", file));
|
||||
const batch = await requestJson("/ingestion-batches", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
renderUploadResult(documentData);
|
||||
showNotice("文档已上传并完成处理。可以前往 Search Lab 检索。", "success");
|
||||
activeBatchId = batch.batch_id;
|
||||
renderBatchResult(batch);
|
||||
scheduleBatchPoll(batch);
|
||||
showNotice(
|
||||
`批次已接收:${batch.accepted_files} 个进入后台队列,${batch.rejected_files} 个被拒绝。`,
|
||||
batch.rejected_files ? "error" : "success",
|
||||
);
|
||||
form.reset();
|
||||
updateFilename();
|
||||
} catch (error) {
|
||||
@ -120,18 +155,62 @@ function initializeUpload() {
|
||||
});
|
||||
}
|
||||
|
||||
function renderUploadResult(documentData) {
|
||||
function scheduleBatchPoll(batch) {
|
||||
window.clearTimeout(batchPollTimer);
|
||||
if (batch.batch_id !== activeBatchId) return;
|
||||
if (terminalBatchStatuses.has(batch.status)) return;
|
||||
batchPollTimer = window.setTimeout(async () => {
|
||||
try {
|
||||
const latest = await requestJson(`/ingestion-batches/${batch.batch_id}`);
|
||||
renderBatchResult(latest);
|
||||
scheduleBatchPoll(latest);
|
||||
} catch (error) {
|
||||
showNotice(`批次状态刷新失败:${error.message}`, "error");
|
||||
scheduleBatchPoll(batch);
|
||||
}
|
||||
}, BATCH_POLL_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function renderBatchResult(batch) {
|
||||
const card = document.querySelector("#upload-result");
|
||||
document.querySelector("#upload-filename").textContent = documentData.filename;
|
||||
document.querySelector("#upload-type").textContent = documentData.media_type;
|
||||
document.querySelector("#upload-size").textContent = formatSize(documentData.size_bytes);
|
||||
document.querySelector("#upload-time").textContent = formatDate(documentData.created_at);
|
||||
document.querySelector("#upload-status").replaceWith(statusPill(documentData.status));
|
||||
document.querySelector("#upload-batch-title").textContent =
|
||||
`批次 ${batch.batch_id.slice(0, 8)} · ${formatDate(batch.created_at)}`;
|
||||
document.querySelector("#upload-total").textContent = `${batch.total_files} 个`;
|
||||
document.querySelector("#upload-completed").textContent =
|
||||
`${batch.completed_files} / ${batch.total_files}`;
|
||||
document.querySelector("#upload-accepted").textContent =
|
||||
`${batch.accepted_files} / ${batch.rejected_files}`;
|
||||
document.querySelector("#upload-status").replaceWith(statusPill(batch.status));
|
||||
const status = card.querySelector(".status-pill");
|
||||
status.id = "upload-status";
|
||||
const items = document.querySelector("#batch-items");
|
||||
items.replaceChildren(...batch.items.map(batchItem));
|
||||
card.hidden = false;
|
||||
}
|
||||
|
||||
function batchItem(item) {
|
||||
const row = document.createElement("article");
|
||||
row.className = "batch-item";
|
||||
|
||||
const order = document.createElement("span");
|
||||
order.className = "batch-item-order";
|
||||
order.textContent = String(item.item_index + 1).padStart(2, "0");
|
||||
|
||||
const body = document.createElement("div");
|
||||
body.className = "batch-item-body";
|
||||
const name = document.createElement("strong");
|
||||
name.textContent = item.filename;
|
||||
const detail = document.createElement("span");
|
||||
const stage = item.stage ? ` · ${stageLabels[item.stage] ?? item.stage}` : "";
|
||||
const attempts = item.max_attempts == null ? "" : ` · 尝试 ${item.attempt}/${item.max_attempts}`;
|
||||
detail.textContent = item.error_message ?? `后台任务${stage}${attempts}`;
|
||||
body.append(name, detail);
|
||||
|
||||
const state = item.job_status ?? item.acceptance_status;
|
||||
row.append(order, body, statusPill(state));
|
||||
return row;
|
||||
}
|
||||
|
||||
async function loadDocuments() {
|
||||
const refresh = document.querySelector("#refresh-documents");
|
||||
setButtonBusy(refresh, true, "加载中…");
|
||||
@ -329,3 +408,4 @@ initializeUpload();
|
||||
initializeSearch();
|
||||
document.querySelector("#refresh-documents").addEventListener("click", loadDocuments);
|
||||
if (page === "documents") loadDocuments();
|
||||
window.addEventListener("pagehide", () => window.clearTimeout(batchPollTimer));
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -15,26 +15,33 @@
|
||||
<div class="app-shell">
|
||||
<aside class="sidebar" aria-label="管理控制台导航">
|
||||
<a class="brand" href="/admin/upload" aria-label="Enterprise Knowledge Platform 首页">
|
||||
<span class="brand-mark" aria-hidden="true">EK</span>
|
||||
<svg class="brand-mark" viewBox="0 0 48 36" aria-hidden="true">
|
||||
<path d="M15 3h17a15 15 0 0 1 0 30H15v-9h17a6 6 0 0 0 0-12H15z" />
|
||||
<circle cx="11" cy="18" r="8" />
|
||||
</svg>
|
||||
<span>
|
||||
<strong>Enterprise Knowledge</strong>
|
||||
<small>Demo Console</small>
|
||||
<strong>地瓜机器人</strong>
|
||||
<small>Knowledge Platform</small>
|
||||
</span>
|
||||
</a>
|
||||
<div class="workspace-label">
|
||||
<span>CONTROL DESK</span>
|
||||
<strong>企业知识运营台</strong>
|
||||
</div>
|
||||
<nav class="primary-nav">
|
||||
<a data-nav="upload" href="/admin/upload">
|
||||
<span aria-hidden="true">+</span>上传文档
|
||||
<span aria-hidden="true">01</span>上传文档
|
||||
</a>
|
||||
<a data-nav="documents" href="/admin/documents">
|
||||
<span aria-hidden="true">▤</span>文档列表
|
||||
<span aria-hidden="true">02</span>文档列表
|
||||
</a>
|
||||
<a data-nav="search" href="/admin/search">
|
||||
<span aria-hidden="true">⌕</span>Search Lab
|
||||
<span aria-hidden="true">03</span>Search Lab
|
||||
</a>
|
||||
</nav>
|
||||
<div class="sidebar-foot">
|
||||
<span class="environment-dot" aria-hidden="true"></span>
|
||||
Demo Environment
|
||||
DEMO ENVIRONMENT
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
@ -44,19 +51,23 @@
|
||||
<p class="eyebrow">KNOWLEDGE OPERATIONS</p>
|
||||
<h1 id="page-title">上传文档</h1>
|
||||
</div>
|
||||
<a class="api-link" href="/api/v1/docs" target="_blank" rel="noreferrer">
|
||||
API 文档 <span aria-hidden="true">↗</span>
|
||||
</a>
|
||||
<div class="topbar-actions">
|
||||
<span class="system-state"><i aria-hidden="true"></i>API FIRST · V1</span>
|
||||
<a class="api-link" href="/api/v1/docs" target="_blank" rel="noreferrer">
|
||||
API 文档 <span aria-hidden="true">↗</span>
|
||||
</a>
|
||||
</div>
|
||||
</header>
|
||||
<div class="signal-line" aria-hidden="true"><span></span><span></span><span></span></div>
|
||||
|
||||
<div id="notice" class="notice" role="status" aria-live="polite" hidden></div>
|
||||
|
||||
<section id="upload-page" class="page" data-page="upload" aria-labelledby="page-title">
|
||||
<div class="page-heading">
|
||||
<div>
|
||||
<p class="section-label">INGESTION</p>
|
||||
<p class="section-label">01 / INGESTION</p>
|
||||
<h2>让知识进入检索链路</h2>
|
||||
<p>上传一个 PDF、DOCX、TXT 或 Markdown 文件,系统将同步完成解析、切分与索引。</p>
|
||||
<p>批量提交 PDF、DOCX、TXT 或 Markdown 文件,系统将逐个接收并在后台独立完成解析、切分与索引。</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -64,14 +75,15 @@
|
||||
<label id="drop-zone" class="drop-zone" for="file-input">
|
||||
<input
|
||||
id="file-input"
|
||||
name="file"
|
||||
name="files"
|
||||
type="file"
|
||||
multiple
|
||||
accept=".pdf,.docx,.txt,.md,application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document,text/plain,text/markdown"
|
||||
required
|
||||
/>
|
||||
<span class="upload-icon" aria-hidden="true">↑</span>
|
||||
<strong>点击选择文件</strong>
|
||||
<span>单次上传一个文件</span>
|
||||
<strong>选择一批知识文件</strong>
|
||||
<span>支持 PDF、DOCX、TXT、Markdown · 每个文件独立处理</span>
|
||||
<span id="selected-file" class="selected-file">尚未选择文件</span>
|
||||
</label>
|
||||
<button id="upload-button" class="primary-button" type="submit">上传并处理</button>
|
||||
@ -79,15 +91,16 @@
|
||||
|
||||
<article id="upload-result" class="result-card" hidden>
|
||||
<div>
|
||||
<p class="section-label">LATEST UPLOAD</p>
|
||||
<h3 id="upload-filename"></h3>
|
||||
<p class="section-label">LATEST BATCH</p>
|
||||
<h3 id="upload-batch-title"></h3>
|
||||
</div>
|
||||
<span id="upload-status" class="status-pill"></span>
|
||||
<dl class="metadata-grid">
|
||||
<div><dt>文件类型</dt><dd id="upload-type"></dd></div>
|
||||
<div><dt>文件大小</dt><dd id="upload-size"></dd></div>
|
||||
<div><dt>上传时间</dt><dd id="upload-time"></dd></div>
|
||||
<div><dt>总文件</dt><dd id="upload-total"></dd></div>
|
||||
<div><dt>已完成</dt><dd id="upload-completed"></dd></div>
|
||||
<div><dt>接收 / 拒绝</dt><dd id="upload-accepted"></dd></div>
|
||||
</dl>
|
||||
<div id="batch-items" class="batch-items" aria-live="polite"></div>
|
||||
<a class="text-link" href="/admin/search">前往 Search Lab 检索 <span>→</span></a>
|
||||
</article>
|
||||
</section>
|
||||
@ -95,7 +108,7 @@
|
||||
<section id="documents-page" class="page" data-page="documents" aria-labelledby="page-title" hidden>
|
||||
<div class="page-heading heading-row">
|
||||
<div>
|
||||
<p class="section-label">DOCUMENTS</p>
|
||||
<p class="section-label">02 / DOCUMENTS</p>
|
||||
<h2>文档列表</h2>
|
||||
<p>查看知识文档的状态、元数据并管理 Demo 数据。</p>
|
||||
</div>
|
||||
@ -115,7 +128,7 @@
|
||||
<th>类型</th>
|
||||
<th>上传时间</th>
|
||||
<th>状态</th>
|
||||
<th><span class="sr-only">操作</span></th>
|
||||
<th aria-label="操作"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="document-rows"></tbody>
|
||||
@ -132,7 +145,7 @@
|
||||
<section id="search-page" class="page" data-page="search" aria-labelledby="page-title" hidden>
|
||||
<div class="page-heading">
|
||||
<div>
|
||||
<p class="section-label">RETRIEVAL</p>
|
||||
<p class="section-label">03 / RETRIEVAL</p>
|
||||
<h2>Search Lab</h2>
|
||||
<p>输入问题,检查 Dense 召回与 Reranker 精排后的 Top-K 来源。</p>
|
||||
</div>
|
||||
|
||||
55
apps/api/src/ekp_api/celery_queue.py
Normal file
55
apps/api/src/ekp_api/celery_queue.py
Normal file
@ -0,0 +1,55 @@
|
||||
"""Celery/RabbitMQ adapter behind the queue-neutral ingestion dispatcher."""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from celery import Celery # type: ignore[import-untyped]
|
||||
|
||||
from ekp_core.config import Settings
|
||||
|
||||
INGESTION_TASK_NAME = "ekp.ingestion.run"
|
||||
RECOVERY_TASK_NAME = "ekp.ingestion.recover"
|
||||
|
||||
|
||||
def create_celery_app(settings: Settings) -> Celery:
|
||||
"""Create a secret-safe broker client without opening a connection eagerly."""
|
||||
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
|
||||
raise ValueError("RABBITMQ_URL is required for asynchronous ingestion")
|
||||
app = Celery(
|
||||
"enterprise-knowledge-platform",
|
||||
broker=settings.rabbitmq_url.get_secret_value(),
|
||||
)
|
||||
app.conf.update(
|
||||
task_ignore_result=True,
|
||||
task_acks_late=True,
|
||||
task_reject_on_worker_lost=True,
|
||||
worker_prefetch_multiplier=1,
|
||||
broker_connection_retry_on_startup=True,
|
||||
timezone="UTC",
|
||||
enable_utc=True,
|
||||
task_routes={
|
||||
INGESTION_TASK_NAME: {"queue": "ingestion"},
|
||||
RECOVERY_TASK_NAME: {"queue": "ingestion"},
|
||||
},
|
||||
beat_schedule={
|
||||
"recover-durable-ingestion-jobs": {
|
||||
"task": RECOVERY_TASK_NAME,
|
||||
"schedule": float(settings.ingestion_recovery_interval_seconds),
|
||||
}
|
||||
},
|
||||
)
|
||||
return app
|
||||
|
||||
|
||||
class CeleryIngestionDispatcher:
|
||||
"""Publish only a durable Job id; source bytes and state stay canonical."""
|
||||
|
||||
def __init__(self, app: Celery) -> None:
|
||||
self._app = app
|
||||
|
||||
def enqueue(self, job_id: UUID) -> None:
|
||||
"""Deliver one idempotent execution hint to the ingestion queue."""
|
||||
self._app.send_task(
|
||||
INGESTION_TASK_NAME,
|
||||
args=[str(job_id)],
|
||||
queue="ingestion",
|
||||
)
|
||||
@ -37,12 +37,13 @@ def create_app(
|
||||
application.state.settings = resolved_settings
|
||||
application.state.api_services = api_services
|
||||
application.include_router(router, prefix=resolved_settings.api_prefix)
|
||||
admin_web_root = (resolved_settings.admin_web_root or ADMIN_WEB_ROOT).resolve()
|
||||
|
||||
# The Demo console is a deliberately thin, same-origin client. It contains
|
||||
# no business logic and calls only the versioned public REST API.
|
||||
application.mount(
|
||||
"/admin/assets",
|
||||
StaticFiles(directory=ADMIN_WEB_ROOT / "assets"),
|
||||
StaticFiles(directory=admin_web_root / "assets"),
|
||||
name="admin-assets",
|
||||
)
|
||||
|
||||
@ -52,7 +53,7 @@ def create_app(
|
||||
@application.get("/admin/search", include_in_schema=False)
|
||||
async def admin_console() -> FileResponse:
|
||||
"""Serve the three-page Demo Admin Console shell."""
|
||||
return FileResponse(ADMIN_WEB_ROOT / "index.html")
|
||||
return FileResponse(admin_web_root / "index.html")
|
||||
|
||||
@application.get("/healthz", response_model=HealthResponse, tags=["system"])
|
||||
async def health() -> HealthResponse:
|
||||
|
||||
@ -4,6 +4,7 @@ from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_api.celery_queue import CeleryIngestionDispatcher, create_celery_app
|
||||
from ekp_core.chunking import StructureAwareChunker
|
||||
from ekp_core.config import Settings
|
||||
from ekp_core.database import create_database_engine, create_session_factory
|
||||
@ -17,6 +18,13 @@ from ekp_core.document_management import DocumentManagementService
|
||||
from ekp_core.document_parsing import DocumentParsingService
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.index_rebuild import resolve_active_index_profile
|
||||
from ekp_core.ingestion_batches import IngestionBatchService
|
||||
from ekp_core.ingestion_jobs import (
|
||||
AsyncIngestionSubmissionService,
|
||||
DerivedDocumentCleanupService,
|
||||
IngestionJobRunner,
|
||||
IngestionJobService,
|
||||
)
|
||||
from ekp_core.search import SearchService
|
||||
from ekp_domain import IndexProfile
|
||||
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
|
||||
@ -42,10 +50,39 @@ class ApiServices:
|
||||
"""Business services injected into REST routes without provider leakage."""
|
||||
|
||||
ingestion: DocumentIndexingPipelineService
|
||||
submissions: AsyncIngestionSubmissionService
|
||||
batches: IngestionBatchService
|
||||
jobs: IngestionJobService
|
||||
documents: DocumentManagementService
|
||||
search: SearchService
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkerServices:
|
||||
"""Provider graph required by Celery without REST search-only services."""
|
||||
|
||||
runner: IngestionJobRunner
|
||||
jobs: IngestionJobService
|
||||
dispatcher: CeleryIngestionDispatcher
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _IngestionComponents:
|
||||
"""Shared composition result used by API and independent worker processes."""
|
||||
|
||||
session_factory: sessionmaker[Session]
|
||||
storage: ObjectStorage
|
||||
vector_store: VectorStore
|
||||
embedding_provider: EmbeddingProvider
|
||||
index_profile: IndexProfile
|
||||
upload: DocumentUploadService
|
||||
parsing: DocumentParsingService
|
||||
chunking: DocumentChunkingService
|
||||
indexing: DocumentIndexingService
|
||||
jobs: IngestionJobService
|
||||
dispatcher: CeleryIngestionDispatcher
|
||||
|
||||
|
||||
def create_index_profile(
|
||||
settings: Settings,
|
||||
session_factory: sessionmaker[Session] | None = None,
|
||||
@ -161,8 +198,31 @@ def create_search_service(
|
||||
)
|
||||
|
||||
|
||||
def create_api_services(settings: Settings) -> ApiServices:
|
||||
"""Compose the synchronous Demo runtime once for all business endpoints."""
|
||||
def create_runtime_search_service(settings: Settings) -> SearchService:
|
||||
"""Compose only read-path providers for the independent MCP Gateway."""
|
||||
engine = create_database_engine(settings)
|
||||
session_factory = create_session_factory(engine)
|
||||
index_profile = create_index_profile(settings, session_factory)
|
||||
embedding_provider = create_embedding_provider(settings)
|
||||
vector_store = create_vector_store(settings)
|
||||
dense_retrieval = create_dense_retrieval_service(
|
||||
settings,
|
||||
embedding_provider,
|
||||
vector_store,
|
||||
index_profile,
|
||||
)
|
||||
return create_search_service(
|
||||
settings,
|
||||
session_factory,
|
||||
dense_retrieval,
|
||||
create_reranker_provider(settings),
|
||||
)
|
||||
|
||||
|
||||
def _create_ingestion_components(settings: Settings) -> _IngestionComponents:
|
||||
"""Compose provider-neutral ingestion stages and the Celery delivery adapter."""
|
||||
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
|
||||
raise ProviderConfigurationError("RABBITMQ_URL is required")
|
||||
engine = create_database_engine(settings)
|
||||
session_factory = create_session_factory(engine)
|
||||
storage = create_object_storage(settings)
|
||||
@ -185,35 +245,103 @@ def create_api_services(settings: Settings) -> ApiServices:
|
||||
max_characters=settings.chunk_max_characters,
|
||||
),
|
||||
)
|
||||
document_pipeline = DocumentPipelineService(
|
||||
upload_service,
|
||||
parsing_service,
|
||||
chunking_service,
|
||||
)
|
||||
indexing_service = DocumentIndexingService(
|
||||
session_factory,
|
||||
embedding_provider,
|
||||
vector_store,
|
||||
index_profile,
|
||||
)
|
||||
ingestion = DocumentIndexingPipelineService(document_pipeline, indexing_service)
|
||||
jobs = IngestionJobService(session_factory, settings.ingestion_max_attempts)
|
||||
dispatcher = CeleryIngestionDispatcher(create_celery_app(settings))
|
||||
return _IngestionComponents(
|
||||
session_factory=session_factory,
|
||||
storage=storage,
|
||||
vector_store=vector_store,
|
||||
embedding_provider=embedding_provider,
|
||||
index_profile=index_profile,
|
||||
upload=upload_service,
|
||||
parsing=parsing_service,
|
||||
chunking=chunking_service,
|
||||
indexing=indexing_service,
|
||||
jobs=jobs,
|
||||
dispatcher=dispatcher,
|
||||
)
|
||||
|
||||
|
||||
def _create_runner(settings: Settings, components: _IngestionComponents) -> IngestionJobRunner:
|
||||
"""Compose retry cleanup and execution policy from non-sensitive settings."""
|
||||
cleanup = DerivedDocumentCleanupService(
|
||||
components.session_factory,
|
||||
components.storage,
|
||||
components.vector_store,
|
||||
components.index_profile.profile_id,
|
||||
)
|
||||
return IngestionJobRunner(
|
||||
components.session_factory,
|
||||
components.parsing,
|
||||
components.chunking,
|
||||
components.indexing,
|
||||
cleanup,
|
||||
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
|
||||
retry_backoff_max_seconds=settings.ingestion_retry_backoff_max_seconds,
|
||||
lease_seconds=settings.ingestion_worker_lease_seconds,
|
||||
)
|
||||
|
||||
|
||||
def create_worker_services(settings: Settings) -> WorkerServices:
|
||||
"""Build the independent worker runtime from the same application boundaries."""
|
||||
components = _create_ingestion_components(settings)
|
||||
return WorkerServices(
|
||||
runner=_create_runner(settings, components),
|
||||
jobs=components.jobs,
|
||||
dispatcher=components.dispatcher,
|
||||
)
|
||||
|
||||
|
||||
def create_api_services(settings: Settings) -> ApiServices:
|
||||
"""Compose REST services while ingestion execution remains in workers."""
|
||||
components = _create_ingestion_components(settings)
|
||||
document_pipeline = DocumentPipelineService(
|
||||
components.upload,
|
||||
components.parsing,
|
||||
components.chunking,
|
||||
)
|
||||
ingestion = DocumentIndexingPipelineService(document_pipeline, components.indexing)
|
||||
submissions = AsyncIngestionSubmissionService(
|
||||
components.upload,
|
||||
components.jobs,
|
||||
components.dispatcher,
|
||||
)
|
||||
batches = IngestionBatchService(
|
||||
components.session_factory,
|
||||
submissions,
|
||||
max_files=settings.batch_max_files,
|
||||
max_total_bytes=settings.batch_max_total_bytes,
|
||||
)
|
||||
dense_retrieval = create_dense_retrieval_service(
|
||||
settings,
|
||||
embedding_provider,
|
||||
vector_store,
|
||||
index_profile,
|
||||
components.embedding_provider,
|
||||
components.vector_store,
|
||||
components.index_profile,
|
||||
)
|
||||
reranker_provider = create_reranker_provider(settings)
|
||||
search = create_search_service(
|
||||
settings,
|
||||
session_factory,
|
||||
components.session_factory,
|
||||
dense_retrieval,
|
||||
reranker_provider,
|
||||
)
|
||||
documents = DocumentManagementService(
|
||||
session_factory,
|
||||
storage,
|
||||
vector_store,
|
||||
active_index_profile_id=index_profile.profile_id,
|
||||
components.session_factory,
|
||||
components.storage,
|
||||
components.vector_store,
|
||||
active_index_profile_id=components.index_profile.profile_id,
|
||||
)
|
||||
return ApiServices(
|
||||
ingestion=ingestion,
|
||||
submissions=submissions,
|
||||
batches=batches,
|
||||
jobs=components.jobs,
|
||||
documents=documents,
|
||||
search=search,
|
||||
)
|
||||
return ApiServices(ingestion=ingestion, documents=documents, search=search)
|
||||
|
||||
@ -6,10 +6,18 @@ from uuid import UUID
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
|
||||
|
||||
from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services
|
||||
from ekp_api.schemas import DocumentResponse, SearchRequest, SearchResponse
|
||||
from ekp_api.schemas import (
|
||||
DocumentResponse,
|
||||
IngestionAcceptedResponse,
|
||||
IngestionBatchResponse,
|
||||
IngestionJobResponse,
|
||||
SearchRequest,
|
||||
SearchResponse,
|
||||
)
|
||||
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
|
||||
from ekp_core.document_management import (
|
||||
DocumentArtifactCleanupError,
|
||||
DocumentIngestionActiveError,
|
||||
DocumentNotFoundError,
|
||||
)
|
||||
from ekp_core.document_upload import (
|
||||
@ -18,7 +26,8 @@ from ekp_core.document_upload import (
|
||||
MediaTypeMismatchError,
|
||||
UnsupportedDocumentTypeError,
|
||||
)
|
||||
from ekp_parser import DocumentParseError
|
||||
from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundError, BatchSource
|
||||
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@ -44,17 +53,17 @@ Services = Annotated[ApiServices, Depends(get_api_services)]
|
||||
|
||||
@router.post(
|
||||
"/documents",
|
||||
response_model=DocumentResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
response_model=IngestionAcceptedResponse,
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
tags=["documents"],
|
||||
)
|
||||
def upload_document(
|
||||
services: Services,
|
||||
file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")],
|
||||
) -> DocumentResponse:
|
||||
"""Upload one file and synchronously run the complete Demo ingestion pipeline."""
|
||||
) -> IngestionAcceptedResponse:
|
||||
"""Persist one immutable source and durable Job, then return immediately."""
|
||||
try:
|
||||
result = services.ingestion.ingest(file.filename or "", file.content_type, file.file)
|
||||
submission = services.submissions.submit(file.filename or "", file.content_type, file.file)
|
||||
except DocumentTooLargeError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
|
||||
@ -65,21 +74,85 @@ def upload_document(
|
||||
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
detail="unsupported document type or media type",
|
||||
) from error
|
||||
except (DocumentUploadError, DocumentParseError) as error:
|
||||
except DocumentUploadError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail="document could not be accepted or parsed",
|
||||
detail="document could not be accepted",
|
||||
) from error
|
||||
except Exception as error:
|
||||
# Provider and persistence exceptions are intentionally not reflected to clients.
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="document processing failed",
|
||||
detail="document acceptance failed",
|
||||
) from error
|
||||
# Reload after every pipeline stage so server-generated timestamps and the
|
||||
# final persisted status, not a detached intermediate ORM object, are returned.
|
||||
document = services.documents.get_document(result.ingestion.document.id)
|
||||
return DocumentResponse.from_document(document)
|
||||
return IngestionAcceptedResponse.from_submission(submission)
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}", response_model=IngestionJobResponse, tags=["jobs"])
|
||||
def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
|
||||
"""Return durable stage, attempt and safe failure state for one Job."""
|
||||
try:
|
||||
job = services.jobs.get(job_id)
|
||||
except IngestionJobNotFoundError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="job not found"
|
||||
) from error
|
||||
return IngestionJobResponse.from_job(job)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/ingestion-batches",
|
||||
response_model=IngestionBatchResponse,
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
tags=["jobs"],
|
||||
)
|
||||
def upload_ingestion_batch(
|
||||
services: Services,
|
||||
files: Annotated[
|
||||
list[UploadFile],
|
||||
File(description="One or more PDF, DOCX, TXT, or Markdown files"),
|
||||
],
|
||||
) -> IngestionBatchResponse:
|
||||
"""Accept a bounded batch while isolating every file's result."""
|
||||
try:
|
||||
view = services.batches.submit(
|
||||
tuple(
|
||||
BatchSource(
|
||||
filename=file.filename or "",
|
||||
media_type=file.content_type,
|
||||
source=file.file,
|
||||
)
|
||||
for file in files
|
||||
)
|
||||
)
|
||||
except BatchFileCountLimitError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
|
||||
detail="batch file count exceeds the configured limit",
|
||||
) from error
|
||||
except Exception as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="batch could not be accepted",
|
||||
) from error
|
||||
return IngestionBatchResponse.from_view(view)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/ingestion-batches/{batch_id}",
|
||||
response_model=IngestionBatchResponse,
|
||||
tags=["jobs"],
|
||||
)
|
||||
def get_ingestion_batch(batch_id: UUID, services: Services) -> IngestionBatchResponse:
|
||||
"""Return aggregate and per-file status for Admin Console polling."""
|
||||
try:
|
||||
view = services.batches.get(batch_id)
|
||||
except BatchNotFoundError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="ingestion batch not found",
|
||||
) from error
|
||||
return IngestionBatchResponse.from_view(view)
|
||||
|
||||
|
||||
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
|
||||
@ -115,6 +188,11 @@ def delete_document(document_id: UUID, services: Services) -> Response:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
|
||||
) from error
|
||||
except DocumentIngestionActiveError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="document ingestion is still active",
|
||||
) from error
|
||||
except DocumentArtifactCleanupError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
|
||||
@ -5,8 +5,17 @@ from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from ekp_core.ingestion_batches import BatchAggregateStatus, BatchItemView, BatchView
|
||||
from ekp_core.ingestion_jobs import IngestionSubmission
|
||||
from ekp_core.search import SearchResult
|
||||
from ekp_domain import Document, DocumentStatus
|
||||
from ekp_domain import (
|
||||
BatchItemAcceptanceStatus,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
IngestionJob,
|
||||
IngestionJobStatus,
|
||||
IngestionStage,
|
||||
)
|
||||
|
||||
|
||||
class DocumentResponse(BaseModel):
|
||||
@ -29,6 +38,138 @@ class DocumentResponse(BaseModel):
|
||||
return cls.model_validate(document)
|
||||
|
||||
|
||||
class IngestionAcceptedResponse(DocumentResponse):
|
||||
"""Single-file acceptance with the durable Job needed for status polling."""
|
||||
|
||||
job_id: UUID
|
||||
job_status: IngestionJobStatus
|
||||
max_attempts: int
|
||||
|
||||
@classmethod
|
||||
def from_submission(cls, submission: IngestionSubmission) -> "IngestionAcceptedResponse":
|
||||
"""Flatten identifiers while retaining the existing document fields."""
|
||||
document = submission.document
|
||||
job = submission.job
|
||||
return cls(
|
||||
id=document.id,
|
||||
filename=document.filename,
|
||||
media_type=document.media_type,
|
||||
size_bytes=document.size_bytes,
|
||||
status=document.status,
|
||||
error_message=document.error_message,
|
||||
created_at=document.created_at,
|
||||
updated_at=document.updated_at,
|
||||
job_id=job.id,
|
||||
job_status=job.status,
|
||||
max_attempts=job.max_attempts,
|
||||
)
|
||||
|
||||
|
||||
class IngestionJobResponse(BaseModel):
|
||||
"""Public queue-independent Job status without provider or broker details."""
|
||||
|
||||
id: UUID
|
||||
document_id: UUID
|
||||
status: IngestionJobStatus
|
||||
stage: IngestionStage | None
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
error_message: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
completed_at: datetime | None
|
||||
|
||||
@classmethod
|
||||
def from_job(cls, job: IngestionJob) -> "IngestionJobResponse":
|
||||
"""Serialize only client-actionable state and safe diagnostics."""
|
||||
return cls(
|
||||
id=job.id,
|
||||
document_id=job.document_id,
|
||||
status=job.status,
|
||||
stage=job.stage,
|
||||
attempt=job.attempt_count,
|
||||
max_attempts=job.max_attempts,
|
||||
error_message=job.error_message,
|
||||
created_at=job.created_at,
|
||||
updated_at=job.updated_at,
|
||||
completed_at=job.completed_at,
|
||||
)
|
||||
|
||||
|
||||
class IngestionBatchItemResponse(BaseModel):
|
||||
"""Per-file acceptance and latest asynchronous processing state."""
|
||||
|
||||
item_index: int
|
||||
filename: str
|
||||
acceptance_status: BatchItemAcceptanceStatus
|
||||
document_id: UUID | None
|
||||
job_id: UUID | None
|
||||
job_status: IngestionJobStatus | None
|
||||
stage: IngestionStage | None
|
||||
attempt: int
|
||||
max_attempts: int | None
|
||||
error_message: str | None
|
||||
|
||||
@classmethod
|
||||
def from_view(cls, view: BatchItemView) -> "IngestionBatchItemResponse":
|
||||
job = view.job
|
||||
return cls(
|
||||
item_index=view.item.item_index,
|
||||
filename=view.item.filename,
|
||||
acceptance_status=view.item.acceptance_status,
|
||||
document_id=view.item.document_id,
|
||||
job_id=view.item.job_id,
|
||||
job_status=job.status if job is not None else None,
|
||||
stage=job.stage if job is not None else None,
|
||||
attempt=job.attempt_count if job is not None else 0,
|
||||
max_attempts=job.max_attempts if job is not None else None,
|
||||
error_message=(job.error_message if job is not None else view.item.error_message),
|
||||
)
|
||||
|
||||
|
||||
class IngestionBatchResponse(BaseModel):
|
||||
"""Batch aggregate with ordered independently processed files."""
|
||||
|
||||
batch_id: UUID
|
||||
status: BatchAggregateStatus
|
||||
total_files: int
|
||||
accepted_files: int
|
||||
rejected_files: int
|
||||
completed_files: int
|
||||
created_at: datetime
|
||||
items: list[IngestionBatchItemResponse]
|
||||
|
||||
@classmethod
|
||||
def from_view(cls, view: BatchView) -> "IngestionBatchResponse":
|
||||
items = [IngestionBatchItemResponse.from_view(item) for item in view.items]
|
||||
accepted = sum(
|
||||
item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED for item in items
|
||||
)
|
||||
rejected = sum(
|
||||
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED for item in items
|
||||
)
|
||||
completed = sum(
|
||||
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED
|
||||
or item.job_status
|
||||
in {
|
||||
IngestionJobStatus.SUCCEEDED,
|
||||
IngestionJobStatus.FAILED,
|
||||
IngestionJobStatus.CANCELLED,
|
||||
}
|
||||
for item in items
|
||||
)
|
||||
return cls(
|
||||
batch_id=view.batch.id,
|
||||
status=view.status,
|
||||
total_files=view.batch.total_files,
|
||||
accepted_files=accepted,
|
||||
rejected_files=rejected,
|
||||
completed_files=completed,
|
||||
created_at=view.batch.created_at,
|
||||
items=items,
|
||||
)
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
"""Validated public search input; normalization remains in the core service."""
|
||||
|
||||
|
||||
51
apps/api/src/ekp_api/worker.py
Normal file
51
apps/api/src/ekp_api/worker.py
Normal file
@ -0,0 +1,51 @@
|
||||
"""Celery worker entry point for durable ingestion Jobs."""
|
||||
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from ekp_api.celery_queue import (
|
||||
INGESTION_TASK_NAME,
|
||||
RECOVERY_TASK_NAME,
|
||||
create_celery_app,
|
||||
)
|
||||
from ekp_api.providers import WorkerServices, create_worker_services
|
||||
from ekp_core.config import get_settings
|
||||
from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs
|
||||
|
||||
_settings = get_settings()
|
||||
celery_app = create_celery_app(_settings)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _worker_services() -> WorkerServices:
|
||||
"""Build one provider/service graph per worker process."""
|
||||
return create_worker_services(_settings)
|
||||
|
||||
|
||||
@celery_app.task( # type: ignore[untyped-decorator]
|
||||
bind=True,
|
||||
name=INGESTION_TASK_NAME,
|
||||
max_retries=None,
|
||||
)
|
||||
def run_ingestion(self: Any, job_id: str) -> None:
|
||||
"""Execute one delivery and ask Celery to redeliver only when required.
|
||||
|
||||
Celery delivery retries are unbounded because valid-lease deferrals are not
|
||||
business Attempts; the durable runner independently enforces max_attempts.
|
||||
"""
|
||||
result = _worker_services().runner.run(UUID(job_id))
|
||||
if result.action == JobRunAction.RETRY:
|
||||
raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None)
|
||||
|
||||
|
||||
@celery_app.task(name=RECOVERY_TASK_NAME) # type: ignore[untyped-decorator]
|
||||
def recover_ingestion_jobs() -> int:
|
||||
"""Redeliver due durable Jobs after API/broker or worker interruptions."""
|
||||
services = _worker_services()
|
||||
return dispatch_pending_jobs(services.jobs, services.dispatcher)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run one worker with the embedded Demo recovery scheduler."""
|
||||
celery_app.worker_main(["worker", "--loglevel=INFO", "--queues=ingestion", "--beat"])
|
||||
5
apps/mcp-server/src/ekp_mcp/__init__.py
Normal file
5
apps/mcp-server/src/ekp_mcp/__init__.py
Normal file
@ -0,0 +1,5 @@
|
||||
"""Enterprise Knowledge MCP gateway."""
|
||||
|
||||
from ekp_mcp.server import create_mcp_http_app, create_mcp_server
|
||||
|
||||
__all__ = ["create_mcp_http_app", "create_mcp_server"]
|
||||
76
apps/mcp-server/src/ekp_mcp/auth.py
Normal file
76
apps/mcp-server/src/ekp_mcp/auth.py
Normal file
@ -0,0 +1,76 @@
|
||||
"""Request-scoped MCP service identity and bearer-token verification."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from secrets import compare_digest
|
||||
|
||||
from mcp.server.auth.middleware.auth_context import get_access_token
|
||||
from mcp.server.auth.provider import AccessToken
|
||||
from mcp.server.mcpserver.exceptions import ToolError
|
||||
from pydantic import SecretStr
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class McpIdentity:
|
||||
"""Authenticated client/service identity passed across the application boundary."""
|
||||
|
||||
client_id: str
|
||||
subject: str
|
||||
scopes: frozenset[str]
|
||||
|
||||
|
||||
class StaticServiceTokenVerifier:
|
||||
"""Validate one runtime-injected, least-privilege Demo service credential.
|
||||
|
||||
The verifier is intentionally small for DEMO-012. It establishes the MCP
|
||||
identity boundary without pretending to be the V1 OAuth or ACL solution.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_client_id",
|
||||
"_expected_token",
|
||||
"_issuer",
|
||||
"_scope",
|
||||
"_subject",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
expected_token: SecretStr,
|
||||
*,
|
||||
client_id: str,
|
||||
subject: str,
|
||||
scope: str,
|
||||
issuer: str,
|
||||
) -> None:
|
||||
self._expected_token = expected_token
|
||||
self._client_id = client_id
|
||||
self._subject = subject
|
||||
self._scope = scope
|
||||
self._issuer = issuer
|
||||
|
||||
async def verify_token(self, token: str) -> AccessToken | None:
|
||||
"""Return scoped identity metadata only for a constant-time token match."""
|
||||
if not compare_digest(token, self._expected_token.get_secret_value()):
|
||||
return None
|
||||
return AccessToken(
|
||||
token=token,
|
||||
client_id=self._client_id,
|
||||
subject=self._subject,
|
||||
scopes=[self._scope],
|
||||
claims={"iss": self._issuer},
|
||||
)
|
||||
|
||||
|
||||
def current_mcp_identity() -> McpIdentity:
|
||||
"""Resolve the SDK-authenticated principal for the active tool request."""
|
||||
access_token = get_access_token()
|
||||
if access_token is None:
|
||||
# HTTP middleware normally rejects this earlier; retaining the guard
|
||||
# prevents accidental unauthenticated in-process tool invocation.
|
||||
raise ToolError("authentication required")
|
||||
subject = access_token.subject or access_token.client_id
|
||||
return McpIdentity(
|
||||
client_id=access_token.client_id,
|
||||
subject=subject,
|
||||
scopes=frozenset(access_token.scopes),
|
||||
)
|
||||
30
apps/mcp-server/src/ekp_mcp/main.py
Normal file
30
apps/mcp-server/src/ekp_mcp/main.py
Normal file
@ -0,0 +1,30 @@
|
||||
"""Command-line entry point for the independent MCP gateway process."""
|
||||
|
||||
from mcp.server.transport_security import TransportSecuritySettings
|
||||
|
||||
from ekp_core.config import get_settings
|
||||
from ekp_mcp.server import create_mcp_server
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run the authenticated Streamable HTTP server from environment settings."""
|
||||
settings = get_settings()
|
||||
server = create_mcp_server(settings)
|
||||
server.run(
|
||||
transport="streamable-http",
|
||||
host=settings.mcp_host,
|
||||
port=settings.mcp_port,
|
||||
streamable_http_path=settings.mcp_path,
|
||||
json_response=True,
|
||||
stateless_http=True,
|
||||
max_request_body_size=settings.mcp_max_request_body_bytes,
|
||||
transport_security=TransportSecuritySettings(
|
||||
enable_dns_rebinding_protection=True,
|
||||
allowed_hosts=settings.mcp_allowed_hosts,
|
||||
allowed_origins=settings.mcp_allowed_origins,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through the console script.
|
||||
main()
|
||||
77
apps/mcp-server/src/ekp_mcp/search.py
Normal file
77
apps/mcp-server/src/ekp_mcp/search.py
Normal file
@ -0,0 +1,77 @@
|
||||
"""MCP-specific application boundary over the provider-neutral SearchService."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ekp_core.search import SearchResult
|
||||
from ekp_mcp.auth import McpIdentity
|
||||
|
||||
|
||||
class KnowledgeSearcher(Protocol):
|
||||
"""Subset of SearchService required by the MCP gateway."""
|
||||
|
||||
def search(self, query: str, top_k: int | None = None) -> SearchResult:
|
||||
"""Return reranked canonical sources for a normalized query."""
|
||||
|
||||
|
||||
class McpSourceResponse(BaseModel):
|
||||
"""Provider-neutral source shape exposed by `search_knowledge`."""
|
||||
|
||||
document_id: UUID
|
||||
filename: str
|
||||
page: int | None
|
||||
section: str | None
|
||||
chunk_id: UUID
|
||||
content: str
|
||||
score: float
|
||||
rerank_score: float
|
||||
|
||||
|
||||
class McpSearchResponse(BaseModel):
|
||||
"""Structured MCP result aligned with the REST search source fields."""
|
||||
|
||||
results: list[McpSourceResponse]
|
||||
|
||||
|
||||
class McpKnowledgeSearch(Protocol):
|
||||
"""Identity-aware search boundary consumed by the protocol server."""
|
||||
|
||||
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
|
||||
"""Return authorized, structured knowledge sources."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class McpSearchApplication:
|
||||
"""Require an identity before delegating to the shared SearchService.
|
||||
|
||||
Document ACL is outside the current Demo scope. Keeping identity explicit
|
||||
here prevents the gateway from becoming an anonymous bypass when the
|
||||
permission resolver is added in the V1 tasks.
|
||||
"""
|
||||
|
||||
searcher: KnowledgeSearcher
|
||||
required_scope: str
|
||||
|
||||
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
|
||||
"""Authorize the Demo scope and serialize canonical search results."""
|
||||
if self.required_scope not in identity.scopes:
|
||||
raise PermissionError("MCP identity lacks the knowledge search scope")
|
||||
result = self.searcher.search(query, top_k)
|
||||
return McpSearchResponse(
|
||||
results=[
|
||||
McpSourceResponse(
|
||||
document_id=source.document_id,
|
||||
filename=source.filename,
|
||||
page=source.page,
|
||||
section=source.section,
|
||||
chunk_id=source.chunk_id,
|
||||
content=source.content,
|
||||
score=source.score,
|
||||
rerank_score=source.rerank_score,
|
||||
)
|
||||
for source in result.results
|
||||
]
|
||||
)
|
||||
133
apps/mcp-server/src/ekp_mcp/server.py
Normal file
133
apps/mcp-server/src/ekp_mcp/server.py
Normal file
@ -0,0 +1,133 @@
|
||||
"""Authenticated Streamable HTTP MCP server exposing Demo business tools."""
|
||||
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Annotated
|
||||
|
||||
import anyio
|
||||
from mcp.server import MCPServer
|
||||
from mcp.server.auth.settings import AuthSettings
|
||||
from mcp.server.mcpserver.exceptions import ToolError
|
||||
from mcp.server.transport_security import TransportSecuritySettings
|
||||
from pydantic import Field
|
||||
from starlette.applications import Starlette
|
||||
|
||||
from ekp_api.providers import ApiServices, create_runtime_search_service
|
||||
from ekp_core.config import Settings, get_settings
|
||||
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
|
||||
from ekp_mcp.auth import StaticServiceTokenVerifier, current_mcp_identity
|
||||
from ekp_mcp.search import McpKnowledgeSearch, McpSearchApplication, McpSearchResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class McpConfigurationError(ValueError):
|
||||
"""Raised when the gateway lacks its dedicated runtime credential."""
|
||||
|
||||
|
||||
def _create_search_application(
|
||||
settings: Settings,
|
||||
api_services: ApiServices | None,
|
||||
) -> McpSearchApplication:
|
||||
searcher = (
|
||||
api_services.search if api_services is not None else create_runtime_search_service(settings)
|
||||
)
|
||||
return McpSearchApplication(searcher, settings.mcp_required_scope)
|
||||
|
||||
|
||||
def create_mcp_server(
|
||||
settings: Settings | None = None,
|
||||
*,
|
||||
api_services: ApiServices | None = None,
|
||||
search_application: McpKnowledgeSearch | None = None,
|
||||
) -> MCPServer[dict[str, object]]:
|
||||
"""Create the authenticated MCP gateway with only `search_knowledge`."""
|
||||
resolved_settings = settings or get_settings()
|
||||
if (
|
||||
resolved_settings.mcp_auth_token is None
|
||||
or len(resolved_settings.mcp_auth_token.get_secret_value()) < 32
|
||||
):
|
||||
raise McpConfigurationError("MCP_AUTH_TOKEN must contain at least 32 characters")
|
||||
|
||||
application = search_application or _create_search_application(
|
||||
resolved_settings,
|
||||
api_services,
|
||||
)
|
||||
token_verifier = StaticServiceTokenVerifier(
|
||||
resolved_settings.mcp_auth_token,
|
||||
client_id=resolved_settings.mcp_client_id,
|
||||
subject=resolved_settings.mcp_service_identity,
|
||||
scope=resolved_settings.mcp_required_scope,
|
||||
issuer=str(resolved_settings.mcp_issuer_url),
|
||||
)
|
||||
server: MCPServer[dict[str, object]] = MCPServer(
|
||||
name="enterprise-knowledge",
|
||||
title="Enterprise Knowledge MCP Server",
|
||||
description="Authenticated enterprise knowledge search backed by canonical sources.",
|
||||
version="0.1.0",
|
||||
token_verifier=token_verifier,
|
||||
auth=AuthSettings(
|
||||
issuer_url=resolved_settings.mcp_issuer_url,
|
||||
resource_server_url=resolved_settings.mcp_public_url,
|
||||
required_scopes=[resolved_settings.mcp_required_scope],
|
||||
),
|
||||
log_level=resolved_settings.log_level,
|
||||
)
|
||||
|
||||
@server.tool(
|
||||
name="search_knowledge",
|
||||
description=(
|
||||
"Search authorized enterprise knowledge and return traceable source passages. "
|
||||
"Use for factual questions that should be grounded in indexed company documents."
|
||||
),
|
||||
structured_output=True,
|
||||
)
|
||||
async def search_knowledge(
|
||||
query: Annotated[str, Field(min_length=1, max_length=1600)],
|
||||
top_k: Annotated[int, Field(ge=1, le=64, strict=True)] = 5,
|
||||
) -> McpSearchResponse:
|
||||
"""Run the shared synchronous search pipeline outside the ASGI event loop."""
|
||||
identity = current_mcp_identity()
|
||||
try:
|
||||
return await anyio.to_thread.run_sync(
|
||||
partial(application.search, identity, query, top_k)
|
||||
)
|
||||
except InvalidRetrievalQueryError as error:
|
||||
raise ToolError("search query must contain non-whitespace text") from error
|
||||
except PermissionError as error:
|
||||
raise ToolError("knowledge search is not authorized") from error
|
||||
except Exception:
|
||||
# Do not reflect provider, database, endpoint, or credential details
|
||||
# into MCP content or logs. Operations can correlate this generic
|
||||
# event with upstream sanitized service telemetry.
|
||||
logger.error("MCP knowledge search failed")
|
||||
raise ToolError("knowledge search failed") from None
|
||||
|
||||
return server
|
||||
|
||||
|
||||
def create_mcp_http_app(
|
||||
settings: Settings | None = None,
|
||||
*,
|
||||
api_services: ApiServices | None = None,
|
||||
search_application: McpKnowledgeSearch | None = None,
|
||||
) -> Starlette:
|
||||
"""Build the stateless Streamable HTTP app used by WorkBuddy and tests."""
|
||||
resolved_settings = settings or get_settings()
|
||||
server = create_mcp_server(
|
||||
resolved_settings,
|
||||
api_services=api_services,
|
||||
search_application=search_application,
|
||||
)
|
||||
return server.streamable_http_app(
|
||||
streamable_http_path=resolved_settings.mcp_path,
|
||||
json_response=True,
|
||||
stateless_http=True,
|
||||
max_request_body_size=resolved_settings.mcp_max_request_body_bytes,
|
||||
host=resolved_settings.mcp_host,
|
||||
transport_security=TransportSecuritySettings(
|
||||
enable_dns_rebinding_protection=True,
|
||||
allowed_hosts=resolved_settings.mcp_allowed_hosts,
|
||||
allowed_origins=resolved_settings.mcp_allowed_origins,
|
||||
),
|
||||
)
|
||||
61
deploy/docker/Dockerfile
Normal file
61
deploy/docker/Dockerfile
Normal file
@ -0,0 +1,61 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# One locked runtime produces three independently deployable process images.
|
||||
ARG PYTHON_IMAGE=python:3.12.14-slim-bookworm@sha256:782412e85d0f0984994c290652577d4018aff08145c85b262bb63dc0c7522254
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1
|
||||
|
||||
FROM ${UV_IMAGE} AS uv-bin
|
||||
|
||||
FROM ${PYTHON_IMAGE} AS runtime
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
UV_COMPILE_BYTECODE=1 \
|
||||
UV_LINK_MODE=copy \
|
||||
VIRTUAL_ENV=/app/.venv \
|
||||
PATH=/app/.venv/bin:${PATH} \
|
||||
HOME=/tmp/ekp \
|
||||
XDG_CACHE_HOME=/tmp/ekp/cache \
|
||||
APP_ENV=production \
|
||||
API_HOST=0.0.0.0 \
|
||||
MCP_HOST=0.0.0.0 \
|
||||
ADMIN_WEB_ROOT=/app/apps/admin-web
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=uv-bin /uv /uvx /usr/local/bin/
|
||||
COPY pyproject.toml uv.lock README.md alembic.ini ./
|
||||
COPY apps ./apps
|
||||
COPY packages ./packages
|
||||
COPY migrations ./migrations
|
||||
|
||||
# --frozen prevents lock changes; --no-dev and --no-editable keep the runtime
|
||||
# independent from source-tree import hooks while retaining migration sources.
|
||||
RUN uv sync --frozen --no-dev --no-editable \
|
||||
&& test -f /app/apps/admin-web/index.html \
|
||||
&& groupadd --gid 10001 ekp \
|
||||
&& useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin ekp \
|
||||
&& mkdir -p /tmp/ekp/cache \
|
||||
&& chown -R 10001:10001 /tmp/ekp \
|
||||
&& rm -rf /root/.cache/uv
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
FROM runtime AS api
|
||||
EXPOSE 8000
|
||||
# This endpoint is intentionally credential-free and suitable for container liveness.
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=2)"]
|
||||
CMD ["uvicorn", "ekp_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
|
||||
|
||||
FROM runtime AS mcp
|
||||
EXPOSE 8766
|
||||
# MCP requests require authentication, so the image-level check only verifies PID 1.
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
CMD ["ekp-mcp"]
|
||||
|
||||
FROM runtime AS worker
|
||||
# Worker health is process-level here; durable Job recovery is validated by the service tests.
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
CMD ["ekp-worker"]
|
||||
36
deploy/docker/docker-bake.hcl
Normal file
36
deploy/docker/docker-bake.hcl
Normal file
@ -0,0 +1,36 @@
|
||||
# Reproducible target names consumed by local and CI buildx invocations.
|
||||
variable "IMAGE_REGISTRY" {
|
||||
default = "knowledge-platform"
|
||||
}
|
||||
|
||||
variable "IMAGE_TAG" {
|
||||
default = "dev"
|
||||
}
|
||||
|
||||
group "default" {
|
||||
targets = ["api", "mcp", "worker"]
|
||||
}
|
||||
|
||||
target "common" {
|
||||
context = "."
|
||||
dockerfile = "deploy/docker/Dockerfile"
|
||||
platforms = ["linux/amd64", "linux/arm64"]
|
||||
}
|
||||
|
||||
target "api" {
|
||||
inherits = ["common"]
|
||||
target = "api"
|
||||
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-api:${IMAGE_TAG}"]
|
||||
}
|
||||
|
||||
target "mcp" {
|
||||
inherits = ["common"]
|
||||
target = "mcp"
|
||||
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-mcp:${IMAGE_TAG}"]
|
||||
}
|
||||
|
||||
target "worker" {
|
||||
inherits = ["common"]
|
||||
target = "worker"
|
||||
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-worker:${IMAGE_TAG}"]
|
||||
}
|
||||
50
deploy/helm/environments/values.example.yaml
Normal file
50
deploy/helm/environments/values.example.yaml
Normal file
@ -0,0 +1,50 @@
|
||||
# Copy this file outside the repository for one environment. It contains only
|
||||
# non-sensitive settings and Kubernetes Secret references; never add values for
|
||||
# DATABASE_URL, RABBITMQ_URL, provider keys, passwords or MCP_AUTH_TOKEN here.
|
||||
existingSecret: knowledge-platform-runtime
|
||||
|
||||
images:
|
||||
api:
|
||||
repository: registry.example.internal/knowledge/enterprise-knowledge-api
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
mcp:
|
||||
repository: registry.example.internal/knowledge/enterprise-knowledge-mcp
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
worker:
|
||||
repository: registry.example.internal/knowledge/enterprise-knowledge-worker
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
|
||||
ingress:
|
||||
enabled: true
|
||||
className: nginx
|
||||
api:
|
||||
host: knowledge.example.internal
|
||||
tlsSecretName: knowledge-platform-api-tls
|
||||
mcp:
|
||||
host: mcp.example.internal
|
||||
tlsSecretName: knowledge-platform-mcp-tls
|
||||
|
||||
config:
|
||||
objectStorage:
|
||||
endpoint: https://s3.example.internal
|
||||
bucket: knowledge-platform
|
||||
vectorStore:
|
||||
uri: oceanbase.example.internal:2881
|
||||
user: knowledge-platform
|
||||
database: knowledge_platform
|
||||
mcp:
|
||||
publicUrl: https://mcp.example.internal/mcp
|
||||
issuerUrl: https://mcp.example.internal
|
||||
allowedHosts:
|
||||
- mcp.example.internal
|
||||
allowedOrigins:
|
||||
- https://mcp.example.internal
|
||||
|
||||
# External RabbitMQ is the production default. If this local dependency is
|
||||
# explicitly enabled for a demo cluster, create its Secret before deployment.
|
||||
rabbitmq:
|
||||
enabled: false
|
||||
existingSecret: knowledge-platform-rabbitmq
|
||||
15
deploy/helm/knowledge-platform/Chart.yaml
Normal file
15
deploy/helm/knowledge-platform/Chart.yaml
Normal file
@ -0,0 +1,15 @@
|
||||
apiVersion: v2
|
||||
name: knowledge-platform
|
||||
description: Enterprise Knowledge Platform public-knowledge deployment
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: "0.1.0"
|
||||
|
||||
# This local dependency is disabled by default. Production deployments should
|
||||
# use an externally managed RabbitMQ service with an independently owned data
|
||||
# lifecycle; the subchart exists only for explicit development/demo use.
|
||||
dependencies:
|
||||
- name: rabbitmq
|
||||
version: 0.1.0
|
||||
repository: file://charts/rabbitmq
|
||||
condition: rabbitmq.enabled
|
||||
@ -0,0 +1,6 @@
|
||||
apiVersion: v2
|
||||
name: rabbitmq
|
||||
description: Minimal single-node RabbitMQ dependency for local demo clusters only
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: "4.1.4"
|
||||
@ -0,0 +1,11 @@
|
||||
{{- define "rabbitmq.fullname" -}}
|
||||
{{- printf "%s-rabbitmq" .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "rabbitmq.image" -}}
|
||||
{{- if .Values.image.digest -}}
|
||||
{{- printf "%s@%s" .Values.image.repository .Values.image.digest -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@ -0,0 +1,16 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "rabbitmq.fullname" . }}
|
||||
labels:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
ports:
|
||||
- name: amqp
|
||||
port: {{ .Values.service.port }}
|
||||
targetPort: amqp
|
||||
selector:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
@ -0,0 +1,8 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "rabbitmq.fullname" . }}
|
||||
labels:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
automountServiceAccountToken: false
|
||||
@ -0,0 +1,109 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "rabbitmq.fullname" . }}
|
||||
labels:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
spec:
|
||||
serviceName: {{ include "rabbitmq.fullname" . }}
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: rabbitmq
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "rabbitmq.fullname" . }}
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
runAsGroup: 999
|
||||
fsGroup: 999
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rabbitmq
|
||||
image: {{ include "rabbitmq.image" . | quote }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
ports:
|
||||
- name: amqp
|
||||
containerPort: 5672
|
||||
env:
|
||||
- name: RABBITMQ_DEFAULT_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.existingSecret }}
|
||||
key: {{ .Values.secretKeys.username }}
|
||||
- name: RABBITMQ_DEFAULT_PASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.existingSecret }}
|
||||
key: {{ .Values.secretKeys.password }}
|
||||
- name: RABBITMQ_ERLANG_COOKIE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.existingSecret }}
|
||||
key: {{ .Values.secretKeys.erlangCookie }}
|
||||
startupProbe:
|
||||
exec:
|
||||
command: ["rabbitmq-diagnostics", "-q", "ping"]
|
||||
periodSeconds: 5
|
||||
failureThreshold: 24
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["rabbitmq-diagnostics", "-q", "check_running"]
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["rabbitmq-diagnostics", "-q", "ping"]
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
resources:
|
||||
{{- toYaml .Values.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/rabbitmq
|
||||
- name: config
|
||||
mountPath: /etc/rabbitmq/conf.d
|
||||
- name: log
|
||||
mountPath: /var/log/rabbitmq
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: config
|
||||
emptyDir: {}
|
||||
- name: log
|
||||
emptyDir: {}
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
{{- if not .Values.persistence.enabled }}
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if .Values.persistence.enabled }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes: [{{ .Values.persistence.accessMode | quote }}]
|
||||
{{- if .Values.persistence.storageClass }}
|
||||
storageClassName: {{ .Values.persistence.storageClass | quote }}
|
||||
{{- end }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.persistence.size }}
|
||||
{{- end }}
|
||||
@ -0,0 +1,69 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "existingSecret", "secretKeys", "image", "service", "persistence", "resources"],
|
||||
"properties": {
|
||||
"global": {"type": "object"},
|
||||
"enabled": {"type": "boolean"},
|
||||
"existingSecret": {"type": "string", "minLength": 1},
|
||||
"secretKeys": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["username", "password", "erlangCookie"],
|
||||
"properties": {
|
||||
"username": {"type": "string", "minLength": 1},
|
||||
"password": {"type": "string", "minLength": 1},
|
||||
"erlangCookie": {"type": "string", "minLength": 1}
|
||||
}
|
||||
},
|
||||
"image": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["repository", "tag", "digest", "pullPolicy"],
|
||||
"properties": {
|
||||
"repository": {"type": "string", "minLength": 1},
|
||||
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
|
||||
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
|
||||
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
|
||||
}
|
||||
},
|
||||
"service": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["port"],
|
||||
"properties": {"port": {"type": "integer", "minimum": 1, "maximum": 65535}}
|
||||
},
|
||||
"persistence": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "storageClass", "accessMode", "size"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"storageClass": {"type": "string"},
|
||||
"accessMode": {"enum": ["ReadWriteOnce", "ReadWriteMany"]},
|
||||
"size": {"type": "string", "minLength": 2}
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["requests", "limits"],
|
||||
"properties": {
|
||||
"requests": {"$ref": "#/definitions/resourcePair"},
|
||||
"limits": {"$ref": "#/definitions/resourcePair"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"resourcePair": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["cpu", "memory"],
|
||||
"properties": {
|
||||
"cpu": {"type": ["string", "number"]},
|
||||
"memory": {"type": "string", "minLength": 2}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
25
deploy/helm/knowledge-platform/charts/rabbitmq/values.yaml
Normal file
25
deploy/helm/knowledge-platform/charts/rabbitmq/values.yaml
Normal file
@ -0,0 +1,25 @@
|
||||
enabled: false
|
||||
existingSecret: knowledge-platform-rabbitmq
|
||||
secretKeys:
|
||||
username: username
|
||||
password: password
|
||||
erlangCookie: erlang-cookie
|
||||
image:
|
||||
repository: rabbitmq
|
||||
tag: 4.1.4-alpine
|
||||
digest: sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2
|
||||
pullPolicy: IfNotPresent
|
||||
service:
|
||||
port: 5672
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: ""
|
||||
accessMode: ReadWriteOnce
|
||||
size: 8Gi
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
12
deploy/helm/knowledge-platform/templates/NOTES.txt
Normal file
12
deploy/helm/knowledge-platform/templates/NOTES.txt
Normal file
@ -0,0 +1,12 @@
|
||||
Enterprise Knowledge Platform {{ .Chart.AppVersion }} is managed by Helm release {{ .Release.Name }}.
|
||||
|
||||
Runtime credentials are read only from Secret {{ .Values.existingSecret }}.
|
||||
{{- if .Values.ingress.enabled }}
|
||||
Admin/API: https://{{ .Values.ingress.api.host }}{{ .Values.ingress.api.path }}
|
||||
MCP: https://{{ .Values.ingress.mcp.host }}{{ .Values.ingress.mcp.path }}
|
||||
{{- else }}
|
||||
Ingress is disabled. Enable it in the environment values file before remote use.
|
||||
{{- end }}
|
||||
|
||||
Inspect rollout state without printing environment variables:
|
||||
kubectl -n {{ .Release.Namespace }} get deploy,job,pod
|
||||
106
deploy/helm/knowledge-platform/templates/_helpers.tpl
Normal file
106
deploy/helm/knowledge-platform/templates/_helpers.tpl
Normal file
@ -0,0 +1,106 @@
|
||||
{{/* Stable resource name helpers keep every workload under one release identity. */}}
|
||||
{{- define "knowledge-platform.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- $name := include "knowledge-platform.name" . -}}
|
||||
{{- if contains $name .Release.Name -}}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.labels" -}}
|
||||
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | quote }}
|
||||
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
{{- default (include "knowledge-platform.fullname" .) .Values.serviceAccount.name -}}
|
||||
{{- else -}}
|
||||
{{- required "serviceAccount.name is required when serviceAccount.create=false" .Values.serviceAccount.name -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* A supplied digest takes precedence over the immutable release tag. */}}
|
||||
{{- define "knowledge-platform.image" -}}
|
||||
{{- if .digest -}}
|
||||
{{- printf "%s@%s" .repository .digest -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s:%s" .repository .tag -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.configMapName" -}}
|
||||
{{- printf "%s-runtime" (include "knowledge-platform.fullname" .) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Cross-field rules complement values.schema.json's structural checks. */}}
|
||||
{{- define "knowledge-platform.validateValues" -}}
|
||||
{{- if lt (float64 .Values.config.ingestion.retryBackoffMaxSeconds) (float64 .Values.config.ingestion.retryBackoffSeconds) -}}
|
||||
{{- fail "config.ingestion.retryBackoffMaxSeconds must be >= retryBackoffSeconds" -}}
|
||||
{{- end -}}
|
||||
{{- if lt (float64 .Values.config.ingestion.batchMaxTotalBytes) (float64 .Values.config.ingestion.maxUploadBytes) -}}
|
||||
{{- fail "config.ingestion.batchMaxTotalBytes must be >= maxUploadBytes" -}}
|
||||
{{- end -}}
|
||||
{{- if ge (float64 .Values.config.chunking.overlapTokens) (float64 .Values.config.chunking.maxTokens) -}}
|
||||
{{- fail "config.chunking.overlapTokens must be < maxTokens" -}}
|
||||
{{- end -}}
|
||||
{{- if gt (float64 .Values.config.retrieval.searchTopK) (float64 .Values.config.retrieval.denseTopN) -}}
|
||||
{{- fail "config.retrieval.searchTopK must be <= denseTopN" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.podSecurityContext" -}}
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.containerSecurityContext" -}}
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.envFrom" -}}
|
||||
- configMapRef:
|
||||
name: {{ include "knowledge-platform.configMapName" . }}
|
||||
- secretRef:
|
||||
name: {{ .Values.existingSecret }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.tmpVolume" -}}
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
sizeLimit: 1Gi
|
||||
{{- end -}}
|
||||
|
||||
{{- define "knowledge-platform.tmpMount" -}}
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
{{- end -}}
|
||||
75
deploy/helm/knowledge-platform/templates/api-deployment.yaml
Normal file
75
deploy/helm/knowledge-platform/templates/api-deployment.yaml
Normal file
@ -0,0 +1,75 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-api
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: api
|
||||
spec:
|
||||
replicas: {{ .Values.api.replicaCount }}
|
||||
minReadySeconds: 5
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 0
|
||||
maxSurge: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: api
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: api
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: api
|
||||
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
|
||||
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8000
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- toYaml .Values.api.probes.startup | nindent 12 }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- toYaml .Values.api.probes.readiness | nindent 12 }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- toYaml .Values.api.probes.liveness | nindent 12 }}
|
||||
resources:
|
||||
{{- toYaml .Values.api.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||
volumes:
|
||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||
17
deploy/helm/knowledge-platform/templates/api-service.yaml
Normal file
17
deploy/helm/knowledge-platform/templates/api-service.yaml
Normal file
@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-api
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: api
|
||||
spec:
|
||||
type: {{ .Values.api.service.type }}
|
||||
ports:
|
||||
- name: http
|
||||
port: {{ .Values.api.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
selector:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: api
|
||||
66
deploy/helm/knowledge-platform/templates/configmap.yaml
Normal file
66
deploy/helm/knowledge-platform/templates/configmap.yaml
Normal file
@ -0,0 +1,66 @@
|
||||
{{- include "knowledge-platform.validateValues" . }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.configMapName" . }}
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
data:
|
||||
# Only the explicitly enumerated non-sensitive settings are rendered here.
|
||||
APP_ENV: "production"
|
||||
APP_NAME: {{ .Values.config.appName | quote }}
|
||||
LOG_LEVEL: {{ .Values.config.logLevel | quote }}
|
||||
API_HOST: "0.0.0.0"
|
||||
API_PORT: "8000"
|
||||
API_PREFIX: "/api/v1"
|
||||
DATABASE_POOL_SIZE: {{ .Values.config.database.poolSize | quote }}
|
||||
DATABASE_MAX_OVERFLOW: {{ .Values.config.database.maxOverflow | quote }}
|
||||
DATABASE_POOL_RECYCLE_SECONDS: {{ .Values.config.database.poolRecycleSeconds | quote }}
|
||||
DATABASE_POOL_TIMEOUT_SECONDS: {{ .Values.config.database.poolTimeoutSeconds | quote }}
|
||||
DATABASE_CONNECT_TIMEOUT_SECONDS: {{ .Values.config.database.connectTimeoutSeconds | quote }}
|
||||
INGESTION_MAX_ATTEMPTS: {{ .Values.config.ingestion.maxAttempts | quote }}
|
||||
INGESTION_RETRY_BACKOFF_SECONDS: {{ .Values.config.ingestion.retryBackoffSeconds | quote }}
|
||||
INGESTION_RETRY_BACKOFF_MAX_SECONDS: {{ .Values.config.ingestion.retryBackoffMaxSeconds | quote }}
|
||||
INGESTION_WORKER_LEASE_SECONDS: {{ .Values.config.ingestion.workerLeaseSeconds | quote }}
|
||||
INGESTION_RECOVERY_INTERVAL_SECONDS: {{ .Values.config.ingestion.recoveryIntervalSeconds | quote }}
|
||||
BATCH_MAX_FILES: {{ .Values.config.ingestion.batchMaxFiles | quote }}
|
||||
BATCH_MAX_TOTAL_BYTES: {{ printf "%.0f" .Values.config.ingestion.batchMaxTotalBytes | quote }}
|
||||
MAX_UPLOAD_BYTES: {{ printf "%.0f" .Values.config.ingestion.maxUploadBytes | quote }}
|
||||
OBJECT_STORAGE_PROVIDER: {{ .Values.config.objectStorage.provider | quote }}
|
||||
BOS_ENDPOINT: {{ .Values.config.objectStorage.endpoint | quote }}
|
||||
BOS_BUCKET: {{ .Values.config.objectStorage.bucket | quote }}
|
||||
BOS_KEY_PREFIX: {{ .Values.config.objectStorage.keyPrefix | quote }}
|
||||
BOS_TIMEOUT_SECONDS: {{ .Values.config.objectStorage.timeoutSeconds | quote }}
|
||||
CHUNK_MAX_TOKENS: {{ .Values.config.chunking.maxTokens | quote }}
|
||||
CHUNK_OVERLAP_TOKENS: {{ .Values.config.chunking.overlapTokens | quote }}
|
||||
CHUNK_MAX_CHARACTERS: {{ .Values.config.chunking.maxCharacters | quote }}
|
||||
EMBEDDING_PROVIDER: {{ .Values.config.embedding.provider | quote }}
|
||||
EMBEDDING_MODEL: {{ .Values.config.embedding.model | quote }}
|
||||
EMBEDDING_ENDPOINT: {{ .Values.config.embedding.endpoint | quote }}
|
||||
EMBEDDING_TIMEOUT_SECONDS: {{ .Values.config.embedding.timeoutSeconds | quote }}
|
||||
VECTOR_STORE_PROVIDER: {{ .Values.config.vectorStore.provider | quote }}
|
||||
OCEANBASE_VECTOR_URI: {{ .Values.config.vectorStore.uri | quote }}
|
||||
OCEANBASE_VECTOR_USER: {{ .Values.config.vectorStore.user | quote }}
|
||||
OCEANBASE_VECTOR_DATABASE: {{ .Values.config.vectorStore.database | quote }}
|
||||
OCEANBASE_VECTOR_TABLE: {{ .Values.config.vectorStore.table | quote }}
|
||||
OCEANBASE_VECTOR_INDEX: {{ .Values.config.vectorStore.index | quote }}
|
||||
OCEANBASE_VECTOR_BATCH_SIZE: {{ .Values.config.vectorStore.batchSize | quote }}
|
||||
OCEANBASE_VECTOR_TIMEOUT_SECONDS: {{ .Values.config.vectorStore.timeoutSeconds | quote }}
|
||||
INDEX_PROFILE_ID: {{ .Values.config.retrieval.indexProfileId | quote }}
|
||||
DENSE_TOP_N: {{ .Values.config.retrieval.denseTopN | quote }}
|
||||
SEARCH_TOP_K: {{ .Values.config.retrieval.searchTopK | quote }}
|
||||
RERANKER_PROVIDER: {{ .Values.config.reranker.provider | quote }}
|
||||
RERANKER_MODEL: {{ .Values.config.reranker.model | quote }}
|
||||
RERANKER_ENDPOINT: {{ .Values.config.reranker.endpoint | quote }}
|
||||
RERANKER_TIMEOUT_SECONDS: {{ .Values.config.reranker.timeoutSeconds | quote }}
|
||||
MCP_HOST: "0.0.0.0"
|
||||
MCP_PORT: "8766"
|
||||
MCP_PATH: "/mcp"
|
||||
MCP_PUBLIC_URL: {{ .Values.config.mcp.publicUrl | quote }}
|
||||
MCP_ISSUER_URL: {{ .Values.config.mcp.issuerUrl | quote }}
|
||||
MCP_CLIENT_ID: {{ .Values.config.mcp.clientId | quote }}
|
||||
MCP_SERVICE_IDENTITY: {{ .Values.config.mcp.serviceIdentity | quote }}
|
||||
MCP_REQUIRED_SCOPE: {{ .Values.config.mcp.requiredScope | quote }}
|
||||
MCP_ALLOWED_HOSTS: {{ .Values.config.mcp.allowedHosts | toJson | quote }}
|
||||
MCP_ALLOWED_ORIGINS: {{ .Values.config.mcp.allowedOrigins | toJson | quote }}
|
||||
MCP_MAX_REQUEST_BODY_BYTES: {{ printf "%.0f" .Values.config.mcp.maxRequestBodyBytes | quote }}
|
||||
42
deploy/helm/knowledge-platform/templates/ingress.yaml
Normal file
42
deploy/helm/knowledge-platform/templates/ingress.yaml
Normal file
@ -0,0 +1,42 @@
|
||||
{{- if .Values.ingress.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
tls:
|
||||
- hosts:
|
||||
- {{ .Values.ingress.api.host | quote }}
|
||||
secretName: {{ .Values.ingress.api.tlsSecretName }}
|
||||
- hosts:
|
||||
- {{ .Values.ingress.mcp.host | quote }}
|
||||
secretName: {{ .Values.ingress.mcp.tlsSecretName }}
|
||||
rules:
|
||||
- host: {{ .Values.ingress.api.host | quote }}
|
||||
http:
|
||||
paths:
|
||||
- path: {{ .Values.ingress.api.path }}
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-api
|
||||
port:
|
||||
number: {{ .Values.api.service.port }}
|
||||
- host: {{ .Values.ingress.mcp.host | quote }}
|
||||
http:
|
||||
paths:
|
||||
- path: {{ .Values.ingress.mcp.path }}
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
||||
port:
|
||||
number: {{ .Values.mcp.service.port }}
|
||||
{{- end }}
|
||||
72
deploy/helm/knowledge-platform/templates/mcp-deployment.yaml
Normal file
72
deploy/helm/knowledge-platform/templates/mcp-deployment.yaml
Normal file
@ -0,0 +1,72 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
spec:
|
||||
replicas: {{ .Values.mcp.replicaCount }}
|
||||
minReadySeconds: 5
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 0
|
||||
maxSurge: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: mcp
|
||||
image: {{ include "knowledge-platform.image" .Values.images.mcp | quote }}
|
||||
imagePullPolicy: {{ .Values.images.mcp.pullPolicy }}
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||
ports:
|
||||
- name: mcp
|
||||
containerPort: 8766
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||
startupProbe:
|
||||
tcpSocket:
|
||||
port: mcp
|
||||
{{- toYaml .Values.mcp.probes.startup | nindent 12 }}
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: mcp
|
||||
{{- toYaml .Values.mcp.probes.readiness | nindent 12 }}
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: mcp
|
||||
{{- toYaml .Values.mcp.probes.liveness | nindent 12 }}
|
||||
resources:
|
||||
{{- toYaml .Values.mcp.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||
volumes:
|
||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||
17
deploy/helm/knowledge-platform/templates/mcp-service.yaml
Normal file
17
deploy/helm/knowledge-platform/templates/mcp-service.yaml
Normal file
@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-mcp
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
spec:
|
||||
type: {{ .Values.mcp.service.type }}
|
||||
ports:
|
||||
- name: mcp
|
||||
port: {{ .Values.mcp.service.port }}
|
||||
targetPort: mcp
|
||||
protocol: TCP
|
||||
selector:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: mcp
|
||||
59
deploy/helm/knowledge-platform/templates/migration-job.yaml
Normal file
59
deploy/helm/knowledge-platform/templates/migration-job.yaml
Normal file
@ -0,0 +1,59 @@
|
||||
{{- if .Values.migration.enabled }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-migration
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: migration
|
||||
annotations:
|
||||
# Helm waits for pre-install/pre-upgrade hooks. A failed migration blocks
|
||||
# the application rollout, and the previous Job remains until the next run.
|
||||
helm.sh/hook: pre-install,pre-upgrade
|
||||
helm.sh/hook-weight: "-10"
|
||||
helm.sh/hook-delete-policy: before-hook-creation
|
||||
spec:
|
||||
backoffLimit: {{ .Values.migration.backoffLimit }}
|
||||
activeDeadlineSeconds: {{ .Values.migration.activeDeadlineSeconds }}
|
||||
ttlSecondsAfterFinished: {{ .Values.migration.ttlSecondsAfterFinished }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: migration
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
# Pre-install hooks run before the release ServiceAccount exists. The
|
||||
# namespace default account is used with token mounting disabled; the
|
||||
# migration needs no Kubernetes API permissions.
|
||||
serviceAccountName: default
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: migration
|
||||
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
|
||||
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
|
||||
command: ["alembic"]
|
||||
args: ["upgrade", "head"]
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||
env:
|
||||
- name: APP_ENV
|
||||
value: production
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.existingSecret }}
|
||||
key: DATABASE_URL
|
||||
resources:
|
||||
{{- toYaml .Values.migration.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||
volumes:
|
||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
@ -0,0 +1,71 @@
|
||||
{{- if .Values.scheduler.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-scheduler
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: scheduler
|
||||
spec:
|
||||
# A single Celery Beat owns periodic durable-Job recovery. Recreate prevents
|
||||
# two schedulers from overlapping during upgrades.
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: scheduler
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: scheduler
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: scheduler
|
||||
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
|
||||
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
|
||||
workingDir: /tmp
|
||||
command: ["celery"]
|
||||
args:
|
||||
- --app=ekp_api.worker:celery_app
|
||||
- beat
|
||||
- --loglevel=INFO
|
||||
- --schedule=/tmp/celerybeat-schedule
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||
envFrom:
|
||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||
startupProbe:
|
||||
exec:
|
||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
{{- toYaml .Values.scheduler.probes.startup | nindent 12 }}
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
{{- toYaml .Values.scheduler.probes.liveness | nindent 12 }}
|
||||
resources:
|
||||
{{- toYaml .Values.scheduler.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||
volumes:
|
||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
13
deploy/helm/knowledge-platform/templates/serviceaccount.yaml
Normal file
13
deploy/helm/knowledge-platform/templates/serviceaccount.yaml
Normal file
@ -0,0 +1,13 @@
|
||||
{{- if .Values.serviceAccount.create }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
{{- with .Values.serviceAccount.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
automountServiceAccountToken: false
|
||||
{{- end }}
|
||||
@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-preflight-references
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
# The Make preflight renders this test hook to discover only resource
|
||||
# names. It never renders or reads Secret data.
|
||||
helm.sh/hook: test
|
||||
data:
|
||||
runtimeSecret: {{ .Values.existingSecret | quote }}
|
||||
ingressEnabled: {{ .Values.ingress.enabled | quote }}
|
||||
apiTlsSecret: {{ .Values.ingress.api.tlsSecretName | quote }}
|
||||
mcpTlsSecret: {{ .Values.ingress.mcp.tlsSecretName | quote }}
|
||||
rabbitmqEnabled: {{ .Values.rabbitmq.enabled | quote }}
|
||||
rabbitmqSecret: {{ .Values.rabbitmq.existingSecret | quote }}
|
||||
@ -0,0 +1,71 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "knowledge-platform.fullname" . }}-worker
|
||||
labels:
|
||||
{{- include "knowledge-platform.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: worker
|
||||
spec:
|
||||
replicas: {{ .Values.worker.replicaCount }}
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 1
|
||||
maxSurge: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: worker
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: worker
|
||||
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
|
||||
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
|
||||
workingDir: /tmp
|
||||
command: ["celery"]
|
||||
args:
|
||||
- --app=ekp_api.worker:celery_app
|
||||
- worker
|
||||
- --loglevel=INFO
|
||||
- --queues=ingestion
|
||||
securityContext:
|
||||
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
|
||||
envFrom:
|
||||
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
|
||||
startupProbe:
|
||||
exec:
|
||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
{{- toYaml .Values.worker.probes.startup | nindent 12 }}
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["python", "-c", "import os; os.kill(1, 0)"]
|
||||
{{- toYaml .Values.worker.probes.liveness | nindent 12 }}
|
||||
resources:
|
||||
{{- toYaml .Values.worker.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
|
||||
volumes:
|
||||
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
|
||||
338
deploy/helm/knowledge-platform/values.schema.json
Normal file
338
deploy/helm/knowledge-platform/values.schema.json
Normal file
@ -0,0 +1,338 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft-07/schema#",
|
||||
"title": "Enterprise Knowledge Platform Helm values",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"existingSecret",
|
||||
"images",
|
||||
"serviceAccount",
|
||||
"api",
|
||||
"mcp",
|
||||
"worker",
|
||||
"scheduler",
|
||||
"migration",
|
||||
"ingress",
|
||||
"config",
|
||||
"rabbitmq"
|
||||
],
|
||||
"properties": {
|
||||
"nameOverride": {"type": "string"},
|
||||
"fullnameOverride": {"type": "string"},
|
||||
"existingSecret": {"$ref": "#/definitions/kubernetesName"},
|
||||
"imagePullSecrets": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["name"],
|
||||
"properties": {"name": {"$ref": "#/definitions/kubernetesName"}}
|
||||
}
|
||||
},
|
||||
"images": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["api", "mcp", "worker"],
|
||||
"properties": {
|
||||
"api": {"$ref": "#/definitions/image"},
|
||||
"mcp": {"$ref": "#/definitions/image"},
|
||||
"worker": {"$ref": "#/definitions/image"}
|
||||
}
|
||||
},
|
||||
"serviceAccount": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["create", "name", "annotations"],
|
||||
"properties": {
|
||||
"create": {"type": "boolean"},
|
||||
"name": {"type": "string"},
|
||||
"annotations": {"$ref": "#/definitions/stringMap"}
|
||||
}
|
||||
},
|
||||
"podAnnotations": {"$ref": "#/definitions/stringMap"},
|
||||
"podLabels": {"$ref": "#/definitions/stringMap"},
|
||||
"api": {"$ref": "#/definitions/httpWorkload"},
|
||||
"mcp": {"$ref": "#/definitions/httpWorkload"},
|
||||
"worker": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["replicaCount", "terminationGracePeriodSeconds", "resources", "probes"],
|
||||
"properties": {
|
||||
"replicaCount": {"type": "integer", "minimum": 1, "maximum": 100},
|
||||
"terminationGracePeriodSeconds": {"type": "integer", "minimum": 30, "maximum": 3600},
|
||||
"resources": {"$ref": "#/definitions/resources"},
|
||||
"probes": {"$ref": "#/definitions/processProbes"}
|
||||
}
|
||||
},
|
||||
"scheduler": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "resources", "probes"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"resources": {"$ref": "#/definitions/resources"},
|
||||
"probes": {"$ref": "#/definitions/processProbes"}
|
||||
}
|
||||
},
|
||||
"migration": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "activeDeadlineSeconds", "backoffLimit", "ttlSecondsAfterFinished", "resources"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"activeDeadlineSeconds": {"type": "integer", "minimum": 60, "maximum": 3600},
|
||||
"backoffLimit": {"type": "integer", "minimum": 0, "maximum": 3},
|
||||
"ttlSecondsAfterFinished": {"type": "integer", "minimum": 0},
|
||||
"resources": {"$ref": "#/definitions/resources"}
|
||||
}
|
||||
},
|
||||
"ingress": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["enabled", "className", "annotations", "api", "mcp"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"className": {"type": "string", "minLength": 1},
|
||||
"annotations": {"$ref": "#/definitions/stringMap"},
|
||||
"api": {"$ref": "#/definitions/ingressRoute"},
|
||||
"mcp": {"$ref": "#/definitions/ingressRoute"}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["appName", "logLevel", "database", "ingestion", "objectStorage", "chunking", "embedding", "vectorStore", "retrieval", "reranker", "mcp"],
|
||||
"properties": {
|
||||
"appName": {"type": "string", "minLength": 1},
|
||||
"logLevel": {"enum": ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]},
|
||||
"database": {"$ref": "#/definitions/databaseConfig"},
|
||||
"ingestion": {"$ref": "#/definitions/ingestionConfig"},
|
||||
"objectStorage": {"$ref": "#/definitions/objectStorageConfig"},
|
||||
"chunking": {"$ref": "#/definitions/chunkingConfig"},
|
||||
"embedding": {"$ref": "#/definitions/embeddingConfig"},
|
||||
"vectorStore": {"$ref": "#/definitions/vectorStoreConfig"},
|
||||
"retrieval": {"$ref": "#/definitions/retrievalConfig"},
|
||||
"reranker": {"$ref": "#/definitions/rerankerConfig"},
|
||||
"mcp": {"$ref": "#/definitions/mcpConfig"}
|
||||
}
|
||||
},
|
||||
"rabbitmq": {
|
||||
"type": "object",
|
||||
"required": ["enabled", "existingSecret"],
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"existingSecret": {"$ref": "#/definitions/kubernetesName"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"kubernetesName": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 253,
|
||||
"pattern": "^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$"
|
||||
},
|
||||
"stringMap": {
|
||||
"type": "object",
|
||||
"additionalProperties": {"type": "string"}
|
||||
},
|
||||
"image": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["repository", "tag", "digest", "pullPolicy"],
|
||||
"properties": {
|
||||
"repository": {"type": "string", "minLength": 1},
|
||||
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
|
||||
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
|
||||
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
|
||||
}
|
||||
},
|
||||
"probeTiming": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["failureThreshold", "periodSeconds"],
|
||||
"properties": {
|
||||
"failureThreshold": {"type": "integer", "minimum": 1},
|
||||
"periodSeconds": {"type": "integer", "minimum": 1},
|
||||
"timeoutSeconds": {"type": "integer", "minimum": 1}
|
||||
}
|
||||
},
|
||||
"httpProbes": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["startup", "readiness", "liveness"],
|
||||
"properties": {
|
||||
"startup": {"$ref": "#/definitions/probeTiming"},
|
||||
"readiness": {"$ref": "#/definitions/probeTiming"},
|
||||
"liveness": {"$ref": "#/definitions/probeTiming"}
|
||||
}
|
||||
},
|
||||
"processProbes": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["startup", "liveness"],
|
||||
"properties": {
|
||||
"startup": {"$ref": "#/definitions/probeTiming"},
|
||||
"liveness": {"$ref": "#/definitions/probeTiming"}
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["requests", "limits"],
|
||||
"properties": {
|
||||
"requests": {"$ref": "#/definitions/resourcePair"},
|
||||
"limits": {"$ref": "#/definitions/resourcePair"}
|
||||
}
|
||||
},
|
||||
"resourcePair": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["cpu", "memory"],
|
||||
"properties": {
|
||||
"cpu": {"type": ["string", "number"]},
|
||||
"memory": {"type": "string", "minLength": 2}
|
||||
}
|
||||
},
|
||||
"httpWorkload": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["replicaCount", "service", "resources", "probes"],
|
||||
"properties": {
|
||||
"replicaCount": {"type": "integer", "minimum": 2, "maximum": 100},
|
||||
"service": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["type", "port"],
|
||||
"properties": {
|
||||
"type": {"enum": ["ClusterIP"]},
|
||||
"port": {"type": "integer", "minimum": 1, "maximum": 65535}
|
||||
}
|
||||
},
|
||||
"resources": {"$ref": "#/definitions/resources"},
|
||||
"probes": {"$ref": "#/definitions/httpProbes"}
|
||||
}
|
||||
},
|
||||
"ingressRoute": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["host", "path", "tlsSecretName"],
|
||||
"properties": {
|
||||
"host": {"type": "string", "minLength": 1},
|
||||
"path": {"type": "string", "pattern": "^/"},
|
||||
"tlsSecretName": {"$ref": "#/definitions/kubernetesName"}
|
||||
}
|
||||
},
|
||||
"databaseConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["poolSize", "maxOverflow", "poolRecycleSeconds", "poolTimeoutSeconds", "connectTimeoutSeconds"],
|
||||
"properties": {
|
||||
"poolSize": {"type": "integer", "minimum": 1, "maximum": 100},
|
||||
"maxOverflow": {"type": "integer", "minimum": 0, "maximum": 100},
|
||||
"poolRecycleSeconds": {"type": "integer", "minimum": 1},
|
||||
"poolTimeoutSeconds": {"type": "number", "exclusiveMinimum": 0},
|
||||
"connectTimeoutSeconds": {"type": "integer", "minimum": 1}
|
||||
}
|
||||
},
|
||||
"ingestionConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["maxAttempts", "retryBackoffSeconds", "retryBackoffMaxSeconds", "workerLeaseSeconds", "recoveryIntervalSeconds", "batchMaxFiles", "batchMaxTotalBytes", "maxUploadBytes"],
|
||||
"properties": {
|
||||
"maxAttempts": {"type": "integer", "minimum": 1, "maximum": 10},
|
||||
"retryBackoffSeconds": {"type": "integer", "minimum": 1},
|
||||
"retryBackoffMaxSeconds": {"type": "integer", "minimum": 1},
|
||||
"workerLeaseSeconds": {"type": "integer", "minimum": 30},
|
||||
"recoveryIntervalSeconds": {"type": "integer", "minimum": 5},
|
||||
"batchMaxFiles": {"type": "integer", "minimum": 1, "maximum": 100},
|
||||
"batchMaxTotalBytes": {"type": "integer", "minimum": 1},
|
||||
"maxUploadBytes": {"type": "integer", "minimum": 1}
|
||||
}
|
||||
},
|
||||
"objectStorageConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["provider", "endpoint", "bucket", "keyPrefix", "timeoutSeconds"],
|
||||
"properties": {
|
||||
"provider": {"const": "bos"},
|
||||
"endpoint": {"type": "string", "minLength": 1},
|
||||
"bucket": {"type": "string", "minLength": 1},
|
||||
"keyPrefix": {"type": "string", "minLength": 1},
|
||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||
}
|
||||
},
|
||||
"chunkingConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["maxTokens", "overlapTokens", "maxCharacters"],
|
||||
"properties": {
|
||||
"maxTokens": {"type": "integer", "minimum": 1},
|
||||
"overlapTokens": {"type": "integer", "minimum": 0},
|
||||
"maxCharacters": {"type": "integer", "minimum": 4}
|
||||
}
|
||||
},
|
||||
"embeddingConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
|
||||
"properties": {
|
||||
"provider": {"const": "qianfan"},
|
||||
"model": {"const": "bge-large-zh"},
|
||||
"endpoint": {"type": "string", "minLength": 1},
|
||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||
}
|
||||
},
|
||||
"vectorStoreConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["provider", "uri", "user", "database", "table", "index", "batchSize", "timeoutSeconds"],
|
||||
"properties": {
|
||||
"provider": {"const": "oceanbase"},
|
||||
"uri": {"type": "string", "minLength": 1},
|
||||
"user": {"type": "string", "minLength": 1},
|
||||
"database": {"type": "string", "minLength": 1},
|
||||
"table": {"type": "string", "minLength": 1},
|
||||
"index": {"type": "string", "minLength": 1},
|
||||
"batchSize": {"type": "integer", "minimum": 1, "maximum": 1000},
|
||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||
}
|
||||
},
|
||||
"retrievalConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["indexProfileId", "denseTopN", "searchTopK"],
|
||||
"properties": {
|
||||
"indexProfileId": {"type": "string", "minLength": 1},
|
||||
"denseTopN": {"type": "integer", "minimum": 1, "maximum": 64},
|
||||
"searchTopK": {"type": "integer", "minimum": 1, "maximum": 64}
|
||||
}
|
||||
},
|
||||
"rerankerConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
|
||||
"properties": {
|
||||
"provider": {"const": "qianfan"},
|
||||
"model": {"const": "bce-reranker-base"},
|
||||
"endpoint": {"type": "string", "minLength": 1},
|
||||
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
|
||||
}
|
||||
},
|
||||
"mcpConfig": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["publicUrl", "issuerUrl", "clientId", "serviceIdentity", "requiredScope", "allowedHosts", "allowedOrigins", "maxRequestBodyBytes"],
|
||||
"properties": {
|
||||
"publicUrl": {"type": "string", "pattern": "^https://"},
|
||||
"issuerUrl": {"type": "string", "pattern": "^https://"},
|
||||
"clientId": {"type": "string", "minLength": 1},
|
||||
"serviceIdentity": {"type": "string", "minLength": 1},
|
||||
"requiredScope": {"const": "knowledge:search"},
|
||||
"allowedHosts": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},
|
||||
"allowedOrigins": {"type": "array", "minItems": 1, "items": {"type": "string", "pattern": "^https://"}},
|
||||
"maxRequestBodyBytes": {"type": "integer", "minimum": 1024, "maximum": 4194304}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
218
deploy/helm/knowledge-platform/values.yaml
Normal file
218
deploy/helm/knowledge-platform/values.yaml
Normal file
@ -0,0 +1,218 @@
|
||||
# DEPLOY-002 safe defaults. Real endpoints, immutable image coordinates and
|
||||
# Secret names are supplied in an environment-specific values file.
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
existingSecret: knowledge-platform-runtime
|
||||
imagePullSecrets: []
|
||||
|
||||
images:
|
||||
api:
|
||||
repository: knowledge-platform/enterprise-knowledge-api
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
mcp:
|
||||
repository: knowledge-platform/enterprise-knowledge-mcp
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
worker:
|
||||
repository: knowledge-platform/enterprise-knowledge-worker
|
||||
tag: "0.1.0"
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
serviceAccount:
|
||||
create: true
|
||||
name: ""
|
||||
annotations: {}
|
||||
|
||||
podAnnotations: {}
|
||||
podLabels: {}
|
||||
|
||||
api:
|
||||
replicaCount: 2
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 80
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
probes:
|
||||
startup:
|
||||
failureThreshold: 30
|
||||
periodSeconds: 2
|
||||
readiness:
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
liveness:
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
|
||||
mcp:
|
||||
replicaCount: 2
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 80
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
probes:
|
||||
startup:
|
||||
failureThreshold: 30
|
||||
periodSeconds: 2
|
||||
readiness:
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
liveness:
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
|
||||
worker:
|
||||
replicaCount: 2
|
||||
terminationGracePeriodSeconds: 60
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
probes:
|
||||
startup:
|
||||
failureThreshold: 30
|
||||
periodSeconds: 2
|
||||
liveness:
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
|
||||
# Celery Beat is separated from scalable workers so only one recovery
|
||||
# scheduler publishes due durable Jobs at a time.
|
||||
scheduler:
|
||||
enabled: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
probes:
|
||||
startup:
|
||||
failureThreshold: 30
|
||||
periodSeconds: 2
|
||||
liveness:
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
|
||||
migration:
|
||||
enabled: true
|
||||
activeDeadlineSeconds: 600
|
||||
backoffLimit: 0
|
||||
ttlSecondsAfterFinished: 86400
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
className: nginx
|
||||
annotations: {}
|
||||
api:
|
||||
host: knowledge.example.invalid
|
||||
path: /
|
||||
tlsSecretName: knowledge-platform-api-tls
|
||||
mcp:
|
||||
host: mcp.example.invalid
|
||||
path: /mcp
|
||||
tlsSecretName: knowledge-platform-mcp-tls
|
||||
|
||||
# Only non-sensitive runtime settings belong here. The templates enumerate
|
||||
# keys explicitly so credentials cannot be added to this ConfigMap by mistake.
|
||||
config:
|
||||
appName: enterprise-knowledge-platform
|
||||
logLevel: INFO
|
||||
database:
|
||||
poolSize: 10
|
||||
maxOverflow: 10
|
||||
poolRecycleSeconds: 600
|
||||
poolTimeoutSeconds: 30
|
||||
connectTimeoutSeconds: 30
|
||||
ingestion:
|
||||
maxAttempts: 3
|
||||
retryBackoffSeconds: 2
|
||||
retryBackoffMaxSeconds: 60
|
||||
workerLeaseSeconds: 300
|
||||
recoveryIntervalSeconds: 30
|
||||
batchMaxFiles: 20
|
||||
batchMaxTotalBytes: 209715200
|
||||
maxUploadBytes: 52428800
|
||||
objectStorage:
|
||||
provider: bos
|
||||
endpoint: https://s3.example.invalid
|
||||
bucket: knowledge-platform
|
||||
keyPrefix: knowledge-platform
|
||||
timeoutSeconds: 30
|
||||
chunking:
|
||||
maxTokens: 450
|
||||
overlapTokens: 75
|
||||
maxCharacters: 2000
|
||||
embedding:
|
||||
provider: qianfan
|
||||
model: bge-large-zh
|
||||
endpoint: https://qianfan.baidubce.com/v2/embeddings
|
||||
timeoutSeconds: 30
|
||||
vectorStore:
|
||||
provider: oceanbase
|
||||
uri: oceanbase.example.invalid:2881
|
||||
user: knowledge-platform
|
||||
database: knowledge_platform
|
||||
table: knowledge_chunks_v1
|
||||
index: vidx_knowledge_chunks_embedding
|
||||
batchSize: 100
|
||||
timeoutSeconds: 30
|
||||
retrieval:
|
||||
indexProfileId: demo-bge-large-zh-structure-v1-oceanbase-v1
|
||||
denseTopN: 20
|
||||
searchTopK: 5
|
||||
reranker:
|
||||
provider: qianfan
|
||||
model: bce-reranker-base
|
||||
endpoint: https://qianfan.baidubce.com/v2/rerank
|
||||
timeoutSeconds: 30
|
||||
mcp:
|
||||
publicUrl: https://mcp.example.invalid/mcp
|
||||
issuerUrl: https://mcp.example.invalid
|
||||
clientId: workbuddy-enterprise
|
||||
serviceIdentity: workbuddy-public-knowledge
|
||||
requiredScope: knowledge:search
|
||||
allowedHosts:
|
||||
- mcp.example.invalid
|
||||
allowedOrigins:
|
||||
- https://mcp.example.invalid
|
||||
maxRequestBodyBytes: 1048576
|
||||
|
||||
# Optional local dependency. Keep disabled for production and provide
|
||||
# RABBITMQ_URL through existingSecret for an externally managed broker.
|
||||
rabbitmq:
|
||||
enabled: false
|
||||
existingSecret: knowledge-platform-rabbitmq
|
||||
12
deploy/workbuddy-mcp.example.json
Normal file
12
deploy/workbuddy-mcp.example.json
Normal file
@ -0,0 +1,12 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"enterprise-knowledge": {
|
||||
"type": "http",
|
||||
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
|
||||
"headers": {
|
||||
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
|
||||
},
|
||||
"description": "企业知识检索"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -11,17 +11,22 @@ Use OpenAPI as executable source of truth once implementation starts. Provider-s
|
||||
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
|
||||
版本化为 `/api/v1/*`。
|
||||
|
||||
`DEMO-015` 已将单文件上传切换为持久化异步 Job;`DEMO-016` 已增加批量提交和聚合查询。
|
||||
|
||||
| Method | Path | 行为 |
|
||||
|---|---|---|
|
||||
| `POST` | `/api/v1/documents` | multipart 单文件上传并同步执行完整入库流水线 |
|
||||
| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
|
||||
| `POST` | `/api/v1/ingestion-batches` | multipart `files` 多文件接收;返回批次与逐文件状态 |
|
||||
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新 Job 状态 |
|
||||
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
|
||||
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
|
||||
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
|
||||
| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
|
||||
| `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 |
|
||||
|
||||
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `201` 和 Document;
|
||||
Demo 同步流水线完成后状态为 `ready`。Document 响应不暴露 `storage_key`、Provider 配置或
|
||||
索引实现。
|
||||
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `202`、Document 字段及
|
||||
`job_id`、`job_status=queued`、`max_attempts`;响应不等待 Parse、Clean、Chunk、Embedding 或
|
||||
Index。Document 响应不暴露 `storage_key`、Provider 配置或索引实现。
|
||||
|
||||
Search 请求:
|
||||
|
||||
@ -52,11 +57,87 @@ Search 响应:
|
||||
```
|
||||
|
||||
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
|
||||
文档不存在返回 `404`,运行时 Provider 未配置返回 `503`。上游 Provider 或持久化异常只返回
|
||||
经过收敛的 `5xx` 信息,不回显凭证、请求正文或厂商响应。
|
||||
文档、Job 或批次不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
|
||||
`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
|
||||
厂商响应。
|
||||
|
||||
## Demo Admin 客户端路由
|
||||
|
||||
DEMO-010 提供 `/admin/upload`、`/admin/documents` 和 `/admin/search` 三个客户端路由,
|
||||
`/admin` 默认展示上传页。它们是同源静态页面,不是新增业务 API,也不改变上述公开 REST
|
||||
契约;所有业务操作仍调用 `/api/v1`。
|
||||
|
||||
## 当前公共知识库部署边界(PLAN-002)
|
||||
|
||||
当前阶段所有 Document 属于同一个公共知识可见域,不新增用户、知识库或 ACL 字段,也不把
|
||||
这些完整未实现能力伪装为默认放行。Admin 写接口只允许部署在企业受控网络或统一接入网关后;
|
||||
Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏路由不构成安全控制。
|
||||
|
||||
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
|
||||
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
|
||||
|
||||
## 异步入库契约(DEMO-015~016 已实现)
|
||||
|
||||
| Method | Path | 行为 |
|
||||
|---|---|---|
|
||||
| `POST` | `/api/v1/documents` | 已实现:单文件接收并返回 Document/Job 标识 |
|
||||
| `POST` | `/api/v1/ingestion-batches` | multipart 多文件接收,返回批次及逐文件 Document/Job 标识 |
|
||||
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新状态 |
|
||||
| `GET` | `/api/v1/jobs/{job_id}` | 已实现:返回持久化 Job 状态 |
|
||||
|
||||
单文件响应示意:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
|
||||
"filename": "退款政策.md",
|
||||
"media_type": "text/markdown",
|
||||
"size_bytes": 1024,
|
||||
"status": "uploaded",
|
||||
"error_message": null,
|
||||
"created_at": "2026-09-02T12:00:00Z",
|
||||
"updated_at": "2026-09-02T12:00:00Z",
|
||||
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
|
||||
"job_status": "queued",
|
||||
"max_attempts": 3
|
||||
}
|
||||
```
|
||||
|
||||
批量字段名为 `files`,成功返回 `202`。响应示意:
|
||||
|
||||
```json
|
||||
{
|
||||
"batch_id": "d21f8012-4c4e-4bb4-89af-e11c0f9f470e",
|
||||
"status": "processing",
|
||||
"total_files": 2,
|
||||
"accepted_files": 1,
|
||||
"rejected_files": 1,
|
||||
"completed_files": 1,
|
||||
"created_at": "2026-09-02T12:00:00Z",
|
||||
"items": [
|
||||
{
|
||||
"item_index": 0,
|
||||
"filename": "退款政策.md",
|
||||
"acceptance_status": "accepted",
|
||||
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
|
||||
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
|
||||
"job_status": "queued",
|
||||
"stage": null,
|
||||
"attempt": 0,
|
||||
"max_attempts": 3,
|
||||
"error_message": null
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Job 公开状态为 `queued | running | retrying | succeeded | failed | cancelled`,并通过 `stage`
|
||||
表示 `parsing | cleaning | chunking | embedding | indexing`。错误字段只返回有界、安全摘要。
|
||||
批次状态为 `processing | completed | partial_failure | failed`;逐文件接收状态为
|
||||
`pending | accepted | rejected`。批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚
|
||||
其他文件。文件数量、单文件大小和请求总大小分别由 `BATCH_MAX_FILES`、`MAX_UPLOAD_BYTES`
|
||||
和 `BATCH_MAX_TOTAL_BYTES` 限制。数量超限返回 `413`;类型、大小或内容不合规作为该文件的
|
||||
`rejected` 安全结果返回,其他合规文件继续接收。
|
||||
|
||||
自动重试是服务端 Job 行为,不要求客户端再次上传,也不新增公开手动重试接口。每次重试前
|
||||
清理派生数据但保留原始文件和稳定的 Document ID;达到次数上限或清理失败后进入 `failed`。
|
||||
|
||||
@ -9,3 +9,43 @@ P0 business tools:
|
||||
- `list_knowledge_bases`
|
||||
|
||||
Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose OceanBase Vector, BOS, model names, parsers or other infrastructure choices.
|
||||
|
||||
## Demo priority slice(DEMO-012,已实现)
|
||||
|
||||
`DEMO-012` 已提升为当前最高实现优先级。首个可运行 Gateway 只注册
|
||||
`search_knowledge`,其输入与 `/api/v1/search` 对齐:
|
||||
|
||||
```json
|
||||
{
|
||||
"query": "产品A退款周期是多少?",
|
||||
"top_k": 5
|
||||
}
|
||||
```
|
||||
|
||||
输出保留业务来源字段:`document_id`、`filename`、`page`、`section`、`chunk_id`、`content`
|
||||
和排序分数。Gateway 必须复用现有 Search Service 或正式 REST API,不得复制检索、来源补全
|
||||
或排序逻辑。
|
||||
|
||||
Demo 验证也必须建立 user/service identity 映射边界,禁止使用共享超级管理员 Token。由于
|
||||
Document ACL 尚未进入 Demo Scope,`DEMO-012` 只能证明身份传递、认证拒绝和业务服务复用,
|
||||
不能声称完成 V1 ACL 授权闭环。错误响应和日志不得包含 Provider Endpoint、凭据、数据库表名
|
||||
或模型名。
|
||||
|
||||
验收至少覆盖:MCP 工具发现、参数校验、成功检索、无结果、认证失败、安全错误收敛,以及
|
||||
WorkBuddy Enterprise 的一次端到端调用。其余 P0 tools 继续留在完整 V1 `EKP-071`。
|
||||
|
||||
实现使用独立 Streamable HTTP Gateway,默认端点为 `http://127.0.0.1:8766/mcp`。请求必须
|
||||
携带 `Authorization: Bearer <token>`;Token 只从运行环境读取,并映射为显式 service identity
|
||||
和 `knowledge:search` scope。当前静态 service credential 是 Demo 兼容层,不替代 V1 OAuth、
|
||||
ACL 或用户委托授权。
|
||||
|
||||
公开错误边界:缺失/无效 Token 为 HTTP `401`,scope 不足为 `403`;Tool 参数不合法返回 MCP
|
||||
参数错误;检索运行失败只返回 `knowledge search failed`。Host/Origin allowlist 默认只允许
|
||||
本机,远程部署必须显式注入公开域名。
|
||||
|
||||
## 当前公共知识库部署边界(PLAN-002)
|
||||
|
||||
集群阶段仍只暴露只读 `search_knowledge`,并为 WorkBuddy 配置稳定的 HTTPS MCP 地址。
|
||||
Credential 由 Kubernetes Secret 在服务端运行时注入,客户端只保存管理员分发的最小
|
||||
`knowledge:search` Token;Pod 重启或滚动升级不得要求重新生成配置。公共知识域不等于匿名
|
||||
访问或超级管理员身份,未来引入非公共知识前必须升级为用户委托身份与 ACL-aware Search。
|
||||
|
||||
@ -25,6 +25,17 @@ canonical Chunk 重建的检索行。共用数据库实例不改变两类数据
|
||||
`VECTOR_INDEX(... pre_filter)` 产生 `VECTOR INDEX ADAPTIVE SCAN (PRE-FILTER)`;版本下限与
|
||||
运行检查据此调整为 4.3.5.4。生产升级和重新验收仍由运维流程控制。
|
||||
|
||||
## Public knowledge delivery boundary
|
||||
|
||||
PLAN-002 将当前交付阶段限定为企业受控网络内的单一公共知识可见域,并加入 Kubernetes/Helm
|
||||
集群交付任务。这是部署范围和实施顺序调整,不是新的架构选型:继续采用 ADR-002 的模块化
|
||||
单体 API、独立 Worker 和 MCP Gateway,以及架构既定的 Celery/RabbitMQ;OceanBase/BOS
|
||||
边界继续遵循 ADR-012。
|
||||
|
||||
ADR-007 仍为 Accepted。当前阶段只是延后用户/组/角色和 Document ACL;开始接收任何非公共
|
||||
知识前,必须先实现 Permission Resolver 和检索前/检索中 ACL 下推。公共知识库也不等于匿名
|
||||
公网开放,Admin 写操作只允许经受控网络或统一接入网关访问,MCP 使用最小只读服务身份。
|
||||
|
||||
The migration retains the Demo's Dense Top20 → Reranker Top5 behavior and current lightweight
|
||||
Parser scope. Top50 retrieval, distance thresholds, dual-query RRF, Marker/Pandoc and LLM-derived
|
||||
index content require separate product and evaluation decisions.
|
||||
|
||||
@ -174,6 +174,46 @@ Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Fail
|
||||
- 软删除;
|
||||
- 来源页码和章节追踪。
|
||||
|
||||
### 4.2.1 批量异步提交与失败恢复
|
||||
|
||||
API 只负责校验并持久化不可变源文件、Document 和 Ingestion Job,然后返回 `202`。批次是
|
||||
提交与聚合查询边界,不是跨文件数据库事务;每个文件独立排队、执行和失败。
|
||||
|
||||
DEMO-016 以 canonical `ingestion_batches` / `ingestion_batch_items` 保存输入顺序和接收引用;
|
||||
处理阶段、Attempt 与最终结果继续以 `ingestion_jobs` / `ingestion_attempts` 为事实源,批次表
|
||||
不复制 Worker 状态。
|
||||
|
||||
```text
|
||||
Batch Request
|
||||
├─ Source A + Document A + Job A ─→ Worker
|
||||
├─ Source B + Document B + Job B ─→ Worker
|
||||
└─ Source C + Document C + Job C ─→ Worker
|
||||
```
|
||||
|
||||
Worker 对可重试错误执行有界退避。重新执行前必须通过独立的派生数据清理服务按以下顺序
|
||||
恢复干净状态:
|
||||
|
||||
```text
|
||||
Vector/Sparse rows
|
||||
→ ChunkIndexState
|
||||
→ Chunks
|
||||
→ Parsed artifact + parsed_storage_key
|
||||
→ reset processing state
|
||||
→ next attempt from immutable source
|
||||
```
|
||||
|
||||
清理服务与 Document 删除语义分离:重试清理保留原始 BOS 对象、Document 和 Job/Attempt
|
||||
历史。清理失败时 Job 终止,不能在未知状态继续执行。Job 通过数据库状态、唯一有效 Attempt
|
||||
和幂等键避免重复投递产生并发执行;具体队列实现保持可替换,不进入业务服务接口。
|
||||
|
||||
公共知识库集群交付阶段由 Celery/RabbitMQ 作为首个生产执行器。API 只调用队列执行器抽象,
|
||||
Celery task 只负责恢复应用上下文并调用 Ingestion Application Service;重试分类、Attempt 状态、
|
||||
清理顺序和业务幂等仍由应用层持有。Broker 消息不是 Job 的 source of truth,Worker 重启或消息
|
||||
重复投递必须以 OceanBase 中的持久化 Job/Attempt 状态收敛。
|
||||
|
||||
DEPLOY-001 将 API/Admin、MCP Gateway 与 Async Worker 构建为三个共享锁定 Python 运行时的
|
||||
非 root 镜像 target;进程边界与上述模块边界一致,不新增服务或把 Provider 配置烘焙进镜像。
|
||||
|
||||
---
|
||||
|
||||
### 4.3 文档结构与 Chunk
|
||||
@ -504,6 +544,17 @@ MinerU
|
||||
|
||||
首要验证客户端:**WorkBuddy Enterprise**。
|
||||
|
||||
### 11.1 DEMO-012 实现边界
|
||||
|
||||
当前已落地独立 Streamable HTTP Gateway,默认只绑定 `127.0.0.1:8766/mcp`,只注册
|
||||
`search_knowledge`。Gateway 与 REST 共享 `SearchService`,同步检索通过工作线程执行,避免
|
||||
阻塞 MCP ASGI 事件循环。
|
||||
|
||||
Demo 使用运行时专用 Bearer Token 映射到 client/service identity 和最小
|
||||
`knowledge:search` scope,并启用 Host/Origin allowlist。该边界用于 WorkBuddy 接入验证,
|
||||
后续 `EKP-070` 仍需用正式 OAuth/企业身份系统和 Permission Resolver 替换静态 Token;
|
||||
`DEMO-012` 不改变 ADR-007 的 ACL 要求。
|
||||
|
||||
---
|
||||
|
||||
## 12. REST API
|
||||
@ -604,6 +655,11 @@ ReindexJob
|
||||
EvaluationJob
|
||||
```
|
||||
|
||||
当前增量实现先提供单个 `IngestionJob` 编排既有 Parse/Clean/Chunk/Embedding/Indexing 阶段,
|
||||
保留阶段级状态和 Attempt 历史;以后拆分为独立阶段 Job 时不得改变公开批次/Job 查询契约。
|
||||
Demo 默认最多 3 次总尝试,只有瞬时 Provider、网络、存储或数据库错误进入自动重试;输入
|
||||
校验和确定性内容错误直接终止。
|
||||
|
||||
GPU 节点优先承担:
|
||||
|
||||
- MinerU / OCR;
|
||||
@ -688,6 +744,30 @@ Observability
|
||||
|
||||
未来根据负载再拆服务。
|
||||
|
||||
### 16.1 当前公共知识库集群交付基线
|
||||
|
||||
PLAN-002 将当前交付范围收敛为企业受控网络内的单一公共知识可见域。该范围不重开 ADR-007:
|
||||
所有现有文档共享同一可见性,因而当前检索不执行事后 ACL 过滤;接收任何非公共知识前,必须
|
||||
先实现 Permission Resolver 和数据库/向量检索前置过滤。Admin 写接口不得直接暴露到匿名公网,
|
||||
MCP 使用运行时 Secret 注入的最小 `knowledge:search` 服务身份。
|
||||
|
||||
集群中 API/Admin、MCP Gateway 和 Async Worker 分别部署并独立扩缩容,业务核心继续保持模块化
|
||||
单体。当前 Parser 及云端 Embedding/Reranker 不需要 GPU Worker,因此它不进入最小交付,但角色
|
||||
边界保留。OceanBase、BOS 和 AI Provider 使用外部服务;RabbitMQ 默认使用外部地址,可为开发/
|
||||
演示显式启用 Chart 依赖。
|
||||
|
||||
Kubernetes/Helm 必须提供:生产镜像、`existingSecret`、串行 Migration Job、Ingress/TLS、
|
||||
readiness/liveness、资源限制、非 root 与只读根文件系统、多副本和滚动更新。仓库的一键入口只
|
||||
编排已有集群和外部服务,不创建云账号或把凭据写入 Values、镜像、命令行及日志。现有
|
||||
`docker-compose.dev.yml` 包含已被 ADR-012 取代的 PostgreSQL/MinIO 假设,只能视为待清理的
|
||||
本地开发资产,不能作为集群模板。详细依赖和验收见
|
||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||
|
||||
DEPLOY-002 的运行拓扑将 Celery Worker 与 Celery Beat 恢复调度器拆成独立 Pod:Worker 可横向
|
||||
扩缩,Beat 固定单副本并使用 `Recreate`,避免每个 Worker 都发布周期恢复消息。二者仍调用同一
|
||||
应用服务与持久化 Job/Attempt 状态,不形成新业务服务,也不改变 ADR-002。Migration 使用 Helm
|
||||
`pre-install,pre-upgrade` Hook;失败会在业务 Deployment 更新前终止原子升级。
|
||||
|
||||
---
|
||||
|
||||
## 17. V1 验收重点
|
||||
|
||||
@ -92,7 +92,7 @@ Demo 阶段优先简单实现,不引入不必要的基础设施。
|
||||
| Vector DB | OceanBase Vector | 当前默认,通过 `VectorStore` 隔离 SDK,不做多数据库运行时切换 |
|
||||
| Reranker | `bce-reranker-base` | 延续当前可调用的千帆 Reranker,Demo 中保留 |
|
||||
| Frontend | 简单 React/Vue 或 FastAPI HTML | 以最小可用为目标 |
|
||||
| Deployment | 本地 / 单机 Docker | 暂不考虑 K8s |
|
||||
| Deployment | Kubernetes/Helm | PLAN-002 起面向企业受控网络交付;本地模式仅用于开发测试 |
|
||||
|
||||
---
|
||||
|
||||
@ -305,7 +305,7 @@ Search Response 示例:
|
||||
|
||||
---
|
||||
|
||||
## 8. P1 可选增强
|
||||
## 8. Demo 后续增强
|
||||
|
||||
### DEMO-011 Hybrid Search
|
||||
|
||||
@ -323,12 +323,26 @@ Reranker
|
||||
|
||||
### DEMO-012 MCP
|
||||
|
||||
只实现一个最小 MCP Tool:`search_knowledge`,内部调用 `/api/search`,用于验证 WorkBuddy 接入。
|
||||
**已完成。** 只实现一个最小 MCP Tool:`search_knowledge`,复用现有
|
||||
Search Service 或 `/api/v1/search`,用于验证标准 MCP、identity 边界和 WorkBuddy 接入。
|
||||
不在本 Task 扩展其他 P0 tools 或声称完成尚未实施的 Document ACL。
|
||||
|
||||
### DEMO-013 OCR / MinerU
|
||||
|
||||
增加扫描 PDF、图片、OCR 和复杂中文 PDF 处理能力。
|
||||
|
||||
### DEMO-015 异步入库 Job 与自动重试清理
|
||||
|
||||
**已完成。** 将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
|
||||
删除旧解析产物、Chunk、索引状态和向量数据,但保留不可变原始文件、Document 和 Attempt
|
||||
历史。清理失败时终止,不在未知脏状态继续执行。通过执行器抽象接入 Celery/RabbitMQ,使
|
||||
API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重复消息。
|
||||
|
||||
### DEMO-016 批量异步上传
|
||||
|
||||
**已完成。** 在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
|
||||
Admin Console 状态展示。批次内文件相互隔离,不包含目录上传、压缩包解包或断点续传。
|
||||
|
||||
---
|
||||
|
||||
## 9. 明确后置能力
|
||||
@ -348,12 +362,13 @@ Reranker
|
||||
- 多 Embedding Benchmark Platform;
|
||||
- 多 Vector DB;
|
||||
- 动态 Provider;
|
||||
- RabbitMQ / Celery;
|
||||
- Kubernetes;
|
||||
- 高可用;
|
||||
- 灾备;
|
||||
- 完整 Observability。
|
||||
|
||||
用户/权限隔离属于完整 V1 后置能力,不因“公共知识库”而取消;本阶段所有已接收知识共享一个
|
||||
可见域,且仅部署在企业受控网络或统一接入网关后。RabbitMQ/Celery 与最小 Kubernetes 可用性
|
||||
已由 PLAN-002 纳入当前交付,跨区域容灾和完整平台级高可用仍不在范围内。
|
||||
|
||||
---
|
||||
|
||||
## 10. Demo Task 清单
|
||||
@ -369,20 +384,45 @@ DEMO-007 Dense Retrieval
|
||||
DEMO-008 Reranker + Source Result
|
||||
DEMO-009 Document/Search REST API
|
||||
DEMO-010 简单 Admin Console
|
||||
DEMO-012 MCP search_knowledge(已提升优先级)
|
||||
DEMO-014 地瓜机器人品牌化前端优化(用户新增)
|
||||
DEMO-015 异步入库 Job + 自动重试前清理(用户新增)
|
||||
DEMO-016 批量异步上传 + 状态展示(用户新增)
|
||||
ARCH-001~006 Provider 架构迁移与真实验收
|
||||
PLAN-002 公共知识库集群交付规划(已完成)
|
||||
DEPLOY-001 API/Admin、MCP Gateway、Async Worker 生产镜像
|
||||
DEPLOY-002 Kubernetes/Helm 一键部署
|
||||
DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
|
||||
```
|
||||
|
||||
实施状态(2026-09-02):`DEMO-001~DEMO-010` 与 `ARCH-001~ARCH-006` 均已按顺序完成并
|
||||
通过当前 Scope 的闭环验收。`DEMO-011~DEMO-013` 仍为可选项,不因本状态记录自动进入范围。
|
||||
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
|
||||
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
|
||||
任务拆分;`DEMO-012`、`DEMO-015~016` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
|
||||
后续严格按 `DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
|
||||
可选项,不因本状态记录自动进入范围。
|
||||
|
||||
可选:
|
||||
|
||||
```text
|
||||
DEMO-011 Hybrid Search
|
||||
DEMO-012 MCP search_knowledge
|
||||
DEMO-013 OCR / MinerU
|
||||
```
|
||||
|
||||
已批准顺序:
|
||||
|
||||
```text
|
||||
PLAN-002 Public Knowledge Cluster Plan
|
||||
→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
|
||||
→ DEMO-016 Batch Async Upload + UI
|
||||
→ DEPLOY-001 Production Images
|
||||
→ DEPLOY-002 Kubernetes/Helm One-click Deploy
|
||||
→ DEPLOY-003 Availability + Remote Acceptance
|
||||
```
|
||||
|
||||
详细 Task 映射、依赖与验收边界见
|
||||
[`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`](../plans/ASYNC_INGESTION_MCP_PLAN.md) 和
|
||||
[`docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||
|
||||
---
|
||||
|
||||
## 11. Demo 测试数据
|
||||
@ -471,10 +511,13 @@ Demo 阶段不设严格生产指标。
|
||||
|
||||
## 14. Codex 开发约束
|
||||
|
||||
Codex 当前只能实现 `DEMO-001 ~ DEMO-010` 和已批准的 `ARCH-001 ~ ARCH-006`。
|
||||
ARCH 迁移任务不授权实现下列被排除的完整 V1 能力。
|
||||
Codex 当前只能实现 `DEMO-001 ~ DEMO-010`、`DEMO-012`、`DEMO-014 ~ DEMO-016`、
|
||||
`ARCH-001 ~ ARCH-006`、`PLAN-001 ~ PLAN-002` 和 `DEPLOY-001 ~ DEPLOY-003`。
|
||||
PLAN-002 只授权实现 Celery/RabbitMQ 异步执行器与最小 Kubernetes/Helm 集群交付,不授权
|
||||
拆分额外微服务或扩大业务能力。
|
||||
|
||||
除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、微服务、Kubernetes 和企业级 Job Queue。
|
||||
除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、
|
||||
额外微服务、跨区域容灾和完整 Observability 平台。
|
||||
|
||||
当本文件与完整 PRD 的实现优先级冲突时:
|
||||
|
||||
@ -503,10 +546,10 @@ docs/demo/DEMO_SCOPE.md
|
||||
>
|
||||
完整 V1 PRD
|
||||
|
||||
只允许实现 DEMO-001 ~ DEMO-010 以及本文件明确列出的 ARCH-001 ~ ARCH-006。
|
||||
Provider 架构迁移任务按顺序完成后,再继续 DEMO-010。
|
||||
只允许实现本文件明确列出的 Demo、Architecture、Plan 与 Deploy Task,并且一次只执行一个
|
||||
Task。当前严格顺序为 DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
|
||||
|
||||
当前 Demo 的目标只有:
|
||||
当前公共知识库阶段的业务目标保持为:
|
||||
|
||||
Upload
|
||||
→ Parse/Clean
|
||||
@ -518,6 +561,14 @@ Upload
|
||||
→ Source Result
|
||||
→ Search UI
|
||||
|
||||
并补齐以下交付能力:
|
||||
|
||||
Async Job / Retry Cleanup
|
||||
→ Batch Async Upload
|
||||
→ Production Images
|
||||
→ Kubernetes/Helm One-click Deploy
|
||||
→ Availability + Remote WorkBuddy Acceptance
|
||||
|
||||
不要提前实现:
|
||||
|
||||
- ACL
|
||||
@ -527,9 +578,9 @@ Upload
|
||||
- Knowledge Compilation
|
||||
- Multi Vector Store
|
||||
- Embedding Benchmark Platform
|
||||
- Celery/RabbitMQ
|
||||
- Kubernetes
|
||||
- 企业级高可用
|
||||
- 额外微服务
|
||||
- 跨区域容灾
|
||||
- 完整 Observability 平台
|
||||
|
||||
先不要修改代码。
|
||||
|
||||
@ -538,7 +589,7 @@ Upload
|
||||
1. 当前仓库状态;
|
||||
2. 为完成 Demo 需要保留哪些现有结构;
|
||||
3. 哪些完整 V1 设计可以暂时不实现;
|
||||
4. DEMO-001 ~ DEMO-010 与 ARCH-001 ~ ARCH-006 的建议执行顺序;
|
||||
4. 当前已批准 Task 的执行顺序;
|
||||
5. 下一步应该实施的第一个 Task。
|
||||
```
|
||||
|
||||
@ -549,10 +600,20 @@ Upload
|
||||
```text
|
||||
Demo
|
||||
↓
|
||||
Hybrid Search
|
||||
↓
|
||||
MCP
|
||||
↓
|
||||
Async Job + Retry Cleanup
|
||||
↓
|
||||
Batch Async Upload
|
||||
↓
|
||||
Production Images
|
||||
↓
|
||||
Kubernetes/Helm One-click Deploy
|
||||
↓
|
||||
Availability + Remote WorkBuddy Acceptance
|
||||
↓
|
||||
Hybrid Search
|
||||
↓
|
||||
OCR / Complex Parsing
|
||||
↓
|
||||
Document Version
|
||||
|
||||
@ -5,9 +5,10 @@ DEMO-010 在 FastAPI 服务中提供一个轻量、同源的管理客户端。
|
||||
|
||||
## 启动与访问
|
||||
|
||||
完成运行时配置后启动 API:
|
||||
完成运行时配置后启动 Worker 与 API:
|
||||
|
||||
```bash
|
||||
make run-worker
|
||||
make run
|
||||
```
|
||||
|
||||
@ -15,17 +16,27 @@ make run
|
||||
|
||||
| 页面 | 地址 | 对应业务接口 |
|
||||
|---|---|---|
|
||||
| 文档上传 | `/admin/upload` | `POST /api/v1/documents` |
|
||||
| 文档上传 | `/admin/upload` | `POST/GET /api/v1/ingestion-batches*` |
|
||||
| 文档列表与详情 | `/admin/documents` | `GET/DELETE /api/v1/documents*` |
|
||||
| Search Lab | `/admin/search` | `POST /api/v1/search` |
|
||||
|
||||
`/admin` 默认展示上传页。OpenAPI 文档仍位于 `/api/v1/docs`。
|
||||
|
||||
## 品牌视觉基线
|
||||
|
||||
DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF3C00`、黑白高对比、中文
|
||||
无衬线标题、紧凑数据字体、低圆角容器和宽松留白。贯穿页面标题的“计算信号轨”是唯一的
|
||||
强调性图形,序号对应上传、文档、检索三步真实操作顺序。
|
||||
|
||||
品牌图形使用仓库内的代码原生 SVG/CSS 绘制,不复制或热链官网图片、字体和脚本;页面
|
||||
运行时不依赖品牌官网。前端设计方法参考 GitHub `anthropics/skills` 的 Apache-2.0
|
||||
`frontend-design` Skill,并以现有 Demo 信息架构、可访问性和业务正确性为优先约束。
|
||||
|
||||
## Demo 操作闭环
|
||||
|
||||
1. 在上传页选择一个 PDF、DOCX、TXT 或 Markdown 文件并提交;
|
||||
2. 等待同步流水线返回 `Ready`,失败时页面显示经过 API 收敛的错误;
|
||||
3. 在文档列表查看状态和安全元数据;
|
||||
1. 在上传页选择一个或多个 PDF、DOCX、TXT 或 Markdown 文件并提交;
|
||||
2. 接口返回 `202` 后,页面自动轮询批次,逐文件显示接收状态、处理阶段、尝试次数和安全错误;
|
||||
3. 批次进入 `Completed`、`Partial failure` 或 `Failed` 后停止轮询,再在文档列表查看元数据;
|
||||
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
|
||||
Rerank Score;
|
||||
5. 如需清理 Demo 数据,在文档列表确认后直接删除。
|
||||
@ -35,8 +46,10 @@ make run
|
||||
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
|
||||
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
|
||||
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
|
||||
- 本任务不包含 ACL/RBAC、审计、异步任务、Document Version、Compiled Knowledge、MCP、
|
||||
Hybrid/BM25 或完整 V1 Admin Console 导航;
|
||||
- Console 只轮询当前批次的聚合接口,不读取 Provider/队列细节,也不提供人工重试按钮;
|
||||
- 批量上传不包含目录上传、压缩包解包、断点续传或跨批次去重;
|
||||
- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
|
||||
或完整 V1 Admin Console 导航;
|
||||
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。
|
||||
|
||||
## 验证
|
||||
|
||||
97
docs/development/ASYNC_INGESTION_SETUP.md
Normal file
97
docs/development/ASYNC_INGESTION_SETUP.md
Normal file
@ -0,0 +1,97 @@
|
||||
# 异步入库与失败恢复说明
|
||||
|
||||
## 范围
|
||||
|
||||
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker,DEMO-016 在其上增加逐文件
|
||||
隔离的批量接收和状态聚合。OceanBase 中的
|
||||
`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
|
||||
不携带文件内容、Provider 凭据或业务状态。
|
||||
|
||||
批次只聚合接收与 Job 状态,不改变每个文件的独立执行和重试边界。
|
||||
|
||||
## 运行时配置
|
||||
|
||||
以下值必须通过环境变量或部署 Secret 注入,不得提交真实值:
|
||||
|
||||
```text
|
||||
RABBITMQ_URL=amqp://<user>:<password>@<host>:5672/<vhost>
|
||||
INGESTION_MAX_ATTEMPTS=3
|
||||
INGESTION_RETRY_BACKOFF_SECONDS=2
|
||||
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
|
||||
INGESTION_WORKER_LEASE_SECONDS=300
|
||||
INGESTION_RECOVERY_INTERVAL_SECONDS=30
|
||||
BATCH_MAX_FILES=20
|
||||
BATCH_MAX_TOTAL_BYTES=209715200
|
||||
```
|
||||
|
||||
Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
|
||||
RabbitMQ 用户应只具备指定 vhost/queue 的最小权限。
|
||||
|
||||
## 启动顺序
|
||||
|
||||
```bash
|
||||
uv run alembic upgrade head
|
||||
make run-worker
|
||||
make run
|
||||
```
|
||||
|
||||
`make run-worker` 在当前 Demo 中同时启动 Worker 和恢复调度器。恢复调度器周期扫描:
|
||||
|
||||
- 尚未成功投递的 `queued` Job;
|
||||
- 已到重试时间的 `retrying` Job;
|
||||
- Worker 丢失后租约已过期的 `running` Job。
|
||||
|
||||
任务投递使用 late acknowledgement、worker-lost reject 和单任务 prefetch。重复消息会先锁定
|
||||
Job 行;有效租约内不会创建第二个 Attempt。租约过期后,旧 Attempt 以安全错误关闭,再按重试
|
||||
规则恢复。
|
||||
|
||||
## 重试与清理
|
||||
|
||||
确定性的内容错误(无法解析、清洗后为空、无 Chunk、派生数据身份不一致)直接进入 `failed`。
|
||||
其余运行时/Provider 错误按指数退避重试,默认最多三次总 Attempt。
|
||||
|
||||
第二次及后续 Attempt 创建前严格执行:
|
||||
|
||||
```text
|
||||
OceanBase Vector rows
|
||||
→ ChunkIndexState
|
||||
→ Chunk
|
||||
→ parsed JSON object
|
||||
→ reset Document processing state
|
||||
```
|
||||
|
||||
不可变源文件、Document、Job 和已结束 Attempt 始终保留。任一清理步骤失败时不创建新 Attempt,
|
||||
Job 直接以 `retry cleanup failed` 终止,禁止在未知脏状态继续执行。
|
||||
|
||||
## API 状态
|
||||
|
||||
`POST /api/v1/documents` 成功返回 `202` 和 `job_id`。查询:
|
||||
|
||||
```text
|
||||
GET /api/v1/jobs/{job_id}
|
||||
```
|
||||
|
||||
Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;阶段为
|
||||
`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
|
||||
返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
|
||||
|
||||
批量提交与查询:
|
||||
|
||||
```text
|
||||
POST /api/v1/ingestion-batches # multipart 字段 files,可重复
|
||||
GET /api/v1/ingestion-batches/{batch_id}
|
||||
```
|
||||
|
||||
批次创建后逐文件返回 `accepted` 或 `rejected`;已接收文件继续通过上述 Job 状态异步推进。
|
||||
批次聚合状态为 `processing | completed | partial_failure | failed`。请求文件数超过
|
||||
`BATCH_MAX_FILES` 时整体返回 `413`;单文件或共享总字节预算超限只拒绝当前文件,不删除此前
|
||||
已接收文件。默认总预算 200 MiB,且配置不得小于单文件 `MAX_UPLOAD_BYTES`。
|
||||
|
||||
## 故障检查
|
||||
|
||||
1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;
|
||||
2. Job 长期为 `running`:检查 Worker,租约到期后恢复调度器会重新投递;
|
||||
3. Job 为 `retrying`:等待 `next_run_at`,客户端不应重新上传同一文件;
|
||||
4. Job 为 `failed` 且错误为 `retry cleanup failed`:停止人工重试,先核对向量、关系数据与 BOS
|
||||
派生对象的一致性;
|
||||
5. 不得通过日志打印 `RABBITMQ_URL`、数据库 URL、BOS/模型凭据或原始 Provider 响应。
|
||||
91
docs/development/CONTAINER_IMAGES.md
Normal file
91
docs/development/CONTAINER_IMAGES.md
Normal file
@ -0,0 +1,91 @@
|
||||
# 生产容器镜像说明
|
||||
|
||||
## 范围
|
||||
|
||||
DEPLOY-001 使用同一个锁定运行时构建三个独立镜像目标:
|
||||
|
||||
| Target | 默认镜像名 | 入口 | 端口 | 镜像健康检查 |
|
||||
|---|---|---|---|---|
|
||||
| `api` | `enterprise-knowledge-api` | Uvicorn + API/Admin | `8000` | `GET /healthz` |
|
||||
| `mcp` | `enterprise-knowledge-mcp` | `ekp-mcp` | `8766` | PID 1 存活 |
|
||||
| `worker` | `enterprise-knowledge-worker` | `ekp-worker` | 无 | PID 1 存活 |
|
||||
|
||||
镜像没有内置数据库迁移、Helm、Ingress 或 Secret 对象;DEPLOY-002 已在独立 Chart 中提供这些
|
||||
编排能力。API 镜像包含 Alembic 和 migrations,由一次性 Migration Job 覆盖命令后复用。详见
|
||||
[`KUBERNETES_DEPLOYMENT.md`](KUBERNETES_DEPLOYMENT.md)。
|
||||
|
||||
## 可重复构建基线
|
||||
|
||||
- Python 基础镜像固定为 `3.12.14-slim-bookworm`,uv 固定为 `0.12.5`,两者同时锁定 manifest
|
||||
digest;
|
||||
- Python 依赖只使用提交的 `uv.lock`,以 `uv sync --frozen --no-dev --no-editable` 安装;
|
||||
- 三个目标共享相同依赖层,不复制不同版本的业务运行时;
|
||||
- 进程均以固定 UID/GID `10001:10001` 运行,不具备 root 权限;
|
||||
- `.dockerignore` 排除 `.env`、Git、虚拟环境、缓存、本地存储、测试和开发产物;
|
||||
- Bake 计划同时声明 `linux/amd64` 与 `linux/arm64`,不下载或打包平台专属模型。
|
||||
|
||||
基础镜像 tag/digest 与应用依赖均已固定;镜像发布系统仍应保存本项目镜像的构建 digest,并让
|
||||
Helm 按 digest 部署。本项目镜像 digest 由实际 Registry 发布产生,不把某个环境的 Registry
|
||||
地址写入仓库。
|
||||
|
||||
## 构建命令
|
||||
|
||||
先确认 Docker Buildx builder 支持两个目标平台:
|
||||
|
||||
```bash
|
||||
make images-print
|
||||
```
|
||||
|
||||
构建全部目标:
|
||||
|
||||
```bash
|
||||
IMAGE_REGISTRY=registry.example.internal/knowledge \
|
||||
IMAGE_TAG=<immutable-release-tag> \
|
||||
make images
|
||||
```
|
||||
|
||||
只有发布流水线才执行推送:
|
||||
|
||||
```bash
|
||||
IMAGE_REGISTRY=registry.example.internal/knowledge \
|
||||
IMAGE_TAG=<immutable-release-tag> \
|
||||
make images-push
|
||||
```
|
||||
|
||||
不要把 Registry 密码放入变量或命令行;构建节点应提前通过其凭据助手登录。禁止使用
|
||||
`latest` 作为集群部署版本。
|
||||
|
||||
## 运行时配置
|
||||
|
||||
镜像只内置以下非敏感进程默认值:`APP_ENV=production`、容器监听地址、静态页面绝对路径和
|
||||
临时缓存目录。OceanBase、BOS、RabbitMQ、Embedding、Reranker 与 MCP Token 必须在容器启动
|
||||
时由 Secret 注入。镜像构建参数、Layer、标签、健康检查和启动命令都不得包含这些值。
|
||||
|
||||
API 的 `/healthz` 只证明进程存活,不读取云端凭据或探测依赖。Kubernetes readiness、资源限制、
|
||||
只读根文件系统所需临时卷及依赖就绪策略由 DEPLOY-002 Chart 配置。MCP endpoint 需要认证,Worker
|
||||
没有 HTTP server,因此这两个镜像的 Docker 健康检查只验证主进程;业务可恢复性仍以持久化
|
||||
Job/Attempt 和 DEPLOY-003 故障验收为准。
|
||||
|
||||
## 本地单目标检查
|
||||
|
||||
需要加载到本地 Docker 时,可直接构建当前平台的一个 target:
|
||||
|
||||
```bash
|
||||
docker build --target api -f deploy/docker/Dockerfile -t ekp-api:test .
|
||||
docker build --target mcp -f deploy/docker/Dockerfile -t ekp-mcp:test .
|
||||
docker build --target worker -f deploy/docker/Dockerfile -t ekp-worker:test .
|
||||
```
|
||||
|
||||
启动时使用 `--env-file` 或编排系统的 Secret 引用,不在命令中展开敏感值。API/Admin smoke test
|
||||
至少检查 `/healthz`、`/admin/upload` 和 `/api/v1/openapi.json`;MCP 与 Worker 的真实依赖启动
|
||||
验收在具备 RabbitMQ 和目标 Provider 的受控环境执行。
|
||||
|
||||
## DEPLOY-001 构建验收
|
||||
|
||||
2026-09-02 已使用 Buildx 分别构建并加载三个 `linux/arm64` target 和三个 `linux/amd64`
|
||||
target。API/Admin 以非 root 用户完成健康、静态页面和 OpenAPI smoke test;MCP/Worker 完成非
|
||||
root 安装入口检查。构建历史与容器文件系统检查未发现 `.env`、Git、测试文件或敏感配置。
|
||||
|
||||
这次验收只生成本地 `deploy001-*` 测试 tag,没有登录或推送 Registry,因此不产生可供集群
|
||||
引用的项目镜像 digest。DEPLOY-002 必须使用操作者指定的不可变 Registry tag/digest,不得把
|
||||
本地测试 tag 当作发布版本。
|
||||
@ -1,3 +1,19 @@
|
||||
# Definition of Done
|
||||
|
||||
A task is done only when scope is implemented, architecture invariants hold, format/lint/typecheck pass, relevant tests pass, migrations/contracts/docs are updated as needed, no secrets are committed, retrieval changes include RAG regression results, and known limitations are documented.
|
||||
|
||||
For asynchronous ingestion or retry tasks, relevant tests must also prove durable Job/Attempt state,
|
||||
bounded retry classification, idempotent duplicate delivery, cleanup-before-retry, preservation of the
|
||||
immutable source, and terminal behavior when cleanup fails. Batch ingestion must cover per-file partial
|
||||
failure and configured limits.
|
||||
|
||||
For MCP tasks, contract tests must cover tool discovery, identity propagation, authorization/error
|
||||
boundaries, REST/application-service parity, infrastructure redaction, and the explicitly targeted MCP
|
||||
client compatibility check.
|
||||
|
||||
For cluster delivery tasks, completion also requires reproducible production images, Secret-only
|
||||
sensitive configuration, serialized migration, readiness/liveness probes, resource and pod-security
|
||||
defaults, idempotent Helm install/upgrade, rolling-restart verification, durable worker recovery, stable
|
||||
HTTPS Admin/MCP endpoints, and remote WorkBuddy acceptance. Manifests, rendered output, logs and images
|
||||
must be checked for credential leakage. Public-knowledge delivery does not complete or waive V1
|
||||
user/ACL tasks.
|
||||
|
||||
143
docs/development/KUBERNETES_DEPLOYMENT.md
Normal file
143
docs/development/KUBERNETES_DEPLOYMENT.md
Normal file
@ -0,0 +1,143 @@
|
||||
# Kubernetes / Helm 部署说明
|
||||
|
||||
## 1. 交付范围
|
||||
|
||||
DEPLOY-002 提供单一 Helm Chart,将以下运行角色部署到已有 Kubernetes 集群:
|
||||
|
||||
- API/Admin:默认两个副本,提供 `/healthz`、`/admin/*` 和 `/api/v1/*`;
|
||||
- MCP Gateway:默认两个副本,提供经 Bearer Token 保护的 `/mcp`;
|
||||
- Async Worker:默认两个副本,只消费 `ingestion` 队列;
|
||||
- Recovery Scheduler:固定单副本,以 `Recreate` 策略运行 Celery Beat;
|
||||
- Migration Job:每次安装/升级前串行执行 `alembic upgrade head`;
|
||||
- 可选 RabbitMQ:只供开发/演示集群显式启用,生产默认使用外部 Broker。
|
||||
|
||||
OceanBase MySQL、BOS、Embedding/Reranker 服务继续使用集群外已有资源。Chart 不创建数据库、
|
||||
BOS Bucket、域名、证书或云账号,也没有扩大公共知识库阶段的业务范围。
|
||||
|
||||
## 2. 前置条件
|
||||
|
||||
操作者需要提前准备:
|
||||
|
||||
1. 可用的 Kubernetes Namespace 和对应 `kubectl` 上下文;
|
||||
2. Helm 3/4 与 Kubernetes 客户端;
|
||||
3. 已推送到目标 Registry 的 API、MCP、Worker 不可变 tag 或 digest;
|
||||
4. Namespace 内的运行时 Secret、TLS Secret,以及私有 Registry 所需的 ImagePullSecret;
|
||||
5. 集群到 OceanBase、BOS、RabbitMQ 和模型服务的网络连通性。
|
||||
|
||||
Chart 默认引用 `knowledge-platform-runtime`,但不会创建它。该 Secret 至少包含以下键:
|
||||
|
||||
```text
|
||||
DATABASE_URL
|
||||
RABBITMQ_URL
|
||||
BOS_ACCESS_KEY_ID
|
||||
BOS_SECRET_ACCESS_KEY
|
||||
BAIDU_QIANFAN_API_KEY
|
||||
OCEANBASE_VECTOR_PASSWORD
|
||||
MCP_AUTH_TOKEN
|
||||
```
|
||||
|
||||
`RERANKER_API_KEY` 可单独提供;未提供时应用按既有契约复用
|
||||
`BAIDU_QIANFAN_API_KEY`。`MCP_AUTH_TOKEN` 至少 32 个字符,只具备
|
||||
`knowledge:search` 服务范围。不要使用 `kubectl --from-literal`、Shell 参数或 Helm
|
||||
`--set` 传递以上值;应由受控 Secret 管理系统或运维流程预先写入 Namespace。
|
||||
|
||||
## 3. 环境 Values
|
||||
|
||||
复制 [`values.example.yaml`](../../deploy/helm/environments/values.example.yaml) 到仓库外的受控
|
||||
路径,填写以下非敏感信息:
|
||||
|
||||
- 三个镜像的 Registry 路径与不可变 tag/digest;
|
||||
- `existingSecret` 和 `imagePullSecrets` 的资源名称;
|
||||
- OceanBase/BOS 的非敏感 Endpoint、数据库标识和 Bucket;
|
||||
- Admin/API 与 MCP 的 Ingress Host、TLS Secret 名称;
|
||||
- 与外部地址一致的 MCP Public URL、Issuer、Allowed Host/Origin;
|
||||
- 副本、资源、超时和批量上传限制。
|
||||
|
||||
`values.schema.json` 拒绝未知顶层字段、`latest` tag、无效 digest、单副本 API/MCP、错误类型和
|
||||
空的关键引用;模板还会校验 Chunk、检索和入库限制之间的大小关系。Values 只允许非敏感配置,
|
||||
模板逐项生成 ConfigMap,不提供任意 `extraEnv` 入口,避免把凭据误写进渲染产物。
|
||||
|
||||
## 4. 安装与幂等升级
|
||||
|
||||
先做离线校验和清单检查:
|
||||
|
||||
```bash
|
||||
make deploy-lint VALUES=/secure/path/environment-values.yaml
|
||||
make deploy-render VALUES=/secure/path/environment-values.yaml
|
||||
```
|
||||
|
||||
安装或升级:
|
||||
|
||||
```bash
|
||||
NAMESPACE=knowledge-platform \
|
||||
RELEASE=knowledge-platform \
|
||||
make deploy VALUES=/secure/path/environment-values.yaml
|
||||
```
|
||||
|
||||
`make deploy` 只接收 Values 文件路径。脚本会先验证 Chart、Namespace、运行时 Secret 及必需
|
||||
键名、可选 RabbitMQ Secret 键名,以及启用 Ingress 时的两个 TLS Secret 引用,然后执行带失败
|
||||
回滚和等待语义的 `helm upgrade --install`;脚本对 Helm 3 使用 `--atomic`,对 Helm 4 使用等价
|
||||
的 `--rollback-on-failure`。它只验证 Secret 资源和键是否存在,不读取或输出 Secret 值。
|
||||
|
||||
Migration 是 `pre-install,pre-upgrade` Hook,固定名称配合
|
||||
`before-hook-creation` 保证同一 Release 内不会并发执行。Helm 等待 Migration 成功后才更新业务
|
||||
工作负载;失败时 Helm 保留上一版本。完成的 Job 最长保留一天供日志追查,并在下次升级前删除。
|
||||
安装命令可以重复执行;数据库迁移必须保持 forward-only 与幂等。
|
||||
|
||||
## 5. Ingress、探针与 Pod 安全
|
||||
|
||||
生产环境启用 Ingress 后,Admin/API 与 MCP 使用独立稳定 HTTPS Host,也可由同一 Ingress
|
||||
Controller 托管。证书只通过预建 TLS Secret 引用,不进入 Values。
|
||||
|
||||
- API 的 startup/readiness/liveness 调用无凭据 `/healthz`;
|
||||
- MCP 的三类探针使用 TCP 端口,不发送或记录 Token;
|
||||
- Worker/Scheduler 使用 PID 1 存活探针,业务恢复正确性由持久化 Job/Attempt 保证;
|
||||
- 全部应用容器固定 UID/GID `10001`、禁止提权、丢弃 Linux capabilities、使用
|
||||
`RuntimeDefault` seccomp 和只读根文件系统;
|
||||
- `/tmp` 使用有大小上限的 `emptyDir`,ServiceAccount 不挂载 API Token;
|
||||
- API/MCP 滚动更新使用 `maxUnavailable: 0`,未 Ready Pod 不进入 Service Endpoint。
|
||||
|
||||
`/healthz` 当前只表示 API 进程存活,不探测所有外部 Provider;真实依赖故障、滚动重启和
|
||||
Worker 恢复行为由 DEPLOY-003 在目标集群验收。
|
||||
|
||||
## 6. 可选 Demo RabbitMQ
|
||||
|
||||
生产默认 `rabbitmq.enabled=false`,`RABBITMQ_URL` 指向外部受管 Broker。仅开发/演示时可开启
|
||||
本地子 Chart;它使用单节点 StatefulSet 和 PVC,并要求预建 Secret 包含:
|
||||
|
||||
```text
|
||||
username
|
||||
password
|
||||
erlang-cookie
|
||||
```
|
||||
|
||||
同时必须让运行时 Secret 中的 `RABBITMQ_URL` 指向该 Service。子 Chart 使用固定
|
||||
`rabbitmq:4.1.4-alpine` tag 与 multi-architecture manifest digest;它不具备生产 RabbitMQ 的
|
||||
高可用、备份或升级保证,禁止作为生产默认值。卸载 Release 时 StatefulSet PVC 默认保留,
|
||||
删除前必须由数据责任人确认。
|
||||
|
||||
## 7. 回滚、卸载与 Secret 轮换
|
||||
|
||||
查看历史并回滚时不要输出 Pod 环境变量:
|
||||
|
||||
```bash
|
||||
helm -n knowledge-platform history knowledge-platform
|
||||
helm -n knowledge-platform rollback knowledge-platform <revision> --wait
|
||||
```
|
||||
|
||||
更新外部 Secret 后,Secret 名称不变;用户和 WorkBuddy 配置无需重新注入。当前应通过受控
|
||||
滚动重启让 Pod 读取新值,目标集群的无中断验证属于 DEPLOY-003。
|
||||
|
||||
卸载只删除本 Release 管理的 Kubernetes 对象:
|
||||
|
||||
```bash
|
||||
helm -n knowledge-platform uninstall knowledge-platform
|
||||
```
|
||||
|
||||
外部 OceanBase、BOS、RabbitMQ、TLS/运行时 Secret 和可选 RabbitMQ PVC 不由卸载命令删除。
|
||||
|
||||
## 8. DEPLOY-002 验收边界
|
||||
|
||||
本任务完成 Chart lint、双路径渲染、严格 Kubernetes Schema、Values 失败关闭、Secret 泄漏扫描
|
||||
以及本地集群两次安装/升级验证。目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实
|
||||
Provider 和远程 WorkBuddy 验收统一属于后续 `DEPLOY-003`,不得提前宣称完成。
|
||||
102
docs/development/MCP_SETUP.md
Normal file
102
docs/development/MCP_SETUP.md
Normal file
@ -0,0 +1,102 @@
|
||||
# DEMO-012 MCP Gateway 使用说明
|
||||
|
||||
## 范围
|
||||
|
||||
DEMO-012 提供独立的 Enterprise Knowledge MCP Gateway,通过标准 Streamable HTTP 暴露一个
|
||||
只读工具:`search_knowledge`。Gateway 直接复用 API 进程使用的 `SearchService` 组合,不复制
|
||||
Dense Retrieval、Reranker 或 Source Enrichment 逻辑。
|
||||
|
||||
当前 Demo 只建立认证和身份传递边界:一个运行时注入的专用 Bearer Token 映射到一个
|
||||
user/service identity,并且只有 `knowledge:search` scope。它不是共享超级管理员 Token,也不
|
||||
代表 V1 OAuth、Document ACL 或完整 MCP P0 工具已经完成。
|
||||
|
||||
## 配置
|
||||
|
||||
必须通过运行环境提供至少 32 字符的随机 Token:
|
||||
|
||||
```bash
|
||||
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
|
||||
make run-mcp
|
||||
```
|
||||
|
||||
默认地址为 `http://127.0.0.1:8766/mcp`。除现有数据库、BOS、Embedding、Vector 和 Reranker
|
||||
配置外,Gateway 支持:
|
||||
|
||||
| 变量 | 默认值 | 说明 |
|
||||
|---|---|---|
|
||||
| `MCP_HOST` | `127.0.0.1` | 监听地址;本地默认不暴露到其他网卡 |
|
||||
| `MCP_PORT` | `8766` | Gateway 端口 |
|
||||
| `MCP_PATH` | `/mcp` | Streamable HTTP 单一端点 |
|
||||
| `MCP_PUBLIC_URL` | `http://127.0.0.1:8766/mcp` | MCP Resource Server URL |
|
||||
| `MCP_ISSUER_URL` | `http://127.0.0.1:8766` | Demo Token 的身份颁发方标识 |
|
||||
| `MCP_CLIENT_ID` | `workbuddy-enterprise` | MCP Client 类型标识 |
|
||||
| `MCP_SERVICE_IDENTITY` | `workbuddy-demo` | 传入业务边界的服务身份 |
|
||||
| `MCP_REQUIRED_SCOPE` | `knowledge:search` | Tool 所需最小 scope |
|
||||
| `MCP_ALLOWED_HOSTS` | 仅本机/test | DNS rebinding Host allowlist,JSON 数组 |
|
||||
| `MCP_ALLOWED_ORIGINS` | 仅本机 | Origin allowlist,JSON 数组 |
|
||||
|
||||
远程部署必须使用 TLS 反向代理,并把公开域名加入 `MCP_ALLOWED_HOSTS`;有 `Origin` 的客户端还
|
||||
必须显式配置 `MCP_ALLOWED_ORIGINS`。Token、真实域名和内部 Endpoint 不得提交 Git。
|
||||
|
||||
## WorkBuddy / CodeBuddy 配置
|
||||
|
||||
WorkBuddy Enterprise 官方客户端支持 `type: "http"`、远程 URL、自定义 Header 和环境变量
|
||||
展开。可复制 `deploy/workbuddy-mcp.example.json` 作为项目配置,内容如下:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"mcpServers": {
|
||||
"enterprise-knowledge": {
|
||||
"type": "http",
|
||||
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
|
||||
"headers": {
|
||||
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
|
||||
},
|
||||
"description": "企业知识检索"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
只提交包含环境变量占位符的配置;`EKP_MCP_TOKEN` 必须由每个运行环境安全注入。连接后应只
|
||||
发现 `mcp__enterprise_knowledge__search_knowledge` 一个工具。
|
||||
|
||||
## 安全和错误语义
|
||||
|
||||
- 缺失或错误 Token 返回 `401`,缺少 scope 返回 `403`;
|
||||
- 未批准 Host 返回 `421`,未批准 Origin 返回 `403`;
|
||||
- Query 长度和 `top_k` 在 MCP schema 层校验;
|
||||
- 预期业务错误返回安全 Tool Error;未知 Provider/数据库错误收敛为
|
||||
`knowledge search failed`;
|
||||
- MCP 输出和 Gateway 日志不得包含 Token、Provider Endpoint、数据库表名、模型名或
|
||||
Index Profile。
|
||||
|
||||
## 验证
|
||||
|
||||
```bash
|
||||
uv run pytest tests/unit/test_mcp_auth.py tests/unit/test_mcp_search.py
|
||||
uv run pytest tests/integration/test_mcp_server.py
|
||||
```
|
||||
|
||||
集成测试使用官方 MCP Python Client 通过 Streamable HTTP 完成 initialize、tools/list 和
|
||||
tools/call,并使用与 WorkBuddy 相同的 Bearer Header 配置验证身份传递、无结果、参数错误、
|
||||
认证失败、Origin 防护和错误脱敏。
|
||||
|
||||
### WorkBuddy 现场验收记录
|
||||
|
||||
2026-09-02 使用 WorkBuddy Desktop `5.3.14` 自带的 CodeBuddy CLI `2.115.0` 完成兼容性
|
||||
验收。客户端直接加载 `deploy/workbuddy-mcp.example.json`,健康检查为 `Connected`;工具
|
||||
发现结果只有 `mcp__enterprise-knowledge__search_knowledge`,随后以字符串 `query` 和
|
||||
`top_k=1` 完成 Tool 调用并读取结构化 `filename`、`section`、`content` 等来源字段。
|
||||
|
||||
现场验收使用 DEMO-012 的真实 Gateway、Bearer 验证、service identity 传递、输入 Schema 和
|
||||
Tool 注册代码。为避免外部数据库与模型波动掩盖客户端兼容性,检索应用使用确定性验收返回;
|
||||
真实 Provider 检索链路由 `ARCH-006` 的受控验收负责。非交互 CLI 需要批准 MCP Tool 调用,
|
||||
本次只在单次会话中允许 `DeferExecuteTool`,没有永久启用跳过权限。测试结束后已删除临时 MCP
|
||||
配置并停止本地 Gateway,未写入真实凭据。
|
||||
|
||||
参考:
|
||||
|
||||
- [MCP Python SDK](https://github.com/modelcontextprotocol/python-sdk)
|
||||
- [MCP Streamable HTTP 规范](https://modelcontextprotocol.io/specification/2025-11-25/basic/transports)
|
||||
- [WorkBuddy Enterprise MCP 使用文档](https://cloud.tencent.cn/document/product/1831/137039)
|
||||
@ -2,19 +2,21 @@
|
||||
|
||||
## 范围
|
||||
|
||||
DEMO-009 将已经完成的同步入库和检索服务公开为版本化 REST API。API 层只负责输入校验、
|
||||
DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
|
||||
Job,DEMO-016 增加批量接收与聚合查询。API 层只负责输入校验、
|
||||
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
|
||||
|
||||
本任务不包含 Admin Console、ACL/RBAC、异步任务、软删除或 Hybrid Search。
|
||||
当前不包含 ACL/RBAC、软删除、目录/压缩包上传、断点续传或 Hybrid Search。
|
||||
|
||||
## 启动前提
|
||||
|
||||
1. 按 `docs/development/BOS_SETUP.md` 配置 BOS;
|
||||
2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding;
|
||||
3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
|
||||
4. 执行 `uv run alembic upgrade head`;
|
||||
5. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
|
||||
6. 执行 `make run`。
|
||||
4. 按 `docs/development/ASYNC_INGESTION_SETUP.md` 配置 RabbitMQ 并启动 Worker;
|
||||
5. 执行 `uv run alembic upgrade head`;
|
||||
6. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
|
||||
7. 执行 `make run`。
|
||||
|
||||
`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时
|
||||
延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。
|
||||
@ -25,13 +27,29 @@ Swagger UI:`/api/v1/docs`
|
||||
|
||||
## 示例
|
||||
|
||||
上传并同步入库:
|
||||
上传并异步排队:
|
||||
|
||||
```bash
|
||||
curl -F 'file=@./policy.md;type=text/markdown' \
|
||||
http://127.0.0.1:8000/api/v1/documents
|
||||
```
|
||||
|
||||
使用响应中的 `job_id` 查询进度:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8000/api/v1/jobs/<job-id>
|
||||
```
|
||||
|
||||
批量上传并查询聚合进度:
|
||||
|
||||
```bash
|
||||
curl -F 'files=@./policy.md;type=text/markdown' \
|
||||
-F 'files=@./manual.pdf;type=application/pdf' \
|
||||
http://127.0.0.1:8000/api/v1/ingestion-batches
|
||||
|
||||
curl http://127.0.0.1:8000/api/v1/ingestion-batches/<batch-id>
|
||||
```
|
||||
|
||||
检索:
|
||||
|
||||
```bash
|
||||
@ -40,8 +58,8 @@ curl -H 'Content-Type: application/json' \
|
||||
http://127.0.0.1:8000/api/v1/search
|
||||
```
|
||||
|
||||
Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase Upsert 完成。大文
|
||||
件或云端调用较慢时,请求耗时会相应增加;异步任务属于 Demo 后置能力。
|
||||
上传只等待不可变源文件、Document 与 Job 持久化,成功返回 `202`。Worker 后台执行 Parse、
|
||||
Clean、Chunk、Embedding 和 OceanBase Upsert;客户端通过 Job 状态判断何时可检索。
|
||||
|
||||
## 删除一致性
|
||||
|
||||
@ -51,6 +69,8 @@ Demo 上传请求会一直等待 Parse、Clean、Chunk、Embedding 和 OceanBase
|
||||
2. 删除 OceanBase canonical 关系表中的 ChunkIndexState、Chunk 和 Document;
|
||||
3. 删除解析 JSON 与不可变原始文件。
|
||||
|
||||
`queued`、`running` 或 `retrying` Job 对应的 Document 返回 `409`,避免删除与 Worker 并发。
|
||||
|
||||
向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口
|
||||
返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1
|
||||
应通过可重试清理任务和审计记录强化。
|
||||
|
||||
@ -11,6 +11,11 @@ Provide deterministic fakes:
|
||||
|
||||
External integrations use separate integration suites/markers.
|
||||
|
||||
Async ingestion tests run with an in-process recording dispatcher and deterministic providers. They
|
||||
must cover persisted Job/Attempt history, safe error summaries, bounded retry, cleanup-before-retry,
|
||||
active-lease duplicate delivery, expired-lease recovery, immutable-source preservation and terminal
|
||||
cleanup failure without requiring a live RabbitMQ broker.
|
||||
|
||||
E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval.
|
||||
|
||||
Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation.
|
||||
|
||||
123
docs/plans/ASYNC_INGESTION_MCP_PLAN.md
Normal file
123
docs/plans/ASYNC_INGESTION_MCP_PLAN.md
Normal file
@ -0,0 +1,123 @@
|
||||
# MCP 与异步批量入库优先级方案
|
||||
|
||||
**Task:** PLAN-001
|
||||
**Status:** Accepted planning baseline
|
||||
**Date:** 2026-09-02
|
||||
|
||||
## 1. 现状判断
|
||||
|
||||
| 能力 | 当前实现 | 结论 |
|
||||
|---|---|---|
|
||||
| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
|
||||
| 批量上传 | 批次 API、逐文件 Job 状态和 Admin Console 自动轮询 | DEMO-016 已实现 |
|
||||
| 失败补偿 | 阶段内补偿 + 重试前清理全部派生数据,保留不可变源和执行历史 | DEMO-015 已实现 |
|
||||
| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
|
||||
| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
|
||||
| MCP | 独立 Gateway、认证边界、`search_knowledge` 与 WorkBuddy 协议级验收 | DEMO-012 已实现 |
|
||||
|
||||
PLAN-001 的异步执行缺口已由 DEMO-015 补齐,批量提交与状态 UI 已由 DEMO-016 补齐。
|
||||
|
||||
## 2. Task 归并结论
|
||||
|
||||
### DEMO-012:MCP `search_knowledge`(提升为最高优先级)
|
||||
|
||||
保留既有 Task ID,不新建重复任务。范围限定为:
|
||||
|
||||
- 提供可运行的标准 MCP Gateway;
|
||||
- 只暴露 `search_knowledge`;
|
||||
- 复用现有 Search Service 或正式 `/api/v1/search`,不复制检索逻辑;
|
||||
- 建立 WorkBuddy 所需的 user/service identity 边界,不使用共享超级管理员凭据;
|
||||
- 完成协议契约、错误收敛、凭据脱敏和 WorkBuddy 兼容性验证。
|
||||
|
||||
该 Task 是 `EKP-070`、`EKP-071`、`EKP-072` 的最小垂直切片,不能据此把三个完整 V1 Task 标记为完成。
|
||||
|
||||
### DEMO-015:异步入库 Job 与自动重试清理(新增)
|
||||
|
||||
从 `EKP-017` 和 `EKP-023` 抽取 Demo 所需的最小基础能力,并复用 DEMO-003~006 的处理服务:
|
||||
|
||||
- 新增持久化 Ingestion Job/Attempt,记录阶段、状态、尝试次数、安全错误和时间戳;
|
||||
- 源文件成功写入 BOS 且 Document 已持久化后立即排队,API 不等待解析和索引;
|
||||
- 同一 Document 同时最多一个有效执行,任务投递和消费具备幂等保护;
|
||||
- 对可重试的瞬时错误自动执行有界重试和退避;Demo 默认最多 3 次总尝试,允许通过非敏感配置调整;
|
||||
- 格式不支持、文件为空、无可提取内容等确定性错误直接失败,不进行无意义重试;
|
||||
- 每次重新执行前,按顺序清理该 Document 的向量行、`ChunkIndexState`、Chunk 和解析产物,再从不可变原始文件重新处理;
|
||||
- 原始文件、Document 标识和审计所需的 Job/Attempt 历史不得被重试清理删除;
|
||||
- 任一清理步骤失败时禁止启动下一次尝试,并记录为终止失败,避免在未知脏状态上重复执行。
|
||||
- 通过执行器抽象接入 Celery/RabbitMQ,使 API 和 Worker 可独立部署;业务应用服务不得直接
|
||||
依赖 Celery API,确保执行器仍可替换;
|
||||
- Worker 重启、消息重复投递和确认超时必须保持持久化 Job 状态一致,不能产生双重有效
|
||||
Attempt 或重复派生数据。
|
||||
|
||||
该 Task 只完成公共知识库入库所需的 Job/Retry 与队列基线,不扩展到完整审计系统、
|
||||
Compilation、OCR、跨区域高可用或通用企业级 Job 平台。
|
||||
|
||||
### DEMO-016:批量异步上传与状态展示(已完成)
|
||||
|
||||
建立在 DEMO-015 之上,不与 Job 基础合并:
|
||||
|
||||
- 新增批量上传 API,一次接收多个 Demo 支持类型文件;
|
||||
- 每个文件独立生成 Document 和 Ingestion Job,单个文件失败不回滚已成功接收的其他文件;
|
||||
- 返回批次 ID、逐文件 Document ID、Job ID 和接收状态,不等待后台处理完成;
|
||||
- 提供批次/Job 查询能力,并在 Admin Console 展示逐文件阶段、重试次数和最终结果;
|
||||
- 文件数量、单文件大小和请求总大小均受可配置限制;
|
||||
- 不在本 Task 引入目录上传、压缩包解包、断点续传或跨批次去重。
|
||||
|
||||
该 Task 对应 `EKP-021` 的批量异步扩展及 `EKP-080`/`EKP-081` 的最小状态 UI 切片,不能据此把完整 V1 Task 标记为完成。
|
||||
|
||||
## 3. 实施顺序与依赖
|
||||
|
||||
```text
|
||||
DEMO-009 + ARCH-006 → DEMO-012(已完成)
|
||||
|
||||
PLAN-002(已完成)
|
||||
└──→ DEMO-003 → DEMO-004 → DEMO-005 → DEMO-006
|
||||
└──→ DEMO-015 Async Job + Celery/RabbitMQ + Retry Cleanup
|
||||
└──→ DEMO-016 Batch Async Upload + UI
|
||||
└──→ DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
||||
```
|
||||
|
||||
推荐严格按以下顺序执行并逐项闭环:
|
||||
|
||||
1. `PLAN-002`(已完成)
|
||||
2. `DEMO-015`(已完成)
|
||||
3. `DEMO-016`(已完成)
|
||||
4. `DEPLOY-001`
|
||||
5. `DEPLOY-002`
|
||||
6. `DEPLOY-003`
|
||||
|
||||
MCP 与异步入库没有代码级前置依赖,因此提高 MCP 优先级不会阻塞后续批量上传。`DEMO-016` 必须等待 `DEMO-015`,否则会把同步流水线简单并发放大,缺少可靠的状态、隔离和恢复边界。
|
||||
|
||||
## 4. 验收边界
|
||||
|
||||
### DEMO-012
|
||||
|
||||
- MCP Inspector/标准客户端能发现并调用 `search_knowledge`;
|
||||
- WorkBuddy 使用已映射的 user/service identity 完成一次端到端检索;
|
||||
- REST 与 MCP 对同一查询复用同一业务逻辑并返回一致的来源字段;
|
||||
- 响应与日志不泄露 OceanBase、BOS、模型名、Endpoint 或凭据。
|
||||
|
||||
### DEMO-015
|
||||
|
||||
- 上传接口在源文件和 Job 持久化后返回 `202`,后台完成现有全流水线;
|
||||
- 注入一次可重试故障后可观察到自动重试,并最终成功;
|
||||
- 第二次尝试开始前,旧解析产物、Chunk、索引状态和向量均已清理,原始源文件仍存在;
|
||||
- 永久错误不重试,达到次数上限后稳定进入 `failed`;
|
||||
- 并发或重复投递不会产生双重有效执行和重复派生数据。
|
||||
- RabbitMQ 消息确认超时或 Worker 重启后,Job 可恢复到终态且不会重复生成派生数据。
|
||||
|
||||
### DEMO-016
|
||||
|
||||
- 一次提交多个文件后立即得到逐文件 Document/Job 标识;
|
||||
- 批次内成功、处理中、重试中和失败状态互不影响且可查询;
|
||||
- Admin Console 无需刷新即可轮询展示逐文件进度和最终状态;
|
||||
- 批次边界、大小限制、部分失败和重试清理均有集成测试。
|
||||
|
||||
## 5. 非目标与 ADR 影响
|
||||
|
||||
- PLAN-002 已把 Celery/RabbitMQ 纳入 `DEMO-015`,并以独立 `DEPLOY-*` Task 纳入最小
|
||||
Kubernetes、多副本和滚动升级能力;跨区域容灾、自动云资源编排和完整 Observability
|
||||
仍为非目标。
|
||||
- 不改变 OceanBase/BOS canonical、向量索引可重建、原始文档不可变、Provider 抽象和 MCP 隐藏基础设施等 Accepted ADR。
|
||||
- 公共知识库只是阶段性单一可见域;ADR-007 保持 Accepted,用户/ACL 实现后置且不得用全局
|
||||
Top-K 后过滤替代。
|
||||
- 本次任务拆分和交付边界不需要新增或重开 Accepted ADR。
|
||||
@ -1,13 +1,12 @@
|
||||
# 项目进度
|
||||
|
||||
当前阶段:Retrieval Demo 闭环完成
|
||||
当前阶段:生产容器镜像闭环完成,准备实施 Kubernetes/Helm 一键部署
|
||||
|
||||
已完成:DEMO-001~DEMO-010、ARCH-001~ARCH-006
|
||||
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001
|
||||
进行中:无
|
||||
阻塞:无
|
||||
|
||||
下一任务:无。`docs/demo/DEMO_SCOPE.md` 当前批准的必做任务已经完成;可选 DEMO-011~013
|
||||
或 V1 backlog 需在明确选定新 Task 后再开始。
|
||||
下一任务:`DEPLOY-002`,完成并合并后执行 `DEPLOY-003`。
|
||||
|
||||
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
|
||||
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
|
||||
@ -250,3 +249,178 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
|
||||
作为底层链路基线。
|
||||
- 已知非阻塞风险保持不变:配置清单中的历史 test 凭据仍应轮换;真实 Provider 瞬时失败的
|
||||
重试、限流与告警留待后续明确 Task。Demo Console 本身没有新增凭据或生产 Endpoint。
|
||||
|
||||
### DEMO-014(已完成)
|
||||
|
||||
- 将 DEMO-010 通用绿色界面升级为地瓜机器人品牌化 Admin Console,使用唯一品牌强调色
|
||||
`#FF3C00`、黑白高对比、无衬线中文排版、低圆角工业组件和与三步操作对应的计算信号轨;
|
||||
没有增加页面、接口或业务能力。
|
||||
- 品牌标记由内联 SVG 与 CSS 原生绘制,不下载、复制或热链官网资产。页面没有新增外部
|
||||
Runtime、字体请求或第三方脚本,继续保持原生 HTML/CSS/JS 和同源 `/api/v1` 调用。
|
||||
- GitHub 调研确认官方 `openai/skills` 当前没有通用前端设计 Skill;选择并安装了 Apache-2.0
|
||||
的 `anthropics/skills/skills/frontend-design`。该 Skill 只影响视觉策划和自检方法,不进入
|
||||
项目运行依赖或仓库产物。
|
||||
- 浏览器使用本地确定性 Provider 再次完成 README 上传 → Ready → 文档列表 → Top5 检索;
|
||||
1440px 桌面端和移动端均无根页面横向溢出、无前端警告或错误。复验中发现隐藏表头将移动
|
||||
页面撑宽至 503px,已改为 `aria-label`,最终根宽度与 463px 视口一致,表格只在容器内滚动。
|
||||
- 完整验证结果:Ruff Format/Lint、Mypy、JavaScript 语法与 Git diff 通过;Unit、Integration
|
||||
和 RAG Evaluation 合计 `155 passed`;Alembic Head 与离线升级通过。本任务不修改 API、
|
||||
检索算法、Provider 或数据库,不重复调用真实资源验收。
|
||||
|
||||
### PLAN-001(已完成)
|
||||
|
||||
- 审计确认当前 `/api/v1/documents` 仅接受一个文件,并在请求线程同步执行解析、Chunk、
|
||||
Embedding 和索引;API 与 Admin Console 均没有批量提交、Job 查询或后台状态轮询。
|
||||
- 当前上传、解析和向量索引具备阶段内失败补偿,影子索引重建具备目标 Profile 的重试安全
|
||||
清理,但正常上传流水线没有持久化 Job/Attempt、自动调度退避或统一的重试前派生数据清理,
|
||||
因而不能视为已实现自动重试。
|
||||
- 保留既有 `DEMO-012` 并提升为最高优先级;新增 `DEMO-015` 承担 Async Job、自动重试和
|
||||
重试前清理,新增 `DEMO-016` 承担批量异步上传和逐文件状态 UI。严格执行顺序为
|
||||
`DEMO-012 → DEMO-015 → DEMO-016`。
|
||||
- 已更新 PRD、Architecture、API/MCP Contract、Execution Plan、Task List、Demo Scope 与 DoD;
|
||||
本任务只形成规划和待实现契约,没有修改业务代码、数据库 Migration 或 Accepted ADR。
|
||||
- 验证结果:Ruff Format/Lint、Mypy、Git diff 与 Alembic Head/离线升级通过;Unit + Integration
|
||||
`150 passed`,RAG Evaluation `5 passed`。首次受限网络运行因无法解析 PyPI 失败,获准联网
|
||||
获取已锁定的构建依赖后全套复验通过。
|
||||
- 下一任务:`DEMO-012`。详细依赖、范围和验收标准见
|
||||
`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`。
|
||||
|
||||
### DEMO-012(已完成)
|
||||
|
||||
- 新增独立 `ekp-mcp` Streamable HTTP Gateway 和 `make run-mcp` 入口,使用官方 MCP Python
|
||||
SDK v2;默认只监听 `127.0.0.1:8766/mcp`,且只注册 `search_knowledge` 一个业务 Tool。
|
||||
- Gateway 直接复用现有 `SearchService`,没有复制 Dense Retrieval、Reranker 或 Source
|
||||
Enrichment;结构化输出与 REST 保持相同来源字段,不返回 Index Profile 或 Provider 信息。
|
||||
- 新增运行时专用 Bearer Token 验证,将凭据映射为显式 client/service identity 和最小
|
||||
`knowledge:search` scope;Token 至少 32 字符并通过 `SecretStr` 管理,不允许写入仓库。
|
||||
当前身份层是 Demo 边界,不宣称完成 V1 OAuth、Document ACL 或用户委托授权。
|
||||
- 启用 Host/Origin allowlist 和请求体限制;缺失/错误认证、scope、参数和检索异常均按安全
|
||||
边界收敛,响应和日志不回显 Token、内部 Endpoint、数据库、模型或索引实现。
|
||||
- 新增官方 MCP Client 的协议级测试,覆盖 initialize、tools/list、tools/call、唯一 Tool、
|
||||
identity 传递、REST 字段一致性、无结果、参数拒绝、认证失败、Origin 防护和错误脱敏;
|
||||
WorkBuddy 的 HTTP + Bearer + 环境变量配置记录于 `docs/development/MCP_SETUP.md`。
|
||||
- 验证结果:Ruff Format/Lint、Mypy、Git diff、Alembic Head/离线升级全部通过;Unit +
|
||||
Integration `165 passed`,RAG Evaluation `5 passed`。本任务未改变检索算法或数据库结构,
|
||||
不重复消耗真实 Embedding/OceanBase/Reranker 资源。
|
||||
- 2026-09-02 补充完成 WorkBuddy 现场验收:本机 WorkBuddy Desktop `5.3.14` 自带的
|
||||
CodeBuddy CLI `2.115.0` 通过仓库交付的 `deploy/workbuddy-mcp.example.json` 建立
|
||||
Streamable HTTP + Bearer 连接,健康检查为 `Connected`,仅发现
|
||||
`mcp__enterprise-knowledge__search_knowledge`,并以 `query` + `top_k=1` 完成真实 Tool
|
||||
调用、读取结构化来源字段。验收使用真实 Gateway、认证、身份传递、Schema 和 Tool
|
||||
注册代码,检索应用返回确定性数据,以隔离 WorkBuddy 兼容性与外部 Provider 波动。
|
||||
- 非交互客户端调用使用仅对当前会话放行 `DeferExecuteTool` 的权限设置,没有持久化放宽权限;
|
||||
临时客户端配置已删除、Gateway 已停止,仓库和用户配置中均未留下测试 Token。至此
|
||||
`DEMO-012` 约定的 WorkBuddy 客户端验收项闭环;公网 HTTPS 部署验证属于部署环境验收,
|
||||
不阻塞本地 Demo Task 完成。
|
||||
- 下一任务:`DEMO-015`。
|
||||
|
||||
### PLAN-002(已完成)
|
||||
|
||||
- 将当前交付目标明确为企业受控网络内的单一公共知识库:本阶段所有知识共享一个可见域,
|
||||
暂缓用户、角色、知识库空间和 Document ACL,但不废弃 ADR-007,也不把未实现项标记完成。
|
||||
- 保持 Accepted ADR 的模块化单体边界,确定 API/Admin、MCP Gateway 和 Async Worker 独立
|
||||
部署;OceanBase/BOS/AI Provider 使用外部服务,当前云端模型链路不启用 GPU Worker。
|
||||
- 将 Celery/RabbitMQ 正式并入 `DEMO-015`,并新增 `DEPLOY-001~003`,严格顺序为
|
||||
`DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
|
||||
- 定义 `make deploy VALUES=...` 的一键部署边界:操作者预先提供集群访问与 Secret,仓库负责
|
||||
生产镜像、串行 Migration、Helm 安装/升级、Ingress/TLS、探针、资源与 Pod 安全配置;不得
|
||||
将凭据写入 Git、镜像、命令行或日志。
|
||||
- 明确现有 `docker-compose.dev.yml` 的 PostgreSQL/MinIO 假设已被 ADR-012 取代,不作为集群
|
||||
交付模板;新增 `docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md` 作为任务范围和验收基线。
|
||||
- 本任务仅修改规划、产品、架构、API/MCP 契约、Demo Scope 和 DoD 文档,没有修改业务代码、
|
||||
数据库 Migration 或 Accepted ADR 状态。
|
||||
- 验证结果:Markdown 相对链接、敏感值模式和 Git diff 检查通过;Ruff Format/Lint、Mypy、
|
||||
Alembic Head/离线升级通过;Unit + Integration `165 passed`,RAG Evaluation `5 passed`。
|
||||
- 下一任务:`DEMO-015`。
|
||||
|
||||
### DEMO-015(已完成)
|
||||
|
||||
- 新增 OceanBase canonical `ingestion_jobs` / `ingestion_attempts` 与 forward-only migration;
|
||||
Document 唯一 Job、Attempt 编号上限和单一运行中 Attempt 均由数据库约束保护。
|
||||
- `POST /api/v1/documents` 改为持久化不可变 Source、Document、Job 后返回 `202`,新增
|
||||
`GET /api/v1/jobs/{job_id}`;活动 Job 的 Document 删除返回 `409`,避免与 Worker 竞态。
|
||||
- 新增队列无关的 Dispatcher 与 Job Runner,首个适配器使用 Celery/RabbitMQ;消息只包含 Job
|
||||
ID,启用 late ack、worker-lost reject、单任务 prefetch 与周期性持久化 Job 恢复扫描。
|
||||
- 瞬时错误按配置执行指数退避和最多三次总 Attempt;Worker 租约有效时重复投递不创建新
|
||||
Attempt,租约过期后关闭旧 Attempt 并恢复。错误记录为固定安全摘要,不保存 Provider 正文。
|
||||
- 第二次及后续 Attempt 前依次清理向量、ChunkIndexState、Chunk 与 parsed JSON,再从不可变
|
||||
源文件重建;清理失败直接终止且不创建新 Attempt。Job 创建失败会补偿 Document 与源对象。
|
||||
- Admin Console 仅将上传提示调整为“已接收并进入队列”;批量选择、Job 轮询和逐文件状态 UI
|
||||
没有提前实现,继续属于 `DEMO-016`。
|
||||
- 验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、依赖锁、Alembic Head/离线 MySQL SQL
|
||||
与 SQLite schema drift 均通过;Unit + Integration `179 passed`,RAG Evaluation `5 passed`;
|
||||
Worker 入口确认注册 `ekp.ingestion.run` 与 `ekp.ingestion.recover`。
|
||||
- 环境限制:当前工作站没有 Docker/RabbitMQ,也没有持久化注入目标 OceanBase URL,因此未做
|
||||
真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
|
||||
重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
|
||||
- 下一任务:`DEMO-016`。
|
||||
|
||||
### DEMO-016(已完成)
|
||||
|
||||
- 新增 canonical `ingestion_batches` / `ingestion_batch_items` 及 forward-only migration,保存
|
||||
输入顺序、逐文件接收结果与 Document/Job 引用;异步阶段和最终结果仍以 Job 为事实源,
|
||||
批次不是跨文件事务。
|
||||
- 新增 `POST /api/v1/ingestion-batches` 与 `GET /api/v1/ingestion-batches/{batch_id}`;批量
|
||||
请求字段为 `files`,成功返回 `202`、聚合状态和逐文件接收/Job 状态。类型、大小或内容错误
|
||||
只拒绝当前文件,错误映射为固定安全摘要,其他文件继续接收。
|
||||
- 新增 `BATCH_MAX_FILES` 与 `BATCH_MAX_TOTAL_BYTES`,复用 `MAX_UPLOAD_BYTES` 约束单文件;
|
||||
上传流共享总字节预算,不把整批文件读入内存。数量超限在创建批次前返回 `413`。
|
||||
- Admin Console 支持多选提交,显示当前批次的接收数量、完成数量、逐文件阶段、Attempt 与安全
|
||||
错误,并每 1.5 秒自动轮询;进入 `completed`、`partial_failure` 或 `failed` 后停止轮询。
|
||||
DOM 数据继续只通过文本节点渲染。
|
||||
- 集成测试覆盖顺序稳定、部分拒绝不回滚、数量/单文件/批次总大小、全成功/部分失败聚合、
|
||||
未知批次、OpenAPI 和前端契约;迁移 drift 增加两张新表。Ruff Format/Lint、Mypy、
|
||||
JavaScript 语法、依赖锁、Alembic Head/离线升级均通过;Unit + Integration `183 passed`,
|
||||
RAG Evaluation `5 passed`。
|
||||
- 浏览器验收确认桌面布局和 390px 窄屏布局正常、无根页面横向溢出、无控制台警告或错误;
|
||||
本地页面不注入云端凭据,批次结果行为由确定性 API 集成测试验证。
|
||||
- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续
|
||||
复用 DEMO-015,Accepted ADR 未变更。
|
||||
- 下一任务:`DEPLOY-001`。
|
||||
|
||||
### DEPLOY-001(已完成)
|
||||
|
||||
- 新增共享生产 Dockerfile 的 `api`、`mcp`、`worker` 三个 target,使用精确 Python/uv 版本、
|
||||
frozen production dependencies、固定非 root UID/GID、exec-form 启动命令和逐 target 健康检查。
|
||||
- 新增 amd64/arm64 Buildx Bake 计划、镜像构建/渲染/推送 Make 入口和 `.dockerignore`;构建上下文
|
||||
排除凭据、本地状态、缓存、测试与开发产物,镜像不包含环境 Endpoint 或 Secret。
|
||||
- Admin 静态目录新增类型化运行时覆盖,使非 editable 安装后的 API 镜像仍能提供同源页面;
|
||||
源码运行继续使用既有默认目录。
|
||||
- 新增不依赖 Docker daemon 的镜像契约测试与中文运行文档;基础 Python/uv 镜像同时固定精确
|
||||
tag 与 manifest digest,Registry 地址和本项目发布 digest 继续由部署环境提供。
|
||||
- 使用 Docker Buildx `0.36.1` + Colima 分别实际构建并加载全部三个 `linux/arm64` 和三个
|
||||
`linux/amd64` target;最终 digest 固定版本再次构建命中相同业务 layer。没有执行 Registry
|
||||
push,也没有在构建参数、history 或镜像文件系统中发现凭据。
|
||||
- API 镜像以 `10001:10001` 启动并进入 healthy,`/healthz`、`/admin/upload`、OpenAPI 及
|
||||
DEMO-016 批量路径 smoke test 通过;MCP/Worker 镜像以同一非 root 用户完成安装入口导入,
|
||||
默认 CMD 与 HEALTHCHECK 元数据核对通过。真实 Provider/Broker 启动仍属于 DEPLOY-003。
|
||||
- Ruff Format/Lint、Mypy、Docker Bake 渲染、JavaScript/依赖锁/Alembic 与完整测试门禁通过;
|
||||
Unit + Integration `187 passed`,RAG Evaluation `5 passed`。下一任务:`DEPLOY-002`。
|
||||
|
||||
### DEPLOY-002(已完成)
|
||||
|
||||
- 新增 `deploy/helm/knowledge-platform` Helm Chart,部署 API/Admin、MCP Gateway、Async Worker、
|
||||
单实例 Recovery Scheduler、串行 Migration Job、Service、Ingress/TLS 和可选 Demo RabbitMQ;
|
||||
API/MCP 默认各两个副本,Worker 可独立扩缩,Beat 使用 `Recreate` 避免重复调度。
|
||||
- 新增严格 `values.schema.json` 与仓库外环境 Values 示例,拒绝未知顶层字段、`latest` tag、
|
||||
非法 digest、API/MCP 单副本和错误类型;跨字段模板校验继续约束批量大小、重试、Chunk 和
|
||||
Top-K 关系。应用和可选 RabbitMQ 镜像均支持 digest,Demo RabbitMQ 已固定 manifest digest。
|
||||
- 敏感配置只通过预建 `existingSecret` 注入;ConfigMap 仅渲染显式白名单内的非敏感字段,
|
||||
没有任意 `extraEnv` 或 Secret 对象。`scripts/deploy.sh` 在部署前校验 Values、Namespace、运行时
|
||||
Secret/必需键、可选 RabbitMQ Secret/键和启用 Ingress 时的 TLS Secret 引用,不读取 Secret
|
||||
值;Helm 3/4 分别使用等价的原子失败回滚参数。
|
||||
- Migration 使用 `pre-install,pre-upgrade` Hook、固定名称和 `before-hook-creation`,安装/升级在
|
||||
Migration 成功前不更新业务 Pod。应用容器固定非 root UID/GID、只读根文件系统、丢弃全部
|
||||
capabilities、禁用 ServiceAccount Token,并为临时目录、RabbitMQ 日志和 Beat 状态提供
|
||||
`emptyDir`;API/MCP/Worker/Scheduler 均配置启动和存活/就绪探针及资源 requests/limits。
|
||||
- 本机使用 K3s `v1.35.0+k3s1`、Helm `4.2.4` 和 DEPLOY-001 ARM64 镜像完成真实安装;Migration
|
||||
成功,API/MCP 为 `2/2`,Worker、Scheduler 与可选 RabbitMQ 全部 Ready。首次安装暴露的
|
||||
RabbitMQ 只读日志目录和 Beat 调度文件权限问题已修复,随后相同 Values 连续升级至 Revision
|
||||
3~5 均成功;API `/healthz` 和 Admin 上传页 smoke test 通过。
|
||||
- 默认、环境示例和 RabbitMQ 三种 Helm lint 均通过;含 Ingress/TLS/RabbitMQ 的 14 个最终
|
||||
资源通过 kubeconform strict 校验。仓库、Helm manifest、Pod 日志三路 Secret 值扫描均为 `0`;
|
||||
Ruff Format/Lint、Mypy、Alembic、Bash 语法与完整门禁通过,Unit + Integration `195 passed`,
|
||||
RAG Evaluation `5 passed`。
|
||||
- 目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实外部依赖监测与远程 WorkBuddy
|
||||
仍属于 `DEPLOY-003`;本任务没有把这些项提前标记完成,也没有改变 Accepted ADR 或公共知识库
|
||||
范围。下一任务:`DEPLOY-003`。
|
||||
|
||||
123
docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md
Normal file
123
docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md
Normal file
@ -0,0 +1,123 @@
|
||||
# 公共知识库集群交付计划
|
||||
|
||||
**Task:** PLAN-002
|
||||
**Status:** Accepted planning baseline
|
||||
**Date:** 2026-09-02
|
||||
|
||||
## 1. 阶段目标与边界
|
||||
|
||||
当前阶段只交付一个面向企业内部受控网络的公共知识库,先保证服务持续可用、可重复部署,
|
||||
并让使用者打开 Admin Console 或连接 WorkBuddy 后即可使用,无需感知后端进程启停。
|
||||
|
||||
“公共”表示本阶段所有已接收知识处于同一个可见域,不代表面向公网匿名开放,也不代表
|
||||
ADR-007 被废弃。用户、用户组、角色、知识库多空间、Document ACL、SSO、OAuth 用户委托等
|
||||
完整 V1 能力继续后置;一旦需要接收非公共知识,必须先实现 Permission Resolver 和检索前/
|
||||
检索中 ACL 下推,禁止先做全局 Top-K 再过滤。
|
||||
|
||||
Admin 写操作只部署在企业受控网络或统一接入网关后。MCP 继续使用运行时注入、只具备
|
||||
`knowledge:search` scope 的服务凭据;静态 Demo credential 不得演变为共享超级管理员 Token。
|
||||
|
||||
## 2. 目标部署拓扑
|
||||
|
||||
```text
|
||||
Ingress / TLS
|
||||
│
|
||||
┌──────────────┴──────────────┐
|
||||
│ │
|
||||
Admin Console + Knowledge API MCP Gateway
|
||||
replicas replicas
|
||||
│ │
|
||||
└──────── Application Services ────────┐
|
||||
│
|
||||
┌─────────────────────────────┼─────────────┐
|
||||
│ │ │
|
||||
OceanBase MySQL BOS RabbitMQ
|
||||
canonical + vector │
|
||||
▼
|
||||
Async Worker(s)
|
||||
```
|
||||
|
||||
沿用 ADR-002:业务核心仍为模块化单体,不拆分额外微服务;API、MCP Gateway 和异步 Worker
|
||||
分别作为可独立扩缩容的进程/容器交付。当前 Parser 与云端 Embedding/Reranker 不需要本地 GPU,
|
||||
因此 GPU Worker 不进入本轮最小部署,未来引入本地 OCR/模型时再按既有架构角色启用。
|
||||
|
||||
OceanBase、BOS 和模型服务使用集群外已有服务。RabbitMQ 默认使用外部地址;Helm Chart 可以
|
||||
提供显式开关安装开发/演示依赖,但生产值不得把队列数据生命周期绑定到应用 Pod。
|
||||
|
||||
## 3. “一键部署”的定义
|
||||
|
||||
在操作者已具备 Kubernetes 访问权限、目标 Namespace 和 Secret 的前提下,仓库提供单一入口:
|
||||
|
||||
```text
|
||||
make deploy VALUES=<environment-values.yaml>
|
||||
```
|
||||
|
||||
该入口负责:
|
||||
|
||||
- 校验非敏感配置和 `existingSecret` 引用,不把凭据写入 Git、镜像、命令行或日志;
|
||||
- 运行一次性、可追踪且串行的数据库 Migration Job;
|
||||
- 安装或升级 API/Admin、MCP Gateway、Async Worker、Service 和可选 RabbitMQ 依赖;
|
||||
- 配置 Ingress/TLS、资源 requests/limits、非 root、只读根文件系统和最小 ServiceAccount;
|
||||
- 配置 readiness/liveness probe、副本数和滚动升级策略;
|
||||
- 支持重复执行和幂等升级,并提供对应卸载/回滚说明。
|
||||
|
||||
本定义不承诺自动创建 OceanBase、BOS、域名证书或云厂商账号,也不允许在仓库内保存这些
|
||||
资源的敏感凭据。
|
||||
|
||||
## 4. Task 顺序与依赖
|
||||
|
||||
```text
|
||||
PLAN-002
|
||||
└──→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
|
||||
└──→ DEMO-016 Batch Async Upload + Status UI
|
||||
└──→ DEPLOY-001 Production Images
|
||||
└──→ DEPLOY-002 Kubernetes/Helm One-click Deploy
|
||||
└──→ DEPLOY-003 Availability + Remote Acceptance
|
||||
```
|
||||
|
||||
### DEMO-015
|
||||
|
||||
实现持久化 Job/Attempt、重试前清理、幂等消费和有界自动重试,并把执行器正式接入
|
||||
Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务直接依赖 Celery API。
|
||||
|
||||
### DEMO-016
|
||||
|
||||
**已完成。** 基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console
|
||||
状态展示;下一交付门槛为 DEPLOY-001 的生产镜像。
|
||||
|
||||
### DEPLOY-001
|
||||
|
||||
**已完成。** 交付 API/Admin、MCP Gateway、Async Worker 的生产容器镜像与启动约定;镜像使用固定依赖、
|
||||
非 root 用户、健康检查和多架构可构建基线,不内置凭据。
|
||||
|
||||
### DEPLOY-002
|
||||
|
||||
**已完成。** 交付 Kubernetes/Helm Chart、严格环境 Values Schema、`existingSecret`、串行
|
||||
Migration Hook、Ingress/TLS、Pod 安全、资源限制、探针和 `make deploy` 入口;Worker 与单实例
|
||||
恢复调度器已分离。现有 `docker-compose.dev.yml` 仍仅作为旧的本地开发文件,因其
|
||||
PostgreSQL/MinIO 假设与 ADR-012 不一致,不得作为集群交付模板。下一交付门槛为 DEPLOY-003
|
||||
的目标集群可用性与远程验收。
|
||||
|
||||
### DEPLOY-003
|
||||
|
||||
在目标集群完成安装、幂等升级、滚动重启、Worker 故障恢复和远程 WorkBuddy 验收;形成运行
|
||||
手册、备份/恢复责任边界和最小日志/指标检查。灾备演练与完整 Observability 平台仍属后置项。
|
||||
|
||||
## 5. 最小验收标准
|
||||
|
||||
- 新 Namespace 可由单一命令完成安装,重复执行不会破坏已有数据;
|
||||
- Migration 在应用升级前串行成功,失败时新版本不继续接管流量;
|
||||
- Admin/API 与 MCP 使用稳定 HTTPS 地址,重启或滚动升级不要求最终用户重新注入配置;
|
||||
- API/MCP 支持至少两个副本,readiness 未通过的 Pod 不接收流量;
|
||||
- Worker 重启后持久化 Job 可继续恢复,重复消息不产生双重有效执行或重复派生数据;
|
||||
- 远程 WorkBuddy 仅发现批准的 `search_knowledge` Tool,并完成一次真实检索;
|
||||
- 清单、镜像、日志和验收产物不包含凭据,敏感值均来自 Secret;
|
||||
- 关键请求、Job 和外部依赖失败具备可定位的结构化日志和最小指标。
|
||||
|
||||
## 6. 明确后置能力
|
||||
|
||||
- 用户、用户组、角色、多租户、知识库空间隔离和 Document ACL;
|
||||
- LDAP/SSO、OAuth 用户委托、公共互联网匿名访问;
|
||||
- GPU Worker、本地模型、OCR/MinerU;
|
||||
- 跨区域容灾、自动云资源编排和完整 Observability 平台;
|
||||
- 未被既有 Demo/V1 明确批准的业务能力。
|
||||
@ -23,6 +23,30 @@ ARCH-001
|
||||
Each architecture task uses an independent branch and is merged into `main` only after its own
|
||||
validation. Provider credentials and internal endpoints are runtime inputs and never task artifacts.
|
||||
|
||||
## Post-Demo priority plan
|
||||
|
||||
- [x] PLAN-001 MCP 与异步批量入库现状审计、任务拆分和契约基线
|
||||
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
|
||||
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
|
||||
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
|
||||
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
|
||||
- [x] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
|
||||
- [x] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
|
||||
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance
|
||||
|
||||
Dependencies and acceptance boundaries are defined in
|
||||
[`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) and
|
||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md). These Demo/Deploy tasks are
|
||||
vertical slices of the V1 backlog below; completing a slice does not automatically complete its
|
||||
corresponding `EKP-*` task. The current public-knowledge phase defers, but does not complete or
|
||||
supersede, the unchecked user/ACL tasks.
|
||||
|
||||
```text
|
||||
DEMO-009 + ARCH-006 → DEMO-012
|
||||
PLAN-002
|
||||
└→ DEMO-003~006 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
||||
```
|
||||
|
||||
## Phase 0
|
||||
- [ ] EKP-001 Repository bootstrap
|
||||
- [ ] EKP-002 Backend Python toolchain
|
||||
|
||||
@ -9,3 +9,26 @@
|
||||
7. Admin Console
|
||||
8. Knowledge Compilation
|
||||
9. Evaluation, security and release hardening
|
||||
|
||||
## 当前增量实施顺序(2026-09-02)
|
||||
|
||||
`DEMO-012` 已完成 MCP `search_knowledge` 与本地 WorkBuddy 验收。根据 PLAN-002,当前阶段按
|
||||
“单一公共知识可见域、企业受控网络、持续可用和一键集群部署”推进,不提前实现用户/权限隔离:
|
||||
|
||||
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
|
||||
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
|
||||
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
|
||||
4. `DEPLOY-001`(已完成):API/Admin、MCP Gateway、Async Worker 生产镜像;
|
||||
5. `DEPLOY-002`(已完成):Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
|
||||
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。
|
||||
|
||||
严格依赖为:
|
||||
|
||||
```text
|
||||
PLAN-002 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
|
||||
```
|
||||
|
||||
`DEMO-016` 不得在缺少持久化状态、幂等消费和恢复边界时直接并发调用同步流水线;集群交付也
|
||||
不得继续使用 `docker-compose.dev.yml` 中已被 ADR-012 取代的 PostgreSQL/MinIO 假设。详细
|
||||
范围和验收标准见 [`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) 与
|
||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||
|
||||
@ -10,6 +10,21 @@
|
||||
|
||||
---
|
||||
|
||||
## 当前阶段交付覆盖(PLAN-002)
|
||||
|
||||
为优先达成服务可用和一键集群部署,当前阶段只交付企业受控网络内的单一公共知识库:所有
|
||||
已接收知识处于同一个可见域,用户打开 Admin Console 或连接 WorkBuddy 即可使用,无需感知
|
||||
后端进程启停。用户、用户组、角色、知识库多空间、Document ACL、LDAP/SSO 和 OAuth 用户
|
||||
委托仍是本 PRD 的后续 V1 要求,本阶段不实现,也不得标记为完成。
|
||||
|
||||
该阶段覆盖不会取代安全原则:公共知识库不等于匿名公网开放;Admin 写操作必须位于受控网络
|
||||
或统一接入网关后,MCP 使用运行时注入的最小 `knowledge:search` 服务凭据。一旦平台需要接收
|
||||
非公共知识,必须先完成 Permission Resolver 与检索前/检索中 ACL 下推,禁止全局 Top-K 后置
|
||||
过滤。实施顺序和集群验收边界见
|
||||
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
|
||||
|
||||
---
|
||||
|
||||
# 1. 项目背景
|
||||
|
||||
企业内部存在大量分散于 PDF、Word、PPT、Excel、Markdown、图片、扫描件及其他文件中的知识资产。
|
||||
@ -447,6 +462,19 @@ V1 必须支持:
|
||||
|
||||
Document processing 必须异步执行。
|
||||
|
||||
源文件与 Document/Job 成功持久化后返回 `202 Accepted`。批量上传中的每个文件拥有独立的
|
||||
Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批次内其他已接收文件。
|
||||
|
||||
当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、
|
||||
断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。
|
||||
|
||||
公共知识库集群交付使用分别可扩缩的 API/Admin、MCP Gateway 和 Async Worker 生产镜像;
|
||||
镜像只包含锁定代码与非敏感默认值,环境凭据必须在部署时注入。
|
||||
|
||||
集群编排由 Helm Chart 统一管理:API/MCP 默认多副本,Worker 与单实例恢复调度器分离;敏感值
|
||||
只引用预建 Kubernetes Secret,数据库 Migration 在安装/升级前串行完成。Ingress/TLS、探针、
|
||||
资源限制和 Pod 安全默认值属于部署契约,不改变 REST/MCP 业务接口。
|
||||
|
||||
**REQ-INGEST-004**
|
||||
|
||||
展示处理阶段:
|
||||
@ -468,10 +496,18 @@ Failed
|
||||
失败必须展示:
|
||||
|
||||
- Failed Stage;
|
||||
- Attempt Count;
|
||||
- 是否仍会自动重试;
|
||||
- Error Code;
|
||||
- Error Message;
|
||||
- Retry。
|
||||
|
||||
**REQ-INGEST-006**
|
||||
|
||||
可重试的瞬时执行失败必须自动进行有界重试。每次重新执行前必须清理该 Document 的旧解析
|
||||
产物、Chunk、Chunk Index State 和向量/稀疏索引数据,再从不可变原始文件开始处理。清理不得
|
||||
删除原始文件、Document 标识或 Job/Attempt 历史;清理失败时不得继续重试。
|
||||
|
||||
---
|
||||
|
||||
# 11. 文件版本和去重
|
||||
@ -1814,6 +1850,9 @@ Cancelled
|
||||
Retry
|
||||
```
|
||||
|
||||
Retry 必须记录 Attempt 历史并区分可重试的瞬时错误与确定性永久错误;重试执行前遵循
|
||||
`REQ-INGEST-006` 的派生数据清理规则。
|
||||
|
||||
---
|
||||
|
||||
# 57. 删除语义
|
||||
|
||||
@ -17,3 +17,11 @@ defaults, and primary keys while excluding those three falsely reported object
|
||||
types. Full constraint comparison still runs against SQLite in integration
|
||||
tests; target-cluster checks separately verify HNSW/PRE-FILTER and the
|
||||
single-active generated unique index.
|
||||
|
||||
DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
|
||||
the latter preserves numbered history and uses a generated nullable active marker plus a unique index
|
||||
to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.
|
||||
|
||||
DEMO-016 adds `ingestion_batches` and `ingestion_batch_items`. They retain input order and per-file
|
||||
acceptance references while Job rows remain the processing state source of truth; a batch is an aggregate,
|
||||
not a cross-file transaction.
|
||||
|
||||
174
migrations/versions/20260902_0007_create_ingestion_jobs.py
Normal file
174
migrations/versions/20260902_0007_create_ingestion_jobs.py
Normal file
@ -0,0 +1,174 @@
|
||||
"""Create durable ingestion jobs and attempt history.
|
||||
|
||||
Revision ID: 20260902_0007
|
||||
Revises: 20260902_0006
|
||||
Create Date: 2026-09-02
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260902_0007"
|
||||
down_revision: str | None = "20260902_0006"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Persist queue-independent Job state and bounded execution attempts."""
|
||||
op.create_table(
|
||||
"ingestion_jobs",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("document_id", sa.Uuid(), nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.Enum(
|
||||
"queued",
|
||||
"running",
|
||||
"retrying",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"cancelled",
|
||||
name="ingestion_job_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"stage",
|
||||
sa.Enum(
|
||||
"parsing",
|
||||
"cleaning",
|
||||
"chunking",
|
||||
"embedding",
|
||||
"indexing",
|
||||
name="ingestion_stage",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column("attempt_count", sa.Integer(), nullable=False),
|
||||
sa.Column("max_attempts", sa.Integer(), nullable=False),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
|
||||
sa.CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
|
||||
sa.CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
|
||||
name="status_valid",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"stage IS NULL OR stage IN "
|
||||
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
||||
name="stage_valid",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["document_id"],
|
||||
["documents.id"],
|
||||
name="fk_ingestion_jobs_document_id_documents",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_jobs"),
|
||||
sa.UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_ingestion_jobs_status_next_run",
|
||||
"ingestion_jobs",
|
||||
["status", "next_run_at"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_table(
|
||||
"ingestion_attempts",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("job_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("attempt_number", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.Enum(
|
||||
"running",
|
||||
"succeeded",
|
||||
"failed",
|
||||
name="ingestion_attempt_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"stage",
|
||||
sa.Enum(
|
||||
"parsing",
|
||||
"cleaning",
|
||||
"chunking",
|
||||
"embedding",
|
||||
"indexing",
|
||||
name="ingestion_attempt_stage",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column("retryable", sa.Boolean(), nullable=True),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"active_marker",
|
||||
sa.Integer(),
|
||||
sa.Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
"started_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('running', 'succeeded', 'failed')",
|
||||
name="status_valid",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"stage IS NULL OR stage IN "
|
||||
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
||||
name="stage_valid",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["job_id"],
|
||||
["ingestion_jobs.id"],
|
||||
name="fk_ingestion_attempts_job_id_ingestion_jobs",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_attempts"),
|
||||
sa.UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_ingestion_attempts_active",
|
||||
"ingestion_attempts",
|
||||
["job_id", "active_marker"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
109
migrations/versions/20260902_0008_create_ingestion_batches.py
Normal file
109
migrations/versions/20260902_0008_create_ingestion_batches.py
Normal file
@ -0,0 +1,109 @@
|
||||
"""Create ingestion batch aggregation metadata.
|
||||
|
||||
Revision ID: 20260902_0008
|
||||
Revises: 20260902_0007
|
||||
Create Date: 2026-09-02
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20260902_0008"
|
||||
down_revision: str | None = "20260902_0007"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Persist ordered per-file acceptance without making a batch transactional."""
|
||||
op.create_table(
|
||||
"ingestion_batches",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("total_files", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint("total_files > 0", name="total_files_positive"),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batches"),
|
||||
)
|
||||
op.create_table(
|
||||
"ingestion_batch_items",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("batch_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("item_index", sa.Integer(), nullable=False),
|
||||
sa.Column("filename", sa.String(length=255), nullable=False),
|
||||
sa.Column(
|
||||
"acceptance_status",
|
||||
sa.Enum(
|
||||
"pending",
|
||||
"accepted",
|
||||
"rejected",
|
||||
name="batch_item_acceptance_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("document_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("job_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("CURRENT_TIMESTAMP"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint("item_index >= 0", name="item_index_non_negative"),
|
||||
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
|
||||
sa.CheckConstraint(
|
||||
"acceptance_status IN ('pending', 'accepted', 'rejected')",
|
||||
name="acceptance_status_valid",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["batch_id"],
|
||||
["ingestion_batches.id"],
|
||||
name="fk_ingestion_batch_items_batch_id_ingestion_batches",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["document_id"],
|
||||
["documents.id"],
|
||||
name="fk_ingestion_batch_items_document_id_documents",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["job_id"],
|
||||
["ingestion_jobs.id"],
|
||||
name="fk_ingestion_batch_items_job_id_ingestion_jobs",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batch_items"),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_ingestion_batch_items_order",
|
||||
"ingestion_batch_items",
|
||||
["batch_id", "item_index"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_index(
|
||||
"uq_ingestion_batch_items_job_id",
|
||||
"ingestion_batch_items",
|
||||
["job_id"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reject downgrade because project migrations are forward-only."""
|
||||
raise NotImplementedError("Database migrations are forward-only")
|
||||
@ -4,7 +4,7 @@ from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import Field, SecretStr, model_validator
|
||||
from pydantic import AnyHttpUrl, Field, SecretStr, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
@ -25,6 +25,7 @@ class Settings(BaseSettings):
|
||||
api_host: str = "0.0.0.0"
|
||||
api_port: int = Field(default=8000, ge=1, le=65535)
|
||||
api_prefix: str = "/api/v1"
|
||||
admin_web_root: Path | None = None
|
||||
# SQLAlchemy URLs commonly embed a password, so configuration rendering
|
||||
# must mask this value and unwrap it only at the database driver boundary.
|
||||
database_url: SecretStr = SecretStr("mysql+pymysql://localhost:2881/ekp")
|
||||
@ -33,6 +34,14 @@ class Settings(BaseSettings):
|
||||
database_pool_recycle_seconds: int = Field(default=600, ge=1)
|
||||
database_pool_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
database_connect_timeout_seconds: int = Field(default=30, ge=1)
|
||||
rabbitmq_url: SecretStr | None = None
|
||||
ingestion_max_attempts: int = Field(default=3, ge=1, le=10)
|
||||
ingestion_retry_backoff_seconds: int = Field(default=2, ge=1, le=3600)
|
||||
ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
|
||||
ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
|
||||
ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
|
||||
batch_max_files: int = Field(default=20, ge=1, le=100)
|
||||
batch_max_total_bytes: int = Field(default=200 * 1024 * 1024, gt=0)
|
||||
object_storage_provider: Literal["local", "bos"] = "bos"
|
||||
object_storage_path: Path = Path("storage")
|
||||
bos_endpoint: str | None = None
|
||||
@ -67,6 +76,22 @@ class Settings(BaseSettings):
|
||||
reranker_api_key: SecretStr | None = None
|
||||
reranker_timeout_seconds: float = Field(default=30.0, gt=0)
|
||||
search_top_k: int = Field(default=5, ge=1, le=64)
|
||||
mcp_host: str = "127.0.0.1"
|
||||
mcp_port: int = Field(default=8766, ge=1, le=65535)
|
||||
mcp_path: str = "/mcp"
|
||||
mcp_public_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766/mcp")
|
||||
mcp_issuer_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766")
|
||||
mcp_auth_token: SecretStr | None = None
|
||||
mcp_client_id: str = Field(default="workbuddy-enterprise", min_length=1, max_length=200)
|
||||
mcp_service_identity: str = Field(default="workbuddy-demo", min_length=1, max_length=200)
|
||||
mcp_required_scope: str = Field(default="knowledge:search", min_length=1, max_length=200)
|
||||
mcp_allowed_hosts: list[str] = ["127.0.0.1:*", "localhost:*", "[::1]:*", "testserver"]
|
||||
mcp_allowed_origins: list[str] = [
|
||||
"http://127.0.0.1:*",
|
||||
"http://localhost:*",
|
||||
"http://[::1]:*",
|
||||
]
|
||||
mcp_max_request_body_bytes: int = Field(default=1024 * 1024, ge=1024, le=4 * 1024 * 1024)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_chunk_window(self) -> "Settings":
|
||||
@ -75,6 +100,23 @@ class Settings(BaseSettings):
|
||||
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
|
||||
if self.search_top_k > self.dense_top_n:
|
||||
raise ValueError("search_top_k must not exceed dense_top_n")
|
||||
if self.ingestion_retry_backoff_max_seconds < self.ingestion_retry_backoff_seconds:
|
||||
raise ValueError(
|
||||
"ingestion_retry_backoff_max_seconds must be at least the initial backoff"
|
||||
)
|
||||
longest_provider_timeout = max(
|
||||
self.bos_timeout_seconds,
|
||||
self.embedding_timeout_seconds,
|
||||
self.oceanbase_vector_timeout_seconds,
|
||||
)
|
||||
if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
|
||||
raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
|
||||
if self.batch_max_total_bytes < self.max_upload_bytes:
|
||||
raise ValueError("batch_max_total_bytes must be at least max_upload_bytes")
|
||||
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
|
||||
raise ValueError("mcp_path must start with '/' and must not end with '/'")
|
||||
if not self.mcp_allowed_hosts:
|
||||
raise ValueError("mcp_allowed_hosts must not be empty")
|
||||
database_url = make_url(self.database_url.get_secret_value())
|
||||
if database_url.get_backend_name() == "sqlite":
|
||||
if self.app_env != "test":
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
"""Application services for building and persisting document chunks."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from hashlib import sha256
|
||||
from uuid import UUID, uuid5
|
||||
@ -11,7 +12,7 @@ from ekp_core.chunking import ChunkDraft, ChunkingStrategy
|
||||
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.parsed_document_artifact import deserialize_parsed_document
|
||||
from ekp_domain import Chunk, Document, DocumentStatus, ParsedDocument
|
||||
from ekp_domain import Chunk, Document, DocumentStatus, IngestionStage, ParsedDocument
|
||||
from ekp_object_storage import BinarySource, ObjectStorage
|
||||
|
||||
MAX_CHUNKING_ERROR_LENGTH = 2000
|
||||
@ -54,10 +55,16 @@ class DocumentChunkingService:
|
||||
self._storage = storage
|
||||
self._strategy = strategy
|
||||
|
||||
def process(self, document_id: UUID) -> tuple[Chunk, ...]:
|
||||
"""Run DEMO-005 while reserving READY for the later index stage."""
|
||||
def process(
|
||||
self,
|
||||
document_id: UUID,
|
||||
stage_callback: Callable[[IngestionStage], None] | None = None,
|
||||
) -> tuple[Chunk, ...]:
|
||||
"""Build chunks while optionally reporting the durable Job stage."""
|
||||
filename, parsed_key = self._start_chunking(document_id)
|
||||
try:
|
||||
if stage_callback is not None:
|
||||
stage_callback(IngestionStage.CHUNKING)
|
||||
with self._storage.open(parsed_key) as source:
|
||||
parsed = deserialize_parsed_document(source.read())
|
||||
if parsed.document_id != document_id or parsed.filename != filename:
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
"""Embedding and vector-index orchestration for canonical document chunks."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from uuid import UUID, uuid5
|
||||
|
||||
@ -7,7 +8,14 @@ from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService
|
||||
from ekp_domain import Chunk, ChunkIndexState, Document, DocumentStatus, IndexProfile
|
||||
from ekp_domain import (
|
||||
Chunk,
|
||||
ChunkIndexState,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
IndexProfile,
|
||||
IngestionStage,
|
||||
)
|
||||
from ekp_embedding import EmbeddingProvider
|
||||
from ekp_object_storage import BinarySource
|
||||
from ekp_vector_store import VectorRecord, VectorStore
|
||||
@ -67,12 +75,16 @@ class DocumentIndexingService:
|
||||
self._vector_store = vector_store
|
||||
self._index_profile = index_profile
|
||||
|
||||
def process(self, document_id: UUID) -> DocumentIndexingResult:
|
||||
"""Index all chunks for one profile, compensating any partial provider write."""
|
||||
def process(
|
||||
self,
|
||||
document_id: UUID,
|
||||
stage_callback: Callable[[IngestionStage], None] | None = None,
|
||||
) -> DocumentIndexingResult:
|
||||
"""Embed/index chunks while optionally reporting durable Job stages."""
|
||||
chunks = self._start_indexing(document_id)
|
||||
upsert_attempted = False
|
||||
try:
|
||||
vectors = self._embed_chunks(chunks)
|
||||
vectors = self._embed_chunks(chunks, stage_callback)
|
||||
records = tuple(
|
||||
VectorRecord(
|
||||
index_record_id=uuid5(
|
||||
@ -93,6 +105,8 @@ class DocumentIndexingService:
|
||||
)
|
||||
for chunk, vector in zip(chunks, vectors, strict=True)
|
||||
)
|
||||
if stage_callback is not None:
|
||||
stage_callback(IngestionStage.INDEXING)
|
||||
upsert_attempted = True
|
||||
self._vector_store.upsert(records)
|
||||
self._record_success(document_id, chunks)
|
||||
@ -152,13 +166,19 @@ class DocumentIndexingService:
|
||||
session.commit()
|
||||
return chunks
|
||||
|
||||
def _embed_chunks(self, chunks: tuple[Chunk, ...]) -> tuple[tuple[float, ...], ...]:
|
||||
"""Respect provider batch limits and validate every returned vector."""
|
||||
def _embed_chunks(
|
||||
self,
|
||||
chunks: tuple[Chunk, ...],
|
||||
stage_callback: Callable[[IngestionStage], None] | None,
|
||||
) -> tuple[tuple[float, ...], ...]:
|
||||
"""Embed bounded batches and renew an optional Job lease before each call."""
|
||||
vectors: list[tuple[float, ...]] = []
|
||||
batch_size = self._embedding_provider.max_batch_size
|
||||
if batch_size <= 0:
|
||||
raise EmbeddingResultError("embedding provider has an invalid batch size")
|
||||
for start in range(0, len(chunks), batch_size):
|
||||
if stage_callback is not None:
|
||||
stage_callback(IngestionStage.EMBEDDING)
|
||||
batch = chunks[start : start + batch_size]
|
||||
embedded = self._embedding_provider.embed([chunk.content for chunk in batch])
|
||||
if len(embedded) != len(batch):
|
||||
|
||||
@ -5,7 +5,7 @@ from uuid import UUID
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_domain import Chunk, ChunkIndexState, Document
|
||||
from ekp_domain import Chunk, ChunkIndexState, Document, IngestionJob, IngestionJobStatus
|
||||
from ekp_object_storage import ObjectStorage
|
||||
from ekp_vector_store import VectorStore
|
||||
|
||||
@ -18,6 +18,10 @@ class DocumentArtifactCleanupError(RuntimeError):
|
||||
"""Raised when deleted metadata leaves an external storage artifact behind."""
|
||||
|
||||
|
||||
class DocumentIngestionActiveError(RuntimeError):
|
||||
"""Raised when direct deletion would race an active asynchronous Job."""
|
||||
|
||||
|
||||
class DocumentManagementService:
|
||||
"""Read canonical documents and coordinate the Demo's direct deletion.
|
||||
|
||||
@ -63,6 +67,22 @@ class DocumentManagementService:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
raise DocumentNotFoundError(str(document_id))
|
||||
active_job_id = session.scalar(
|
||||
select(IngestionJob.id)
|
||||
.where(
|
||||
IngestionJob.document_id == document_id,
|
||||
IngestionJob.status.in_(
|
||||
(
|
||||
IngestionJobStatus.QUEUED,
|
||||
IngestionJobStatus.RUNNING,
|
||||
IngestionJobStatus.RETRYING,
|
||||
)
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if active_job_id is not None:
|
||||
raise DocumentIngestionActiveError(str(document_id))
|
||||
|
||||
source_keys = tuple(
|
||||
key
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
"""Application services for parsing and persisting derived document content."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from io import BytesIO
|
||||
from uuid import UUID
|
||||
|
||||
@ -8,7 +9,7 @@ from sqlalchemy.orm import Session, sessionmaker
|
||||
from ekp_core.document_cleaner import clean_parsed_document
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.parsed_document_artifact import serialize_parsed_document
|
||||
from ekp_domain import Document, DocumentStatus, ParsedDocument
|
||||
from ekp_domain import Document, DocumentStatus, IngestionStage, ParsedDocument
|
||||
from ekp_object_storage import BinarySource, ObjectStorage
|
||||
from ekp_parser import DocumentParseError, Parser
|
||||
|
||||
@ -48,15 +49,23 @@ class DocumentParsingService:
|
||||
self._storage = storage
|
||||
self._parser = parser
|
||||
|
||||
def process(self, document_id: UUID) -> ParsedDocument:
|
||||
"""Run DEMO-004 synchronously and persist success or failure state."""
|
||||
def process(
|
||||
self,
|
||||
document_id: UUID,
|
||||
stage_callback: Callable[[IngestionStage], None] | None = None,
|
||||
) -> ParsedDocument:
|
||||
"""Parse and clean while optionally reporting durable Job stage boundaries."""
|
||||
filename, source_key = self._start_processing(document_id)
|
||||
parsed_key = f"parsed/{document_id}/document.json"
|
||||
parsed_object_stored = False
|
||||
|
||||
try:
|
||||
if stage_callback is not None:
|
||||
stage_callback(IngestionStage.PARSING)
|
||||
with self._storage.open(source_key) as source:
|
||||
parsed = self._parser.parse(document_id, filename, source)
|
||||
if stage_callback is not None:
|
||||
stage_callback(IngestionStage.CLEANING)
|
||||
cleaned = clean_parsed_document(parsed)
|
||||
if not cleaned.blocks:
|
||||
raise EmptyParsedDocumentError(
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
"""Validated, transactional source-document upload application service."""
|
||||
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_domain import Document
|
||||
@ -132,6 +133,26 @@ class DocumentUploadService:
|
||||
|
||||
return document
|
||||
|
||||
def rollback_acceptance(self, document_id: UUID) -> None:
|
||||
"""Compensate a just-uploaded source when durable Job creation fails.
|
||||
|
||||
This narrow method is only valid before parsing starts; normal document
|
||||
deletion remains coordinated by ``DocumentManagementService``.
|
||||
"""
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
return
|
||||
storage_key = document.storage_key
|
||||
session.execute(delete(Document).where(Document.id == document.id))
|
||||
session.commit()
|
||||
try:
|
||||
self._storage.delete(storage_key)
|
||||
except Exception as cleanup_error:
|
||||
raise UploadRollbackError(
|
||||
f"job creation failed and source cleanup failed for {storage_key}"
|
||||
) from cleanup_error
|
||||
|
||||
|
||||
def _validate_filename(filename: str) -> str:
|
||||
"""Accept a plain basename that can be traced without path ambiguity."""
|
||||
|
||||
270
packages/core/src/ekp_core/ingestion_batches.py
Normal file
270
packages/core/src/ekp_core/ingestion_batches.py
Normal file
@ -0,0 +1,270 @@
|
||||
"""Batch submission and aggregate status over independent ingestion Jobs."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from typing import Protocol
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.document_upload import (
|
||||
DocumentTooLargeError,
|
||||
DocumentUploadError,
|
||||
EmptyDocumentError,
|
||||
MediaTypeMismatchError,
|
||||
UnsupportedDocumentTypeError,
|
||||
)
|
||||
from ekp_core.ingestion_jobs import AsyncIngestionSubmissionService
|
||||
from ekp_domain import (
|
||||
BatchItemAcceptanceStatus,
|
||||
IngestionBatch,
|
||||
IngestionBatchItem,
|
||||
IngestionJob,
|
||||
IngestionJobStatus,
|
||||
)
|
||||
from ekp_object_storage import BinarySource
|
||||
|
||||
SAFE_UNSUPPORTED_ERROR = "unsupported document type or media type"
|
||||
SAFE_FILE_SIZE_ERROR = "document exceeds the configured upload limit"
|
||||
SAFE_BATCH_SIZE_ERROR = "batch exceeds the configured total upload limit"
|
||||
SAFE_EMPTY_ERROR = "document must contain content"
|
||||
SAFE_ACCEPTANCE_ERROR = "document could not be accepted"
|
||||
|
||||
|
||||
class BatchNotFoundError(LookupError):
|
||||
"""Raised when a requested ingestion batch does not exist."""
|
||||
|
||||
|
||||
class BatchFileCountLimitError(ValueError):
|
||||
"""Raised before persistence when the multipart file count is invalid."""
|
||||
|
||||
|
||||
class BatchAggregateStatus(StrEnum):
|
||||
"""Computed batch state derived from item acceptance and durable Jobs."""
|
||||
|
||||
PROCESSING = "processing"
|
||||
COMPLETED = "completed"
|
||||
PARTIAL_FAILURE = "partial_failure"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class BatchTotalSizeLimitError(ValueError):
|
||||
"""Raised while streaming the file that crosses the shared batch budget."""
|
||||
|
||||
|
||||
class _ReadableSource(Protocol):
|
||||
def read(self, size: int = -1, /) -> bytes:
|
||||
"""Read bytes from an uploaded file."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchSource:
|
||||
"""Provider-neutral input for one ordered multipart file."""
|
||||
|
||||
filename: str
|
||||
media_type: str | None
|
||||
source: _ReadableSource
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchItemView:
|
||||
"""Detached batch item plus its current optional Job state."""
|
||||
|
||||
item: IngestionBatchItem
|
||||
job: IngestionJob | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchView:
|
||||
"""Detached aggregate used by REST without exposing persistence sessions."""
|
||||
|
||||
batch: IngestionBatch
|
||||
items: tuple[BatchItemView, ...]
|
||||
|
||||
@property
|
||||
def status(self) -> BatchAggregateStatus:
|
||||
"""Aggregate terminal results without copying Job state into batch rows."""
|
||||
if any(
|
||||
view.item.acceptance_status == BatchItemAcceptanceStatus.PENDING
|
||||
or (
|
||||
view.item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED
|
||||
and (
|
||||
view.job is None
|
||||
or view.job.status
|
||||
in {
|
||||
IngestionJobStatus.QUEUED,
|
||||
IngestionJobStatus.RUNNING,
|
||||
IngestionJobStatus.RETRYING,
|
||||
}
|
||||
)
|
||||
)
|
||||
for view in self.items
|
||||
):
|
||||
return BatchAggregateStatus.PROCESSING
|
||||
succeeded = sum(
|
||||
view.job is not None and view.job.status == IngestionJobStatus.SUCCEEDED
|
||||
for view in self.items
|
||||
)
|
||||
failed = len(self.items) - succeeded
|
||||
if failed == 0:
|
||||
return BatchAggregateStatus.COMPLETED
|
||||
if succeeded == 0:
|
||||
return BatchAggregateStatus.FAILED
|
||||
return BatchAggregateStatus.PARTIAL_FAILURE
|
||||
|
||||
|
||||
class _BatchByteBudget:
|
||||
"""Count bytes across independently streamed files without buffering them."""
|
||||
|
||||
def __init__(self, limit_bytes: int) -> None:
|
||||
self._limit_bytes = limit_bytes
|
||||
self._consumed_bytes = 0
|
||||
|
||||
def wrap(self, source: _ReadableSource) -> BinarySource:
|
||||
return _BatchLimitedSource(source, self)
|
||||
|
||||
def consume(self, size_bytes: int) -> None:
|
||||
self._consumed_bytes += size_bytes
|
||||
if self._consumed_bytes > self._limit_bytes:
|
||||
raise BatchTotalSizeLimitError(SAFE_BATCH_SIZE_ERROR)
|
||||
|
||||
|
||||
class _BatchLimitedSource:
|
||||
"""Apply one shared request budget through the ObjectStorage stream contract."""
|
||||
|
||||
def __init__(self, source: _ReadableSource, budget: _BatchByteBudget) -> None:
|
||||
self._source = source
|
||||
self._budget = budget
|
||||
|
||||
def read(self, size: int = -1, /) -> bytes:
|
||||
chunk = self._source.read(size)
|
||||
self._budget.consume(len(chunk))
|
||||
return chunk
|
||||
|
||||
|
||||
class IngestionBatchService:
|
||||
"""Accept files independently and persist ordered aggregate status."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker[Session],
|
||||
submissions: AsyncIngestionSubmissionService,
|
||||
*,
|
||||
max_files: int,
|
||||
max_total_bytes: int,
|
||||
) -> None:
|
||||
if max_files <= 0 or max_total_bytes <= 0:
|
||||
raise ValueError("batch limits must be positive")
|
||||
self._session_factory = session_factory
|
||||
self._submissions = submissions
|
||||
self._max_files = max_files
|
||||
self._max_total_bytes = max_total_bytes
|
||||
|
||||
def submit(self, sources: tuple[BatchSource, ...]) -> BatchView:
|
||||
"""Create a batch and independently accept or reject every ordered file."""
|
||||
if not sources or len(sources) > self._max_files:
|
||||
raise BatchFileCountLimitError(
|
||||
f"batch must contain between 1 and {self._max_files} files"
|
||||
)
|
||||
batch_id, item_ids = self._create_pending_batch(sources)
|
||||
budget = _BatchByteBudget(self._max_total_bytes)
|
||||
for item_id, source in zip(item_ids, sources, strict=True):
|
||||
try:
|
||||
submission = self._submissions.submit(
|
||||
source.filename,
|
||||
source.media_type,
|
||||
budget.wrap(source.source),
|
||||
)
|
||||
except Exception as error:
|
||||
self._record_rejected(item_id, _safe_acceptance_error(error))
|
||||
continue
|
||||
self._record_accepted(
|
||||
item_id,
|
||||
document_id=submission.document.id,
|
||||
job_id=submission.job.id,
|
||||
)
|
||||
return self.get(batch_id)
|
||||
|
||||
def get(self, batch_id: UUID) -> BatchView:
|
||||
"""Return ordered items and fresh Job state for polling clients."""
|
||||
with self._session_factory() as session:
|
||||
batch = session.get(IngestionBatch, batch_id)
|
||||
if batch is None:
|
||||
raise BatchNotFoundError(str(batch_id))
|
||||
items = tuple(
|
||||
session.scalars(
|
||||
select(IngestionBatchItem)
|
||||
.where(IngestionBatchItem.batch_id == batch_id)
|
||||
.order_by(IngestionBatchItem.item_index)
|
||||
).all()
|
||||
)
|
||||
views = tuple(
|
||||
BatchItemView(
|
||||
item=item,
|
||||
job=session.get(IngestionJob, item.job_id) if item.job_id is not None else None,
|
||||
)
|
||||
for item in items
|
||||
)
|
||||
return BatchView(batch=batch, items=views)
|
||||
|
||||
def _create_pending_batch(
|
||||
self, sources: tuple[BatchSource, ...]
|
||||
) -> tuple[UUID, tuple[UUID, ...]]:
|
||||
"""Persist every input position before starting external storage writes."""
|
||||
with self._session_factory() as session:
|
||||
batch = IngestionBatch(total_files=len(sources))
|
||||
session.add(batch)
|
||||
session.flush()
|
||||
items = tuple(
|
||||
IngestionBatchItem(
|
||||
batch_id=batch.id,
|
||||
item_index=index,
|
||||
filename=_display_filename(source.filename),
|
||||
)
|
||||
for index, source in enumerate(sources)
|
||||
)
|
||||
session.add_all(items)
|
||||
session.commit()
|
||||
return batch.id, tuple(item.id for item in items)
|
||||
|
||||
def _record_accepted(self, item_id: UUID, *, document_id: UUID, job_id: UUID) -> None:
|
||||
with self._session_factory() as session:
|
||||
item = session.get(IngestionBatchItem, item_id)
|
||||
if item is None:
|
||||
raise BatchNotFoundError(str(item_id))
|
||||
item.acceptance_status = BatchItemAcceptanceStatus.ACCEPTED
|
||||
item.document_id = document_id
|
||||
item.job_id = job_id
|
||||
item.error_message = None
|
||||
session.commit()
|
||||
|
||||
def _record_rejected(self, item_id: UUID, error_message: str) -> None:
|
||||
with self._session_factory() as session:
|
||||
item = session.get(IngestionBatchItem, item_id)
|
||||
if item is None:
|
||||
raise BatchNotFoundError(str(item_id))
|
||||
item.acceptance_status = BatchItemAcceptanceStatus.REJECTED
|
||||
item.error_message = error_message
|
||||
session.commit()
|
||||
|
||||
|
||||
def _display_filename(filename: str) -> str:
|
||||
"""Store a bounded safe label even when the upload filename is invalid."""
|
||||
display = filename.replace("\x00", "<EFBFBD>").strip()
|
||||
return display[:255] or "<unnamed>"
|
||||
|
||||
|
||||
def _safe_acceptance_error(error: Exception) -> str:
|
||||
"""Map per-file failures without persisting exception text or credentials."""
|
||||
if isinstance(error, (UnsupportedDocumentTypeError, MediaTypeMismatchError)):
|
||||
return SAFE_UNSUPPORTED_ERROR
|
||||
if isinstance(error, DocumentTooLargeError):
|
||||
return SAFE_FILE_SIZE_ERROR
|
||||
if isinstance(error, BatchTotalSizeLimitError):
|
||||
return SAFE_BATCH_SIZE_ERROR
|
||||
if isinstance(error, EmptyDocumentError):
|
||||
return SAFE_EMPTY_ERROR
|
||||
if isinstance(error, DocumentUploadError):
|
||||
return SAFE_ACCEPTANCE_ERROR
|
||||
return SAFE_ACCEPTANCE_ERROR
|
||||
588
packages/core/src/ekp_core/ingestion_jobs.py
Normal file
588
packages/core/src/ekp_core/ingestion_jobs.py
Normal file
@ -0,0 +1,588 @@
|
||||
"""Durable asynchronous ingestion orchestration and retry cleanup."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from enum import StrEnum
|
||||
from math import ceil
|
||||
from typing import Protocol
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.document_chunking import (
|
||||
DocumentAlreadyChunkedError,
|
||||
DocumentChunkingService,
|
||||
DocumentNotParsedError,
|
||||
EmptyChunkSetError,
|
||||
ParsedDocumentMismatchError,
|
||||
)
|
||||
from ekp_core.document_chunking import (
|
||||
DocumentNotFoundError as ChunkingDocumentNotFoundError,
|
||||
)
|
||||
from ekp_core.document_indexing import (
|
||||
DocumentAlreadyIndexedError,
|
||||
DocumentHasNoChunksError,
|
||||
DocumentIndexingService,
|
||||
)
|
||||
from ekp_core.document_indexing import (
|
||||
DocumentNotFoundError as IndexingDocumentNotFoundError,
|
||||
)
|
||||
from ekp_core.document_parsing import (
|
||||
DocumentAlreadyParsedError,
|
||||
DocumentParsingService,
|
||||
EmptyParsedDocumentError,
|
||||
)
|
||||
from ekp_core.document_parsing import (
|
||||
DocumentNotFoundError as ParsingDocumentNotFoundError,
|
||||
)
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.parsed_document_artifact import ParsedDocumentArtifactError
|
||||
from ekp_domain import (
|
||||
Chunk,
|
||||
ChunkIndexState,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
IngestionAttempt,
|
||||
IngestionAttemptStatus,
|
||||
IngestionJob,
|
||||
IngestionJobStatus,
|
||||
IngestionStage,
|
||||
)
|
||||
from ekp_object_storage import BinarySource, ObjectStorage
|
||||
from ekp_parser import DocumentParseError
|
||||
from ekp_vector_store import VectorStore
|
||||
|
||||
SAFE_TRANSIENT_ERROR = "temporary ingestion failure"
|
||||
SAFE_PERMANENT_ERROR = "document content could not be processed"
|
||||
SAFE_CLEANUP_ERROR = "retry cleanup failed"
|
||||
SAFE_LEASE_ERROR = "worker execution lease expired"
|
||||
|
||||
|
||||
class IngestionJobNotFoundError(LookupError):
|
||||
"""Raised when a requested ingestion Job does not exist."""
|
||||
|
||||
|
||||
class IngestionCleanupError(RuntimeError):
|
||||
"""Raised when derived data cannot be restored to a known clean state."""
|
||||
|
||||
|
||||
class JobDispatcher(Protocol):
|
||||
"""Replaceable message-delivery boundary used by submission services."""
|
||||
|
||||
def enqueue(self, job_id: UUID) -> None:
|
||||
"""Request asynchronous execution of one already-persisted Job."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class IngestionSubmission:
|
||||
"""Stable identifiers returned after source and Job persistence."""
|
||||
|
||||
document: Document
|
||||
job: IngestionJob
|
||||
|
||||
|
||||
class JobRunAction(StrEnum):
|
||||
"""Instruction returned to a queue adapter after one delivery."""
|
||||
|
||||
COMPLETE = "complete"
|
||||
RETRY = "retry"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class JobRunResult:
|
||||
"""Queue-neutral result for acknowledging or rescheduling a delivery."""
|
||||
|
||||
action: JobRunAction
|
||||
status: IngestionJobStatus
|
||||
retry_delay_seconds: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Claim:
|
||||
"""Internal durable claim outcome used before external provider work."""
|
||||
|
||||
run: bool
|
||||
cleanup_required: bool = False
|
||||
result: JobRunResult | None = None
|
||||
|
||||
|
||||
class IngestionJobService:
|
||||
"""Create and query Job state independently from the broker implementation."""
|
||||
|
||||
def __init__(self, session_factory: sessionmaker[Session], max_attempts: int) -> None:
|
||||
if max_attempts <= 0:
|
||||
raise ValueError("max_attempts must be positive")
|
||||
self._session_factory = session_factory
|
||||
self._max_attempts = max_attempts
|
||||
|
||||
def create(self, document_id: UUID) -> IngestionJob:
|
||||
"""Create exactly one queued ingestion Job for a canonical Document."""
|
||||
with self._session_factory() as session:
|
||||
job = IngestionJob(document_id=document_id, max_attempts=self._max_attempts)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
return job
|
||||
|
||||
def get(self, job_id: UUID) -> IngestionJob:
|
||||
"""Return one detached Job or a stable not-found error."""
|
||||
with self._session_factory() as session:
|
||||
job = session.get(IngestionJob, job_id)
|
||||
if job is None:
|
||||
raise IngestionJobNotFoundError(str(job_id))
|
||||
return job
|
||||
|
||||
def list_dispatchable(self, limit: int = 100) -> tuple[UUID, ...]:
|
||||
"""Find due queued/retrying Jobs and expired running leases for recovery."""
|
||||
if not 1 <= limit <= 1000:
|
||||
raise ValueError("dispatch limit must be between 1 and 1000")
|
||||
now = datetime.now(UTC)
|
||||
with self._session_factory() as session:
|
||||
jobs = session.scalars(
|
||||
select(IngestionJob)
|
||||
.where(
|
||||
(IngestionJob.status == IngestionJobStatus.QUEUED)
|
||||
| (
|
||||
(IngestionJob.status == IngestionJobStatus.RETRYING)
|
||||
& ((IngestionJob.next_run_at.is_(None)) | (IngestionJob.next_run_at <= now))
|
||||
)
|
||||
| (
|
||||
(IngestionJob.status == IngestionJobStatus.RUNNING)
|
||||
& (
|
||||
(IngestionJob.lease_expires_at.is_(None))
|
||||
| (IngestionJob.lease_expires_at <= now)
|
||||
)
|
||||
)
|
||||
)
|
||||
.order_by(IngestionJob.created_at, IngestionJob.id)
|
||||
.limit(limit)
|
||||
).all()
|
||||
return tuple(job.id for job in jobs)
|
||||
|
||||
|
||||
class AsyncIngestionSubmissionService:
|
||||
"""Persist immutable source and Job before best-effort broker delivery."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
upload_service: DocumentUploadService,
|
||||
jobs: IngestionJobService,
|
||||
dispatcher: JobDispatcher,
|
||||
) -> None:
|
||||
self._upload_service = upload_service
|
||||
self._jobs = jobs
|
||||
self._dispatcher = dispatcher
|
||||
|
||||
def submit(
|
||||
self,
|
||||
filename: str,
|
||||
media_type: str | None,
|
||||
source: BinarySource,
|
||||
) -> IngestionSubmission:
|
||||
"""Accept one source and return without executing expensive stages inline.
|
||||
|
||||
Broker delivery is deliberately best effort after the canonical commit.
|
||||
A periodic dispatcher recovers queued Jobs when the broker was temporarily
|
||||
unavailable, so the request never treats a message as the source of truth.
|
||||
"""
|
||||
document = self._upload_service.upload(filename, media_type, source)
|
||||
try:
|
||||
job = self._jobs.create(document.id)
|
||||
except Exception:
|
||||
self._upload_service.rollback_acceptance(document.id)
|
||||
raise
|
||||
try:
|
||||
self._dispatcher.enqueue(job.id)
|
||||
except Exception:
|
||||
# Keeping the Job queued is intentional. The recovery dispatcher can
|
||||
# safely redeliver it without asking the client to upload again.
|
||||
pass
|
||||
return IngestionSubmission(document=document, job=job)
|
||||
|
||||
|
||||
class DerivedDocumentCleanupService:
|
||||
"""Remove only rebuildable derivatives before a retry begins."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker[Session],
|
||||
storage: ObjectStorage,
|
||||
vector_store: VectorStore,
|
||||
active_index_profile_id: str,
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._storage = storage
|
||||
self._vector_store = vector_store
|
||||
self._active_index_profile_id = active_index_profile_id
|
||||
|
||||
def cleanup(self, document_id: UUID) -> None:
|
||||
"""Clean vector, relational and parsed artifacts while preserving source/Job."""
|
||||
try:
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
raise IngestionCleanupError("document metadata is missing")
|
||||
parsed_key = document.parsed_storage_key
|
||||
profile_ids = set(
|
||||
session.scalars(
|
||||
select(ChunkIndexState.index_profile_id)
|
||||
.join(Chunk, Chunk.chunk_id == ChunkIndexState.chunk_id)
|
||||
.where(Chunk.document_id == document_id)
|
||||
.distinct()
|
||||
).all()
|
||||
)
|
||||
profile_ids.add(self._active_index_profile_id)
|
||||
|
||||
# External vector rows are deleted before their canonical evidence.
|
||||
for profile_id in sorted(profile_ids):
|
||||
self._vector_store.delete_document(document_id, profile_id)
|
||||
|
||||
with self._session_factory() as session:
|
||||
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
|
||||
session.execute(
|
||||
delete(ChunkIndexState).where(ChunkIndexState.chunk_id.in_(chunk_ids))
|
||||
)
|
||||
session.execute(delete(Chunk).where(Chunk.document_id == document_id))
|
||||
session.commit()
|
||||
|
||||
if parsed_key is not None and self._storage.exists(parsed_key):
|
||||
self._storage.delete(parsed_key)
|
||||
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
raise IngestionCleanupError("document metadata is missing")
|
||||
document.parsed_storage_key = None
|
||||
document.status = DocumentStatus.UPLOADED
|
||||
document.error_message = None
|
||||
session.commit()
|
||||
except IngestionCleanupError:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise IngestionCleanupError(SAFE_CLEANUP_ERROR) from error
|
||||
|
||||
|
||||
class IngestionJobRunner:
|
||||
"""Execute one durable attempt and return queue-neutral retry instructions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker[Session],
|
||||
parsing: DocumentParsingService,
|
||||
chunking: DocumentChunkingService,
|
||||
indexing: DocumentIndexingService,
|
||||
cleanup: DerivedDocumentCleanupService,
|
||||
*,
|
||||
retry_backoff_seconds: int,
|
||||
retry_backoff_max_seconds: int,
|
||||
lease_seconds: int,
|
||||
) -> None:
|
||||
if retry_backoff_seconds <= 0 or retry_backoff_max_seconds <= 0 or lease_seconds <= 0:
|
||||
raise ValueError("retry and lease durations must be positive")
|
||||
self._session_factory = session_factory
|
||||
self._parsing = parsing
|
||||
self._chunking = chunking
|
||||
self._indexing = indexing
|
||||
self._cleanup = cleanup
|
||||
self._retry_backoff_seconds = retry_backoff_seconds
|
||||
self._retry_backoff_max_seconds = retry_backoff_max_seconds
|
||||
self._lease_seconds = lease_seconds
|
||||
|
||||
def run(self, job_id: UUID) -> JobRunResult:
|
||||
"""Claim, optionally clean, execute and durably finalize one delivery."""
|
||||
claim = self._claim(job_id)
|
||||
if not claim.run:
|
||||
assert claim.result is not None
|
||||
return claim.result
|
||||
|
||||
if claim.cleanup_required:
|
||||
try:
|
||||
document_id = self._job_document_id(job_id)
|
||||
self._cleanup.cleanup(document_id)
|
||||
except Exception:
|
||||
self._finish_cleanup_failure(job_id)
|
||||
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.FAILED)
|
||||
|
||||
attempt_number, document_id = self._start_attempt(job_id)
|
||||
|
||||
def stage_callback(stage: IngestionStage) -> None:
|
||||
"""Bind this attempt while pipeline stages remain Job-agnostic."""
|
||||
self._record_stage(job_id, attempt_number, stage)
|
||||
|
||||
try:
|
||||
self._parsing.process(document_id, stage_callback=stage_callback)
|
||||
self._chunking.process(document_id, stage_callback=stage_callback)
|
||||
self._indexing.process(document_id, stage_callback=stage_callback)
|
||||
except Exception as error:
|
||||
return self._finish_failure(job_id, attempt_number, document_id, error)
|
||||
|
||||
self._finish_success(job_id, attempt_number, document_id)
|
||||
return JobRunResult(JobRunAction.COMPLETE, IngestionJobStatus.SUCCEEDED)
|
||||
|
||||
def _claim(self, job_id: UUID) -> _Claim:
|
||||
"""Serialize deliveries and recover only after an execution lease expires."""
|
||||
now = datetime.now(UTC)
|
||||
with self._session_factory() as session:
|
||||
job = session.scalar(
|
||||
select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
|
||||
)
|
||||
if job is None:
|
||||
raise IngestionJobNotFoundError(str(job_id))
|
||||
if job.status in {
|
||||
IngestionJobStatus.SUCCEEDED,
|
||||
IngestionJobStatus.FAILED,
|
||||
IngestionJobStatus.CANCELLED,
|
||||
}:
|
||||
return _Claim(
|
||||
run=False,
|
||||
result=JobRunResult(JobRunAction.COMPLETE, job.status),
|
||||
)
|
||||
if job.status == IngestionJobStatus.RUNNING:
|
||||
lease_expiry = _as_utc(job.lease_expires_at)
|
||||
if lease_expiry is not None and lease_expiry > now:
|
||||
delay = max(1, ceil((lease_expiry - now).total_seconds()))
|
||||
return _Claim(
|
||||
run=False,
|
||||
result=JobRunResult(
|
||||
JobRunAction.RETRY,
|
||||
IngestionJobStatus.RUNNING,
|
||||
delay,
|
||||
),
|
||||
)
|
||||
self._expire_active_attempt(session, job, now)
|
||||
if job.attempt_count >= job.max_attempts:
|
||||
job.status = IngestionJobStatus.FAILED
|
||||
job.lease_expires_at = None
|
||||
job.completed_at = now
|
||||
document = session.get(Document, job.document_id)
|
||||
if document is not None:
|
||||
document.status = DocumentStatus.FAILED
|
||||
document.error_message = SAFE_LEASE_ERROR
|
||||
session.commit()
|
||||
return _Claim(
|
||||
run=False,
|
||||
result=JobRunResult(JobRunAction.COMPLETE, job.status),
|
||||
)
|
||||
job.status = IngestionJobStatus.RETRYING
|
||||
next_run = _as_utc(job.next_run_at)
|
||||
if (
|
||||
job.status == IngestionJobStatus.RETRYING
|
||||
and next_run is not None
|
||||
and next_run > now
|
||||
):
|
||||
delay = max(1, ceil((next_run - now).total_seconds()))
|
||||
return _Claim(
|
||||
run=False,
|
||||
result=JobRunResult(
|
||||
JobRunAction.RETRY,
|
||||
IngestionJobStatus.RETRYING,
|
||||
delay,
|
||||
),
|
||||
)
|
||||
|
||||
cleanup_required = job.attempt_count > 0
|
||||
job.status = IngestionJobStatus.RUNNING
|
||||
job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
|
||||
job.next_run_at = None
|
||||
session.commit()
|
||||
return _Claim(run=True, cleanup_required=cleanup_required)
|
||||
|
||||
def _expire_active_attempt(self, session: Session, job: IngestionJob, now: datetime) -> None:
|
||||
"""Close an abandoned Attempt before a replacement can be created."""
|
||||
attempt = session.scalar(
|
||||
select(IngestionAttempt)
|
||||
.where(
|
||||
IngestionAttempt.job_id == job.id,
|
||||
IngestionAttempt.status == IngestionAttemptStatus.RUNNING,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if attempt is not None:
|
||||
attempt.status = IngestionAttemptStatus.FAILED
|
||||
attempt.retryable = True
|
||||
attempt.error_message = SAFE_LEASE_ERROR
|
||||
attempt.finished_at = now
|
||||
job.error_message = SAFE_LEASE_ERROR
|
||||
|
||||
def _job_document_id(self, job_id: UUID) -> UUID:
|
||||
with self._session_factory() as session:
|
||||
job = session.get(IngestionJob, job_id)
|
||||
if job is None:
|
||||
raise IngestionJobNotFoundError(str(job_id))
|
||||
return job.document_id
|
||||
|
||||
def _start_attempt(self, job_id: UUID) -> tuple[int, UUID]:
|
||||
"""Create the next numbered Attempt only after retry cleanup succeeds."""
|
||||
now = datetime.now(UTC)
|
||||
with self._session_factory() as session:
|
||||
job = session.scalar(
|
||||
select(IngestionJob).where(IngestionJob.id == job_id).with_for_update()
|
||||
)
|
||||
if job is None:
|
||||
raise IngestionJobNotFoundError(str(job_id))
|
||||
attempt_number = job.attempt_count + 1
|
||||
job.attempt_count = attempt_number
|
||||
job.stage = IngestionStage.PARSING
|
||||
job.error_message = None
|
||||
job.lease_expires_at = now + timedelta(seconds=self._lease_seconds)
|
||||
session.add(
|
||||
IngestionAttempt(
|
||||
job_id=job.id,
|
||||
attempt_number=attempt_number,
|
||||
stage=IngestionStage.PARSING,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return attempt_number, job.document_id
|
||||
|
||||
def _record_stage(self, job_id: UUID, attempt_number: int, stage: IngestionStage) -> None:
|
||||
"""Persist progress and renew the worker lease at every stage boundary."""
|
||||
with self._session_factory() as session:
|
||||
job = session.get(IngestionJob, job_id)
|
||||
attempt = session.scalar(
|
||||
select(IngestionAttempt).where(
|
||||
IngestionAttempt.job_id == job_id,
|
||||
IngestionAttempt.attempt_number == attempt_number,
|
||||
)
|
||||
)
|
||||
if job is None or attempt is None:
|
||||
raise IngestionJobNotFoundError(str(job_id))
|
||||
job.stage = stage
|
||||
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=self._lease_seconds)
|
||||
attempt.stage = stage
|
||||
session.commit()
|
||||
|
||||
def _finish_success(self, job_id: UUID, attempt_number: int, document_id: UUID) -> None:
|
||||
"""Atomically close Job/Attempt after the document becomes ready."""
|
||||
now = datetime.now(UTC)
|
||||
with self._session_factory() as session:
|
||||
job, attempt = self._load_job_attempt(session, job_id, attempt_number)
|
||||
document = session.get(Document, document_id)
|
||||
if document is None:
|
||||
raise IngestionJobNotFoundError(str(job_id))
|
||||
attempt.status = IngestionAttemptStatus.SUCCEEDED
|
||||
attempt.retryable = False
|
||||
attempt.finished_at = now
|
||||
job.status = IngestionJobStatus.SUCCEEDED
|
||||
job.error_message = None
|
||||
job.lease_expires_at = None
|
||||
job.completed_at = now
|
||||
document.status = DocumentStatus.READY
|
||||
document.error_message = None
|
||||
session.commit()
|
||||
|
||||
def _finish_failure(
|
||||
self,
|
||||
job_id: UUID,
|
||||
attempt_number: int,
|
||||
document_id: UUID,
|
||||
error: Exception,
|
||||
) -> JobRunResult:
|
||||
"""Classify safely, bound retry count, and schedule exponential backoff."""
|
||||
retryable = _is_retryable(error)
|
||||
safe_error = SAFE_TRANSIENT_ERROR if retryable else SAFE_PERMANENT_ERROR
|
||||
now = datetime.now(UTC)
|
||||
with self._session_factory() as session:
|
||||
job, attempt = self._load_job_attempt(session, job_id, attempt_number)
|
||||
document = session.get(Document, document_id)
|
||||
attempt.status = IngestionAttemptStatus.FAILED
|
||||
attempt.retryable = retryable
|
||||
attempt.error_message = safe_error
|
||||
attempt.finished_at = now
|
||||
job.error_message = safe_error
|
||||
job.lease_expires_at = None
|
||||
if retryable and job.attempt_count < job.max_attempts:
|
||||
delay = min(
|
||||
self._retry_backoff_seconds * (2 ** (job.attempt_count - 1)),
|
||||
self._retry_backoff_max_seconds,
|
||||
)
|
||||
job.status = IngestionJobStatus.RETRYING
|
||||
job.next_run_at = now + timedelta(seconds=delay)
|
||||
if document is not None:
|
||||
document.status = DocumentStatus.PROCESSING
|
||||
document.error_message = safe_error
|
||||
session.commit()
|
||||
return JobRunResult(JobRunAction.RETRY, job.status, delay)
|
||||
|
||||
job.status = IngestionJobStatus.FAILED
|
||||
job.next_run_at = None
|
||||
job.completed_at = now
|
||||
if document is not None:
|
||||
document.status = DocumentStatus.FAILED
|
||||
document.error_message = safe_error
|
||||
session.commit()
|
||||
return JobRunResult(JobRunAction.COMPLETE, job.status)
|
||||
|
||||
def _finish_cleanup_failure(self, job_id: UUID) -> None:
|
||||
"""Fail closed without creating a new Attempt on unknown dirty state."""
|
||||
now = datetime.now(UTC)
|
||||
with self._session_factory() as session:
|
||||
job = session.get(IngestionJob, job_id)
|
||||
if job is None:
|
||||
raise IngestionJobNotFoundError(str(job_id))
|
||||
document = session.get(Document, job.document_id)
|
||||
job.status = IngestionJobStatus.FAILED
|
||||
job.error_message = SAFE_CLEANUP_ERROR
|
||||
job.next_run_at = None
|
||||
job.lease_expires_at = None
|
||||
job.completed_at = now
|
||||
if document is not None:
|
||||
document.status = DocumentStatus.FAILED
|
||||
document.error_message = SAFE_CLEANUP_ERROR
|
||||
session.commit()
|
||||
|
||||
@staticmethod
|
||||
def _load_job_attempt(
|
||||
session: Session, job_id: UUID, attempt_number: int
|
||||
) -> tuple[IngestionJob, IngestionAttempt]:
|
||||
job = session.get(IngestionJob, job_id)
|
||||
attempt = session.scalar(
|
||||
select(IngestionAttempt).where(
|
||||
IngestionAttempt.job_id == job_id,
|
||||
IngestionAttempt.attempt_number == attempt_number,
|
||||
)
|
||||
)
|
||||
if job is None or attempt is None:
|
||||
raise IngestionJobNotFoundError(str(job_id))
|
||||
return job, attempt
|
||||
|
||||
|
||||
def dispatch_pending_jobs(jobs: IngestionJobService, dispatcher: JobDispatcher) -> int:
|
||||
"""Best-effort redelivery loop used by Celery beat after broker/API failures."""
|
||||
dispatched = 0
|
||||
for job_id in jobs.list_dispatchable():
|
||||
try:
|
||||
dispatcher.enqueue(job_id)
|
||||
except Exception:
|
||||
continue
|
||||
dispatched += 1
|
||||
return dispatched
|
||||
|
||||
|
||||
def _is_retryable(error: Exception) -> bool:
|
||||
"""Treat known content/state violations as permanent and providers as transient."""
|
||||
permanent_errors = (
|
||||
DocumentParseError,
|
||||
EmptyParsedDocumentError,
|
||||
EmptyChunkSetError,
|
||||
ParsedDocumentMismatchError,
|
||||
ParsedDocumentArtifactError,
|
||||
DocumentAlreadyParsedError,
|
||||
DocumentAlreadyChunkedError,
|
||||
DocumentAlreadyIndexedError,
|
||||
DocumentNotParsedError,
|
||||
DocumentHasNoChunksError,
|
||||
ParsingDocumentNotFoundError,
|
||||
ChunkingDocumentNotFoundError,
|
||||
IndexingDocumentNotFoundError,
|
||||
)
|
||||
return not isinstance(error, permanent_errors)
|
||||
|
||||
|
||||
def _as_utc(value: datetime | None) -> datetime | None:
|
||||
"""Normalize SQLite's naive DateTime values to the UTC runtime contract."""
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=UTC)
|
||||
return value.astimezone(UTC)
|
||||
@ -8,16 +8,36 @@ from ekp_domain.index_profile import (
|
||||
IndexProfileStatus,
|
||||
StoredIndexProfile,
|
||||
)
|
||||
from ekp_domain.ingestion_batch import (
|
||||
BatchItemAcceptanceStatus,
|
||||
IngestionBatch,
|
||||
IngestionBatchItem,
|
||||
)
|
||||
from ekp_domain.ingestion_job import (
|
||||
IngestionAttempt,
|
||||
IngestionAttemptStatus,
|
||||
IngestionJob,
|
||||
IngestionJobStatus,
|
||||
IngestionStage,
|
||||
)
|
||||
from ekp_domain.parsed_document import ParsedBlock, ParsedBlockType, ParsedDocument
|
||||
|
||||
__all__ = [
|
||||
"Base",
|
||||
"BatchItemAcceptanceStatus",
|
||||
"Chunk",
|
||||
"ChunkIndexState",
|
||||
"Document",
|
||||
"DocumentStatus",
|
||||
"IndexProfile",
|
||||
"IndexProfileStatus",
|
||||
"IngestionAttempt",
|
||||
"IngestionAttemptStatus",
|
||||
"IngestionJob",
|
||||
"IngestionJobStatus",
|
||||
"IngestionStage",
|
||||
"IngestionBatch",
|
||||
"IngestionBatchItem",
|
||||
"ParsedBlock",
|
||||
"ParsedBlockType",
|
||||
"ParsedDocument",
|
||||
|
||||
96
packages/domain/src/ekp_domain/ingestion_batch.py
Normal file
96
packages/domain/src/ekp_domain/ingestion_batch.py
Normal file
@ -0,0 +1,96 @@
|
||||
"""Canonical batch submission metadata for per-file ingestion tracking."""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
Uuid,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy import Enum as SqlEnum
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ekp_domain.document import Base
|
||||
|
||||
|
||||
class BatchItemAcceptanceStatus(StrEnum):
|
||||
"""Source acceptance state, independent from its asynchronous Job state."""
|
||||
|
||||
PENDING = "pending"
|
||||
ACCEPTED = "accepted"
|
||||
REJECTED = "rejected"
|
||||
|
||||
|
||||
class IngestionBatch(Base):
|
||||
"""Stable aggregation identifier for one bounded multipart submission."""
|
||||
|
||||
__tablename__ = "ingestion_batches"
|
||||
__table_args__ = (CheckConstraint("total_files > 0", name="total_files_positive"),)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
|
||||
total_files: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class IngestionBatchItem(Base):
|
||||
"""One ordered input and its optional accepted Document/Job references."""
|
||||
|
||||
__tablename__ = "ingestion_batch_items"
|
||||
__table_args__ = (
|
||||
CheckConstraint("item_index >= 0", name="item_index_non_negative"),
|
||||
CheckConstraint("length(filename) > 0", name="filename_not_empty"),
|
||||
CheckConstraint(
|
||||
"acceptance_status IN ('pending', 'accepted', 'rejected')",
|
||||
name="acceptance_status_valid",
|
||||
),
|
||||
Index("uq_ingestion_batch_items_order", "batch_id", "item_index", unique=True),
|
||||
Index("uq_ingestion_batch_items_job_id", "job_id", unique=True),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
|
||||
batch_id: Mapped[UUID] = mapped_column(
|
||||
Uuid(as_uuid=True),
|
||||
ForeignKey("ingestion_batches.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
item_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
filename: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
acceptance_status: Mapped[BatchItemAcceptanceStatus] = mapped_column(
|
||||
SqlEnum(
|
||||
BatchItemAcceptanceStatus,
|
||||
name="batch_item_acceptance_status",
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
validate_strings=True,
|
||||
values_callable=lambda values: [item.value for item in values],
|
||||
),
|
||||
nullable=False,
|
||||
default=BatchItemAcceptanceStatus.PENDING,
|
||||
)
|
||||
document_id: Mapped[UUID | None] = mapped_column(
|
||||
Uuid(as_uuid=True),
|
||||
ForeignKey("documents.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
job_id: Mapped[UUID | None] = mapped_column(
|
||||
Uuid(as_uuid=True),
|
||||
ForeignKey("ingestion_jobs.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
166
packages/domain/src/ekp_domain/ingestion_job.py
Normal file
166
packages/domain/src/ekp_domain/ingestion_job.py
Normal file
@ -0,0 +1,166 @@
|
||||
"""Durable ingestion Job and Attempt state for asynchronous processing."""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
Computed,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
Uuid,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy import Enum as SqlEnum
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ekp_domain.document import Base
|
||||
|
||||
|
||||
class IngestionJobStatus(StrEnum):
|
||||
"""Public lifecycle states for one document ingestion Job."""
|
||||
|
||||
QUEUED = "queued"
|
||||
RUNNING = "running"
|
||||
RETRYING = "retrying"
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
class IngestionStage(StrEnum):
|
||||
"""Observable stages within one ingestion attempt."""
|
||||
|
||||
PARSING = "parsing"
|
||||
CLEANING = "cleaning"
|
||||
CHUNKING = "chunking"
|
||||
EMBEDDING = "embedding"
|
||||
INDEXING = "indexing"
|
||||
|
||||
|
||||
class IngestionAttemptStatus(StrEnum):
|
||||
"""Terminal or active state of one bounded execution attempt."""
|
||||
|
||||
RUNNING = "running"
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
def _enum_column(enum_type: type[StrEnum], name: str) -> SqlEnum:
|
||||
"""Build consistent portable string enums for OceanBase and SQLite tests."""
|
||||
return SqlEnum(
|
||||
enum_type,
|
||||
name=name,
|
||||
native_enum=False,
|
||||
create_constraint=False,
|
||||
validate_strings=True,
|
||||
values_callable=lambda values: [item.value for item in values],
|
||||
)
|
||||
|
||||
|
||||
class IngestionJob(Base):
|
||||
"""Canonical orchestration state; broker messages are only delivery hints."""
|
||||
|
||||
__tablename__ = "ingestion_jobs"
|
||||
__table_args__ = (
|
||||
CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
|
||||
CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
|
||||
CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
|
||||
CheckConstraint(
|
||||
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
|
||||
name="status_valid",
|
||||
),
|
||||
CheckConstraint(
|
||||
"stage IS NULL OR stage IN "
|
||||
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
||||
name="stage_valid",
|
||||
),
|
||||
UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
|
||||
Index("ix_ingestion_jobs_status_next_run", "status", "next_run_at"),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
|
||||
document_id: Mapped[UUID] = mapped_column(
|
||||
Uuid(as_uuid=True),
|
||||
ForeignKey("documents.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
status: Mapped[IngestionJobStatus] = mapped_column(
|
||||
_enum_column(IngestionJobStatus, "ingestion_job_status"),
|
||||
nullable=False,
|
||||
default=IngestionJobStatus.QUEUED,
|
||||
)
|
||||
stage: Mapped[IngestionStage | None] = mapped_column(
|
||||
_enum_column(IngestionStage, "ingestion_stage"),
|
||||
nullable=True,
|
||||
)
|
||||
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class IngestionAttempt(Base):
|
||||
"""Immutable-numbered execution history with at most one active row per Job."""
|
||||
|
||||
__tablename__ = "ingestion_attempts"
|
||||
__table_args__ = (
|
||||
CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
|
||||
CheckConstraint(
|
||||
"status IN ('running', 'succeeded', 'failed')",
|
||||
name="status_valid",
|
||||
),
|
||||
CheckConstraint(
|
||||
"stage IS NULL OR stage IN "
|
||||
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
|
||||
name="stage_valid",
|
||||
),
|
||||
UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
|
||||
# NULL values may repeat, while every running attempt maps to 1. This
|
||||
# provides portable partial uniqueness on OceanBase MySQL and SQLite.
|
||||
Index("uq_ingestion_attempts_active", "job_id", "active_marker", unique=True),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid4)
|
||||
job_id: Mapped[UUID] = mapped_column(
|
||||
Uuid(as_uuid=True),
|
||||
ForeignKey("ingestion_jobs.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
attempt_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
status: Mapped[IngestionAttemptStatus] = mapped_column(
|
||||
_enum_column(IngestionAttemptStatus, "ingestion_attempt_status"),
|
||||
nullable=False,
|
||||
default=IngestionAttemptStatus.RUNNING,
|
||||
)
|
||||
stage: Mapped[IngestionStage | None] = mapped_column(
|
||||
_enum_column(IngestionStage, "ingestion_attempt_stage"),
|
||||
nullable=True,
|
||||
)
|
||||
retryable: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
active_marker: Mapped[int | None] = mapped_column(
|
||||
Integer,
|
||||
Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
|
||||
nullable=True,
|
||||
)
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
@ -11,7 +11,9 @@ requires-python = ">=3.12,<3.14"
|
||||
dependencies = [
|
||||
"alembic>=1.18,<2",
|
||||
"bce-python-sdk>=0.9.72,<0.10",
|
||||
"celery>=5.5,<6",
|
||||
"fastapi>=0.115,<1",
|
||||
"mcp>=2,<3",
|
||||
"pymysql>=1.1,<2",
|
||||
"pymupdf>=1.26,<2",
|
||||
"pydantic-settings>=2.7,<3",
|
||||
@ -25,6 +27,8 @@ dependencies = [
|
||||
[project.scripts]
|
||||
ekp-index-rebuild = "ekp_api.index_rebuild:main"
|
||||
ekp-live-acceptance = "ekp_api.live_acceptance:main"
|
||||
ekp-mcp = "ekp_mcp.main:main"
|
||||
ekp-worker = "ekp_api.worker:main"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
@ -38,6 +42,7 @@ dev = [
|
||||
[tool.setuptools.packages.find]
|
||||
where = [
|
||||
"apps/api/src",
|
||||
"apps/mcp-server/src",
|
||||
"packages/core/src",
|
||||
"packages/domain/src",
|
||||
"packages/providers/object_storage/src",
|
||||
@ -57,6 +62,7 @@ select = ["ASYNC", "B", "E4", "E7", "E9", "F", "I", "UP"]
|
||||
[tool.ruff.lint.isort]
|
||||
known-first-party = [
|
||||
"ekp_api",
|
||||
"ekp_mcp",
|
||||
"ekp_core",
|
||||
"ekp_domain",
|
||||
"ekp_embedding",
|
||||
@ -71,6 +77,7 @@ python_version = "3.12"
|
||||
strict = true
|
||||
mypy_path = [
|
||||
"apps/api/src",
|
||||
"apps/mcp-server/src",
|
||||
"packages/core/src",
|
||||
"packages/domain/src",
|
||||
"packages/providers/object_storage/src",
|
||||
|
||||
130
scripts/deploy.sh
Executable file
130
scripts/deploy.sh
Executable file
@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# DEPLOY-002 accepts a values file path only. Secret contents remain in
|
||||
# Kubernetes and are never expanded into this process or Helm arguments.
|
||||
values_file="${1:-}"
|
||||
chart_dir="${CHART:-deploy/helm/knowledge-platform}"
|
||||
release_name="${RELEASE:-knowledge-platform}"
|
||||
namespace="${NAMESPACE:-knowledge-platform}"
|
||||
deploy_timeout="${DEPLOY_TIMEOUT:-10m}"
|
||||
|
||||
if [[ -z "${values_file}" || ! -f "${values_file}" ]]; then
|
||||
echo "VALUES must point to an existing environment values file" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
for tool in helm kubectl; do
|
||||
if ! command -v "${tool}" >/dev/null 2>&1; then
|
||||
echo "required deployment tool is missing: ${tool}" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
# Helm 4 renamed --atomic to --rollback-on-failure. Select the equivalent flag
|
||||
# by major version while preserving compatibility with supported Helm 3.
|
||||
helm_version="$(helm version --template '{{.Version}}')"
|
||||
if [[ "${helm_version}" =~ ^v?([0-9]+) ]] && (( BASH_REMATCH[1] >= 4 )); then
|
||||
rollback_flag=(--rollback-on-failure)
|
||||
else
|
||||
rollback_flag=(--atomic)
|
||||
fi
|
||||
|
||||
helm lint "${chart_dir}" --values "${values_file}"
|
||||
|
||||
# The test-hook ConfigMap contains resource names only. Its fixed, controlled
|
||||
# shape lets awk extract names without adding a YAML parser or reading Secrets.
|
||||
preflight_yaml="$({
|
||||
helm template "${release_name}" "${chart_dir}" \
|
||||
--namespace "${namespace}" \
|
||||
--values "${values_file}" \
|
||||
--show-only templates/tests/preflight-references.yaml
|
||||
})"
|
||||
runtime_secret="$({
|
||||
printf '%s\n' "${preflight_yaml}" | awk '$1 == "runtimeSecret:" {gsub(/\"/, "", $2); print $2}'
|
||||
})"
|
||||
ingress_enabled="$({
|
||||
printf '%s\n' "${preflight_yaml}" | awk '$1 == "ingressEnabled:" {gsub(/\"/, "", $2); print $2}'
|
||||
})"
|
||||
rabbitmq_enabled="$({
|
||||
printf '%s\n' "${preflight_yaml}" | awk '$1 == "rabbitmqEnabled:" {gsub(/\"/, "", $2); print $2}'
|
||||
})"
|
||||
|
||||
if [[ -z "${runtime_secret}" || -z "${ingress_enabled}" || -z "${rabbitmq_enabled}" ]]; then
|
||||
echo "failed to resolve Helm Secret references" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
secret_has_key() {
|
||||
local secret_name="$1"
|
||||
local key_name="$2"
|
||||
local present
|
||||
present="$({
|
||||
kubectl -n "${namespace}" get secret "${secret_name}" \
|
||||
-o go-template="{{with .data}}{{if index . \"${key_name}\"}}present{{end}}{{end}}"
|
||||
})"
|
||||
[[ "${present}" == "present" ]]
|
||||
}
|
||||
|
||||
if ! kubectl get namespace "${namespace}" >/dev/null; then
|
||||
echo "target namespace does not exist: ${namespace}" >&2
|
||||
exit 2
|
||||
fi
|
||||
if ! kubectl -n "${namespace}" get secret "${runtime_secret}" >/dev/null; then
|
||||
echo "referenced runtime Secret does not exist: ${runtime_secret}" >&2
|
||||
exit 2
|
||||
fi
|
||||
required_runtime_keys=(
|
||||
DATABASE_URL
|
||||
RABBITMQ_URL
|
||||
BOS_ACCESS_KEY_ID
|
||||
BOS_SECRET_ACCESS_KEY
|
||||
BAIDU_QIANFAN_API_KEY
|
||||
OCEANBASE_VECTOR_PASSWORD
|
||||
MCP_AUTH_TOKEN
|
||||
)
|
||||
for key in "${required_runtime_keys[@]}"; do
|
||||
if ! secret_has_key "${runtime_secret}" "${key}"; then
|
||||
echo "referenced runtime Secret is missing required key: ${key}" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${ingress_enabled}" == "true" ]]; then
|
||||
for key in apiTlsSecret mcpTlsSecret; do
|
||||
tls_secret="$({
|
||||
printf '%s\n' "${preflight_yaml}" | awk -v field="${key}:" '$1 == field {gsub(/\"/, "", $2); print $2}'
|
||||
})"
|
||||
if ! kubectl -n "${namespace}" get secret "${tls_secret}" >/dev/null; then
|
||||
echo "referenced TLS Secret does not exist: ${tls_secret}" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ "${rabbitmq_enabled}" == "true" ]]; then
|
||||
rabbitmq_secret="$({
|
||||
printf '%s\n' "${preflight_yaml}" | awk '$1 == "rabbitmqSecret:" {gsub(/\"/, "", $2); print $2}'
|
||||
})"
|
||||
if ! kubectl -n "${namespace}" get secret "${rabbitmq_secret}" >/dev/null; then
|
||||
echo "referenced RabbitMQ Secret does not exist: ${rabbitmq_secret}" >&2
|
||||
exit 2
|
||||
fi
|
||||
for key in username password erlang-cookie; do
|
||||
if ! secret_has_key "${rabbitmq_secret}" "${key}"; then
|
||||
echo "referenced RabbitMQ Secret is missing required key: ${key}" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# The version-specific rollback flag keeps the previous release active when a
|
||||
# pre-upgrade migration or rollout fails. Helm serializes operations for this
|
||||
# release and waits for hook/workload readiness before returning success.
|
||||
helm upgrade --install "${release_name}" "${chart_dir}" \
|
||||
--namespace "${namespace}" \
|
||||
--values "${values_file}" \
|
||||
"${rollback_flag[@]}" \
|
||||
--wait \
|
||||
--timeout "${deploy_timeout}" \
|
||||
--history-max 10
|
||||
@ -35,11 +35,18 @@ def test_admin_console_contains_only_demo_pages_and_versioned_api_calls(
|
||||
with TestClient(create_app(Settings(app_env="test"))) as client:
|
||||
html = client.get("/admin").text
|
||||
script = client.get("/admin/assets/app.js").text
|
||||
stylesheet = client.get("/admin/assets/styles.css").text
|
||||
|
||||
assert all(label in html for label in ("上传文档", "文档列表", "Search Lab"))
|
||||
assert "地瓜机器人" in html
|
||||
assert all(page_id in html for page_id in ("upload-page", "documents-page", "search-page"))
|
||||
assert all(label not in html for label in ("权限管理", "任务管理", "Compiled Knowledge"))
|
||||
assert 'const API_BASE = "/api/v1"' in script
|
||||
assert "innerHTML" not in script
|
||||
assert all(endpoint in script for endpoint in ('"/documents"', '"/search"'))
|
||||
assert all(endpoint in script for endpoint in ('"/ingestion-batches"', '"/search"'))
|
||||
assert "multiple" in html
|
||||
assert "BATCH_POLL_INTERVAL_MS" in script
|
||||
assert all(field in script for field in ("Page", "Section", "Dense Score", "Rerank Score"))
|
||||
assert "--brand: #FF3C00" in stylesheet
|
||||
assert "prefers-reduced-motion" in stylesheet
|
||||
assert "https://" not in html
|
||||
|
||||
431
tests/integration/test_async_ingestion.py
Normal file
431
tests/integration/test_async_ingestion.py
Normal file
@ -0,0 +1,431 @@
|
||||
"""Durable Job, bounded retry and cleanup-before-retry integration tests."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ekp_core.chunking import StructureAwareChunker
|
||||
from ekp_core.database import create_session_factory
|
||||
from ekp_core.document_chunking import DocumentChunkingService
|
||||
from ekp_core.document_indexing import DocumentIndexingService
|
||||
from ekp_core.document_parsing import DocumentParsingService
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.ingestion_jobs import (
|
||||
AsyncIngestionSubmissionService,
|
||||
DerivedDocumentCleanupService,
|
||||
IngestionJobRunner,
|
||||
IngestionJobService,
|
||||
JobRunAction,
|
||||
dispatch_pending_jobs,
|
||||
)
|
||||
from ekp_domain import (
|
||||
Base,
|
||||
Chunk,
|
||||
Document,
|
||||
IndexProfile,
|
||||
IngestionAttempt,
|
||||
IngestionAttemptStatus,
|
||||
IngestionJob,
|
||||
IngestionJobStatus,
|
||||
ParsedDocument,
|
||||
)
|
||||
from ekp_embedding import EmbeddingProviderError, FakeEmbeddingProvider
|
||||
from ekp_object_storage import LocalObjectStorage
|
||||
from ekp_parser import ParserRouter
|
||||
from ekp_vector_store import FakeVectorStore
|
||||
|
||||
|
||||
class RecordingDispatcher:
|
||||
"""Capture delivery requests while preserving the dispatcher contract."""
|
||||
|
||||
def __init__(self, *, fail: bool = False) -> None:
|
||||
self.fail = fail
|
||||
self.job_ids: list[UUID] = []
|
||||
|
||||
def enqueue(self, job_id: UUID) -> None:
|
||||
if self.fail:
|
||||
raise RuntimeError("simulated broker outage containing secret=must-not-persist")
|
||||
self.job_ids.append(job_id)
|
||||
|
||||
|
||||
class FailingJobService(IngestionJobService):
|
||||
"""Inject metadata failure after immutable source upload."""
|
||||
|
||||
def create(self, document_id: UUID) -> IngestionJob:
|
||||
del document_id
|
||||
raise RuntimeError("simulated Job persistence failure")
|
||||
|
||||
|
||||
class FailOnceEmbeddingProvider:
|
||||
"""Inject one transient provider failure after parse/chunk persistence."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._delegate = FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
|
||||
self.calls = 0
|
||||
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
return self._delegate.model_name
|
||||
|
||||
@property
|
||||
def dimensions(self) -> int:
|
||||
return self._delegate.dimensions
|
||||
|
||||
@property
|
||||
def max_batch_size(self) -> int:
|
||||
return self._delegate.max_batch_size
|
||||
|
||||
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
|
||||
self.calls += 1
|
||||
if self.calls == 1:
|
||||
raise EmbeddingProviderError("temporary failure token=must-not-persist")
|
||||
return self._delegate.embed(texts)
|
||||
|
||||
|
||||
class AlwaysFailEmbeddingProvider(FailOnceEmbeddingProvider):
|
||||
"""Inject a transient error on every attempt to verify the retry bound."""
|
||||
|
||||
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
|
||||
del texts
|
||||
self.calls += 1
|
||||
raise EmbeddingProviderError("temporary failure token=must-not-persist")
|
||||
|
||||
|
||||
class RetryObservingParser:
|
||||
"""Verify cleanup finished before the second provider execution begins."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker[Session],
|
||||
storage: LocalObjectStorage,
|
||||
vector_store: FakeVectorStore,
|
||||
) -> None:
|
||||
self._delegate = ParserRouter()
|
||||
self._session_factory = session_factory
|
||||
self._storage = storage
|
||||
self._vector_store = vector_store
|
||||
self.calls = 0
|
||||
self.retry_started_clean = False
|
||||
|
||||
def parse(
|
||||
self,
|
||||
document_id: UUID,
|
||||
filename: str,
|
||||
source: BinaryIO,
|
||||
) -> ParsedDocument:
|
||||
self.calls += 1
|
||||
if self.calls == 2:
|
||||
with self._session_factory() as session:
|
||||
document = session.get(Document, document_id)
|
||||
chunks = session.scalars(
|
||||
select(Chunk).where(Chunk.document_id == document_id)
|
||||
).all()
|
||||
# The source stream is already open here, proving it survived cleanup.
|
||||
self.retry_started_clean = (
|
||||
document is not None
|
||||
and document.parsed_storage_key is None
|
||||
and not chunks
|
||||
and not self._vector_store.records
|
||||
and self._storage.exists(document.storage_key)
|
||||
)
|
||||
return self._delegate.parse(document_id, filename, source)
|
||||
|
||||
|
||||
class CleanupFailingVectorStore(FakeVectorStore):
|
||||
"""Inject a retry cleanup failure without affecting the first attempt."""
|
||||
|
||||
fail_delete = False
|
||||
|
||||
def delete_document(self, document_id: UUID, index_profile_id: str) -> None:
|
||||
if self.fail_delete:
|
||||
raise RuntimeError("cleanup provider failed secret=must-not-persist")
|
||||
super().delete_document(document_id, index_profile_id)
|
||||
|
||||
|
||||
def _build_runner(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
embedding: FailOnceEmbeddingProvider | FakeEmbeddingProvider | None = None,
|
||||
vector_store: FakeVectorStore | None = None,
|
||||
max_attempts: int = 3,
|
||||
) -> tuple[
|
||||
DocumentUploadService,
|
||||
IngestionJobService,
|
||||
IngestionJobRunner,
|
||||
sessionmaker[Session],
|
||||
LocalObjectStorage,
|
||||
FakeVectorStore,
|
||||
RetryObservingParser,
|
||||
]:
|
||||
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
session_factory = create_session_factory(engine)
|
||||
storage = LocalObjectStorage(tmp_path / "storage")
|
||||
selected_store = vector_store or FakeVectorStore()
|
||||
selected_embedding = embedding or FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
|
||||
profile = IndexProfile(
|
||||
profile_id="test-profile",
|
||||
chunk_strategy_version="structure_v1",
|
||||
embedding_model=selected_embedding.model_name,
|
||||
vector_dimension=selected_embedding.dimensions,
|
||||
)
|
||||
parser = RetryObservingParser(session_factory, storage, selected_store)
|
||||
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
|
||||
parsing = DocumentParsingService(session_factory, storage, parser)
|
||||
chunking = DocumentChunkingService(
|
||||
session_factory,
|
||||
storage,
|
||||
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
|
||||
)
|
||||
indexing = DocumentIndexingService(
|
||||
session_factory,
|
||||
selected_embedding,
|
||||
selected_store,
|
||||
profile,
|
||||
)
|
||||
jobs = IngestionJobService(session_factory, max_attempts=max_attempts)
|
||||
cleanup = DerivedDocumentCleanupService(
|
||||
session_factory,
|
||||
storage,
|
||||
selected_store,
|
||||
profile.profile_id,
|
||||
)
|
||||
runner = IngestionJobRunner(
|
||||
session_factory,
|
||||
parsing,
|
||||
chunking,
|
||||
indexing,
|
||||
cleanup,
|
||||
retry_backoff_seconds=1,
|
||||
retry_backoff_max_seconds=4,
|
||||
lease_seconds=30,
|
||||
)
|
||||
return upload, jobs, runner, session_factory, storage, selected_store, parser
|
||||
|
||||
|
||||
def _submit(
|
||||
upload: DocumentUploadService,
|
||||
jobs: IngestionJobService,
|
||||
payload: bytes = b"# Policy\n\nRefunds take seven days.",
|
||||
) -> tuple[UUID, UUID, RecordingDispatcher]:
|
||||
dispatcher = RecordingDispatcher()
|
||||
result = AsyncIngestionSubmissionService(upload, jobs, dispatcher).submit(
|
||||
"policy.md", "text/markdown", BytesIO(payload)
|
||||
)
|
||||
return result.document.id, result.job.id, dispatcher
|
||||
|
||||
|
||||
def _make_retry_due(session_factory: sessionmaker[Session], job_id: UUID) -> None:
|
||||
with session_factory() as session:
|
||||
job = session.get(IngestionJob, job_id)
|
||||
assert job is not None
|
||||
job.next_run_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
session.commit()
|
||||
|
||||
|
||||
def test_transient_failure_cleans_derivatives_before_retry(tmp_path: Path) -> None:
|
||||
embedding = FailOnceEmbeddingProvider()
|
||||
upload, jobs, runner, session_factory, storage, vector_store, parser = _build_runner(
|
||||
tmp_path, embedding=embedding
|
||||
)
|
||||
document_id, job_id, dispatcher = _submit(upload, jobs)
|
||||
|
||||
first = runner.run(job_id)
|
||||
assert first.action == JobRunAction.RETRY
|
||||
assert first.retry_delay_seconds == 1
|
||||
assert dispatcher.job_ids == [job_id]
|
||||
with session_factory() as session:
|
||||
stored_job = session.get(IngestionJob, job_id)
|
||||
assert stored_job is not None
|
||||
assert stored_job.status == IngestionJobStatus.RETRYING
|
||||
assert stored_job.attempt_count == 1
|
||||
assert "must-not-persist" not in (stored_job.error_message or "")
|
||||
|
||||
_make_retry_due(session_factory, job_id)
|
||||
second = runner.run(job_id)
|
||||
|
||||
assert second.status == IngestionJobStatus.SUCCEEDED
|
||||
assert parser.retry_started_clean
|
||||
with session_factory() as session:
|
||||
stored_job = session.get(IngestionJob, job_id)
|
||||
attempts = session.scalars(
|
||||
select(IngestionAttempt)
|
||||
.where(IngestionAttempt.job_id == job_id)
|
||||
.order_by(IngestionAttempt.attempt_number)
|
||||
).all()
|
||||
assert stored_job is not None
|
||||
assert stored_job.attempt_count == 2
|
||||
assert [attempt.status for attempt in attempts] == [
|
||||
IngestionAttemptStatus.FAILED,
|
||||
IngestionAttemptStatus.SUCCEEDED,
|
||||
]
|
||||
document = session.get(Document, document_id)
|
||||
assert document is not None
|
||||
assert storage.exists(document.storage_key)
|
||||
assert document.parsed_storage_key is not None
|
||||
assert storage.exists(document.parsed_storage_key)
|
||||
assert vector_store.records
|
||||
|
||||
|
||||
def test_permanent_content_failure_does_not_retry(tmp_path: Path) -> None:
|
||||
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
|
||||
_, job_id, _ = _submit(upload, jobs, payload=b" \n\t")
|
||||
|
||||
result = runner.run(job_id)
|
||||
|
||||
assert result.action == JobRunAction.COMPLETE
|
||||
assert result.status == IngestionJobStatus.FAILED
|
||||
with session_factory() as session:
|
||||
job = session.get(IngestionJob, job_id)
|
||||
assert job is not None
|
||||
assert job.attempt_count == 1
|
||||
assert job.error_message == "document content could not be processed"
|
||||
|
||||
|
||||
def test_cleanup_failure_is_terminal_and_preserves_source(tmp_path: Path) -> None:
|
||||
embedding = FailOnceEmbeddingProvider()
|
||||
vector_store = CleanupFailingVectorStore()
|
||||
upload, jobs, runner, session_factory, storage, _, _ = _build_runner(
|
||||
tmp_path,
|
||||
embedding=embedding,
|
||||
vector_store=vector_store,
|
||||
)
|
||||
document_id, job_id, _ = _submit(upload, jobs)
|
||||
assert runner.run(job_id).action == JobRunAction.RETRY
|
||||
_make_retry_due(session_factory, job_id)
|
||||
vector_store.fail_delete = True
|
||||
|
||||
result = runner.run(job_id)
|
||||
|
||||
assert result.status == IngestionJobStatus.FAILED
|
||||
with session_factory() as session:
|
||||
job = session.get(IngestionJob, job_id)
|
||||
assert job is not None
|
||||
assert job.attempt_count == 1 # Cleanup failed before attempt 2 was created.
|
||||
assert job.error_message == "retry cleanup failed"
|
||||
document = session.get(Document, document_id)
|
||||
assert document is not None
|
||||
assert storage.exists(document.storage_key)
|
||||
|
||||
|
||||
def test_duplicate_delivery_defers_while_active_lease_is_valid(tmp_path: Path) -> None:
|
||||
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
|
||||
_, job_id, _ = _submit(upload, jobs)
|
||||
with session_factory() as session:
|
||||
job = session.get(IngestionJob, job_id)
|
||||
assert job is not None
|
||||
job.status = IngestionJobStatus.RUNNING
|
||||
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=30)
|
||||
session.commit()
|
||||
|
||||
result = runner.run(job_id)
|
||||
|
||||
assert result.action == JobRunAction.RETRY
|
||||
assert result.status == IngestionJobStatus.RUNNING
|
||||
with session_factory() as session:
|
||||
assert session.scalars(select(IngestionAttempt)).all() == []
|
||||
|
||||
|
||||
def test_expired_worker_lease_is_recovered_as_a_new_attempt(tmp_path: Path) -> None:
|
||||
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
|
||||
_, job_id, _ = _submit(upload, jobs)
|
||||
with session_factory() as session:
|
||||
job = session.get(IngestionJob, job_id)
|
||||
assert job is not None
|
||||
job.status = IngestionJobStatus.RUNNING
|
||||
job.attempt_count = 1
|
||||
job.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
session.add(IngestionAttempt(job_id=job_id, attempt_number=1))
|
||||
session.commit()
|
||||
|
||||
result = runner.run(job_id)
|
||||
|
||||
assert result.status == IngestionJobStatus.SUCCEEDED
|
||||
with session_factory() as session:
|
||||
attempts = session.scalars(
|
||||
select(IngestionAttempt)
|
||||
.where(IngestionAttempt.job_id == job_id)
|
||||
.order_by(IngestionAttempt.attempt_number)
|
||||
).all()
|
||||
assert [attempt.status for attempt in attempts] == [
|
||||
IngestionAttemptStatus.FAILED,
|
||||
IngestionAttemptStatus.SUCCEEDED,
|
||||
]
|
||||
assert attempts[0].error_message == "worker execution lease expired"
|
||||
|
||||
|
||||
def test_transient_failures_stop_at_configured_attempt_bound(tmp_path: Path) -> None:
|
||||
embedding = AlwaysFailEmbeddingProvider()
|
||||
upload, jobs, runner, session_factory, _, _, _ = _build_runner(
|
||||
tmp_path,
|
||||
embedding=embedding,
|
||||
max_attempts=2,
|
||||
)
|
||||
_, job_id, _ = _submit(upload, jobs)
|
||||
|
||||
assert runner.run(job_id).action == JobRunAction.RETRY
|
||||
_make_retry_due(session_factory, job_id)
|
||||
result = runner.run(job_id)
|
||||
|
||||
assert result.action == JobRunAction.COMPLETE
|
||||
assert result.status == IngestionJobStatus.FAILED
|
||||
assert runner.run(job_id).status == IngestionJobStatus.FAILED
|
||||
with session_factory() as session:
|
||||
job = session.get(IngestionJob, job_id)
|
||||
attempts = session.scalars(
|
||||
select(IngestionAttempt).where(IngestionAttempt.job_id == job_id)
|
||||
).all()
|
||||
assert job is not None
|
||||
assert job.attempt_count == job.max_attempts == 2
|
||||
assert len(attempts) == 2
|
||||
|
||||
|
||||
def test_recovery_dispatches_persisted_job_after_broker_outage(tmp_path: Path) -> None:
|
||||
upload, jobs, _, _, _, _, _ = _build_runner(tmp_path)
|
||||
failing = RecordingDispatcher(fail=True)
|
||||
submission = AsyncIngestionSubmissionService(upload, jobs, failing).submit(
|
||||
"policy.md", "text/markdown", BytesIO(b"policy")
|
||||
)
|
||||
recovered = RecordingDispatcher()
|
||||
|
||||
assert dispatch_pending_jobs(jobs, recovered) == 1
|
||||
assert recovered.job_ids == [submission.job.id]
|
||||
|
||||
|
||||
def test_job_persistence_failure_compensates_document_and_source(tmp_path: Path) -> None:
|
||||
upload, _, _, session_factory, _, _, _ = _build_runner(tmp_path)
|
||||
failing_jobs = FailingJobService(session_factory, max_attempts=3)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Job persistence failure"):
|
||||
AsyncIngestionSubmissionService(upload, failing_jobs, RecordingDispatcher()).submit(
|
||||
"policy.md", "text/markdown", BytesIO(b"policy")
|
||||
)
|
||||
|
||||
with session_factory() as session:
|
||||
assert session.scalars(select(Document)).all() == []
|
||||
assert session.scalars(select(IngestionJob)).all() == []
|
||||
assert not [path for path in tmp_path.rglob("*") if path.is_file()]
|
||||
|
||||
|
||||
def test_database_rejects_two_active_attempts_for_one_job(tmp_path: Path) -> None:
|
||||
upload, jobs, _, session_factory, _, _, _ = _build_runner(tmp_path)
|
||||
_, job_id, _ = _submit(upload, jobs)
|
||||
|
||||
with session_factory() as session:
|
||||
session.add_all(
|
||||
[
|
||||
IngestionAttempt(job_id=job_id, attempt_number=1),
|
||||
IngestionAttempt(job_id=job_id, attempt_number=2),
|
||||
]
|
||||
)
|
||||
with pytest.raises(IntegrityError):
|
||||
session.commit()
|
||||
session.rollback()
|
||||
@ -23,9 +23,16 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
|
||||
command.check(config)
|
||||
|
||||
inspector = inspect(create_engine(database_url))
|
||||
assert {"documents", "chunks", "chunk_index_states", "index_profiles"}.issubset(
|
||||
inspector.get_table_names()
|
||||
)
|
||||
assert {
|
||||
"documents",
|
||||
"chunks",
|
||||
"chunk_index_states",
|
||||
"index_profiles",
|
||||
"ingestion_jobs",
|
||||
"ingestion_attempts",
|
||||
"ingestion_batches",
|
||||
"ingestion_batch_items",
|
||||
}.issubset(inspector.get_table_names())
|
||||
assert "unrelated_provider_table" in inspector.get_table_names()
|
||||
assert {column["name"] for column in inspector.get_columns("documents")} == {
|
||||
"id",
|
||||
@ -70,3 +77,46 @@ def test_document_migration_matches_metadata(tmp_path: Path) -> None:
|
||||
"activated_at",
|
||||
"active_marker",
|
||||
}
|
||||
assert {column["name"] for column in inspector.get_columns("ingestion_jobs")} == {
|
||||
"id",
|
||||
"document_id",
|
||||
"status",
|
||||
"stage",
|
||||
"attempt_count",
|
||||
"max_attempts",
|
||||
"error_message",
|
||||
"next_run_at",
|
||||
"lease_expires_at",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"completed_at",
|
||||
}
|
||||
assert {column["name"] for column in inspector.get_columns("ingestion_attempts")} == {
|
||||
"id",
|
||||
"job_id",
|
||||
"attempt_number",
|
||||
"status",
|
||||
"stage",
|
||||
"retryable",
|
||||
"error_message",
|
||||
"active_marker",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
}
|
||||
assert {column["name"] for column in inspector.get_columns("ingestion_batches")} == {
|
||||
"id",
|
||||
"total_files",
|
||||
"created_at",
|
||||
}
|
||||
assert {column["name"] for column in inspector.get_columns("ingestion_batch_items")} == {
|
||||
"id",
|
||||
"batch_id",
|
||||
"item_index",
|
||||
"filename",
|
||||
"acceptance_status",
|
||||
"document_id",
|
||||
"job_id",
|
||||
"error_message",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
}
|
||||
|
||||
@ -19,8 +19,15 @@ from ekp_core.document_indexing import DocumentIndexingPipelineService, Document
|
||||
from ekp_core.document_management import DocumentManagementService
|
||||
from ekp_core.document_parsing import DocumentParsingService
|
||||
from ekp_core.document_upload import DocumentUploadService
|
||||
from ekp_core.ingestion_batches import IngestionBatchService
|
||||
from ekp_core.ingestion_jobs import (
|
||||
AsyncIngestionSubmissionService,
|
||||
DerivedDocumentCleanupService,
|
||||
IngestionJobRunner,
|
||||
IngestionJobService,
|
||||
)
|
||||
from ekp_core.search import SearchService
|
||||
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile
|
||||
from ekp_domain import Base, Chunk, ChunkIndexState, Document, IndexProfile, IngestionBatch
|
||||
from ekp_embedding import FakeEmbeddingProvider
|
||||
from ekp_object_storage import LocalObjectStorage
|
||||
from ekp_parser import ParserRouter
|
||||
@ -39,13 +46,28 @@ REQUIRED_SOURCE_FIELDS = {
|
||||
}
|
||||
|
||||
|
||||
class RecordingDispatcher:
|
||||
"""Capture durable Job deliveries without starting a real broker."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.job_ids: list[UUID] = []
|
||||
|
||||
def enqueue(self, job_id: UUID) -> None:
|
||||
self.job_ids.append(job_id)
|
||||
|
||||
|
||||
def _build_api_services(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
max_upload_bytes: int = 1024 * 1024,
|
||||
batch_max_files: int = 20,
|
||||
batch_max_total_bytes: int = 2 * 1024 * 1024,
|
||||
) -> tuple[
|
||||
ApiServices,
|
||||
sessionmaker[Session],
|
||||
LocalObjectStorage,
|
||||
FakeVectorStore,
|
||||
IngestionJobRunner,
|
||||
]:
|
||||
"""Build the production service graph with offline providers and shared SQLite."""
|
||||
engine = create_engine(
|
||||
@ -67,7 +89,11 @@ def _build_api_services(
|
||||
embedding_model=embedding.model_name,
|
||||
vector_dimension=embedding.dimensions,
|
||||
)
|
||||
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
|
||||
upload = DocumentUploadService(
|
||||
session_factory,
|
||||
storage,
|
||||
max_upload_bytes=max_upload_bytes,
|
||||
)
|
||||
parsing = DocumentParsingService(session_factory, storage, ParserRouter())
|
||||
chunking = DocumentChunkingService(
|
||||
session_factory,
|
||||
@ -82,6 +108,30 @@ def _build_api_services(
|
||||
profile,
|
||||
)
|
||||
ingestion = DocumentIndexingPipelineService(pipeline, indexing)
|
||||
jobs = IngestionJobService(session_factory, max_attempts=3)
|
||||
dispatcher = RecordingDispatcher()
|
||||
submissions = AsyncIngestionSubmissionService(upload, jobs, dispatcher)
|
||||
batches = IngestionBatchService(
|
||||
session_factory,
|
||||
submissions,
|
||||
max_files=batch_max_files,
|
||||
max_total_bytes=batch_max_total_bytes,
|
||||
)
|
||||
runner = IngestionJobRunner(
|
||||
session_factory,
|
||||
parsing,
|
||||
chunking,
|
||||
indexing,
|
||||
DerivedDocumentCleanupService(
|
||||
session_factory,
|
||||
storage,
|
||||
vector_store,
|
||||
profile.profile_id,
|
||||
),
|
||||
retry_backoff_seconds=1,
|
||||
retry_backoff_max_seconds=4,
|
||||
lease_seconds=30,
|
||||
)
|
||||
dense = DenseRetrievalService(embedding, vector_store, profile, default_top_n=20)
|
||||
search = SearchService(
|
||||
session_factory,
|
||||
@ -96,10 +146,18 @@ def _build_api_services(
|
||||
active_index_profile_id=profile.profile_id,
|
||||
)
|
||||
return (
|
||||
ApiServices(ingestion=ingestion, documents=documents, search=search),
|
||||
ApiServices(
|
||||
ingestion=ingestion,
|
||||
submissions=submissions,
|
||||
batches=batches,
|
||||
jobs=jobs,
|
||||
documents=documents,
|
||||
search=search,
|
||||
),
|
||||
session_factory,
|
||||
storage,
|
||||
vector_store,
|
||||
runner,
|
||||
)
|
||||
|
||||
|
||||
@ -107,7 +165,7 @@ def test_document_and_search_rest_flow(
|
||||
tmp_path: Path,
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
services, session_factory, storage, vector_store = _build_api_services(tmp_path)
|
||||
services, session_factory, storage, vector_store, runner = _build_api_services(tmp_path)
|
||||
app = create_app(Settings(app_env="test"), api_services=services)
|
||||
|
||||
with TestClient(app) as client:
|
||||
@ -121,19 +179,33 @@ def test_document_and_search_rest_flow(
|
||||
)
|
||||
},
|
||||
)
|
||||
assert upload_response.status_code == 201
|
||||
assert upload_response.status_code == 202
|
||||
uploaded = upload_response.json()
|
||||
document_id = UUID(uploaded["id"])
|
||||
assert uploaded["status"] == "ready"
|
||||
job_id = UUID(uploaded["job_id"])
|
||||
assert uploaded["status"] == "uploaded"
|
||||
assert uploaded["job_status"] == "queued"
|
||||
assert uploaded["filename"] == "退款政策.md"
|
||||
assert "storage_key" not in uploaded
|
||||
|
||||
queued = client.get(f"/api/v1/jobs/{job_id}")
|
||||
assert queued.status_code == 200
|
||||
assert queued.json()["attempt"] == 0
|
||||
assert queued.json()["status"] == "queued"
|
||||
assert client.delete(f"/api/v1/documents/{document_id}").status_code == 409
|
||||
|
||||
runner.run(job_id)
|
||||
|
||||
list_response = client.get("/api/v1/documents")
|
||||
detail_response = client.get(f"/api/v1/documents/{document_id}")
|
||||
assert list_response.status_code == 200
|
||||
assert [item["id"] for item in list_response.json()] == [str(document_id)]
|
||||
assert detail_response.status_code == 200
|
||||
assert detail_response.json() == uploaded
|
||||
assert detail_response.json()["status"] == "ready"
|
||||
completed = client.get(f"/api/v1/jobs/{job_id}").json()
|
||||
assert completed["status"] == "succeeded"
|
||||
assert completed["stage"] == "indexing"
|
||||
assert completed["attempt"] == 1
|
||||
|
||||
search_response = client.post(
|
||||
"/api/v1/search",
|
||||
@ -173,7 +245,7 @@ def test_rest_validation_not_found_and_openapi_contract(
|
||||
tmp_path: Path,
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
services, _, _, _ = _build_api_services(tmp_path)
|
||||
services, _, _, _, _ = _build_api_services(tmp_path)
|
||||
app = create_app(Settings(app_env="test"), api_services=services)
|
||||
missing_id = uuid4()
|
||||
|
||||
@ -182,6 +254,9 @@ def test_rest_validation_not_found_and_openapi_contract(
|
||||
assert set(openapi["paths"]) >= {
|
||||
"/api/v1/documents",
|
||||
"/api/v1/documents/{document_id}",
|
||||
"/api/v1/ingestion-batches",
|
||||
"/api/v1/ingestion-batches/{batch_id}",
|
||||
"/api/v1/jobs/{job_id}",
|
||||
"/api/v1/search",
|
||||
}
|
||||
assert set(openapi["paths"]["/api/v1/documents"]) >= {"get", "post"}
|
||||
@ -204,3 +279,141 @@ def test_rest_validation_not_found_and_openapi_contract(
|
||||
assert boolean_top_k.status_code == 422
|
||||
assert missing_get.status_code == 404
|
||||
assert missing_delete.status_code == 404
|
||||
|
||||
|
||||
def test_batch_upload_isolates_files_and_reports_terminal_status(
|
||||
tmp_path: Path,
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
"""A rejected file must not roll back valid siblings in the same batch."""
|
||||
services, _, _, _, runner = _build_api_services(tmp_path)
|
||||
app = create_app(Settings(app_env="test"), api_services=services)
|
||||
|
||||
with TestClient(app) as client:
|
||||
accepted = client.post(
|
||||
"/api/v1/ingestion-batches",
|
||||
files=[
|
||||
("files", ("alpha.md", b"# Alpha\n\nFirst policy.", "text/markdown")),
|
||||
("files", ("blocked.csv", b"secret=must-not-persist", "text/csv")),
|
||||
("files", ("beta.txt", b"Second policy.", "text/plain")),
|
||||
],
|
||||
)
|
||||
|
||||
assert accepted.status_code == 202
|
||||
body = accepted.json()
|
||||
assert body["status"] == "processing"
|
||||
assert body["total_files"] == 3
|
||||
assert body["accepted_files"] == 2
|
||||
assert body["rejected_files"] == 1
|
||||
assert body["completed_files"] == 1
|
||||
assert [item["filename"] for item in body["items"]] == [
|
||||
"alpha.md",
|
||||
"blocked.csv",
|
||||
"beta.txt",
|
||||
]
|
||||
assert [item["acceptance_status"] for item in body["items"]] == [
|
||||
"accepted",
|
||||
"rejected",
|
||||
"accepted",
|
||||
]
|
||||
assert body["items"][1]["error_message"] == ("unsupported document type or media type")
|
||||
assert "must-not-persist" not in str(body)
|
||||
|
||||
for item in body["items"]:
|
||||
if item["job_id"] is not None:
|
||||
runner.run(UUID(item["job_id"]))
|
||||
|
||||
terminal = client.get(f"/api/v1/ingestion-batches/{body['batch_id']}")
|
||||
|
||||
assert terminal.status_code == 200
|
||||
terminal_body = terminal.json()
|
||||
assert terminal_body["status"] == "partial_failure"
|
||||
assert terminal_body["completed_files"] == 3
|
||||
assert [item["job_status"] for item in terminal_body["items"]] == [
|
||||
"succeeded",
|
||||
None,
|
||||
"succeeded",
|
||||
]
|
||||
|
||||
|
||||
def test_batch_upload_enforces_count_and_shared_byte_limits(
|
||||
tmp_path: Path,
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
"""Request limits are bounded while already accepted siblings remain durable."""
|
||||
services, session_factory, _, _, _ = _build_api_services(
|
||||
tmp_path,
|
||||
max_upload_bytes=10,
|
||||
batch_max_files=2,
|
||||
batch_max_total_bytes=15,
|
||||
)
|
||||
app = create_app(Settings(app_env="test"), api_services=services)
|
||||
|
||||
with TestClient(app) as client:
|
||||
too_many = client.post(
|
||||
"/api/v1/ingestion-batches",
|
||||
files=[
|
||||
("files", ("one.txt", b"1", "text/plain")),
|
||||
("files", ("two.txt", b"2", "text/plain")),
|
||||
("files", ("three.txt", b"3", "text/plain")),
|
||||
],
|
||||
)
|
||||
bounded = client.post(
|
||||
"/api/v1/ingestion-batches",
|
||||
files=[
|
||||
("files", ("first.txt", b"1234567890", "text/plain")),
|
||||
("files", ("second.txt", b"abcdefghij", "text/plain")),
|
||||
],
|
||||
)
|
||||
oversized = client.post(
|
||||
"/api/v1/ingestion-batches",
|
||||
files=[("files", ("large.txt", b"12345678901", "text/plain"))],
|
||||
)
|
||||
|
||||
assert too_many.status_code == 413
|
||||
assert too_many.json()["detail"] == "batch file count exceeds the configured limit"
|
||||
assert bounded.status_code == 202
|
||||
body = bounded.json()
|
||||
assert body["accepted_files"] == 1
|
||||
assert body["rejected_files"] == 1
|
||||
assert body["items"][1]["error_message"] == ("batch exceeds the configured total upload limit")
|
||||
assert oversized.status_code == 202
|
||||
assert oversized.json()["status"] == "failed"
|
||||
assert oversized.json()["accepted_files"] == 0
|
||||
assert oversized.json()["items"][0]["error_message"] == (
|
||||
"document exceeds the configured upload limit"
|
||||
)
|
||||
|
||||
# A count-rejected request creates no batch; the two accepted requests do.
|
||||
with session_factory() as session:
|
||||
assert len(session.scalars(select(IngestionBatch)).all()) == 2
|
||||
|
||||
|
||||
def test_batch_upload_completed_and_missing_contract(
|
||||
tmp_path: Path,
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
"""All-success batches become completed and unknown IDs stay explicit."""
|
||||
services, _, _, _, runner = _build_api_services(tmp_path)
|
||||
app = create_app(Settings(app_env="test"), api_services=services)
|
||||
missing_id = uuid4()
|
||||
|
||||
with TestClient(app) as client:
|
||||
accepted = client.post(
|
||||
"/api/v1/ingestion-batches",
|
||||
files=[
|
||||
("files", ("one.md", b"# One\n\nContent.", "text/markdown")),
|
||||
("files", ("two.txt", b"More content.", "text/plain")),
|
||||
],
|
||||
).json()
|
||||
for item in accepted["items"]:
|
||||
runner.run(UUID(item["job_id"]))
|
||||
|
||||
completed = client.get(f"/api/v1/ingestion-batches/{accepted['batch_id']}")
|
||||
missing = client.get(f"/api/v1/ingestion-batches/{missing_id}")
|
||||
|
||||
assert completed.status_code == 200
|
||||
assert completed.json()["status"] == "completed"
|
||||
assert completed.json()["completed_files"] == 2
|
||||
assert missing.status_code == 404
|
||||
assert missing.json()["detail"] == "ingestion batch not found"
|
||||
|
||||
227
tests/integration/test_mcp_server.py
Normal file
227
tests/integration/test_mcp_server.py
Normal file
@ -0,0 +1,227 @@
|
||||
"""Protocol-level tests for authenticated WorkBuddy-compatible MCP search."""
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass, field
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx2
|
||||
from mcp import ClientSession
|
||||
from mcp.client.streamable_http import streamable_http_client
|
||||
from mcp_types import CallToolResult
|
||||
from pydantic import AnyHttpUrl, SecretStr
|
||||
|
||||
from ekp_core.config import Settings
|
||||
from ekp_mcp.auth import McpIdentity
|
||||
from ekp_mcp.search import McpSearchResponse, McpSourceResponse
|
||||
from ekp_mcp.server import McpConfigurationError, create_mcp_http_app, create_mcp_server
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RecordingSearchApplication:
|
||||
"""Capture the authenticated identity observed at the business boundary."""
|
||||
|
||||
calls: list[tuple[McpIdentity, str, int]] = field(default_factory=list)
|
||||
failure: Exception | None = None
|
||||
empty: bool = False
|
||||
|
||||
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
|
||||
self.calls.append((identity, query, top_k))
|
||||
if self.failure is not None:
|
||||
raise self.failure
|
||||
if self.empty:
|
||||
return McpSearchResponse(results=[])
|
||||
return McpSearchResponse(
|
||||
results=[
|
||||
McpSourceResponse(
|
||||
document_id=uuid4(),
|
||||
filename="退款政策.md",
|
||||
page=2,
|
||||
section="退款政策",
|
||||
chunk_id=uuid4(),
|
||||
content="产品A退款周期为七天。",
|
||||
score=0.8,
|
||||
rerank_score=0.9,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _settings() -> Settings:
|
||||
return Settings(
|
||||
app_env="test",
|
||||
database_url=SecretStr("sqlite+pysqlite:///:memory:"),
|
||||
mcp_auth_token=SecretStr("workbuddy-test-token-at-least-32-chars"),
|
||||
mcp_public_url=AnyHttpUrl("http://testserver/mcp"),
|
||||
mcp_issuer_url=AnyHttpUrl("https://identity.example.test"),
|
||||
mcp_service_identity="workbuddy-team-a",
|
||||
)
|
||||
|
||||
|
||||
async def _exercise_authenticated_gateway(
|
||||
search_application: RecordingSearchApplication,
|
||||
arguments: dict[str, object] | None = None,
|
||||
) -> tuple[list[str], object]:
|
||||
app = create_mcp_http_app(_settings(), search_application=search_application)
|
||||
transport = httpx2.ASGITransport(app=app)
|
||||
async with app.router.lifespan_context(app):
|
||||
async with httpx2.AsyncClient(
|
||||
transport=transport,
|
||||
base_url="http://testserver",
|
||||
headers={"Authorization": "Bearer workbuddy-test-token-at-least-32-chars"},
|
||||
) as client:
|
||||
async with streamable_http_client(
|
||||
"http://testserver/mcp",
|
||||
http_client=client,
|
||||
) as (read_stream, write_stream):
|
||||
async with ClientSession(read_stream, write_stream) as session:
|
||||
await session.initialize()
|
||||
tools = await session.list_tools()
|
||||
result = await session.call_tool(
|
||||
"search_knowledge",
|
||||
arguments or {"query": "产品A退款周期", "top_k": 1},
|
||||
)
|
||||
return [tool.name for tool in tools.tools], result
|
||||
|
||||
|
||||
def test_streamable_http_discovers_only_search_and_propagates_identity() -> None:
|
||||
search_application = RecordingSearchApplication()
|
||||
|
||||
tool_names, result = asyncio.run(_exercise_authenticated_gateway(search_application))
|
||||
|
||||
assert tool_names == ["search_knowledge"]
|
||||
assert isinstance(result, CallToolResult)
|
||||
assert result.is_error is False
|
||||
structured = result.structured_content
|
||||
assert structured is not None
|
||||
assert structured["results"][0]["filename"] == "退款政策.md"
|
||||
assert set(structured["results"][0]) == {
|
||||
"document_id",
|
||||
"filename",
|
||||
"page",
|
||||
"section",
|
||||
"chunk_id",
|
||||
"content",
|
||||
"score",
|
||||
"rerank_score",
|
||||
}
|
||||
identity, query, top_k = search_application.calls[0]
|
||||
assert identity.client_id == "workbuddy-enterprise"
|
||||
assert identity.subject == "workbuddy-team-a"
|
||||
assert identity.scopes == frozenset({"knowledge:search"})
|
||||
assert (query, top_k) == ("产品A退款周期", 1)
|
||||
|
||||
|
||||
def test_streamable_http_returns_an_empty_structured_result() -> None:
|
||||
search_application = RecordingSearchApplication(empty=True)
|
||||
|
||||
_, result = asyncio.run(_exercise_authenticated_gateway(search_application))
|
||||
|
||||
assert isinstance(result, CallToolResult)
|
||||
assert result.is_error is False
|
||||
assert result.structured_content == {"results": []}
|
||||
|
||||
|
||||
def test_streamable_http_rejects_boolean_top_k_before_search() -> None:
|
||||
search_application = RecordingSearchApplication()
|
||||
|
||||
_, result = asyncio.run(
|
||||
_exercise_authenticated_gateway(
|
||||
search_application,
|
||||
{"query": "退款周期", "top_k": True},
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(result, CallToolResult)
|
||||
assert result.is_error is True
|
||||
assert search_application.calls == []
|
||||
|
||||
|
||||
def test_streamable_http_rejects_missing_and_wrong_bearer_tokens() -> None:
|
||||
async def exercise() -> None:
|
||||
app = create_mcp_http_app(_settings(), search_application=RecordingSearchApplication())
|
||||
request = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": "2025-11-25",
|
||||
"capabilities": {},
|
||||
"clientInfo": {"name": "workbuddy-compatibility-test", "version": "1.0"},
|
||||
},
|
||||
}
|
||||
async with app.router.lifespan_context(app):
|
||||
async with httpx2.AsyncClient(
|
||||
transport=httpx2.ASGITransport(app=app),
|
||||
base_url="http://testserver",
|
||||
) as client:
|
||||
missing = await client.post(
|
||||
"/mcp",
|
||||
json=request,
|
||||
headers={"Accept": "application/json, text/event-stream"},
|
||||
)
|
||||
wrong = await client.post(
|
||||
"/mcp",
|
||||
json=request,
|
||||
headers={
|
||||
"Accept": "application/json, text/event-stream",
|
||||
"Authorization": "Bearer wrong-token",
|
||||
},
|
||||
)
|
||||
assert missing.status_code == 401
|
||||
assert wrong.status_code == 401
|
||||
assert "workbuddy-test-token-at-least-32-chars" not in missing.text + wrong.text
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
def test_tool_failure_is_sanitized_for_client_and_logs() -> None:
|
||||
search_application = RecordingSearchApplication(
|
||||
failure=RuntimeError(
|
||||
"provider https://internal.example.test failed with token secret-provider-token"
|
||||
)
|
||||
)
|
||||
|
||||
with patch("ekp_mcp.server.logger.error") as log_error:
|
||||
_, result = asyncio.run(_exercise_authenticated_gateway(search_application))
|
||||
|
||||
assert isinstance(result, CallToolResult)
|
||||
assert result.is_error is True
|
||||
rendered = repr(result)
|
||||
assert "knowledge search failed" in rendered
|
||||
assert "internal.example.test" not in rendered
|
||||
assert "secret-provider-token" not in rendered
|
||||
log_error.assert_called_once_with("MCP knowledge search failed")
|
||||
|
||||
|
||||
def test_transport_rejects_unapproved_origin() -> None:
|
||||
async def exercise() -> None:
|
||||
app = create_mcp_http_app(_settings(), search_application=RecordingSearchApplication())
|
||||
async with app.router.lifespan_context(app):
|
||||
async with httpx2.AsyncClient(
|
||||
transport=httpx2.ASGITransport(app=app),
|
||||
base_url="http://testserver",
|
||||
) as client:
|
||||
response = await client.post(
|
||||
"/mcp",
|
||||
json={"jsonrpc": "2.0", "id": 1, "method": "ping"},
|
||||
headers={
|
||||
"Accept": "application/json, text/event-stream",
|
||||
"Authorization": "Bearer workbuddy-test-token-at-least-32-chars",
|
||||
"Origin": "https://unapproved.example.test",
|
||||
},
|
||||
)
|
||||
assert response.status_code == 403
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
def test_gateway_requires_a_strong_runtime_token() -> None:
|
||||
settings = _settings().model_copy(update={"mcp_auth_token": SecretStr("short")})
|
||||
|
||||
try:
|
||||
create_mcp_server(settings, search_application=RecordingSearchApplication())
|
||||
except McpConfigurationError as error:
|
||||
assert "32 characters" in str(error)
|
||||
else: # pragma: no cover - defensive assertion for fail-open regressions.
|
||||
raise AssertionError("MCP gateway accepted a weak runtime token")
|
||||
57
tests/unit/test_celery_queue.py
Normal file
57
tests/unit/test_celery_queue.py
Normal file
@ -0,0 +1,57 @@
|
||||
"""Unit tests for the Celery adapter without opening a broker connection."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from pydantic import SecretStr
|
||||
|
||||
from ekp_api.celery_queue import (
|
||||
INGESTION_TASK_NAME,
|
||||
RECOVERY_TASK_NAME,
|
||||
CeleryIngestionDispatcher,
|
||||
create_celery_app,
|
||||
)
|
||||
from ekp_core.config import Settings
|
||||
|
||||
|
||||
def test_celery_app_enables_late_ack_and_recovery_schedule(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
settings = Settings(rabbitmq_url=SecretStr("amqp://user:test@queue.example/ekp"))
|
||||
|
||||
app = create_celery_app(settings)
|
||||
|
||||
assert app.conf.task_acks_late is True
|
||||
assert app.conf.task_reject_on_worker_lost is True
|
||||
assert app.conf.worker_prefetch_multiplier == 1
|
||||
assert app.conf.task_routes[INGESTION_TASK_NAME]["queue"] == "ingestion"
|
||||
assert app.conf.task_routes[RECOVERY_TASK_NAME]["queue"] == "ingestion"
|
||||
assert app.conf.beat_schedule["recover-durable-ingestion-jobs"]["schedule"] == 30.0
|
||||
|
||||
|
||||
def test_celery_dispatcher_sends_only_job_identifier(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
app = create_celery_app(Settings(rabbitmq_url=SecretStr("amqp://user:test@queue/ekp")))
|
||||
send_task = Mock()
|
||||
monkeypatch.setattr(app, "send_task", send_task)
|
||||
job_id = uuid4()
|
||||
|
||||
CeleryIngestionDispatcher(app).enqueue(job_id)
|
||||
|
||||
send_task.assert_called_once_with(
|
||||
INGESTION_TASK_NAME,
|
||||
args=[str(job_id)],
|
||||
queue="ingestion",
|
||||
)
|
||||
|
||||
|
||||
def test_celery_app_requires_runtime_broker_secret(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="RABBITMQ_URL"):
|
||||
create_celery_app(Settings())
|
||||
with pytest.raises(ValueError, match="RABBITMQ_URL"):
|
||||
create_celery_app(Settings(rabbitmq_url=SecretStr("")))
|
||||
@ -14,12 +14,20 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
|
||||
assert settings.app_env == "development"
|
||||
assert settings.api_prefix == "/api/v1"
|
||||
assert settings.api_port == 8000
|
||||
assert settings.admin_web_root is None
|
||||
assert settings.database_url.get_secret_value().startswith("mysql+pymysql://")
|
||||
assert settings.database_pool_size == 10
|
||||
assert settings.database_max_overflow == 10
|
||||
assert settings.database_pool_recycle_seconds == 600
|
||||
assert settings.database_pool_timeout_seconds == 30
|
||||
assert settings.database_connect_timeout_seconds == 30
|
||||
assert settings.rabbitmq_url is None
|
||||
assert settings.ingestion_max_attempts == 3
|
||||
assert settings.ingestion_retry_backoff_seconds == 2
|
||||
assert settings.ingestion_retry_backoff_max_seconds == 60
|
||||
assert settings.ingestion_worker_lease_seconds == 300
|
||||
assert settings.batch_max_files == 20
|
||||
assert settings.batch_max_total_bytes == 200 * 1024 * 1024
|
||||
assert settings.object_storage_provider == "bos"
|
||||
assert settings.object_storage_path == Path("storage")
|
||||
assert settings.bos_endpoint is None
|
||||
@ -52,6 +60,14 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
|
||||
assert settings.reranker_endpoint.endswith("/v2/rerank")
|
||||
assert settings.reranker_api_key is None
|
||||
assert settings.search_top_k == 5
|
||||
assert settings.mcp_host == "127.0.0.1"
|
||||
assert settings.mcp_port == 8766
|
||||
assert settings.mcp_path == "/mcp"
|
||||
assert settings.mcp_auth_token is None
|
||||
assert settings.mcp_client_id == "workbuddy-enterprise"
|
||||
assert settings.mcp_service_identity == "workbuddy-demo"
|
||||
assert settings.mcp_required_scope == "knowledge:search"
|
||||
assert "localhost:*" in settings.mcp_allowed_hosts
|
||||
|
||||
|
||||
def test_settings_reject_invalid_port(isolated_settings_environment: None) -> None:
|
||||
@ -81,6 +97,8 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) -
|
||||
bos_secret_access_key=SecretStr("bos-secret-key"),
|
||||
oceanbase_vector_password=SecretStr("oceanbase-secret"),
|
||||
reranker_api_key=SecretStr("reranker-secret"),
|
||||
mcp_auth_token=SecretStr("mcp-secret"),
|
||||
rabbitmq_url=SecretStr("amqp://user:rabbit-secret@queue.example/ekp"),
|
||||
)
|
||||
|
||||
rendered = repr(settings)
|
||||
@ -90,6 +108,8 @@ def test_settings_redact_provider_secrets(isolated_settings_environment: None) -
|
||||
assert "bos-secret-key" not in rendered
|
||||
assert "oceanbase-secret" not in rendered
|
||||
assert "reranker-secret" not in rendered
|
||||
assert "mcp-secret" not in rendered
|
||||
assert "rabbit-secret" not in rendered
|
||||
|
||||
|
||||
def test_settings_restrict_sqlite_to_test_environment(
|
||||
@ -110,3 +130,40 @@ def test_settings_reject_non_oceanbase_runtime_dialect(
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match="OceanBase"):
|
||||
Settings(database_url=SecretStr("postgresql+psycopg://localhost/ekp"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mcp_path", ["mcp", "/mcp/"])
|
||||
def test_settings_reject_invalid_mcp_path(
|
||||
isolated_settings_environment: None,
|
||||
mcp_path: str,
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match="mcp_path"):
|
||||
Settings(mcp_path=mcp_path)
|
||||
|
||||
|
||||
def test_settings_reject_empty_mcp_allowed_hosts(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match="mcp_allowed_hosts"):
|
||||
Settings(mcp_allowed_hosts=[])
|
||||
|
||||
|
||||
def test_settings_reject_retry_backoff_larger_than_cap(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match="initial backoff"):
|
||||
Settings(ingestion_retry_backoff_seconds=10, ingestion_retry_backoff_max_seconds=5)
|
||||
|
||||
|
||||
def test_settings_require_worker_lease_longer_than_provider_timeout(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match="worker_lease"):
|
||||
Settings(ingestion_worker_lease_seconds=30, embedding_timeout_seconds=30)
|
||||
|
||||
|
||||
def test_settings_require_batch_limit_to_fit_one_file(
|
||||
isolated_settings_environment: None,
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match="batch_max_total_bytes"):
|
||||
Settings(max_upload_bytes=100, batch_max_total_bytes=99)
|
||||
|
||||
154
tests/unit/test_helm_deployment.py
Normal file
154
tests/unit/test_helm_deployment.py
Normal file
@ -0,0 +1,154 @@
|
||||
"""Static and rendered contracts for the DEPLOY-002 Helm delivery."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CHART = ROOT / "deploy" / "helm" / "knowledge-platform"
|
||||
EXAMPLE_VALUES = ROOT / "deploy" / "helm" / "environments" / "values.example.yaml"
|
||||
|
||||
|
||||
def _helm(*arguments: str, expect_success: bool = True) -> subprocess.CompletedProcess[str]:
|
||||
"""Run Helm when installed; static checks still run on Python-only hosts."""
|
||||
executable = shutil.which("helm")
|
||||
if executable is None:
|
||||
pytest.skip("Helm is required for rendered chart assertions")
|
||||
result = subprocess.run(
|
||||
[executable, *arguments],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if expect_success and result.returncode != 0:
|
||||
pytest.fail(f"Helm failed: {result.stderr or result.stdout}")
|
||||
return result
|
||||
|
||||
|
||||
def _render(*extra_arguments: str) -> str:
|
||||
"""Render deterministic manifests without contacting a cluster."""
|
||||
result = _helm(
|
||||
"template",
|
||||
"knowledge-platform",
|
||||
str(CHART),
|
||||
"--namespace",
|
||||
"knowledge-platform",
|
||||
"--values",
|
||||
str(EXAMPLE_VALUES),
|
||||
*extra_arguments,
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def test_chart_has_machine_validated_values_schemas() -> None:
|
||||
"""Both the application and optional broker expose valid JSON Schemas."""
|
||||
for schema_path in (
|
||||
CHART / "values.schema.json",
|
||||
CHART / "charts" / "rabbitmq" / "values.schema.json",
|
||||
):
|
||||
schema = json.loads(schema_path.read_text())
|
||||
assert schema["$schema"] == "https://json-schema.org/draft-07/schema#"
|
||||
assert schema["type"] == "object"
|
||||
|
||||
|
||||
def test_default_chart_renders_secret_only_sensitive_configuration() -> None:
|
||||
"""Credentials are references; the chart never creates or embeds a Secret."""
|
||||
rendered = _render()
|
||||
|
||||
assert "kind: Secret\n" not in rendered
|
||||
assert "secretRef:\n name: knowledge-platform-runtime" in rendered
|
||||
assert "key: DATABASE_URL" in rendered
|
||||
assert "helm.sh/hook: pre-install,pre-upgrade" in rendered
|
||||
assert 'args: ["upgrade", "head"]' in rendered
|
||||
assert "value: mysql+pymysql://" not in rendered
|
||||
assert "change-me" not in rendered
|
||||
|
||||
|
||||
def test_workloads_render_availability_and_restricted_pod_defaults() -> None:
|
||||
"""API/MCP are multi-replica and every application process is non-root."""
|
||||
rendered = _render()
|
||||
|
||||
assert "name: knowledge-platform-api" in rendered
|
||||
assert "name: knowledge-platform-mcp" in rendered
|
||||
assert rendered.count("replicas: 2") >= 3
|
||||
assert rendered.count("readOnlyRootFilesystem: true") >= 5
|
||||
assert rendered.count("allowPrivilegeEscalation: false") >= 5
|
||||
assert rendered.count("automountServiceAccountToken: false") >= 6
|
||||
assert "maxUnavailable: 0" in rendered
|
||||
assert "readinessProbe:" in rendered
|
||||
assert "livenessProbe:" in rendered
|
||||
assert "startupProbe:" in rendered
|
||||
|
||||
|
||||
def test_scheduler_is_singleton_and_workers_do_not_embed_beat() -> None:
|
||||
"""Scaled workers cannot accidentally create multiple periodic schedulers."""
|
||||
rendered = _render()
|
||||
|
||||
scheduler = rendered[rendered.index("name: knowledge-platform-scheduler") :]
|
||||
scheduler = scheduler[: scheduler.index("---", 1)]
|
||||
worker = rendered[rendered.index("name: knowledge-platform-worker") :]
|
||||
worker = worker[: worker.index("---", 1)]
|
||||
assert "replicas: 1" in scheduler
|
||||
assert "type: Recreate" in scheduler
|
||||
assert "- beat" in scheduler
|
||||
assert "- worker" in worker
|
||||
assert "- beat" not in worker
|
||||
|
||||
|
||||
def test_ingress_and_optional_rabbitmq_are_explicitly_renderable() -> None:
|
||||
"""TLS routes and the persistent demo broker remain opt-in values."""
|
||||
rendered = _render("--set", "rabbitmq.enabled=true")
|
||||
|
||||
assert "kind: Ingress" in rendered
|
||||
assert "secretName: knowledge-platform-api-tls" in rendered
|
||||
assert "secretName: knowledge-platform-mcp-tls" in rendered
|
||||
assert "kind: StatefulSet" in rendered
|
||||
assert "volumeClaimTemplates:" in rendered
|
||||
assert "name: RABBITMQ_DEFAULT_PASS" in rendered
|
||||
assert "valueFrom:" in rendered
|
||||
|
||||
|
||||
def test_values_schema_rejects_latest_image_tags() -> None:
|
||||
"""Cluster releases require a non-latest immutable tag or digest."""
|
||||
result = _helm(
|
||||
"lint",
|
||||
str(CHART),
|
||||
"--set",
|
||||
"images.api.tag=latest",
|
||||
expect_success=False,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "/images/api/tag" in (result.stdout + result.stderr)
|
||||
|
||||
|
||||
def test_cross_field_validation_rejects_unsafe_ingestion_limits() -> None:
|
||||
"""Rendered settings preserve the application's own configuration rules."""
|
||||
result = _helm(
|
||||
"template",
|
||||
"knowledge-platform",
|
||||
str(CHART),
|
||||
"--set",
|
||||
"config.ingestion.batchMaxTotalBytes=1",
|
||||
expect_success=False,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "batchMaxTotalBytes must be >= maxUploadBytes" in (result.stdout + result.stderr)
|
||||
|
||||
|
||||
def test_deploy_entrypoint_uses_atomic_upgrade_and_never_reads_secret_values() -> None:
|
||||
"""The one-command wrapper checks references and performs an atomic wait."""
|
||||
script = (ROOT / "scripts" / "deploy.sh").read_text()
|
||||
|
||||
assert "helm upgrade --install" in script
|
||||
assert "--atomic" in script
|
||||
assert "--rollback-on-failure" in script
|
||||
assert "--wait" in script
|
||||
assert 'kubectl -n "${namespace}" get secret' in script
|
||||
assert 'awk \'$1 == "runtimeSecret:"' in script
|
||||
assert "DATABASE_URL}'" not in script
|
||||
38
tests/unit/test_mcp_auth.py
Normal file
38
tests/unit/test_mcp_auth.py
Normal file
@ -0,0 +1,38 @@
|
||||
"""Tests for the DEMO-012 service-token identity boundary."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from pydantic import SecretStr
|
||||
|
||||
from ekp_mcp.auth import StaticServiceTokenVerifier
|
||||
|
||||
|
||||
def test_static_service_token_maps_to_scoped_identity_without_secret_repr() -> None:
|
||||
verifier = StaticServiceTokenVerifier(
|
||||
SecretStr("runtime-only-token"),
|
||||
client_id="workbuddy-enterprise",
|
||||
subject="workbuddy-team-a",
|
||||
scope="knowledge:search",
|
||||
issuer="https://identity.example.test",
|
||||
)
|
||||
|
||||
access_token = asyncio.run(verifier.verify_token("runtime-only-token"))
|
||||
|
||||
assert access_token is not None
|
||||
assert access_token.client_id == "workbuddy-enterprise"
|
||||
assert access_token.subject == "workbuddy-team-a"
|
||||
assert access_token.scopes == ["knowledge:search"]
|
||||
assert access_token.claims == {"iss": "https://identity.example.test"}
|
||||
assert "runtime-only-token" not in repr(verifier)
|
||||
|
||||
|
||||
def test_static_service_token_rejects_wrong_token() -> None:
|
||||
verifier = StaticServiceTokenVerifier(
|
||||
SecretStr("expected"),
|
||||
client_id="workbuddy-enterprise",
|
||||
subject="workbuddy-team-a",
|
||||
scope="knowledge:search",
|
||||
issuer="https://identity.example.test",
|
||||
)
|
||||
|
||||
assert asyncio.run(verifier.verify_token("wrong")) is None
|
||||
72
tests/unit/test_mcp_search.py
Normal file
72
tests/unit/test_mcp_search.py
Normal file
@ -0,0 +1,72 @@
|
||||
"""Tests for MCP identity authorization and SearchService result mapping."""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from ekp_api.schemas import SearchResponse
|
||||
from ekp_core.search import SearchResult, SourceResult
|
||||
from ekp_mcp.auth import McpIdentity
|
||||
from ekp_mcp.search import McpSearchApplication
|
||||
|
||||
|
||||
class StubSearcher:
|
||||
"""Return one canonical source while recording the delegated query."""
|
||||
|
||||
def __init__(self, result: SearchResult) -> None:
|
||||
self.result = result
|
||||
self.calls: list[tuple[str, int | None]] = []
|
||||
|
||||
def search(self, query: str, top_k: int | None = None) -> SearchResult:
|
||||
self.calls.append((query, top_k))
|
||||
return self.result
|
||||
|
||||
|
||||
def _search_result() -> SearchResult:
|
||||
return SearchResult(
|
||||
query="退款周期",
|
||||
index_profile_id="internal-profile-must-not-leak",
|
||||
results=(
|
||||
SourceResult(
|
||||
document_id=uuid4(),
|
||||
filename="退款政策.md",
|
||||
page=2,
|
||||
section="退款政策",
|
||||
chunk_id=uuid4(),
|
||||
content="退款周期为七天。",
|
||||
score=0.8,
|
||||
rerank_score=0.9,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_mcp_search_requires_scope_and_hides_index_profile() -> None:
|
||||
searcher = StubSearcher(_search_result())
|
||||
application = McpSearchApplication(searcher, "knowledge:search")
|
||||
identity = McpIdentity(
|
||||
client_id="workbuddy-enterprise",
|
||||
subject="workbuddy-team-a",
|
||||
scopes=frozenset({"knowledge:search"}),
|
||||
)
|
||||
|
||||
response = application.search(identity, "退款周期", 1)
|
||||
|
||||
assert searcher.calls == [("退款周期", 1)]
|
||||
payload = response.model_dump(mode="json")
|
||||
rest_payload = SearchResponse.from_result(searcher.result).model_dump(mode="json")
|
||||
assert payload["results"][0]["filename"] == "退款政策.md"
|
||||
assert payload == rest_payload
|
||||
assert "index_profile_id" not in payload
|
||||
|
||||
|
||||
def test_mcp_search_rejects_identity_without_required_scope() -> None:
|
||||
application = McpSearchApplication(StubSearcher(_search_result()), "knowledge:search")
|
||||
identity = McpIdentity(
|
||||
client_id="unprivileged-client",
|
||||
subject="unprivileged-service",
|
||||
scopes=frozenset(),
|
||||
)
|
||||
|
||||
with pytest.raises(PermissionError, match="knowledge search scope"):
|
||||
application.search(identity, "退款周期", 1)
|
||||
65
tests/unit/test_production_images.py
Normal file
65
tests/unit/test_production_images.py
Normal file
@ -0,0 +1,65 @@
|
||||
"""Static production-image contract tests that do not require a Docker daemon."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
DOCKERFILE = PROJECT_ROOT / "deploy/docker/Dockerfile"
|
||||
BAKE_FILE = PROJECT_ROOT / "deploy/docker/docker-bake.hcl"
|
||||
|
||||
|
||||
def test_runtime_is_locked_non_root_and_secret_free() -> None:
|
||||
"""Guard the supply-chain and least-privilege choices in the shared runtime."""
|
||||
dockerfile = DOCKERFILE.read_text()
|
||||
|
||||
assert "python:3.12.14-slim-bookworm" in dockerfile
|
||||
assert "ghcr.io/astral-sh/uv:0.12.5" in dockerfile
|
||||
assert dockerfile.count("@sha256:") == 2
|
||||
assert "uv sync --frozen --no-dev --no-editable" in dockerfile
|
||||
assert "USER 10001:10001" in dockerfile
|
||||
assert "COPY . " not in dockerfile
|
||||
assert all(
|
||||
sensitive not in dockerfile
|
||||
for sensitive in ("DATABASE_URL=", "RABBITMQ_URL=", "API_KEY=", "PASSWORD=")
|
||||
)
|
||||
|
||||
|
||||
def test_three_process_targets_have_healthchecks_and_exec_commands() -> None:
|
||||
"""Every independently deployed process must have an explicit runtime contract."""
|
||||
dockerfile = DOCKERFILE.read_text()
|
||||
|
||||
assert dockerfile.count("HEALTHCHECK ") == 3
|
||||
assert "FROM runtime AS api" in dockerfile
|
||||
assert "FROM runtime AS mcp" in dockerfile
|
||||
assert "FROM runtime AS worker" in dockerfile
|
||||
assert 'CMD ["uvicorn", "ekp_api.main:app"' in dockerfile
|
||||
assert 'CMD ["ekp-mcp"]' in dockerfile
|
||||
assert 'CMD ["ekp-worker"]' in dockerfile
|
||||
|
||||
|
||||
def test_bake_plan_covers_both_architectures_and_all_images() -> None:
|
||||
"""The checked-in build plan must stay suitable for amd64 and arm64 clusters."""
|
||||
bake = BAKE_FILE.read_text()
|
||||
|
||||
assert 'platforms = ["linux/amd64", "linux/arm64"]' in bake
|
||||
assert 'context = "."' in bake
|
||||
assert 'targets = ["api", "mcp", "worker"]' in bake
|
||||
assert bake.count('inherits = ["common"]') == 3
|
||||
assert all(
|
||||
image in bake
|
||||
for image in (
|
||||
"enterprise-knowledge-api",
|
||||
"enterprise-knowledge-mcp",
|
||||
"enterprise-knowledge-worker",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_docker_context_excludes_credentials_and_local_state() -> None:
|
||||
"""Prevent common developer files from entering any build layer."""
|
||||
ignored = (PROJECT_ROOT / ".dockerignore").read_text().splitlines()
|
||||
|
||||
assert ".env" in ignored
|
||||
assert ".git" in ignored
|
||||
assert ".venv" in ignored
|
||||
assert "storage" in ignored
|
||||
assert "tests" in ignored
|
||||
16
tests/unit/test_workbuddy_mcp_config.py
Normal file
16
tests/unit/test_workbuddy_mcp_config.py
Normal file
@ -0,0 +1,16 @@
|
||||
"""Validate the checked-in WorkBuddy configuration without embedding credentials."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
WORKBUDDY_CONFIG = Path(__file__).resolve().parents[2] / "deploy/workbuddy-mcp.example.json"
|
||||
|
||||
|
||||
def test_workbuddy_http_config_uses_runtime_bearer_token() -> None:
|
||||
payload = json.loads(WORKBUDDY_CONFIG.read_text(encoding="utf-8"))
|
||||
|
||||
server = payload["mcpServers"]["enterprise-knowledge"]
|
||||
assert server["type"] == "http"
|
||||
assert server["url"].endswith("/mcp}")
|
||||
assert server["headers"] == {"Authorization": "Bearer ${EKP_MCP_TOKEN}"}
|
||||
assert "runtime-secret" not in WORKBUDDY_CONFIG.read_text(encoding="utf-8")
|
||||
421
uv.lock
generated
421
uv.lock
generated
@ -28,6 +28,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/20/89/e62cc37b69ad357cc8ecd6e7367f5245f523d3cbb338a66197212bdf6749/alembic-1.19.1-py3-none-any.whl", hash = "sha256:b39018cb3d9413a19cbd54cf3c02ad33998641f0538eb77413a488a21c3e14be", size = 265946, upload-time = "2026-08-08T16:32:03.153Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "amqp"
|
||||
version = "5.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "vine" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "annotated-doc"
|
||||
version = "0.0.5"
|
||||
@ -59,6 +71,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "attrs"
|
||||
version = "26.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bce-python-sdk"
|
||||
version = "0.9.79"
|
||||
@ -74,6 +95,72 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/de/52b763b183e23f4dd73811a2dd2664824ae7fc6ceb1aa8dcd3f1102b1094/bce_python_sdk-0.9.79-py3-none-any.whl", hash = "sha256:71799ac8740505e0759d30873f6f1a478fa8f83aedf425d511f1419a5f30082e", size = 440268, upload-time = "2026-09-01T05:48:50.459Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "billiard"
|
||||
version = "4.2.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "celery"
|
||||
version = "5.6.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "billiard" },
|
||||
{ name = "click" },
|
||||
{ name = "click-didyoumean" },
|
||||
{ name = "click-plugins" },
|
||||
{ name = "click-repl" },
|
||||
{ name = "kombu" },
|
||||
{ name = "python-dateutil" },
|
||||
{ name = "tzlocal" },
|
||||
{ name = "vine" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cffi"
|
||||
version = "2.1.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pycparser", marker = "implementation_name != 'PyPy'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/55/41/4c7042f317b9217502988f0873af87e16ad606dc20f84e546e3e6ce9764c/cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971", size = 184764, upload-time = "2026-08-03T21:20:03.141Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/43/1f/1c3d90d91811c8f86ced9ed637956c54bfe5b79ca98fe976d7f8c8979f6b/cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c", size = 214722, upload-time = "2026-08-03T21:20:04.377Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/37/6f/3b5ce4c3b2192d250f04908f2bfd91ef34552ec8f7716a5d4abdb8d67bb2/cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125", size = 222369, upload-time = "2026-08-03T21:20:05.544Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/02/10/4b3c75dde3d9663c9e02ba05c2668b954f671d4bbe346413ca8c696b295a/cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264", size = 210175, upload-time = "2026-08-03T21:20:06.75Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/df/62/14f74b9543e605d17701dc797b815958b8bb70b7624ce1b832ddad48ed6c/cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3", size = 208670, upload-time = "2026-08-03T21:20:08.04Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/95/86342356ff5953b3fb06f7ef7c5bee212d45e770abc7218d451b9148313c/cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2", size = 221824, upload-time = "2026-08-03T21:20:09.274Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/ff/7b3429ff53aafe931ed8a5fc69f481bbef7ba6de87ddcbb63d08f483f613/cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b", size = 225148, upload-time = "2026-08-03T21:20:10.7Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/34/34/a95870b9221e09cf4f2ce3178b1a210abdfe63a1bd357da940418d7b8d15/cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7", size = 223564, upload-time = "2026-08-03T21:20:12.165Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/70/ea/839b50531021a647fb5e929f72cf97bc1ff702b5472166164b5b6e76b851/cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac", size = 175263, upload-time = "2026-08-03T21:20:13.559Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/60/a6/8b149b2c3f2e11aaa1618ef64500b45f50f22c57a977a4dff1aff1f91042/cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d", size = 185688, upload-time = "2026-08-03T21:20:14.69Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/01/9a/11f687cb39d6a3504060d5242f04f48c735afb4d3d533958a20594890cb2/cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973", size = 180078, upload-time = "2026-08-03T21:20:15.917Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "click"
|
||||
version = "8.5.0"
|
||||
@ -83,6 +170,43 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "click-didyoumean"
|
||||
version = "0.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "click" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "click-plugins"
|
||||
version = "1.1.1.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "click" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "click-repl"
|
||||
version = "0.3.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "click" },
|
||||
{ name = "prompt-toolkit" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colorama"
|
||||
version = "0.4.6"
|
||||
@ -159,6 +283,43 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/b2/6166c789ebacfef08b5752996f79d1e472644e9b393ed7c87b1997686910/crc32c-2.9-cp313-cp313-win_arm64.whl", hash = "sha256:94e9646a05951c7cf0657fee48eec85bf669cbf9b7bec09eff02ae12da2e9ca7", size = 63356, upload-time = "2026-08-21T03:56:25.304Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cryptography"
|
||||
version = "50.0.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cffi", marker = "platform_python_implementation != 'PyPy'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bb/ad/5d6702db60b1e40b41ef513b6967ff5848f307d50f8449baf1634f5908f1/cryptography-50.0.1.tar.gz", hash = "sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20", size = 880381, upload-time = "2026-08-25T19:45:45.499Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/19/797e2aaac9df6a66f1550f49979dc1b1e39ecd2077501c30efa81e8d5d67/cryptography-50.0.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986", size = 4010153, upload-time = "2026-08-25T19:44:03.155Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/90/34/9ce9a62ed9dc82ca9fd6a34445b6904af56e5f38b3eae2ed32e49c36053d/cryptography-50.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f", size = 4723133, upload-time = "2026-08-25T19:44:05.461Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/57/26/e6d4fc8512a51a5f9ee7bfdbfb853bce1197087df40c9ad993ad370b846f/cryptography-50.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef", size = 4712478, upload-time = "2026-08-25T19:44:07.375Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/de/d3cdc2815697aae84126cbd6a030ca7b6b452e28a88b501b836bd3aa7a86/cryptography-50.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8", size = 4730726, upload-time = "2026-08-25T19:44:09.294Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/55/32/38c0d344b98c06d34b5df8946565a9c0d6dbf32c8e0730a7f05f0a3c6cab/cryptography-50.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45", size = 5353524, upload-time = "2026-08-25T19:44:11.96Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/1b/82f0f0d8858d4432be1af790477edf62aef90324041aa07c57e57bef1af7/cryptography-50.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad", size = 4746720, upload-time = "2026-08-25T19:44:14.051Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/29/ba/042ca458b8c64348c768284b5d23e69b92ed53d057ab779fee628564676d/cryptography-50.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49", size = 4361866, upload-time = "2026-08-25T19:44:16.167Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/39/3b/e96c1ef71edef71057c7e3c3d982ce8fda554e0c52d0cc19c18845cde3eb/cryptography-50.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f", size = 4730028, upload-time = "2026-08-25T19:44:18.085Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/38/45abd72ef63f2e7d0754a6cacf97bd8b69512ace7f6130d24c39ece65da2/cryptography-50.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527", size = 5308405, upload-time = "2026-08-25T19:44:20.197Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/66/6ccca4722987ddedaa7fc9c3f4708af7431f5535666c174350830888c6b7/cryptography-50.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a", size = 4746230, upload-time = "2026-08-25T19:44:22.376Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/0e/b1f92e013228111413f2e6743948b80bc24dfd3c1b87ba98ceea16f5df89/cryptography-50.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959", size = 4862596, upload-time = "2026-08-25T19:44:24.472Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/22/c3654cccc856e9d682817b04ac3ee79731cb09ca6f95996a95c904de2883/cryptography-50.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b", size = 5014082, upload-time = "2026-08-25T19:44:26.709Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/8b/cb12b1b60c91b074ca6bf0fdd59aa8f10d8bc5f73af8faece86ef0421b37/cryptography-50.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648", size = 3842826, upload-time = "2026-08-25T19:44:28.784Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/a9/ee16a903f13755e914d1eecc482fe64d1f10761c3960e5d8fa6837377aff/cryptography-50.0.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0", size = 4035307, upload-time = "2026-08-25T19:44:58.305Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/a5/9ec7e81e8526c0d7a387d73386b2daed3f39e10d81a85930bd1b6bfba65c/cryptography-50.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23", size = 4751900, upload-time = "2026-08-25T19:45:00.401Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/3c/0e77bd5ffcf078e9dd27d3074aad6c030d9b10d0bf69329d573c927a188c/cryptography-50.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733", size = 4738357, upload-time = "2026-08-25T19:45:02.786Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/3a/3c5f80daa4dcd47323c7af8a2fcb90de27a33564d4fcac69846c0972691a/cryptography-50.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88", size = 4758474, upload-time = "2026-08-25T19:45:04.889Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/2b/214cf0cf93db9628c3c20c896b229f327f6fb1b20e4b3743d8ad3f00af8b/cryptography-50.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054", size = 5375862, upload-time = "2026-08-25T19:45:07.163Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/51/3f9701867a46b6c1740c9b52fc4d3bed6cbdcfedcc9b6e64305c07f39cff/cryptography-50.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5", size = 4772942, upload-time = "2026-08-25T19:45:09.396Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/5c/13ea642e08e2544d0f5396122055f4820cfacb3203562197b5967125ea97/cryptography-50.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361", size = 4383347, upload-time = "2026-08-25T19:45:11.659Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/d5/7d1fe1cb93f91c428093ff234e128c89ba8ea61a6f26aab406081f9b996e/cryptography-50.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71", size = 4758050, upload-time = "2026-08-25T19:45:13.745Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/04/557fc5ead96a829e0bc812a3b9dc4a52a2f27e4f7f5950da7ff27653a805/cryptography-50.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80", size = 5332955, upload-time = "2026-08-25T19:45:16.193Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/eb/5d7124083e8d8cda8f5b348f544b71ad6f707ad63193758ef4d8e569da02/cryptography-50.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239", size = 4772694, upload-time = "2026-08-25T19:45:18.315Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/63/8e/f1f955e0921dd2b6d22eae7e8d24a4c4b638d10735ffbf6a71f99eb0fcb8/cryptography-50.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558", size = 4888413, upload-time = "2026-08-25T19:45:20.4Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1f/ab/89e2b798d2c3925f82e2bb72d5979f3d2f6da2dd22ef4a8cd8b70d920039/cryptography-50.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e", size = 5044355, upload-time = "2026-08-25T19:45:22.353Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/99/89/87ef49ffe383ef4e147d27b7bf2088fb0b54ea409dd87b5a89442e5828a5/cryptography-50.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2", size = 3875429, upload-time = "2026-08-25T19:45:24.418Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "enterprise-knowledge-platform"
|
||||
version = "0.1.0"
|
||||
@ -166,7 +327,9 @@ source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "alembic" },
|
||||
{ name = "bce-python-sdk" },
|
||||
{ name = "celery" },
|
||||
{ name = "fastapi" },
|
||||
{ name = "mcp" },
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "pymupdf" },
|
||||
{ name = "pymysql" },
|
||||
@ -190,7 +353,9 @@ dev = [
|
||||
requires-dist = [
|
||||
{ name = "alembic", specifier = ">=1.18,<2" },
|
||||
{ name = "bce-python-sdk", specifier = ">=0.9.72,<0.10" },
|
||||
{ name = "celery", specifier = ">=5.5,<6" },
|
||||
{ name = "fastapi", specifier = ">=0.115,<1" },
|
||||
{ name = "mcp", specifier = ">=2,<3" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.7,<3" },
|
||||
{ name = "pymupdf", specifier = ">=1.26,<2" },
|
||||
{ name = "pymysql", specifier = ">=1.1,<2" },
|
||||
@ -347,6 +512,48 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema"
|
||||
version = "4.26.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "attrs" },
|
||||
{ name = "jsonschema-specifications" },
|
||||
{ name = "referencing" },
|
||||
{ name = "rpds-py" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonschema-specifications"
|
||||
version = "2025.9.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "referencing" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kombu"
|
||||
version = "5.6.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "amqp" },
|
||||
{ name = "packaging" },
|
||||
{ name = "tzdata" },
|
||||
{ name = "vine" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "librt"
|
||||
version = "0.15.0"
|
||||
@ -483,6 +690,44 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mcp"
|
||||
version = "2.1.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "anyio" },
|
||||
{ name = "httpx2" },
|
||||
{ name = "jsonschema" },
|
||||
{ name = "mcp-types" },
|
||||
{ name = "opentelemetry-api" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pyjwt", extra = ["crypto"] },
|
||||
{ name = "python-multipart" },
|
||||
{ name = "pywin32", marker = "sys_platform == 'win32'" },
|
||||
{ name = "sse-starlette" },
|
||||
{ name = "starlette" },
|
||||
{ name = "typing-extensions" },
|
||||
{ name = "typing-inspection" },
|
||||
{ name = "uvicorn", marker = "sys_platform != 'emscripten'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d4/6e/21fb8e5d579dbe21d96ea4d5034200d46d8bdf2261053b5bd041f3c2f612/mcp-2.1.1.tar.gz", hash = "sha256:50b7ba1ebbe117008ea7bdd288234043e69c20b403d6851d19661e6d431a75ef", size = 3984589, upload-time = "2026-08-25T16:14:02.376Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/50/af/8644cc5fa26a59afd2df2e98eeb19e72926887fa4b7441aba4ff661140db/mcp-2.1.1-py3-none-any.whl", hash = "sha256:1c6c31c5d6471c58db76af3af8af67f46d11d01f0a59077d0a308cbdb3d3e915", size = 357912, upload-time = "2026-08-25T16:13:59.024Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mcp-types"
|
||||
version = "2.1.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pydantic" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6a/dd/1c4417dc0b722c23a1669032d5f044e41170fe5d4773b488a50fcce98c32/mcp_types-2.1.1.tar.gz", hash = "sha256:77dcbe48fba73cca71a673f2646a5f037a017b7a0a07ac89cec1113028890eda", size = 66674, upload-time = "2026-08-25T16:14:03.861Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/71/d0/242e63c510f4a17381f55b1549a3f94f5687a0595984febd2b6f87a687a0/mcp_types-2.1.1-py3-none-any.whl", hash = "sha256:26f9f7f03f2a5730717a5b98e2ab7eb640ac352d05a00cdc725c311864778295", size = 69656, upload-time = "2026-08-25T16:14:00.667Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mypy"
|
||||
version = "1.20.2"
|
||||
@ -551,6 +796,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/21/35b31dde1b283b79de828b80f876afd8c94e28fe1e9c375f89e261cc4c0d/numpy-2.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:bd68ece1553d2023c09a4226d9e41c586ad2d20594d1a456186c33513d2cb3f2", size = 10396725, upload-time = "2026-08-09T13:46:00.478Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-api"
|
||||
version = "1.44.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "26.3"
|
||||
@ -578,6 +835,27 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prompt-toolkit"
|
||||
version = "3.0.53"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "wcwidth" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pycparser"
|
||||
version = "3.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pycryptodome"
|
||||
version = "3.23.0"
|
||||
@ -691,6 +969,20 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyjwt"
|
||||
version = "2.13.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
crypto = [
|
||||
{ name = "cryptography" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pymupdf"
|
||||
version = "1.28.2"
|
||||
@ -764,6 +1056,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-dateutil"
|
||||
version = "2.9.0.post0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "six" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-docx"
|
||||
version = "1.2.0"
|
||||
@ -795,6 +1099,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pywin32"
|
||||
version = "312"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", size = 6371877, upload-time = "2026-06-04T07:49:28.836Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", size = 6914841, upload-time = "2026-06-04T07:49:31.85Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", size = 6727901, upload-time = "2026-06-04T07:49:34.244Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", size = 6370184, upload-time = "2026-06-04T07:49:36.253Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", size = 6914298, upload-time = "2026-06-04T07:49:38.876Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", size = 6727640, upload-time = "2026-06-04T07:49:41.083Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyyaml"
|
||||
version = "6.0.3"
|
||||
@ -823,6 +1140,58 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "referencing"
|
||||
version = "0.37.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "attrs" },
|
||||
{ name = "rpds-py" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rpds-py"
|
||||
version = "2026.6.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.16.5"
|
||||
@ -893,6 +1262,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/bc/2a07cef49046e6cf5d1a8b1de553aaef8491b4170dbfe254c8687c764408/sqlglot-30.17.0-py3-none-any.whl", hash = "sha256:84435ac283a60173da31b5fd7d11a725037a1c3fd6ed1e21fb065de74ddb579f", size = 741795, upload-time = "2026-08-12T19:36:48.699Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sse-starlette"
|
||||
version = "3.4.8"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "anyio" },
|
||||
{ name = "starlette" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f8/00/b42a44342a054d58cb1115d7c8aa9cb4290dd9442f9c1b91a4b8173dba22/sse_starlette-3.4.8.tar.gz", hash = "sha256:ed89ffbb75cbf78a5fe2f2109cd584792ee7f9dfac96f791db546df8f15f3f9c", size = 32548, upload-time = "2026-08-05T11:19:49.982Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/3a/764912c58293d95b6dcdf4cc255f9d10de310580ced547b082eb9d72018c/sse_starlette-3.4.8-py3-none-any.whl", hash = "sha256:6e82314c786709a3cd9520f2285cf9fff90e181e598e8a357b0cf80f66afba0d", size = 16516, upload-time = "2026-08-05T11:19:48.748Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "starlette"
|
||||
version = "1.6.0"
|
||||
@ -936,6 +1318,27 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tzdata"
|
||||
version = "2026.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tzlocal"
|
||||
version = "5.4.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "tzdata", marker = "sys_platform == 'win32'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/81/5b/879b2f932adfa7a053c360d50bc896c977fa6426109185f7c12ebdd0cb9d/tzlocal-5.4.4.tar.gz", hash = "sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4", size = 31170, upload-time = "2026-06-29T08:03:40.026Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/a4/017a7a6cbe387d961a688ec31364ae60a5c4e22c96ae9921b79a947c855d/tzlocal-5.4.4-py3-none-any.whl", hash = "sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15", size = 18115, upload-time = "2026-06-29T08:03:38.666Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "uvicorn"
|
||||
version = "0.52.4"
|
||||
@ -979,6 +1382,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "vine"
|
||||
version = "5.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "watchfiles"
|
||||
version = "1.2.0"
|
||||
@ -1029,6 +1441,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wcwidth"
|
||||
version = "0.8.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/36/57/ed58088fafdf4c55a0ad6bde846502567645424d7ebf325230b9237f4085/wcwidth-0.8.3.tar.gz", hash = "sha256:d128512515fbf4612e0ff21fd6380399210318b7b54a9af59dff8454cf9730eb", size = 1458450, upload-time = "2026-08-28T18:10:06.875Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/0e/57f6bb3024a597b2e8ec4aee710ffe62ddc95af2e2bb1ee7a7abdc22c68c/wcwidth-0.8.3-py3-none-any.whl", hash = "sha256:d5b73dba6158a595ec9370350e7f2637bcac8d6c5e4fde34f30fcffb6103a5e4", size = 331669, upload-time = "2026-08-28T18:10:04.909Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "websockets"
|
||||
version = "17.1"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user