Compare commits

..

42 Commits

Author SHA1 Message Date
d-robotics
03dd5ffbfd merge: complete DEPLOY-002 Kubernetes Helm delivery 2026-09-03 13:07:19 +08:00
d-robotics
5100d98e3c deploy: add Kubernetes Helm delivery 2026-09-03 12:21:54 +08:00
d-robotics
d727d0a118 merge: complete DEPLOY-001 production images 2026-09-03 11:01:58 +08:00
d-robotics
72dc248bbc build: add production process images 2026-09-02 21:16:33 +08:00
d-robotics
a1eae16fb1 merge: complete DEMO-016 batch async upload 2026-09-02 20:43:26 +08:00
d-robotics
0fad825eb0 feat: add batch async ingestion 2026-09-02 20:42:52 +08:00
d-robotics
a8dfbe0cfa merge: DEMO-015 durable async ingestion 2026-09-02 20:19:28 +08:00
d-robotics
7938c0b9cf feat: add durable async ingestion jobs 2026-09-02 20:18:58 +08:00
d-robotics
4d73465888 merge: PLAN-002 public cluster delivery plan 2026-09-02 19:43:15 +08:00
d-robotics
f47634e104 docs: plan public knowledge cluster delivery 2026-09-02 19:41:13 +08:00
d-robotics
6f16c971e4 merge: close DEMO-012 WorkBuddy acceptance 2026-09-02 17:11:29 +08:00
d-robotics
123864b80d docs: close WorkBuddy MCP acceptance 2026-09-02 17:10:56 +08:00
d-robotics
9d53bdbe4b merge: complete DEMO-012 MCP search gateway 2026-09-02 16:40:00 +08:00
d-robotics
77119f4a80 feat: add authenticated knowledge MCP gateway 2026-09-02 16:39:26 +08:00
d-robotics
6af4ab465d merge: PLAN-001 async ingestion and MCP priority 2026-09-02 16:01:20 +08:00
d-robotics
e3b5b4d50f docs: plan async ingestion and prioritize MCP 2026-09-02 16:00:42 +08:00
d-robotics
fe9d85b6da merge: complete DEMO-014 branded admin console 2026-09-02 15:28:59 +08:00
d-robotics
f16c738035 feat: align DEMO-014 console with D-Robotics brand 2026-09-02 15:27:40 +08:00
d-robotics
8fba051da4 merge: complete DEMO-010 admin console 2026-09-02 13:49:38 +08:00
d-robotics
c13ae39543 feat: complete DEMO-010 admin console 2026-09-02 13:48:31 +08:00
d-robotics
9931224b46 merge: ARCH-006 OceanBase live acceptance 2026-09-02 13:09:08 +08:00
d-robotics
2558264d21 feat: complete ARCH-006 OceanBase live acceptance 2026-09-02 13:08:20 +08:00
d-robotics
c6421cc9d4 merge: ARCH-005 runtime cutover 2026-09-02 11:29:01 +08:00
d-robotics
df4b0ae91e feat: complete ARCH-005 runtime cutover 2026-09-02 11:28:13 +08:00
d-robotics
4ce74604ba merge: ARCH-004 BCE reranker alignment 2026-09-02 11:14:03 +08:00
d-robotics
0d21fae173 feat: align ARCH-004 reranker with BCE 2026-09-02 11:13:27 +08:00
d-robotics
92d098cc25 merge: ARCH-003 OceanBase vector adapter 2026-09-02 11:10:05 +08:00
d-robotics
6efc33e868 feat: add ARCH-003 OceanBase vector adapter 2026-09-02 11:09:04 +08:00
d-robotics
b98b04af17 merge: ARCH-002 BOS object storage adapter 2026-09-01 20:49:04 +08:00
d-robotics
c95bd8a118 feat: add ARCH-002 BOS object storage adapter 2026-09-01 20:48:41 +08:00
d-robotics
d5b1d4fe4e merge: ARCH-001 provider migration baseline 2026-09-01 20:21:53 +08:00
d-robotics
48e6732d4f docs: establish ARCH-001 provider migration baseline 2026-09-01 20:21:21 +08:00
d-robotics
03f0f2b0e3 merge: DEMO-009 document and search APIs 2026-09-01 16:11:21 +08:00
d-robotics
cb1cdfd07d feat: expose DEMO-009 document and search APIs 2026-09-01 16:10:14 +08:00
d-robotics
ac054eaf94 merge: DEMO-008 reranking and source results 2026-09-01 15:49:39 +08:00
d-robotics
f696aeec2b feat: implement DEMO-008 reranking and source results 2026-09-01 15:48:34 +08:00
d-robotics
a0aa30c41c merge: DEMO-007 dense retrieval 2026-09-01 15:28:57 +08:00
d-robotics
43b649c46a feat: implement DEMO-007 dense retrieval 2026-09-01 15:27:53 +08:00
d-robotics
be69b16d18 feat: implement DEMO-006 embedding and Mochow indexing 2026-09-01 14:52:31 +08:00
d-robotics
3349c97f59 feat: implement DEMO-005 chunk pipeline 2026-09-01 14:31:30 +08:00
d-robotics
934c755901 feat: implement DEMO-004 parser and cleaner 2026-09-01 14:17:20 +08:00
d-robotics
3c0440abc8 feat: add validated document upload service 2026-09-01 14:02:53 +08:00
174 changed files with 18375 additions and 215 deletions

21
.dockerignore Normal file
View File

@ -0,0 +1,21 @@
# Keep credentials, local state, caches, and developer artifacts out of every image context.
.env
.env.*
!.env.example
.git
.github
.idea
.vscode
.DS_Store
.mypy_cache
.pytest_cache
.ruff_cache
.venv
__pycache__
*.py[cod]
*.sqlite*
dist
storage
tests
docs
docker-compose.dev.yml

View File

@ -4,35 +4,86 @@ LOG_LEVEL=INFO
API_HOST=0.0.0.0
API_PORT=8000
API_PREFIX=/api/v1
# Optional outside source checkouts; production image fixes this to /app/apps/admin-web.
# ADMIN_WEB_ROOT=/absolute/path/to/apps/admin-web
DATABASE_URL=postgresql+psycopg://ekp:ekp@localhost:5432/ekp
REDIS_URL=redis://localhost:6379/0
RABBITMQ_URL=amqp://guest:guest@localhost:5672//
DATABASE_URL=mysql+pymysql://ekp:change-me@localhost:2881/ekp
DATABASE_POOL_SIZE=10
DATABASE_MAX_OVERFLOW=10
DATABASE_POOL_RECYCLE_SECONDS=600
DATABASE_POOL_TIMEOUT_SECONDS=30
DATABASE_CONNECT_TIMEOUT_SECONDS=30
# DEMO-015 queue. Inject the real credential at runtime; never commit its value.
RABBITMQ_URL=
INGESTION_MAX_ATTEMPTS=3
INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
BATCH_MAX_FILES=20
BATCH_MAX_TOTAL_BYTES=209715200
OBJECT_STORAGE_PROVIDER=local
OBJECT_STORAGE_PROVIDER=bos
OBJECT_STORAGE_PATH=storage
BOS_ENDPOINT=
BOS_ACCESS_KEY_ID=
BOS_SECRET_ACCESS_KEY=
BOS_BUCKET=
BOS_KEY_PREFIX=knowledge-platform
BOS_TIMEOUT_SECONDS=30
MAX_UPLOAD_BYTES=52428800
CHUNK_MAX_TOKENS=450
CHUNK_OVERLAP_TOKENS=75
CHUNK_MAX_CHARACTERS=2000
# Reserved for a future S3/MinIO adapter; DEMO-002 uses local storage.
OBJECT_STORAGE_ENDPOINT=http://localhost:9000
OBJECT_STORAGE_ACCESS_KEY=
OBJECT_STORAGE_SECRET_KEY=
OBJECT_STORAGE_BUCKET=ekp
VECTOR_STORE_PROVIDER=mochow
MOCHOW_ENDPOINT=
MOCHOW_API_KEY=
MOCHOW_DATABASE=
MOCHOW_TABLE=
VECTOR_STORE_PROVIDER=oceanbase
OCEANBASE_VECTOR_URI=
OCEANBASE_VECTOR_USER=
OCEANBASE_VECTOR_PASSWORD=
OCEANBASE_VECTOR_DATABASE=
OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1
OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding
OCEANBASE_VECTOR_BATCH_SIZE=100
OCEANBASE_VECTOR_TIMEOUT_SECONDS=30
EMBEDDING_PROVIDER=qianfan
EMBEDDING_MODEL=bge-large-zh
EMBEDDING_ENDPOINT=
EMBEDDING_API_KEY=
EMBEDDING_ENDPOINT=https://qianfan.baidubce.com/v2/embeddings
BAIDU_QIANFAN_API_KEY=
EMBEDDING_TIMEOUT_SECONDS=30
INDEX_PROFILE_ID=demo-bge-large-zh-structure-v1-oceanbase-v1
DENSE_TOP_N=20
RERANKER_PROVIDER=qianfan
RERANKER_MODEL=
RERANKER_ENDPOINT=
RERANKER_MODEL=bce-reranker-base
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
RERANKER_API_KEY=
RERANKER_TIMEOUT_SECONDS=30
SEARCH_TOP_K=5
# DEMO-012 MCP gateway. Inject MCP_AUTH_TOKEN at runtime; never commit its value.
MCP_HOST=127.0.0.1
MCP_PORT=8766
MCP_PATH=/mcp
MCP_PUBLIC_URL=http://127.0.0.1:8766/mcp
MCP_ISSUER_URL=http://127.0.0.1:8766
MCP_AUTH_TOKEN=
MCP_CLIENT_ID=workbuddy-enterprise
MCP_SERVICE_IDENTITY=workbuddy-demo
MCP_REQUIRED_SCOPE=knowledge:search
MCP_ALLOWED_HOSTS='["127.0.0.1:*","localhost:*","[::1]:*","testserver"]'
MCP_ALLOWED_ORIGINS='["http://127.0.0.1:*","http://localhost:*","http://[::1]:*"]'
MCP_MAX_REQUEST_BODY_BYTES=1048576
# Controlled ARCH-006 acceptance; keep disabled outside an authorized test run.
EKP_RUN_LIVE_TESTS=0
EKP_CREDENTIALS_AUTHORIZED=0
EKP_LIVE_RESOURCE_ACK=0
EKP_LIVE_MIN_RECALL_AT_1=0.90
EKP_LIVE_MIN_RECALL_AT_5=1.0
EKP_LIVE_MIN_MRR=0.90
EKP_LIVE_MIN_NDCG_AT_5=0.90
EKP_LIVE_MAX_P95_MS=10000
LLM_PROVIDER=qianfan
LLM_MODEL=

View File

@ -19,8 +19,8 @@ Do not redesign accepted architecture unless the task explicitly asks for an arc
1. API First.
2. V1 uses a modular monolith plus independent async/GPU workers and an MCP gateway.
3. PostgreSQL + object storage hold canonical data.
4. Vector databases are rebuildable indexes, not sources of truth.
3. OceanBase relational tables + object storage hold canonical data.
4. OceanBase vector tables are rebuildable indexes, not sources of truth, even when co-located with canonical relational tables.
5. Business services depend on `VectorStore` abstractions, never directly on Mochow.
6. Embedding, reranking, parsing, OCR, object storage, and LLMs use provider interfaces.
7. Original source documents are immutable.

View File

@ -1,4 +1,4 @@
.PHONY: help install run format format-check lint typecheck test test-unit test-integration migration-check eval up down
.PHONY: help install run run-mcp run-worker images images-print images-push deploy deploy-lint deploy-render format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
export UV_CACHE_DIR
@ -7,6 +7,14 @@ help:
@echo "Enterprise Knowledge Platform"
@echo " install Install development dependencies"
@echo " run Start the development API"
@echo " run-mcp Start the authenticated MCP gateway"
@echo " run-worker Start the ingestion worker and recovery scheduler"
@echo " images Build production images for the current builder"
@echo " images-print Render the multi-architecture image build plan"
@echo " images-push Build and push amd64/arm64 production images"
@echo " deploy Install or upgrade Kubernetes with VALUES=<file>"
@echo " deploy-lint Validate Helm values and templates"
@echo " deploy-render Render non-secret Kubernetes manifests"
@echo " format Format code"
@echo " format-check Check formatting without modifying files"
@echo " lint Run linters"
@ -16,6 +24,7 @@ help:
@echo " test-integration Run integration tests"
@echo " migration-check Validate the forward migration path"
@echo " eval Run RAG evaluation"
@echo " live-acceptance Run controlled target-provider acceptance"
@echo " up Start local dependencies"
@echo " down Stop local dependencies"
@ -23,6 +32,24 @@ install:
uv sync --all-groups
run:
uv run uvicorn ekp_api.main:app --host "$${API_HOST:-0.0.0.0}" --port "$${API_PORT:-8000}"
run-mcp:
uv run ekp-mcp
run-worker:
uv run ekp-worker
images:
docker buildx bake -f deploy/docker/docker-bake.hcl
images-print:
docker buildx bake -f deploy/docker/docker-bake.hcl --print
images-push:
docker buildx bake -f deploy/docker/docker-bake.hcl --push
deploy:
./scripts/deploy.sh "$(VALUES)"
deploy-lint:
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
helm lint deploy/helm/knowledge-platform --values "$(VALUES)"
deploy-render:
@test -n "$(VALUES)" || (echo "VALUES must point to an environment values file" >&2; exit 2)
helm template "$${RELEASE:-knowledge-platform}" deploy/helm/knowledge-platform --namespace "$${NAMESPACE:-knowledge-platform}" --values "$(VALUES)"
format:
uv run ruff format .
format-check:
@ -30,7 +57,7 @@ format-check:
lint:
uv run ruff check .
typecheck:
uv run mypy apps/api/src packages tests
uv run mypy apps/api/src apps/mcp-server/src packages tests
test:
uv run pytest tests/unit tests/integration
test-unit:
@ -42,6 +69,8 @@ migration-check:
uv run alembic upgrade head --sql >/dev/null
eval:
uv run pytest tests/evaluation
live-acceptance:
uv run ekp-live-acceptance
up:
docker compose -f docker-compose.dev.yml up -d
down:

View File

@ -19,8 +19,9 @@ Enterprise internal knowledge infrastructure.
- API First + Admin Console
- Modular Monolith + async workers + MCP gateway
- PostgreSQL + object storage as canonical source of truth
- Mochow as the initial retrieval index, behind a replaceable adapter
- OceanBase MySQL relational tables + object storage as canonical source of truth
- Co-located OceanBase Vector tables as rebuildable retrieval indexes behind an adapter
- Baidu BOS as target object storage; local storage remains for tests/development
- Embedding / reranker / parser / LLM providers are replaceable
- Raw source is immutable
- Compiled Knowledge is derived and traceable
@ -45,19 +46,40 @@ The current implementation target is the retrieval demo defined in
Prerequisites:
- `uv`
- Docker (for PostgreSQL and later integration dependencies)
- Docker (for optional local integration dependencies)
Install dependencies and run the API:
```bash
cp .env.example .env
make install
make run-worker
make run
```
Run the independent authenticated MCP Gateway after injecting a dedicated
runtime token of at least 32 characters:
```bash
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
make run-mcp
```
The Streamable HTTP endpoint defaults to `http://127.0.0.1:8766/mcp` and exposes
only `search_knowledge`. See `docs/development/MCP_SETUP.md` for WorkBuddy configuration,
identity boundaries, remote allowlists, and protocol verification.
The health endpoint is available at `http://localhost:8000/healthz`, and the
versioned OpenAPI document is available at
`http://localhost:8000/api/v1/openapi.json`.
The Demo Admin Console is available at `http://localhost:8000/admin` and contains
only document upload, document management, and Search Lab pages. See
`docs/development/ADMIN_CONSOLE.md` for the operation and scope boundaries.
DEPLOY-001 provides locked, non-root `api`, `mcp`, and `worker` production image targets with an
amd64/arm64 Buildx Bake plan. DEPLOY-002 adds the Kubernetes/Helm Chart, runtime Secret references,
serialized Migration, Ingress/TLS and the `make deploy VALUES=<file>` entrypoint. See
`docs/development/CONTAINER_IMAGES.md` and `docs/development/KUBERNETES_DEPLOYMENT.md`.
Run the development checks with:
@ -65,7 +87,64 @@ Run the development checks with:
make format-check lint typecheck test migration-check eval
```
Document metadata is canonical in PostgreSQL. Demo source files use the local
Controlled target-provider acceptance uses 32 synthetic queries and is disabled by
default. It requires rotated secrets, dedicated test resources, and all safety
acknowledgements described in `docs/development/LIVE_ACCEPTANCE.md`; only then run
`make live-acceptance`.
Document metadata is canonical in OceanBase MySQL relational tables. Demo source files use the local
`storage/` directory through the provider-neutral `ObjectStorage` interface;
existing objects cannot be overwritten. Upload endpoints are introduced by
DEMO-003, not by the current storage foundation.
existing objects cannot be overwritten. The upload service accepts PDF, DOCX,
TXT, and MD sources, streams at most `MAX_UPLOAD_BYTES`, and compensates stored
bytes if metadata persistence fails. The ingestion service then routes PDF to
PyMuPDF, DOCX to python-docx, and UTF-8 TXT/Markdown to lightweight parsers. It
normalizes whitespace, retains available page/section context, and stores a
versioned parsed JSON artifact under `parsed/{document_id}/document.json`.
The `structure_v1` chunk strategy keeps heading/section boundaries, combines
paragraphs, and creates configurable 450-token windows with a 75-token overlap.
Chunk metadata, including deterministic IDs, filename, page range, section,
index, and strategy version, is stored atomically in canonical relational tables. The lightweight
token counter is replaceable so DEMO-006 can inject model-specific behavior.
Documents remain `processing` until embedding/indexing completes; pipeline
failures become `failed` without deleting the immutable original. DEMO-015 now
accepts a source plus durable Job with HTTP `202`, while a Celery/RabbitMQ worker
runs the pipeline. Bounded transient retries clean every derived artifact before
starting the next Attempt; broker redelivery and expired worker leases converge on
OceanBase Job state. See `docs/development/ASYNC_INGESTION_SETUP.md`.
DEMO-006 added a provider-neutral `EmbeddingProvider` and `VectorStore`, plus a
validated Qianfan `bge-large-zh` adapter. ARCH-005 now wires that boundary to OceanBase.
Chunk vectors are upserted with source metadata under a versioned index profile;
OceanBase relational tables record canonical index state while vector rows remain rebuildable. A
document becomes `ready` only after every chunk is indexed. See
`docs/development/OCEANBASE_VECTOR_SETUP.md` for the required environment variables
and pre-provisioned table schema.
The accepted provider migration is tracked by `ARCH-001` through `ARCH-006`:
Baidu BOS, OceanBase Vector, and Qianfan `bce-reranker-base` replace the initial
demo providers, and ARCH-006 aligns canonical ownership with the verified OceanBase target. BOS and OceanBase
are now the runtime defaults; LocalObjectStorage and FakeVectorStore remain available
for explicit local/offline tests. ARCH-005 also adds the controlled shadow rebuild
workflow documented in `docs/development/INDEX_REBUILD.md`.
DEMO-007 adds query embedding and profile-scoped Dense Top-N retrieval. The
default is Top 20; the in-memory fake uses
exact cosine search for deterministic offline regression. Returned objects are
raw dense candidates only.
DEMO-008 adds a provider-neutral reranker, and ARCH-004 aligns its validated Qianfan
adapter with the `bce-reranker-base` baseline. The final Top20-to-Top5 pipeline and
source results are rehydrated from canonical relational chunks and retain both
Dense and reranker scores; stale vector content fails closed. See
`docs/development/RERANKER_SETUP.md` for configuration and validation limits.
DEMO-009 exposes document and search REST endpoints under `/api/v1`; DEMO-015
changes upload to asynchronous acceptance and adds Job status. DEMO-016 adds bounded multi-file
submission, batch/per-file status, and automatic Admin Console polling. Upload, list, detail, direct delete,
and reranked source search are documented in
`docs/development/REST_API_SETUP.md`; provider and storage internals remain
outside the public response contract.
DEMO-012 adds the independent authenticated MCP Gateway. It uses the official
MCP Python SDK, maps a runtime-only Bearer credential to a least-privilege service
identity, and delegates `search_knowledge` to the same `SearchService` used by REST.
No provider, storage, database, model, or index-profile details are exposed.

View File

@ -0,0 +1,411 @@
// The Admin Console owns presentation state only. Batch acceptance, Job state,
// and retrieval remain behind the versioned public REST contract.
const API_BASE = "/api/v1";
const BATCH_POLL_INTERVAL_MS = 1500;
const terminalBatchStatuses = new Set(["completed", "partial_failure", "failed"]);
let batchPollTimer;
let activeBatchId;
const statusLabels = {
uploaded: "Uploaded",
processing: "Processing",
ready: "Ready",
failed: "Failed",
pending: "Pending",
accepted: "Accepted",
rejected: "Rejected",
queued: "Queued",
running: "Running",
retrying: "Retrying",
succeeded: "Succeeded",
completed: "Completed",
partial_failure: "Partial failure",
};
const stageLabels = {
parsing: "解析",
cleaning: "清理",
chunking: "切分",
embedding: "向量化",
indexing: "索引",
};
const pageTitles = {
upload: "上传文档",
documents: "文档列表",
search: "Search Lab",
};
function currentPage() {
const route = window.location.pathname.split("/").filter(Boolean).at(-1);
return Object.hasOwn(pageTitles, route) ? route : "upload";
}
function showPage(page) {
document.querySelectorAll("[data-page]").forEach((section) => {
section.hidden = section.dataset.page !== page;
});
document.querySelectorAll("[data-nav]").forEach((link) => {
const active = link.dataset.nav === page;
link.classList.toggle("active", active);
if (active) link.setAttribute("aria-current", "page");
else link.removeAttribute("aria-current");
});
document.querySelector("#page-title").textContent = pageTitles[page];
}
function showNotice(message, kind = "success") {
const notice = document.querySelector("#notice");
notice.textContent = message;
notice.className = `notice ${kind}`;
notice.hidden = false;
window.clearTimeout(showNotice.timeout);
showNotice.timeout = window.setTimeout(() => {
notice.hidden = true;
}, 6000);
}
async function requestJson(path, options = {}) {
const response = await fetch(`${API_BASE}${path}`, options);
if (!response.ok) {
let message = `请求失败(HTTP ${response.status})`;
try {
const body = await response.json();
if (body.detail) message = body.detail;
} catch {
// A non-JSON upstream error is represented by the stable HTTP message above.
}
throw new Error(message);
}
return response.status === 204 ? null : response.json();
}
function formatDate(value) {
if (!value) return "—";
return new Intl.DateTimeFormat("zh-CN", {
dateStyle: "medium",
timeStyle: "short",
}).format(new Date(value));
}
function formatSize(bytes) {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`;
return `${(bytes / 1024 ** 2).toFixed(1)} MB`;
}
function statusPill(status) {
const pill = document.createElement("span");
pill.className = `status-pill status-${status}`;
pill.textContent = statusLabels[status] ?? status;
return pill;
}
function setButtonBusy(button, busy, busyText) {
if (!button.dataset.label) button.dataset.label = button.textContent;
button.disabled = busy;
button.textContent = busy ? busyText : button.dataset.label;
}
function initializeUpload() {
const form = document.querySelector("#upload-form");
const input = document.querySelector("#file-input");
const selected = document.querySelector("#selected-file");
const button = document.querySelector("#upload-button");
const updateFilename = () => {
const files = Array.from(input.files);
if (!files.length) {
selected.textContent = "尚未选择文件";
} else if (files.length === 1) {
selected.textContent = files[0].name;
} else {
selected.textContent = `已选择 ${files.length} 个文件`;
}
};
input.addEventListener("change", updateFilename);
form.addEventListener("submit", async (event) => {
event.preventDefault();
if (!input.files.length) return;
window.clearTimeout(batchPollTimer);
activeBatchId = undefined;
setButtonBusy(button, true, "提交中…");
try {
const formData = new FormData();
Array.from(input.files).forEach((file) => formData.append("files", file));
const batch = await requestJson("/ingestion-batches", {
method: "POST",
body: formData,
});
activeBatchId = batch.batch_id;
renderBatchResult(batch);
scheduleBatchPoll(batch);
showNotice(
`批次已接收:${batch.accepted_files} 个进入后台队列,${batch.rejected_files} 个被拒绝。`,
batch.rejected_files ? "error" : "success",
);
form.reset();
updateFilename();
} catch (error) {
showNotice(`上传失败:${error.message}`, "error");
} finally {
setButtonBusy(button, false, "");
}
});
}
function scheduleBatchPoll(batch) {
window.clearTimeout(batchPollTimer);
if (batch.batch_id !== activeBatchId) return;
if (terminalBatchStatuses.has(batch.status)) return;
batchPollTimer = window.setTimeout(async () => {
try {
const latest = await requestJson(`/ingestion-batches/${batch.batch_id}`);
renderBatchResult(latest);
scheduleBatchPoll(latest);
} catch (error) {
showNotice(`批次状态刷新失败:${error.message}`, "error");
scheduleBatchPoll(batch);
}
}, BATCH_POLL_INTERVAL_MS);
}
function renderBatchResult(batch) {
const card = document.querySelector("#upload-result");
document.querySelector("#upload-batch-title").textContent =
`批次 ${batch.batch_id.slice(0, 8)} · ${formatDate(batch.created_at)}`;
document.querySelector("#upload-total").textContent = `${batch.total_files} 个`;
document.querySelector("#upload-completed").textContent =
`${batch.completed_files} / ${batch.total_files}`;
document.querySelector("#upload-accepted").textContent =
`${batch.accepted_files} / ${batch.rejected_files}`;
document.querySelector("#upload-status").replaceWith(statusPill(batch.status));
const status = card.querySelector(".status-pill");
status.id = "upload-status";
const items = document.querySelector("#batch-items");
items.replaceChildren(...batch.items.map(batchItem));
card.hidden = false;
}
function batchItem(item) {
const row = document.createElement("article");
row.className = "batch-item";
const order = document.createElement("span");
order.className = "batch-item-order";
order.textContent = String(item.item_index + 1).padStart(2, "0");
const body = document.createElement("div");
body.className = "batch-item-body";
const name = document.createElement("strong");
name.textContent = item.filename;
const detail = document.createElement("span");
const stage = item.stage ? ` · ${stageLabels[item.stage] ?? item.stage}` : "";
const attempts = item.max_attempts == null ? "" : ` · 尝试 ${item.attempt}/${item.max_attempts}`;
detail.textContent = item.error_message ?? `后台任务${stage}${attempts}`;
body.append(name, detail);
const state = item.job_status ?? item.acceptance_status;
row.append(order, body, statusPill(state));
return row;
}
async function loadDocuments() {
const refresh = document.querySelector("#refresh-documents");
setButtonBusy(refresh, true, "加载中…");
try {
const documents = await requestJson("/documents");
renderDocuments(documents);
} catch (error) {
showNotice(`文档列表加载失败:${error.message}`, "error");
} finally {
setButtonBusy(refresh, false, "");
}
}
function renderDocuments(documents) {
const rows = document.querySelector("#document-rows");
const empty = document.querySelector("#document-empty");
rows.replaceChildren();
empty.hidden = documents.length !== 0;
document.querySelector("#document-count").textContent = `${documents.length} 个文档`;
document.querySelector("#document-refresh-time").textContent =
`更新于 ${new Intl.DateTimeFormat("zh-CN", { timeStyle: "short" }).format(new Date())}`;
documents.forEach((documentData) => {
const row = document.createElement("tr");
row.append(
tableCell(documentData.filename, "filename-cell"),
tableCell(documentData.media_type),
tableCell(formatDate(documentData.created_at)),
);
const statusCell = document.createElement("td");
statusCell.append(statusPill(documentData.status));
row.append(statusCell);
const actions = document.createElement("td");
actions.className = "row-actions";
const view = actionButton("查看", () => showDocumentDetail(documentData));
const remove = actionButton("删除", () => deleteDocument(documentData), "danger-link");
actions.append(view, remove);
row.append(actions);
rows.append(row);
});
}
function tableCell(value, className = "") {
const cell = document.createElement("td");
cell.textContent = value;
cell.className = className;
return cell;
}
function actionButton(label, handler, className = "") {
const button = document.createElement("button");
button.type = "button";
button.className = `table-action ${className}`;
button.textContent = label;
button.addEventListener("click", handler);
return button;
}
function showDocumentDetail(documentData) {
const dialog = document.querySelector("#document-dialog");
document.querySelector("#dialog-filename").textContent = documentData.filename;
const metadata = document.querySelector("#dialog-metadata");
metadata.replaceChildren();
[
["文档 ID", documentData.id],
["文件类型", documentData.media_type],
["文件大小", formatSize(documentData.size_bytes)],
["当前状态", statusLabels[documentData.status] ?? documentData.status],
["上传时间", formatDate(documentData.created_at)],
["更新时间", formatDate(documentData.updated_at)],
["错误信息", documentData.error_message ?? "无"],
].forEach(([term, value]) => {
const wrapper = document.createElement("div");
const dt = document.createElement("dt");
const dd = document.createElement("dd");
dt.textContent = term;
dd.textContent = value;
wrapper.append(dt, dd);
metadata.append(wrapper);
});
dialog.showModal();
}
async function deleteDocument(documentData) {
if (!window.confirm(`确认删除“${documentData.filename}”及其索引数据?`)) return;
try {
await requestJson(`/documents/${documentData.id}`, { method: "DELETE" });
showNotice(`已删除 ${documentData.filename}`, "success");
await loadDocuments();
} catch (error) {
showNotice(`删除失败:${error.message}`, "error");
}
}
function initializeSearch() {
const form = document.querySelector("#search-form");
const button = document.querySelector("#search-button");
form.addEventListener("submit", async (event) => {
event.preventDefault();
const query = document.querySelector("#search-query").value.trim();
const topK = Number(document.querySelector("#search-top-k").value);
if (!query) return;
setButtonBusy(button, true, "检索中…");
try {
const startedAt = performance.now();
const response = await requestJson("/search", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ query, top_k: topK }),
});
renderSearchResults(response.results, performance.now() - startedAt);
} catch (error) {
showNotice(`检索失败:${error.message}`, "error");
} finally {
setButtonBusy(button, false, "");
}
});
}
function renderSearchResults(results, elapsedMs) {
const container = document.querySelector("#search-results");
container.replaceChildren();
document.querySelector("#search-summary").textContent =
`${results.length} 条结果 · ${(elapsedMs / 1000).toFixed(2)} 秒`;
if (!results.length) {
const empty = document.createElement("div");
empty.className = "empty-state search-empty";
const strong = document.createElement("strong");
strong.textContent = "没有找到匹配内容";
const detail = document.createElement("span");
detail.textContent = "可以换一种问法,或先确认文档状态为 Ready。";
empty.append(strong, detail);
container.append(empty);
return;
}
results.forEach((result, index) => {
const card = document.createElement("article");
card.className = "search-result-card";
const rank = document.createElement("span");
rank.className = "rank";
rank.textContent = String(index + 1).padStart(2, "0");
const body = document.createElement("div");
body.className = "result-body";
const heading = document.createElement("div");
heading.className = "result-title-row";
const title = document.createElement("h4");
title.textContent = result.filename;
const sourceMeta = document.createElement("div");
sourceMeta.className = "source-meta";
sourceMeta.append(
metadataItem("Page", result.page == null ? "—" : String(result.page)),
metadataItem("Section", result.section ?? "—"),
);
heading.append(title, sourceMeta);
const content = document.createElement("p");
content.className = "chunk-content";
content.textContent = result.content;
const scores = document.createElement("div");
scores.className = "score-row";
scores.append(
scoreItem("Dense Score", result.score),
scoreItem("Rerank Score", result.rerank_score),
);
body.append(heading, content, scores);
card.append(rank, body);
container.append(card);
});
}
function scoreItem(label, value) {
const item = document.createElement("span");
const labelNode = document.createElement("small");
const valueNode = document.createElement("strong");
labelNode.textContent = label;
valueNode.textContent = Number(value).toFixed(4);
item.append(labelNode, valueNode);
return item;
}
function metadataItem(label, value) {
const item = document.createElement("span");
const labelNode = document.createElement("small");
const valueNode = document.createElement("strong");
labelNode.textContent = label;
valueNode.textContent = value;
item.append(labelNode, valueNode);
return item;
}
const page = currentPage();
showPage(page);
initializeUpload();
initializeSearch();
document.querySelector("#refresh-documents").addEventListener("click", loadDocuments);
if (page === "documents") loadDocuments();
window.addEventListener("pagehide", () => window.clearTimeout(batchPollTimer));

File diff suppressed because it is too large Load Diff

201
apps/admin-web/index.html Normal file
View File

@ -0,0 +1,201 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta
name="description"
content="Enterprise Knowledge Platform Demo 管理控制台"
/>
<title>Enterprise Knowledge Platform</title>
<link rel="stylesheet" href="/admin/assets/styles.css" />
<script type="module" src="/admin/assets/app.js"></script>
</head>
<body>
<div class="app-shell">
<aside class="sidebar" aria-label="管理控制台导航">
<a class="brand" href="/admin/upload" aria-label="Enterprise Knowledge Platform 首页">
<svg class="brand-mark" viewBox="0 0 48 36" aria-hidden="true">
<path d="M15 3h17a15 15 0 0 1 0 30H15v-9h17a6 6 0 0 0 0-12H15z" />
<circle cx="11" cy="18" r="8" />
</svg>
<span>
<strong>地瓜机器人</strong>
<small>Knowledge Platform</small>
</span>
</a>
<div class="workspace-label">
<span>CONTROL DESK</span>
<strong>企业知识运营台</strong>
</div>
<nav class="primary-nav">
<a data-nav="upload" href="/admin/upload">
<span aria-hidden="true">01</span>上传文档
</a>
<a data-nav="documents" href="/admin/documents">
<span aria-hidden="true">02</span>文档列表
</a>
<a data-nav="search" href="/admin/search">
<span aria-hidden="true">03</span>Search Lab
</a>
</nav>
<div class="sidebar-foot">
<span class="environment-dot" aria-hidden="true"></span>
DEMO ENVIRONMENT
</div>
</aside>
<main class="workspace">
<header class="topbar">
<div>
<p class="eyebrow">KNOWLEDGE OPERATIONS</p>
<h1 id="page-title">上传文档</h1>
</div>
<div class="topbar-actions">
<span class="system-state"><i aria-hidden="true"></i>API FIRST · V1</span>
<a class="api-link" href="/api/v1/docs" target="_blank" rel="noreferrer">
API 文档 <span aria-hidden="true">↗</span>
</a>
</div>
</header>
<div class="signal-line" aria-hidden="true"><span></span><span></span><span></span></div>
<div id="notice" class="notice" role="status" aria-live="polite" hidden></div>
<section id="upload-page" class="page" data-page="upload" aria-labelledby="page-title">
<div class="page-heading">
<div>
<p class="section-label">01 / INGESTION</p>
<h2>让知识进入检索链路</h2>
<p>批量提交 PDF、DOCX、TXT 或 Markdown 文件,系统将逐个接收并在后台独立完成解析、切分与索引。</p>
</div>
</div>
<form id="upload-form" class="upload-panel">
<label id="drop-zone" class="drop-zone" for="file-input">
<input
id="file-input"
name="files"
type="file"
multiple
accept=".pdf,.docx,.txt,.md,application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document,text/plain,text/markdown"
required
/>
<span class="upload-icon" aria-hidden="true">↑</span>
<strong>选择一批知识文件</strong>
<span>支持 PDF、DOCX、TXT、Markdown · 每个文件独立处理</span>
<span id="selected-file" class="selected-file">尚未选择文件</span>
</label>
<button id="upload-button" class="primary-button" type="submit">上传并处理</button>
</form>
<article id="upload-result" class="result-card" hidden>
<div>
<p class="section-label">LATEST BATCH</p>
<h3 id="upload-batch-title"></h3>
</div>
<span id="upload-status" class="status-pill"></span>
<dl class="metadata-grid">
<div><dt>总文件</dt><dd id="upload-total"></dd></div>
<div><dt>已完成</dt><dd id="upload-completed"></dd></div>
<div><dt>接收 / 拒绝</dt><dd id="upload-accepted"></dd></div>
</dl>
<div id="batch-items" class="batch-items" aria-live="polite"></div>
<a class="text-link" href="/admin/search">前往 Search Lab 检索 <span>→</span></a>
</article>
</section>
<section id="documents-page" class="page" data-page="documents" aria-labelledby="page-title" hidden>
<div class="page-heading heading-row">
<div>
<p class="section-label">02 / DOCUMENTS</p>
<h2>文档列表</h2>
<p>查看知识文档的状态、元数据并管理 Demo 数据。</p>
</div>
<button id="refresh-documents" class="secondary-button" type="button">刷新列表</button>
</div>
<div class="table-card">
<div class="table-meta">
<span id="document-count">0 个文档</span>
<span id="document-refresh-time">尚未加载</span>
</div>
<div class="table-scroll">
<table>
<thead>
<tr>
<th>文件名</th>
<th>类型</th>
<th>上传时间</th>
<th>状态</th>
<th aria-label="操作"></th>
</tr>
</thead>
<tbody id="document-rows"></tbody>
</table>
</div>
<div id="document-empty" class="empty-state" hidden>
<strong>还没有文档</strong>
<span>上传第一个文件后,它会出现在这里。</span>
<a class="primary-button compact" href="/admin/upload">上传文档</a>
</div>
</div>
</section>
<section id="search-page" class="page" data-page="search" aria-labelledby="page-title" hidden>
<div class="page-heading">
<div>
<p class="section-label">03 / RETRIEVAL</p>
<h2>Search Lab</h2>
<p>输入问题,检查 Dense 召回与 Reranker 精排后的 Top-K 来源。</p>
</div>
</div>
<form id="search-form" class="search-panel">
<label for="search-query">检索问题</label>
<div class="search-controls">
<textarea
id="search-query"
name="query"
rows="3"
maxlength="1600"
placeholder="例如:产品 A 的退款周期是多少?"
required
></textarea>
<label class="top-k-control" for="search-top-k">
Top-K
<select id="search-top-k" name="top_k">
<option value="3">3</option>
<option value="5" selected>5</option>
<option value="10">10</option>
</select>
</label>
<button id="search-button" class="primary-button" type="submit">开始检索</button>
</div>
</form>
<div class="results-heading">
<div>
<p class="section-label">RESULTS</p>
<h3>来源结果</h3>
</div>
<span id="search-summary">等待检索</span>
</div>
<div id="search-results" class="search-results" aria-live="polite">
<div class="empty-state search-empty">
<strong>检索结果会显示在这里</strong>
<span>结果包含排名、文件、页码、章节、Chunk 内容和两阶段分数。</span>
</div>
</div>
</section>
</main>
</div>
<dialog id="document-dialog" class="document-dialog">
<form method="dialog"><button class="icon-button" aria-label="关闭详情">×</button></form>
<p class="section-label">DOCUMENT DETAIL</p>
<h2 id="dialog-filename"></h2>
<dl id="dialog-metadata" class="dialog-metadata"></dl>
</dialog>
</body>
</html>

View File

@ -0,0 +1,55 @@
"""Celery/RabbitMQ adapter behind the queue-neutral ingestion dispatcher."""
from uuid import UUID
from celery import Celery # type: ignore[import-untyped]
from ekp_core.config import Settings
INGESTION_TASK_NAME = "ekp.ingestion.run"
RECOVERY_TASK_NAME = "ekp.ingestion.recover"
def create_celery_app(settings: Settings) -> Celery:
"""Create a secret-safe broker client without opening a connection eagerly."""
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
raise ValueError("RABBITMQ_URL is required for asynchronous ingestion")
app = Celery(
"enterprise-knowledge-platform",
broker=settings.rabbitmq_url.get_secret_value(),
)
app.conf.update(
task_ignore_result=True,
task_acks_late=True,
task_reject_on_worker_lost=True,
worker_prefetch_multiplier=1,
broker_connection_retry_on_startup=True,
timezone="UTC",
enable_utc=True,
task_routes={
INGESTION_TASK_NAME: {"queue": "ingestion"},
RECOVERY_TASK_NAME: {"queue": "ingestion"},
},
beat_schedule={
"recover-durable-ingestion-jobs": {
"task": RECOVERY_TASK_NAME,
"schedule": float(settings.ingestion_recovery_interval_seconds),
}
},
)
return app
class CeleryIngestionDispatcher:
"""Publish only a durable Job id; source bytes and state stay canonical."""
def __init__(self, app: Celery) -> None:
self._app = app
def enqueue(self, job_id: UUID) -> None:
"""Deliver one idempotent execution hint to the ingestion queue."""
self._app.send_task(
INGESTION_TASK_NAME,
args=[str(job_id)],
queue="ingestion",
)

View File

@ -0,0 +1,67 @@
"""Controlled CLI for the ARCH-005 shadow index cutover workflow."""
import argparse
import json
import sys
from collections.abc import Sequence
from dataclasses import asdict
from ekp_api.providers import (
create_embedding_provider,
create_index_profile,
create_vector_store,
)
from ekp_core.config import Settings
from ekp_core.database import create_database_engine, create_session_factory
from ekp_core.index_rebuild import IndexBuildSummary, IndexRebuildService
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Build, verify, and activate a versioned shadow vector index."
)
parser.add_argument(
"command",
choices=("build", "verify", "activate"),
help="Lifecycle step to execute for INDEX_PROFILE_ID.",
)
return parser
def _create_service(settings: Settings) -> IndexRebuildService:
"""Compose the command from the same provider settings as the API runtime."""
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
return IndexRebuildService(
session_factory=session_factory,
embedding_provider=create_embedding_provider(settings),
vector_store=create_vector_store(settings),
target_profile=create_index_profile(settings),
)
def _execute(service: IndexRebuildService, command: str) -> IndexBuildSummary:
"""Run exactly one explicit lifecycle step."""
if command == "build":
return service.rebuild()
if command == "verify":
return service.verify()
if command == "activate":
return service.activate()
raise ValueError("unsupported index rebuild command")
def main(argv: Sequence[str] | None = None) -> int:
"""Return a shell-safe status without printing endpoints, SQL, or credentials."""
args = _parser().parse_args(argv)
try:
summary = _execute(_create_service(Settings()), args.command)
except Exception:
print("index profile command failed; inspect sanitized service logs", file=sys.stderr)
return 1
print(json.dumps(asdict(summary), sort_keys=True, separators=(",", ":")))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,279 @@
"""Controlled end-to-end acceptance for the target provider chain."""
import json
import math
import os
import sys
from collections.abc import Sequence
from dataclasses import asdict, dataclass
from io import BytesIO
from pathlib import Path
from time import perf_counter
from typing import cast
from uuid import uuid4
from ekp_api.providers import ApiServices, create_api_services
from ekp_core.config import Settings
from ekp_core.search import SourceResult
MIN_GOLDEN_QUERIES = 30
MAX_GOLDEN_QUERIES = 50
DEFAULT_MIN_RECALL_AT_1 = 0.90
DEFAULT_MIN_RECALL_AT_5 = 1.0
DEFAULT_MIN_MRR = 0.90
DEFAULT_MIN_NDCG_AT_5 = 0.90
DEFAULT_MAX_P95_MS = 10_000.0
class LiveAcceptanceError(RuntimeError):
"""Raised for a safe precondition or acceptance failure."""
@dataclass(frozen=True, slots=True)
class GoldenQuery:
"""One anonymized expected source and section."""
query_id: str
query: str
expected_document: str
expected_sections: tuple[str, ...]
@dataclass(frozen=True, slots=True)
class AcceptanceReport:
"""Provider-neutral quality and latency metrics without retrieved content."""
query_count: int
recall_at_1: float
recall_at_3: float
recall_at_5: float
mrr: float
ndcg_at_5: float
latency_p50_ms: float
latency_p95_ms: float
passed: bool
@dataclass(frozen=True, slots=True)
class AcceptanceThresholds:
"""Explicit gates for controlled live acceptance."""
min_recall_at_1: float = DEFAULT_MIN_RECALL_AT_1
min_recall_at_5: float = DEFAULT_MIN_RECALL_AT_5
min_mrr: float = DEFAULT_MIN_MRR
min_ndcg_at_5: float = DEFAULT_MIN_NDCG_AT_5
max_p95_ms: float = DEFAULT_MAX_P95_MS
def __post_init__(self) -> None:
quality_values = (
self.min_recall_at_1,
self.min_recall_at_5,
self.min_mrr,
self.min_ndcg_at_5,
)
if any(not 0 <= value <= 1 for value in quality_values):
raise ValueError("live quality thresholds must be between zero and one")
if self.max_p95_ms <= 0:
raise ValueError("live latency threshold must be positive")
def _project_root() -> Path:
return Path(__file__).resolve().parents[4]
def _load_golden_queries(path: Path) -> tuple[GoldenQuery, ...]:
"""Load only the minimal fields needed by the live evaluator."""
records: list[GoldenQuery] = []
for line in path.read_text(encoding="utf-8").splitlines():
if not line:
continue
raw: object = json.loads(line)
if not isinstance(raw, dict):
raise LiveAcceptanceError("Golden Query record must be an object")
documents = raw.get("expected_documents")
sections = raw.get("expected_sections")
if (
not isinstance(raw.get("id"), str)
or not isinstance(raw.get("query"), str)
or not isinstance(documents, list)
or len(documents) != 1
or not isinstance(documents[0], str)
or not isinstance(sections, list)
or not sections
or any(not isinstance(section, str) for section in sections)
):
raise LiveAcceptanceError("Golden Query record violates the live contract")
records.append(
GoldenQuery(
query_id=cast(str, raw["id"]),
query=cast(str, raw["query"]),
expected_document=documents[0],
expected_sections=tuple(cast(list[str], sections)),
)
)
if not MIN_GOLDEN_QUERIES <= len(records) <= MAX_GOLDEN_QUERIES:
raise LiveAcceptanceError("live Golden Dataset must contain 30 to 50 queries")
if len({record.query_id for record in records}) != len(records):
raise LiveAcceptanceError("live Golden Query IDs must be unique")
return tuple(records)
def _matching_rank(
results: Sequence[SourceResult],
*,
expected_filename: str,
expected_sections: tuple[str, ...],
) -> int | None:
"""Return the one-based rank of the first matching canonical source."""
for rank, result in enumerate(results, start=1):
if result.filename != expected_filename or result.section is None:
continue
if any(section in result.section for section in expected_sections):
return rank
return None
def _percentile(values: Sequence[float], percentile: float) -> float:
"""Use nearest-rank percentiles for a small, deterministic acceptance set."""
if not values:
raise ValueError("latency sample must not be empty")
ordered = sorted(values)
index = max(0, math.ceil(percentile * len(ordered)) - 1)
return ordered[index]
def _build_report(
ranks: Sequence[int | None],
latencies_ms: Sequence[float],
thresholds: AcceptanceThresholds,
) -> AcceptanceReport:
"""Calculate binary-relevance metrics and apply explicit acceptance gates."""
if not ranks or len(ranks) != len(latencies_ms):
raise ValueError("live ranks and latencies must have equal non-zero length")
query_count = len(ranks)
def recall_at(limit: int) -> float:
return sum(rank is not None and rank <= limit for rank in ranks) / query_count
reciprocal_ranks = [0.0 if rank is None else 1.0 / rank for rank in ranks]
discounted_gains = [
0.0 if rank is None or rank > 5 else 1.0 / math.log2(rank + 1) for rank in ranks
]
recall_at_1 = recall_at(1)
recall_at_3 = recall_at(3)
recall_at_5 = recall_at(5)
mrr = sum(reciprocal_ranks) / query_count
ndcg_at_5 = sum(discounted_gains) / query_count
p50 = _percentile(latencies_ms, 0.50)
p95 = _percentile(latencies_ms, 0.95)
passed = (
recall_at_1 >= thresholds.min_recall_at_1
and recall_at_5 >= thresholds.min_recall_at_5
and mrr >= thresholds.min_mrr
and ndcg_at_5 >= thresholds.min_ndcg_at_5
and p95 <= thresholds.max_p95_ms
)
return AcceptanceReport(
query_count=query_count,
recall_at_1=recall_at_1,
recall_at_3=recall_at_3,
recall_at_5=recall_at_5,
mrr=mrr,
ndcg_at_5=ndcg_at_5,
latency_p50_ms=p50,
latency_p95_ms=p95,
passed=passed,
)
def _thresholds_from_environment() -> AcceptanceThresholds:
return AcceptanceThresholds(
min_recall_at_1=float(os.getenv("EKP_LIVE_MIN_RECALL_AT_1", str(DEFAULT_MIN_RECALL_AT_1))),
min_recall_at_5=float(os.getenv("EKP_LIVE_MIN_RECALL_AT_5", str(DEFAULT_MIN_RECALL_AT_5))),
min_mrr=float(os.getenv("EKP_LIVE_MIN_MRR", str(DEFAULT_MIN_MRR))),
min_ndcg_at_5=float(os.getenv("EKP_LIVE_MIN_NDCG_AT_5", str(DEFAULT_MIN_NDCG_AT_5))),
max_p95_ms=float(os.getenv("EKP_LIVE_MAX_P95_MS", str(DEFAULT_MAX_P95_MS))),
)
def _require_safe_live_settings(settings: Settings) -> None:
"""Refuse external writes unless rotation and resource authorization are explicit."""
required_flags = (
"EKP_RUN_LIVE_TESTS",
"EKP_CREDENTIALS_AUTHORIZED",
"EKP_LIVE_RESOURCE_ACK",
)
if any(os.getenv(name) != "1" for name in required_flags):
raise LiveAcceptanceError("live acceptance safety flags are not enabled")
if settings.app_env != "test":
raise LiveAcceptanceError("live acceptance requires APP_ENV=test")
if settings.object_storage_provider != "bos" or settings.vector_store_provider != "oceanbase":
raise LiveAcceptanceError("live acceptance requires the target provider baseline")
if not settings.bos_key_prefix.startswith("live-acceptance/"):
raise LiveAcceptanceError("BOS_KEY_PREFIX must use a dedicated live-acceptance prefix")
def _cleanup_run_documents(services: ApiServices, filename_prefix: str) -> None:
"""Delete only documents created under this random acceptance-run prefix."""
documents = tuple(
document
for document in services.documents.list_documents()
if document.filename.startswith(filename_prefix)
)
for document in documents:
services.documents.delete_document(document.id)
def _run_live(settings: Settings) -> AcceptanceReport:
"""Upload synthetic sources, execute every query, and clean run-owned data."""
_require_safe_live_settings(settings)
project_root = _project_root()
golden = _load_golden_queries(project_root / "fixtures/evaluation/golden.jsonl")
services = create_api_services(settings)
filename_prefix = f"live-{uuid4().hex[:12]}-"
uploaded_names: dict[str, str] = {}
cleanup_error: Exception | None = None
try:
source_names = sorted({record.expected_document for record in golden})
for source_name in source_names:
live_name = f"{filename_prefix}{source_name}"
payload = (project_root / "fixtures/documents" / source_name).read_bytes()
services.ingestion.ingest(live_name, "text/markdown", BytesIO(payload))
uploaded_names[source_name] = live_name
ranks: list[int | None] = []
latencies_ms: list[float] = []
for record in golden:
started = perf_counter()
result = services.search.search(record.query, top_k=5)
latencies_ms.append((perf_counter() - started) * 1000)
ranks.append(
_matching_rank(
result.results,
expected_filename=uploaded_names[record.expected_document],
expected_sections=record.expected_sections,
)
)
return _build_report(ranks, latencies_ms, _thresholds_from_environment())
finally:
try:
_cleanup_run_documents(services, filename_prefix)
except Exception as error:
cleanup_error = error
if cleanup_error is not None:
raise LiveAcceptanceError("live acceptance cleanup failed") from cleanup_error
def main() -> int:
"""Print only aggregate metrics and return non-zero for unsafe or failed runs."""
try:
report = _run_live(Settings())
except Exception:
print("live acceptance failed; no provider details were emitted", file=sys.stderr)
return 1
print(json.dumps(asdict(report), sort_keys=True, separators=(",", ":")))
return 0 if report.passed else 2
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -1,10 +1,18 @@
"""FastAPI application entry point."""
from pathlib import Path
from fastapi import FastAPI
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from ekp_api.providers import ApiServices
from ekp_api.routes import router
from ekp_core.config import Settings, get_settings
ADMIN_WEB_ROOT = Path(__file__).resolve().parents[3] / "admin-web"
class HealthResponse(BaseModel):
"""Health endpoint response."""
@ -14,7 +22,10 @@ class HealthResponse(BaseModel):
environment: str
def create_app(settings: Settings | None = None) -> FastAPI:
def create_app(
settings: Settings | None = None,
api_services: ApiServices | None = None,
) -> FastAPI:
"""Create the API application with explicit, testable settings."""
resolved_settings = settings or get_settings()
application = FastAPI(
@ -23,6 +34,26 @@ def create_app(settings: Settings | None = None) -> FastAPI:
docs_url=f"{resolved_settings.api_prefix}/docs",
openapi_url=f"{resolved_settings.api_prefix}/openapi.json",
)
application.state.settings = resolved_settings
application.state.api_services = api_services
application.include_router(router, prefix=resolved_settings.api_prefix)
admin_web_root = (resolved_settings.admin_web_root or ADMIN_WEB_ROOT).resolve()
# The Demo console is a deliberately thin, same-origin client. It contains
# no business logic and calls only the versioned public REST API.
application.mount(
"/admin/assets",
StaticFiles(directory=admin_web_root / "assets"),
name="admin-assets",
)
@application.get("/admin", include_in_schema=False)
@application.get("/admin/upload", include_in_schema=False)
@application.get("/admin/documents", include_in_schema=False)
@application.get("/admin/search", include_in_schema=False)
async def admin_console() -> FileResponse:
"""Serve the three-page Demo Admin Console shell."""
return FileResponse(admin_web_root / "index.html")
@application.get("/healthz", response_model=HealthResponse, tags=["system"])
async def health() -> HealthResponse:

View File

@ -0,0 +1,347 @@
"""Runtime provider and application-service composition from safe settings."""
from dataclasses import dataclass
from sqlalchemy.orm import Session, sessionmaker
from ekp_api.celery_queue import CeleryIngestionDispatcher, create_celery_app
from ekp_core.chunking import StructureAwareChunker
from ekp_core.config import Settings
from ekp_core.database import create_database_engine, create_session_factory
from ekp_core.dense_retrieval import DenseRetrievalService
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
from ekp_core.document_indexing import (
DocumentIndexingPipelineService,
DocumentIndexingService,
)
from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.index_rebuild import resolve_active_index_profile
from ekp_core.ingestion_batches import IngestionBatchService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
)
from ekp_core.search import SearchService
from ekp_domain import IndexProfile
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
from ekp_object_storage import ObjectStorage, create_bos_object_storage
from ekp_object_storage.local import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_reranker import QianfanRerankerProvider, RerankerProvider
from ekp_vector_store import (
VectorStore,
create_oceanbase_vector_store,
)
BGE_LARGE_ZH_DIMENSIONS = 1024
DEMO_CHUNK_STRATEGY_VERSION = "structure_v1"
class ProviderConfigurationError(ValueError):
"""Raised when a runtime provider lacks required injected configuration."""
@dataclass(frozen=True, slots=True)
class ApiServices:
"""Business services injected into REST routes without provider leakage."""
ingestion: DocumentIndexingPipelineService
submissions: AsyncIngestionSubmissionService
batches: IngestionBatchService
jobs: IngestionJobService
documents: DocumentManagementService
search: SearchService
@dataclass(frozen=True, slots=True)
class WorkerServices:
"""Provider graph required by Celery without REST search-only services."""
runner: IngestionJobRunner
jobs: IngestionJobService
dispatcher: CeleryIngestionDispatcher
@dataclass(frozen=True, slots=True)
class _IngestionComponents:
"""Shared composition result used by API and independent worker processes."""
session_factory: sessionmaker[Session]
storage: ObjectStorage
vector_store: VectorStore
embedding_provider: EmbeddingProvider
index_profile: IndexProfile
upload: DocumentUploadService
parsing: DocumentParsingService
chunking: DocumentChunkingService
indexing: DocumentIndexingService
jobs: IngestionJobService
dispatcher: CeleryIngestionDispatcher
def create_index_profile(
settings: Settings,
session_factory: sessionmaker[Session] | None = None,
) -> IndexProfile:
"""Resolve the canonical active profile or the fresh-deployment baseline."""
configured = IndexProfile(
profile_id=settings.index_profile_id,
chunk_strategy_version=DEMO_CHUNK_STRATEGY_VERSION,
embedding_model=settings.embedding_model,
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
)
if session_factory is None:
return configured
return resolve_active_index_profile(session_factory, configured)
def create_embedding_provider(settings: Settings) -> QianfanEmbeddingProvider:
"""Create Qianfan only when its API key is supplied by the runtime."""
if settings.baidu_qianfan_api_key is None:
raise ProviderConfigurationError("BAIDU_QIANFAN_API_KEY is required")
return QianfanEmbeddingProvider(
api_key=settings.baidu_qianfan_api_key.get_secret_value(),
endpoint=settings.embedding_endpoint,
timeout_seconds=settings.embedding_timeout_seconds,
)
def create_object_storage(settings: Settings) -> ObjectStorage:
"""Create local or BOS storage while keeping credentials out of core services."""
if settings.object_storage_provider == "local":
return LocalObjectStorage(settings.object_storage_path)
if settings.bos_endpoint is None:
raise ProviderConfigurationError("BOS_ENDPOINT is required")
if settings.bos_access_key_id is None:
raise ProviderConfigurationError("BOS_ACCESS_KEY_ID is required")
if settings.bos_secret_access_key is None:
raise ProviderConfigurationError("BOS_SECRET_ACCESS_KEY is required")
if settings.bos_bucket is None:
raise ProviderConfigurationError("BOS_BUCKET is required")
return create_bos_object_storage(
endpoint=settings.bos_endpoint,
access_key_id=settings.bos_access_key_id.get_secret_value(),
secret_access_key=settings.bos_secret_access_key.get_secret_value(),
bucket_name=settings.bos_bucket,
key_prefix=settings.bos_key_prefix,
timeout_seconds=settings.bos_timeout_seconds,
)
def create_vector_store(settings: Settings) -> VectorStore:
"""Create the OceanBase adapter from explicit, secret-safe settings."""
if settings.oceanbase_vector_uri is None:
raise ProviderConfigurationError("OCEANBASE_VECTOR_URI is required")
if settings.oceanbase_vector_user is None:
raise ProviderConfigurationError("OCEANBASE_VECTOR_USER is required")
if settings.oceanbase_vector_password is None:
raise ProviderConfigurationError("OCEANBASE_VECTOR_PASSWORD is required")
if settings.oceanbase_vector_database is None:
raise ProviderConfigurationError("OCEANBASE_VECTOR_DATABASE is required")
return create_oceanbase_vector_store(
uri=settings.oceanbase_vector_uri,
user=settings.oceanbase_vector_user,
password=settings.oceanbase_vector_password.get_secret_value(),
database_name=settings.oceanbase_vector_database,
table_name=settings.oceanbase_vector_table,
index_name=settings.oceanbase_vector_index,
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
batch_size=settings.oceanbase_vector_batch_size,
timeout_seconds=settings.oceanbase_vector_timeout_seconds,
)
def create_dense_retrieval_service(
settings: Settings,
embedding_provider: EmbeddingProvider,
vector_store: VectorStore,
index_profile: IndexProfile,
) -> DenseRetrievalService:
"""Compose the profile-scoped dense stage reused by ingestion search APIs."""
return DenseRetrievalService(
embedding_provider=embedding_provider,
vector_store=vector_store,
index_profile=index_profile,
default_top_n=settings.dense_top_n,
)
def create_reranker_provider(settings: Settings) -> QianfanRerankerProvider:
"""Create the Demo BCE reranker using an explicit or shared Qianfan key."""
secret = settings.reranker_api_key or settings.baidu_qianfan_api_key
if secret is None:
raise ProviderConfigurationError("RERANKER_API_KEY or BAIDU_QIANFAN_API_KEY is required")
return QianfanRerankerProvider(
api_key=secret.get_secret_value(),
endpoint=settings.reranker_endpoint,
model_name=settings.reranker_model,
timeout_seconds=settings.reranker_timeout_seconds,
)
def create_search_service(
settings: Settings,
session_factory: sessionmaker[Session],
dense_retriever: DenseRetrievalService,
reranker_provider: RerankerProvider,
) -> SearchService:
"""Compose reranked search independently from REST response schemas."""
return SearchService(
session_factory=session_factory,
dense_retriever=dense_retriever,
reranker_provider=reranker_provider,
default_top_k=settings.search_top_k,
)
def create_runtime_search_service(settings: Settings) -> SearchService:
"""Compose only read-path providers for the independent MCP Gateway."""
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
index_profile = create_index_profile(settings, session_factory)
embedding_provider = create_embedding_provider(settings)
vector_store = create_vector_store(settings)
dense_retrieval = create_dense_retrieval_service(
settings,
embedding_provider,
vector_store,
index_profile,
)
return create_search_service(
settings,
session_factory,
dense_retrieval,
create_reranker_provider(settings),
)
def _create_ingestion_components(settings: Settings) -> _IngestionComponents:
"""Compose provider-neutral ingestion stages and the Celery delivery adapter."""
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
raise ProviderConfigurationError("RABBITMQ_URL is required")
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
storage = create_object_storage(settings)
index_profile = create_index_profile(settings, session_factory)
embedding_provider = create_embedding_provider(settings)
vector_store = create_vector_store(settings)
upload_service = DocumentUploadService(
session_factory,
storage,
max_upload_bytes=settings.max_upload_bytes,
)
parsing_service = DocumentParsingService(session_factory, storage, ParserRouter())
chunking_service = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(
max_tokens=settings.chunk_max_tokens,
overlap_tokens=settings.chunk_overlap_tokens,
max_characters=settings.chunk_max_characters,
),
)
indexing_service = DocumentIndexingService(
session_factory,
embedding_provider,
vector_store,
index_profile,
)
jobs = IngestionJobService(session_factory, settings.ingestion_max_attempts)
dispatcher = CeleryIngestionDispatcher(create_celery_app(settings))
return _IngestionComponents(
session_factory=session_factory,
storage=storage,
vector_store=vector_store,
embedding_provider=embedding_provider,
index_profile=index_profile,
upload=upload_service,
parsing=parsing_service,
chunking=chunking_service,
indexing=indexing_service,
jobs=jobs,
dispatcher=dispatcher,
)
def _create_runner(settings: Settings, components: _IngestionComponents) -> IngestionJobRunner:
"""Compose retry cleanup and execution policy from non-sensitive settings."""
cleanup = DerivedDocumentCleanupService(
components.session_factory,
components.storage,
components.vector_store,
components.index_profile.profile_id,
)
return IngestionJobRunner(
components.session_factory,
components.parsing,
components.chunking,
components.indexing,
cleanup,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
retry_backoff_max_seconds=settings.ingestion_retry_backoff_max_seconds,
lease_seconds=settings.ingestion_worker_lease_seconds,
)
def create_worker_services(settings: Settings) -> WorkerServices:
"""Build the independent worker runtime from the same application boundaries."""
components = _create_ingestion_components(settings)
return WorkerServices(
runner=_create_runner(settings, components),
jobs=components.jobs,
dispatcher=components.dispatcher,
)
def create_api_services(settings: Settings) -> ApiServices:
"""Compose REST services while ingestion execution remains in workers."""
components = _create_ingestion_components(settings)
document_pipeline = DocumentPipelineService(
components.upload,
components.parsing,
components.chunking,
)
ingestion = DocumentIndexingPipelineService(document_pipeline, components.indexing)
submissions = AsyncIngestionSubmissionService(
components.upload,
components.jobs,
components.dispatcher,
)
batches = IngestionBatchService(
components.session_factory,
submissions,
max_files=settings.batch_max_files,
max_total_bytes=settings.batch_max_total_bytes,
)
dense_retrieval = create_dense_retrieval_service(
settings,
components.embedding_provider,
components.vector_store,
components.index_profile,
)
reranker_provider = create_reranker_provider(settings)
search = create_search_service(
settings,
components.session_factory,
dense_retrieval,
reranker_provider,
)
documents = DocumentManagementService(
components.session_factory,
components.storage,
components.vector_store,
active_index_profile_id=components.index_profile.profile_id,
)
return ApiServices(
ingestion=ingestion,
submissions=submissions,
batches=batches,
jobs=components.jobs,
documents=documents,
search=search,
)

View File

@ -0,0 +1,224 @@
"""Versioned document and search routes for the retrieval Demo."""
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
from ekp_api.providers import ApiServices, ProviderConfigurationError, create_api_services
from ekp_api.schemas import (
DocumentResponse,
IngestionAcceptedResponse,
IngestionBatchResponse,
IngestionJobResponse,
SearchRequest,
SearchResponse,
)
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
from ekp_core.document_management import (
DocumentArtifactCleanupError,
DocumentIngestionActiveError,
DocumentNotFoundError,
)
from ekp_core.document_upload import (
DocumentTooLargeError,
DocumentUploadError,
MediaTypeMismatchError,
UnsupportedDocumentTypeError,
)
from ekp_core.ingestion_batches import BatchFileCountLimitError, BatchNotFoundError, BatchSource
from ekp_core.ingestion_jobs import IngestionJobNotFoundError
router = APIRouter()
def get_api_services(request: Request) -> ApiServices:
"""Resolve provider composition lazily so health checks need no cloud credentials."""
configured = getattr(request.app.state, "api_services", None)
if isinstance(configured, ApiServices):
return configured
try:
services = create_api_services(request.app.state.settings)
except ProviderConfigurationError as error:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="API providers are not fully configured",
) from error
request.app.state.api_services = services
return services
Services = Annotated[ApiServices, Depends(get_api_services)]
@router.post(
"/documents",
response_model=IngestionAcceptedResponse,
status_code=status.HTTP_202_ACCEPTED,
tags=["documents"],
)
def upload_document(
services: Services,
file: Annotated[UploadFile, File(description="One PDF, DOCX, TXT, or Markdown file")],
) -> IngestionAcceptedResponse:
"""Persist one immutable source and durable Job, then return immediately."""
try:
submission = services.submissions.submit(file.filename or "", file.content_type, file.file)
except DocumentTooLargeError as error:
raise HTTPException(
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
detail="document exceeds the configured upload limit",
) from error
except (UnsupportedDocumentTypeError, MediaTypeMismatchError) as error:
raise HTTPException(
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
detail="unsupported document type or media type",
) from error
except DocumentUploadError as error:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail="document could not be accepted",
) from error
except Exception as error:
# Provider and persistence exceptions are intentionally not reflected to clients.
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="document acceptance failed",
) from error
return IngestionAcceptedResponse.from_submission(submission)
@router.get("/jobs/{job_id}", response_model=IngestionJobResponse, tags=["jobs"])
def get_ingestion_job(job_id: UUID, services: Services) -> IngestionJobResponse:
"""Return durable stage, attempt and safe failure state for one Job."""
try:
job = services.jobs.get(job_id)
except IngestionJobNotFoundError as error:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="job not found"
) from error
return IngestionJobResponse.from_job(job)
@router.post(
"/ingestion-batches",
response_model=IngestionBatchResponse,
status_code=status.HTTP_202_ACCEPTED,
tags=["jobs"],
)
def upload_ingestion_batch(
services: Services,
files: Annotated[
list[UploadFile],
File(description="One or more PDF, DOCX, TXT, or Markdown files"),
],
) -> IngestionBatchResponse:
"""Accept a bounded batch while isolating every file's result."""
try:
view = services.batches.submit(
tuple(
BatchSource(
filename=file.filename or "",
media_type=file.content_type,
source=file.file,
)
for file in files
)
)
except BatchFileCountLimitError as error:
raise HTTPException(
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
detail="batch file count exceeds the configured limit",
) from error
except Exception as error:
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="batch could not be accepted",
) from error
return IngestionBatchResponse.from_view(view)
@router.get(
"/ingestion-batches/{batch_id}",
response_model=IngestionBatchResponse,
tags=["jobs"],
)
def get_ingestion_batch(batch_id: UUID, services: Services) -> IngestionBatchResponse:
"""Return aggregate and per-file status for Admin Console polling."""
try:
view = services.batches.get(batch_id)
except BatchNotFoundError as error:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="ingestion batch not found",
) from error
return IngestionBatchResponse.from_view(view)
@router.get("/documents", response_model=list[DocumentResponse], tags=["documents"])
def list_documents(services: Services) -> list[DocumentResponse]:
"""List canonical document metadata for the Demo admin page."""
return [
DocumentResponse.from_document(document) for document in services.documents.list_documents()
]
@router.get("/documents/{document_id}", response_model=DocumentResponse, tags=["documents"])
def get_document(document_id: UUID, services: Services) -> DocumentResponse:
"""Return one document's status and safe processing diagnostic."""
try:
document = services.documents.get_document(document_id)
except DocumentNotFoundError as error:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
) from error
return DocumentResponse.from_document(document)
@router.delete(
"/documents/{document_id}",
status_code=status.HTTP_204_NO_CONTENT,
tags=["documents"],
)
def delete_document(document_id: UUID, services: Services) -> Response:
"""Directly remove Demo metadata, source objects, and vector rows."""
try:
services.documents.delete_document(document_id)
except DocumentNotFoundError as error:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="document not found"
) from error
except DocumentIngestionActiveError as error:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="document ingestion is still active",
) from error
except DocumentArtifactCleanupError as error:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="document metadata was deleted but artifact cleanup failed",
) from error
except Exception as error:
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="document deletion failed",
) from error
return Response(status_code=status.HTTP_204_NO_CONTENT)
@router.post("/search", response_model=SearchResponse, tags=["search"])
def search_knowledge(payload: SearchRequest, services: Services) -> SearchResponse:
"""Return reranked Top-K sources without leaking provider or index details."""
try:
result = services.search.search(payload.query, payload.top_k)
except InvalidRetrievalQueryError as error:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail="search query must contain non-whitespace text",
) from error
except Exception as error:
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="search providers failed",
) from error
return SearchResponse.from_result(result)

View File

@ -0,0 +1,215 @@
"""Provider-neutral REST request and response schemas for the Demo API."""
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
from ekp_core.ingestion_batches import BatchAggregateStatus, BatchItemView, BatchView
from ekp_core.ingestion_jobs import IngestionSubmission
from ekp_core.search import SearchResult
from ekp_domain import (
BatchItemAcceptanceStatus,
Document,
DocumentStatus,
IngestionJob,
IngestionJobStatus,
IngestionStage,
)
class DocumentResponse(BaseModel):
"""Public document metadata without internal storage or provider fields."""
model_config = ConfigDict(from_attributes=True)
id: UUID
filename: str
media_type: str
size_bytes: int
status: DocumentStatus
error_message: str | None
created_at: datetime
updated_at: datetime
@classmethod
def from_document(cls, document: Document) -> "DocumentResponse":
"""Serialize a detached canonical model through the explicit API shape."""
return cls.model_validate(document)
class IngestionAcceptedResponse(DocumentResponse):
"""Single-file acceptance with the durable Job needed for status polling."""
job_id: UUID
job_status: IngestionJobStatus
max_attempts: int
@classmethod
def from_submission(cls, submission: IngestionSubmission) -> "IngestionAcceptedResponse":
"""Flatten identifiers while retaining the existing document fields."""
document = submission.document
job = submission.job
return cls(
id=document.id,
filename=document.filename,
media_type=document.media_type,
size_bytes=document.size_bytes,
status=document.status,
error_message=document.error_message,
created_at=document.created_at,
updated_at=document.updated_at,
job_id=job.id,
job_status=job.status,
max_attempts=job.max_attempts,
)
class IngestionJobResponse(BaseModel):
"""Public queue-independent Job status without provider or broker details."""
id: UUID
document_id: UUID
status: IngestionJobStatus
stage: IngestionStage | None
attempt: int
max_attempts: int
error_message: str | None
created_at: datetime
updated_at: datetime
completed_at: datetime | None
@classmethod
def from_job(cls, job: IngestionJob) -> "IngestionJobResponse":
"""Serialize only client-actionable state and safe diagnostics."""
return cls(
id=job.id,
document_id=job.document_id,
status=job.status,
stage=job.stage,
attempt=job.attempt_count,
max_attempts=job.max_attempts,
error_message=job.error_message,
created_at=job.created_at,
updated_at=job.updated_at,
completed_at=job.completed_at,
)
class IngestionBatchItemResponse(BaseModel):
"""Per-file acceptance and latest asynchronous processing state."""
item_index: int
filename: str
acceptance_status: BatchItemAcceptanceStatus
document_id: UUID | None
job_id: UUID | None
job_status: IngestionJobStatus | None
stage: IngestionStage | None
attempt: int
max_attempts: int | None
error_message: str | None
@classmethod
def from_view(cls, view: BatchItemView) -> "IngestionBatchItemResponse":
job = view.job
return cls(
item_index=view.item.item_index,
filename=view.item.filename,
acceptance_status=view.item.acceptance_status,
document_id=view.item.document_id,
job_id=view.item.job_id,
job_status=job.status if job is not None else None,
stage=job.stage if job is not None else None,
attempt=job.attempt_count if job is not None else 0,
max_attempts=job.max_attempts if job is not None else None,
error_message=(job.error_message if job is not None else view.item.error_message),
)
class IngestionBatchResponse(BaseModel):
"""Batch aggregate with ordered independently processed files."""
batch_id: UUID
status: BatchAggregateStatus
total_files: int
accepted_files: int
rejected_files: int
completed_files: int
created_at: datetime
items: list[IngestionBatchItemResponse]
@classmethod
def from_view(cls, view: BatchView) -> "IngestionBatchResponse":
items = [IngestionBatchItemResponse.from_view(item) for item in view.items]
accepted = sum(
item.acceptance_status == BatchItemAcceptanceStatus.ACCEPTED for item in items
)
rejected = sum(
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED for item in items
)
completed = sum(
item.acceptance_status == BatchItemAcceptanceStatus.REJECTED
or item.job_status
in {
IngestionJobStatus.SUCCEEDED,
IngestionJobStatus.FAILED,
IngestionJobStatus.CANCELLED,
}
for item in items
)
return cls(
batch_id=view.batch.id,
status=view.status,
total_files=view.batch.total_files,
accepted_files=accepted,
rejected_files=rejected,
completed_files=completed,
created_at=view.batch.created_at,
items=items,
)
class SearchRequest(BaseModel):
"""Validated public search input; normalization remains in the core service."""
query: str = Field(min_length=1, max_length=1600)
top_k: int = Field(default=5, ge=1, le=64, strict=True)
class SourceResponse(BaseModel):
"""Traceable source returned after dense retrieval and reranking."""
document_id: UUID
filename: str
page: int | None
section: str | None
chunk_id: UUID
content: str
score: float
rerank_score: float
class SearchResponse(BaseModel):
"""Minimal response contract consumed by the DEMO-010 Search Lab."""
results: list[SourceResponse]
@classmethod
def from_result(cls, result: SearchResult) -> "SearchResponse":
"""Hide index-profile internals while preserving every required source field."""
return cls(
results=[
SourceResponse(
document_id=source.document_id,
filename=source.filename,
page=source.page,
section=source.section,
chunk_id=source.chunk_id,
content=source.content,
score=source.score,
rerank_score=source.rerank_score,
)
for source in result.results
]
)

View File

@ -0,0 +1,51 @@
"""Celery worker entry point for durable ingestion Jobs."""
from functools import lru_cache
from typing import Any
from uuid import UUID
from ekp_api.celery_queue import (
INGESTION_TASK_NAME,
RECOVERY_TASK_NAME,
create_celery_app,
)
from ekp_api.providers import WorkerServices, create_worker_services
from ekp_core.config import get_settings
from ekp_core.ingestion_jobs import JobRunAction, dispatch_pending_jobs
_settings = get_settings()
celery_app = create_celery_app(_settings)
@lru_cache
def _worker_services() -> WorkerServices:
"""Build one provider/service graph per worker process."""
return create_worker_services(_settings)
@celery_app.task( # type: ignore[untyped-decorator]
bind=True,
name=INGESTION_TASK_NAME,
max_retries=None,
)
def run_ingestion(self: Any, job_id: str) -> None:
"""Execute one delivery and ask Celery to redeliver only when required.
Celery delivery retries are unbounded because valid-lease deferrals are not
business Attempts; the durable runner independently enforces max_attempts.
"""
result = _worker_services().runner.run(UUID(job_id))
if result.action == JobRunAction.RETRY:
raise self.retry(countdown=result.retry_delay_seconds or 1, max_retries=None)
@celery_app.task(name=RECOVERY_TASK_NAME) # type: ignore[untyped-decorator]
def recover_ingestion_jobs() -> int:
"""Redeliver due durable Jobs after API/broker or worker interruptions."""
services = _worker_services()
return dispatch_pending_jobs(services.jobs, services.dispatcher)
def main() -> None:
"""Run one worker with the embedded Demo recovery scheduler."""
celery_app.worker_main(["worker", "--loglevel=INFO", "--queues=ingestion", "--beat"])

View File

@ -0,0 +1,5 @@
"""Enterprise Knowledge MCP gateway."""
from ekp_mcp.server import create_mcp_http_app, create_mcp_server
__all__ = ["create_mcp_http_app", "create_mcp_server"]

View File

@ -0,0 +1,76 @@
"""Request-scoped MCP service identity and bearer-token verification."""
from dataclasses import dataclass
from secrets import compare_digest
from mcp.server.auth.middleware.auth_context import get_access_token
from mcp.server.auth.provider import AccessToken
from mcp.server.mcpserver.exceptions import ToolError
from pydantic import SecretStr
@dataclass(frozen=True, slots=True)
class McpIdentity:
"""Authenticated client/service identity passed across the application boundary."""
client_id: str
subject: str
scopes: frozenset[str]
class StaticServiceTokenVerifier:
"""Validate one runtime-injected, least-privilege Demo service credential.
The verifier is intentionally small for DEMO-012. It establishes the MCP
identity boundary without pretending to be the V1 OAuth or ACL solution.
"""
__slots__ = (
"_client_id",
"_expected_token",
"_issuer",
"_scope",
"_subject",
)
def __init__(
self,
expected_token: SecretStr,
*,
client_id: str,
subject: str,
scope: str,
issuer: str,
) -> None:
self._expected_token = expected_token
self._client_id = client_id
self._subject = subject
self._scope = scope
self._issuer = issuer
async def verify_token(self, token: str) -> AccessToken | None:
"""Return scoped identity metadata only for a constant-time token match."""
if not compare_digest(token, self._expected_token.get_secret_value()):
return None
return AccessToken(
token=token,
client_id=self._client_id,
subject=self._subject,
scopes=[self._scope],
claims={"iss": self._issuer},
)
def current_mcp_identity() -> McpIdentity:
"""Resolve the SDK-authenticated principal for the active tool request."""
access_token = get_access_token()
if access_token is None:
# HTTP middleware normally rejects this earlier; retaining the guard
# prevents accidental unauthenticated in-process tool invocation.
raise ToolError("authentication required")
subject = access_token.subject or access_token.client_id
return McpIdentity(
client_id=access_token.client_id,
subject=subject,
scopes=frozenset(access_token.scopes),
)

View File

@ -0,0 +1,30 @@
"""Command-line entry point for the independent MCP gateway process."""
from mcp.server.transport_security import TransportSecuritySettings
from ekp_core.config import get_settings
from ekp_mcp.server import create_mcp_server
def main() -> None:
"""Run the authenticated Streamable HTTP server from environment settings."""
settings = get_settings()
server = create_mcp_server(settings)
server.run(
transport="streamable-http",
host=settings.mcp_host,
port=settings.mcp_port,
streamable_http_path=settings.mcp_path,
json_response=True,
stateless_http=True,
max_request_body_size=settings.mcp_max_request_body_bytes,
transport_security=TransportSecuritySettings(
enable_dns_rebinding_protection=True,
allowed_hosts=settings.mcp_allowed_hosts,
allowed_origins=settings.mcp_allowed_origins,
),
)
if __name__ == "__main__": # pragma: no cover - exercised through the console script.
main()

View File

@ -0,0 +1,77 @@
"""MCP-specific application boundary over the provider-neutral SearchService."""
from dataclasses import dataclass
from typing import Protocol
from uuid import UUID
from pydantic import BaseModel
from ekp_core.search import SearchResult
from ekp_mcp.auth import McpIdentity
class KnowledgeSearcher(Protocol):
"""Subset of SearchService required by the MCP gateway."""
def search(self, query: str, top_k: int | None = None) -> SearchResult:
"""Return reranked canonical sources for a normalized query."""
class McpSourceResponse(BaseModel):
"""Provider-neutral source shape exposed by `search_knowledge`."""
document_id: UUID
filename: str
page: int | None
section: str | None
chunk_id: UUID
content: str
score: float
rerank_score: float
class McpSearchResponse(BaseModel):
"""Structured MCP result aligned with the REST search source fields."""
results: list[McpSourceResponse]
class McpKnowledgeSearch(Protocol):
"""Identity-aware search boundary consumed by the protocol server."""
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
"""Return authorized, structured knowledge sources."""
@dataclass(frozen=True, slots=True)
class McpSearchApplication:
"""Require an identity before delegating to the shared SearchService.
Document ACL is outside the current Demo scope. Keeping identity explicit
here prevents the gateway from becoming an anonymous bypass when the
permission resolver is added in the V1 tasks.
"""
searcher: KnowledgeSearcher
required_scope: str
def search(self, identity: McpIdentity, query: str, top_k: int) -> McpSearchResponse:
"""Authorize the Demo scope and serialize canonical search results."""
if self.required_scope not in identity.scopes:
raise PermissionError("MCP identity lacks the knowledge search scope")
result = self.searcher.search(query, top_k)
return McpSearchResponse(
results=[
McpSourceResponse(
document_id=source.document_id,
filename=source.filename,
page=source.page,
section=source.section,
chunk_id=source.chunk_id,
content=source.content,
score=source.score,
rerank_score=source.rerank_score,
)
for source in result.results
]
)

View File

@ -0,0 +1,133 @@
"""Authenticated Streamable HTTP MCP server exposing Demo business tools."""
import logging
from functools import partial
from typing import Annotated
import anyio
from mcp.server import MCPServer
from mcp.server.auth.settings import AuthSettings
from mcp.server.mcpserver.exceptions import ToolError
from mcp.server.transport_security import TransportSecuritySettings
from pydantic import Field
from starlette.applications import Starlette
from ekp_api.providers import ApiServices, create_runtime_search_service
from ekp_core.config import Settings, get_settings
from ekp_core.dense_retrieval import InvalidRetrievalQueryError
from ekp_mcp.auth import StaticServiceTokenVerifier, current_mcp_identity
from ekp_mcp.search import McpKnowledgeSearch, McpSearchApplication, McpSearchResponse
logger = logging.getLogger(__name__)
class McpConfigurationError(ValueError):
"""Raised when the gateway lacks its dedicated runtime credential."""
def _create_search_application(
settings: Settings,
api_services: ApiServices | None,
) -> McpSearchApplication:
searcher = (
api_services.search if api_services is not None else create_runtime_search_service(settings)
)
return McpSearchApplication(searcher, settings.mcp_required_scope)
def create_mcp_server(
settings: Settings | None = None,
*,
api_services: ApiServices | None = None,
search_application: McpKnowledgeSearch | None = None,
) -> MCPServer[dict[str, object]]:
"""Create the authenticated MCP gateway with only `search_knowledge`."""
resolved_settings = settings or get_settings()
if (
resolved_settings.mcp_auth_token is None
or len(resolved_settings.mcp_auth_token.get_secret_value()) < 32
):
raise McpConfigurationError("MCP_AUTH_TOKEN must contain at least 32 characters")
application = search_application or _create_search_application(
resolved_settings,
api_services,
)
token_verifier = StaticServiceTokenVerifier(
resolved_settings.mcp_auth_token,
client_id=resolved_settings.mcp_client_id,
subject=resolved_settings.mcp_service_identity,
scope=resolved_settings.mcp_required_scope,
issuer=str(resolved_settings.mcp_issuer_url),
)
server: MCPServer[dict[str, object]] = MCPServer(
name="enterprise-knowledge",
title="Enterprise Knowledge MCP Server",
description="Authenticated enterprise knowledge search backed by canonical sources.",
version="0.1.0",
token_verifier=token_verifier,
auth=AuthSettings(
issuer_url=resolved_settings.mcp_issuer_url,
resource_server_url=resolved_settings.mcp_public_url,
required_scopes=[resolved_settings.mcp_required_scope],
),
log_level=resolved_settings.log_level,
)
@server.tool(
name="search_knowledge",
description=(
"Search authorized enterprise knowledge and return traceable source passages. "
"Use for factual questions that should be grounded in indexed company documents."
),
structured_output=True,
)
async def search_knowledge(
query: Annotated[str, Field(min_length=1, max_length=1600)],
top_k: Annotated[int, Field(ge=1, le=64, strict=True)] = 5,
) -> McpSearchResponse:
"""Run the shared synchronous search pipeline outside the ASGI event loop."""
identity = current_mcp_identity()
try:
return await anyio.to_thread.run_sync(
partial(application.search, identity, query, top_k)
)
except InvalidRetrievalQueryError as error:
raise ToolError("search query must contain non-whitespace text") from error
except PermissionError as error:
raise ToolError("knowledge search is not authorized") from error
except Exception:
# Do not reflect provider, database, endpoint, or credential details
# into MCP content or logs. Operations can correlate this generic
# event with upstream sanitized service telemetry.
logger.error("MCP knowledge search failed")
raise ToolError("knowledge search failed") from None
return server
def create_mcp_http_app(
settings: Settings | None = None,
*,
api_services: ApiServices | None = None,
search_application: McpKnowledgeSearch | None = None,
) -> Starlette:
"""Build the stateless Streamable HTTP app used by WorkBuddy and tests."""
resolved_settings = settings or get_settings()
server = create_mcp_server(
resolved_settings,
api_services=api_services,
search_application=search_application,
)
return server.streamable_http_app(
streamable_http_path=resolved_settings.mcp_path,
json_response=True,
stateless_http=True,
max_request_body_size=resolved_settings.mcp_max_request_body_bytes,
host=resolved_settings.mcp_host,
transport_security=TransportSecuritySettings(
enable_dns_rebinding_protection=True,
allowed_hosts=resolved_settings.mcp_allowed_hosts,
allowed_origins=resolved_settings.mcp_allowed_origins,
),
)

61
deploy/docker/Dockerfile Normal file
View File

@ -0,0 +1,61 @@
# syntax=docker/dockerfile:1.7
# One locked runtime produces three independently deployable process images.
ARG PYTHON_IMAGE=python:3.12.14-slim-bookworm@sha256:782412e85d0f0984994c290652577d4018aff08145c85b262bb63dc0c7522254
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1
FROM ${UV_IMAGE} AS uv-bin
FROM ${PYTHON_IMAGE} AS runtime
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
VIRTUAL_ENV=/app/.venv \
PATH=/app/.venv/bin:${PATH} \
HOME=/tmp/ekp \
XDG_CACHE_HOME=/tmp/ekp/cache \
APP_ENV=production \
API_HOST=0.0.0.0 \
MCP_HOST=0.0.0.0 \
ADMIN_WEB_ROOT=/app/apps/admin-web
WORKDIR /app
COPY --from=uv-bin /uv /uvx /usr/local/bin/
COPY pyproject.toml uv.lock README.md alembic.ini ./
COPY apps ./apps
COPY packages ./packages
COPY migrations ./migrations
# --frozen prevents lock changes; --no-dev and --no-editable keep the runtime
# independent from source-tree import hooks while retaining migration sources.
RUN uv sync --frozen --no-dev --no-editable \
&& test -f /app/apps/admin-web/index.html \
&& groupadd --gid 10001 ekp \
&& useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin ekp \
&& mkdir -p /tmp/ekp/cache \
&& chown -R 10001:10001 /tmp/ekp \
&& rm -rf /root/.cache/uv
USER 10001:10001
FROM runtime AS api
EXPOSE 8000
# This endpoint is intentionally credential-free and suitable for container liveness.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=2)"]
CMD ["uvicorn", "ekp_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
FROM runtime AS mcp
EXPOSE 8766
# MCP requests require authentication, so the image-level check only verifies PID 1.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import os; os.kill(1, 0)"]
CMD ["ekp-mcp"]
FROM runtime AS worker
# Worker health is process-level here; durable Job recovery is validated by the service tests.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import os; os.kill(1, 0)"]
CMD ["ekp-worker"]

View File

@ -0,0 +1,36 @@
# Reproducible target names consumed by local and CI buildx invocations.
variable "IMAGE_REGISTRY" {
default = "knowledge-platform"
}
variable "IMAGE_TAG" {
default = "dev"
}
group "default" {
targets = ["api", "mcp", "worker"]
}
target "common" {
context = "."
dockerfile = "deploy/docker/Dockerfile"
platforms = ["linux/amd64", "linux/arm64"]
}
target "api" {
inherits = ["common"]
target = "api"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-api:${IMAGE_TAG}"]
}
target "mcp" {
inherits = ["common"]
target = "mcp"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-mcp:${IMAGE_TAG}"]
}
target "worker" {
inherits = ["common"]
target = "worker"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-worker:${IMAGE_TAG}"]
}

View File

@ -0,0 +1,50 @@
# Copy this file outside the repository for one environment. It contains only
# non-sensitive settings and Kubernetes Secret references; never add values for
# DATABASE_URL, RABBITMQ_URL, provider keys, passwords or MCP_AUTH_TOKEN here.
existingSecret: knowledge-platform-runtime
images:
api:
repository: registry.example.internal/knowledge/enterprise-knowledge-api
tag: "0.1.0"
digest: ""
mcp:
repository: registry.example.internal/knowledge/enterprise-knowledge-mcp
tag: "0.1.0"
digest: ""
worker:
repository: registry.example.internal/knowledge/enterprise-knowledge-worker
tag: "0.1.0"
digest: ""
ingress:
enabled: true
className: nginx
api:
host: knowledge.example.internal
tlsSecretName: knowledge-platform-api-tls
mcp:
host: mcp.example.internal
tlsSecretName: knowledge-platform-mcp-tls
config:
objectStorage:
endpoint: https://s3.example.internal
bucket: knowledge-platform
vectorStore:
uri: oceanbase.example.internal:2881
user: knowledge-platform
database: knowledge_platform
mcp:
publicUrl: https://mcp.example.internal/mcp
issuerUrl: https://mcp.example.internal
allowedHosts:
- mcp.example.internal
allowedOrigins:
- https://mcp.example.internal
# External RabbitMQ is the production default. If this local dependency is
# explicitly enabled for a demo cluster, create its Secret before deployment.
rabbitmq:
enabled: false
existingSecret: knowledge-platform-rabbitmq

View File

@ -0,0 +1,15 @@
apiVersion: v2
name: knowledge-platform
description: Enterprise Knowledge Platform public-knowledge deployment
type: application
version: 0.1.0
appVersion: "0.1.0"
# This local dependency is disabled by default. Production deployments should
# use an externally managed RabbitMQ service with an independently owned data
# lifecycle; the subchart exists only for explicit development/demo use.
dependencies:
- name: rabbitmq
version: 0.1.0
repository: file://charts/rabbitmq
condition: rabbitmq.enabled

View File

@ -0,0 +1,6 @@
apiVersion: v2
name: rabbitmq
description: Minimal single-node RabbitMQ dependency for local demo clusters only
type: application
version: 0.1.0
appVersion: "4.1.4"

View File

@ -0,0 +1,11 @@
{{- define "rabbitmq.fullname" -}}
{{- printf "%s-rabbitmq" .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "rabbitmq.image" -}}
{{- if .Values.image.digest -}}
{{- printf "%s@%s" .Values.image.repository .Values.image.digest -}}
{{- else -}}
{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}}
{{- end -}}
{{- end -}}

View File

@ -0,0 +1,16 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "rabbitmq.fullname" . }}
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
clusterIP: None
ports:
- name: amqp
port: {{ .Values.service.port }}
targetPort: amqp
selector:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}

View File

@ -0,0 +1,8 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "rabbitmq.fullname" . }}
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
automountServiceAccountToken: false

View File

@ -0,0 +1,109 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ include "rabbitmq.fullname" . }}
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
serviceName: {{ include "rabbitmq.fullname" . }}
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
labels:
app.kubernetes.io/name: rabbitmq
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
serviceAccountName: {{ include "rabbitmq.fullname" . }}
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 999
runAsGroup: 999
fsGroup: 999
seccompProfile:
type: RuntimeDefault
containers:
- name: rabbitmq
image: {{ include "rabbitmq.image" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
ports:
- name: amqp
containerPort: 5672
env:
- name: RABBITMQ_DEFAULT_USER
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: {{ .Values.secretKeys.username }}
- name: RABBITMQ_DEFAULT_PASS
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: {{ .Values.secretKeys.password }}
- name: RABBITMQ_ERLANG_COOKIE
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: {{ .Values.secretKeys.erlangCookie }}
startupProbe:
exec:
command: ["rabbitmq-diagnostics", "-q", "ping"]
periodSeconds: 5
failureThreshold: 24
readinessProbe:
exec:
command: ["rabbitmq-diagnostics", "-q", "check_running"]
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
livenessProbe:
exec:
command: ["rabbitmq-diagnostics", "-q", "ping"]
periodSeconds: 20
timeoutSeconds: 5
failureThreshold: 3
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: data
mountPath: /var/lib/rabbitmq
- name: config
mountPath: /etc/rabbitmq/conf.d
- name: log
mountPath: /var/log/rabbitmq
- name: tmp
mountPath: /tmp
volumes:
- name: config
emptyDir: {}
- name: log
emptyDir: {}
- name: tmp
emptyDir: {}
{{- if not .Values.persistence.enabled }}
- name: data
emptyDir: {}
{{- end }}
{{- if .Values.persistence.enabled }}
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [{{ .Values.persistence.accessMode | quote }}]
{{- if .Values.persistence.storageClass }}
storageClassName: {{ .Values.persistence.storageClass | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.persistence.size }}
{{- end }}

View File

@ -0,0 +1,69 @@
{
"$schema": "https://json-schema.org/draft-07/schema#",
"type": "object",
"additionalProperties": false,
"required": ["enabled", "existingSecret", "secretKeys", "image", "service", "persistence", "resources"],
"properties": {
"global": {"type": "object"},
"enabled": {"type": "boolean"},
"existingSecret": {"type": "string", "minLength": 1},
"secretKeys": {
"type": "object",
"additionalProperties": false,
"required": ["username", "password", "erlangCookie"],
"properties": {
"username": {"type": "string", "minLength": 1},
"password": {"type": "string", "minLength": 1},
"erlangCookie": {"type": "string", "minLength": 1}
}
},
"image": {
"type": "object",
"additionalProperties": false,
"required": ["repository", "tag", "digest", "pullPolicy"],
"properties": {
"repository": {"type": "string", "minLength": 1},
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
}
},
"service": {
"type": "object",
"additionalProperties": false,
"required": ["port"],
"properties": {"port": {"type": "integer", "minimum": 1, "maximum": 65535}}
},
"persistence": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "storageClass", "accessMode", "size"],
"properties": {
"enabled": {"type": "boolean"},
"storageClass": {"type": "string"},
"accessMode": {"enum": ["ReadWriteOnce", "ReadWriteMany"]},
"size": {"type": "string", "minLength": 2}
}
},
"resources": {
"type": "object",
"additionalProperties": false,
"required": ["requests", "limits"],
"properties": {
"requests": {"$ref": "#/definitions/resourcePair"},
"limits": {"$ref": "#/definitions/resourcePair"}
}
}
},
"definitions": {
"resourcePair": {
"type": "object",
"additionalProperties": false,
"required": ["cpu", "memory"],
"properties": {
"cpu": {"type": ["string", "number"]},
"memory": {"type": "string", "minLength": 2}
}
}
}
}

View File

@ -0,0 +1,25 @@
enabled: false
existingSecret: knowledge-platform-rabbitmq
secretKeys:
username: username
password: password
erlangCookie: erlang-cookie
image:
repository: rabbitmq
tag: 4.1.4-alpine
digest: sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2
pullPolicy: IfNotPresent
service:
port: 5672
persistence:
enabled: true
storageClass: ""
accessMode: ReadWriteOnce
size: 8Gi
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi

View File

@ -0,0 +1,12 @@
Enterprise Knowledge Platform {{ .Chart.AppVersion }} is managed by Helm release {{ .Release.Name }}.
Runtime credentials are read only from Secret {{ .Values.existingSecret }}.
{{- if .Values.ingress.enabled }}
Admin/API: https://{{ .Values.ingress.api.host }}{{ .Values.ingress.api.path }}
MCP: https://{{ .Values.ingress.mcp.host }}{{ .Values.ingress.mcp.path }}
{{- else }}
Ingress is disabled. Enable it in the environment values file before remote use.
{{- end }}
Inspect rollout state without printing environment variables:
kubectl -n {{ .Release.Namespace }} get deploy,job,pod

View File

@ -0,0 +1,106 @@
{{/* Stable resource name helpers keep every workload under one release identity. */}}
{{- define "knowledge-platform.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "knowledge-platform.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := include "knowledge-platform.name" . -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- define "knowledge-platform.labels" -}}
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | quote }}
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end -}}
{{- define "knowledge-platform.selectorLabels" -}}
app.kubernetes.io/name: {{ include "knowledge-platform.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end -}}
{{- define "knowledge-platform.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{- default (include "knowledge-platform.fullname" .) .Values.serviceAccount.name -}}
{{- else -}}
{{- required "serviceAccount.name is required when serviceAccount.create=false" .Values.serviceAccount.name -}}
{{- end -}}
{{- end -}}
{{/* A supplied digest takes precedence over the immutable release tag. */}}
{{- define "knowledge-platform.image" -}}
{{- if .digest -}}
{{- printf "%s@%s" .repository .digest -}}
{{- else -}}
{{- printf "%s:%s" .repository .tag -}}
{{- end -}}
{{- end -}}
{{- define "knowledge-platform.configMapName" -}}
{{- printf "%s-runtime" (include "knowledge-platform.fullname" .) -}}
{{- end -}}
{{/* Cross-field rules complement values.schema.json's structural checks. */}}
{{- define "knowledge-platform.validateValues" -}}
{{- if lt (float64 .Values.config.ingestion.retryBackoffMaxSeconds) (float64 .Values.config.ingestion.retryBackoffSeconds) -}}
{{- fail "config.ingestion.retryBackoffMaxSeconds must be >= retryBackoffSeconds" -}}
{{- end -}}
{{- if lt (float64 .Values.config.ingestion.batchMaxTotalBytes) (float64 .Values.config.ingestion.maxUploadBytes) -}}
{{- fail "config.ingestion.batchMaxTotalBytes must be >= maxUploadBytes" -}}
{{- end -}}
{{- if ge (float64 .Values.config.chunking.overlapTokens) (float64 .Values.config.chunking.maxTokens) -}}
{{- fail "config.chunking.overlapTokens must be < maxTokens" -}}
{{- end -}}
{{- if gt (float64 .Values.config.retrieval.searchTopK) (float64 .Values.config.retrieval.denseTopN) -}}
{{- fail "config.retrieval.searchTopK must be <= denseTopN" -}}
{{- end -}}
{{- end -}}
{{- define "knowledge-platform.podSecurityContext" -}}
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
{{- end -}}
{{- define "knowledge-platform.containerSecurityContext" -}}
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
capabilities:
drop:
- ALL
{{- end -}}
{{- define "knowledge-platform.envFrom" -}}
- configMapRef:
name: {{ include "knowledge-platform.configMapName" . }}
- secretRef:
name: {{ .Values.existingSecret }}
{{- end -}}
{{- define "knowledge-platform.tmpVolume" -}}
- name: tmp
emptyDir:
sizeLimit: 1Gi
{{- end -}}
{{- define "knowledge-platform.tmpMount" -}}
- name: tmp
mountPath: /tmp
{{- end -}}

View File

@ -0,0 +1,75 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-api
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
replicas: {{ .Values.api.replicaCount }}
minReadySeconds: 5
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: api
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: api
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: api
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
ports:
- name: http
containerPort: 8000
protocol: TCP
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
httpGet:
path: /healthz
port: http
{{- toYaml .Values.api.probes.startup | nindent 12 }}
readinessProbe:
httpGet:
path: /healthz
port: http
{{- toYaml .Values.api.probes.readiness | nindent 12 }}
livenessProbe:
httpGet:
path: /healthz
port: http
{{- toYaml .Values.api.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.api.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}

View File

@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "knowledge-platform.fullname" . }}-api
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
type: {{ .Values.api.service.type }}
ports:
- name: http
port: {{ .Values.api.service.port }}
targetPort: http
protocol: TCP
selector:
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: api

View File

@ -0,0 +1,66 @@
{{- include "knowledge-platform.validateValues" . }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "knowledge-platform.configMapName" . }}
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
data:
# Only the explicitly enumerated non-sensitive settings are rendered here.
APP_ENV: "production"
APP_NAME: {{ .Values.config.appName | quote }}
LOG_LEVEL: {{ .Values.config.logLevel | quote }}
API_HOST: "0.0.0.0"
API_PORT: "8000"
API_PREFIX: "/api/v1"
DATABASE_POOL_SIZE: {{ .Values.config.database.poolSize | quote }}
DATABASE_MAX_OVERFLOW: {{ .Values.config.database.maxOverflow | quote }}
DATABASE_POOL_RECYCLE_SECONDS: {{ .Values.config.database.poolRecycleSeconds | quote }}
DATABASE_POOL_TIMEOUT_SECONDS: {{ .Values.config.database.poolTimeoutSeconds | quote }}
DATABASE_CONNECT_TIMEOUT_SECONDS: {{ .Values.config.database.connectTimeoutSeconds | quote }}
INGESTION_MAX_ATTEMPTS: {{ .Values.config.ingestion.maxAttempts | quote }}
INGESTION_RETRY_BACKOFF_SECONDS: {{ .Values.config.ingestion.retryBackoffSeconds | quote }}
INGESTION_RETRY_BACKOFF_MAX_SECONDS: {{ .Values.config.ingestion.retryBackoffMaxSeconds | quote }}
INGESTION_WORKER_LEASE_SECONDS: {{ .Values.config.ingestion.workerLeaseSeconds | quote }}
INGESTION_RECOVERY_INTERVAL_SECONDS: {{ .Values.config.ingestion.recoveryIntervalSeconds | quote }}
BATCH_MAX_FILES: {{ .Values.config.ingestion.batchMaxFiles | quote }}
BATCH_MAX_TOTAL_BYTES: {{ printf "%.0f" .Values.config.ingestion.batchMaxTotalBytes | quote }}
MAX_UPLOAD_BYTES: {{ printf "%.0f" .Values.config.ingestion.maxUploadBytes | quote }}
OBJECT_STORAGE_PROVIDER: {{ .Values.config.objectStorage.provider | quote }}
BOS_ENDPOINT: {{ .Values.config.objectStorage.endpoint | quote }}
BOS_BUCKET: {{ .Values.config.objectStorage.bucket | quote }}
BOS_KEY_PREFIX: {{ .Values.config.objectStorage.keyPrefix | quote }}
BOS_TIMEOUT_SECONDS: {{ .Values.config.objectStorage.timeoutSeconds | quote }}
CHUNK_MAX_TOKENS: {{ .Values.config.chunking.maxTokens | quote }}
CHUNK_OVERLAP_TOKENS: {{ .Values.config.chunking.overlapTokens | quote }}
CHUNK_MAX_CHARACTERS: {{ .Values.config.chunking.maxCharacters | quote }}
EMBEDDING_PROVIDER: {{ .Values.config.embedding.provider | quote }}
EMBEDDING_MODEL: {{ .Values.config.embedding.model | quote }}
EMBEDDING_ENDPOINT: {{ .Values.config.embedding.endpoint | quote }}
EMBEDDING_TIMEOUT_SECONDS: {{ .Values.config.embedding.timeoutSeconds | quote }}
VECTOR_STORE_PROVIDER: {{ .Values.config.vectorStore.provider | quote }}
OCEANBASE_VECTOR_URI: {{ .Values.config.vectorStore.uri | quote }}
OCEANBASE_VECTOR_USER: {{ .Values.config.vectorStore.user | quote }}
OCEANBASE_VECTOR_DATABASE: {{ .Values.config.vectorStore.database | quote }}
OCEANBASE_VECTOR_TABLE: {{ .Values.config.vectorStore.table | quote }}
OCEANBASE_VECTOR_INDEX: {{ .Values.config.vectorStore.index | quote }}
OCEANBASE_VECTOR_BATCH_SIZE: {{ .Values.config.vectorStore.batchSize | quote }}
OCEANBASE_VECTOR_TIMEOUT_SECONDS: {{ .Values.config.vectorStore.timeoutSeconds | quote }}
INDEX_PROFILE_ID: {{ .Values.config.retrieval.indexProfileId | quote }}
DENSE_TOP_N: {{ .Values.config.retrieval.denseTopN | quote }}
SEARCH_TOP_K: {{ .Values.config.retrieval.searchTopK | quote }}
RERANKER_PROVIDER: {{ .Values.config.reranker.provider | quote }}
RERANKER_MODEL: {{ .Values.config.reranker.model | quote }}
RERANKER_ENDPOINT: {{ .Values.config.reranker.endpoint | quote }}
RERANKER_TIMEOUT_SECONDS: {{ .Values.config.reranker.timeoutSeconds | quote }}
MCP_HOST: "0.0.0.0"
MCP_PORT: "8766"
MCP_PATH: "/mcp"
MCP_PUBLIC_URL: {{ .Values.config.mcp.publicUrl | quote }}
MCP_ISSUER_URL: {{ .Values.config.mcp.issuerUrl | quote }}
MCP_CLIENT_ID: {{ .Values.config.mcp.clientId | quote }}
MCP_SERVICE_IDENTITY: {{ .Values.config.mcp.serviceIdentity | quote }}
MCP_REQUIRED_SCOPE: {{ .Values.config.mcp.requiredScope | quote }}
MCP_ALLOWED_HOSTS: {{ .Values.config.mcp.allowedHosts | toJson | quote }}
MCP_ALLOWED_ORIGINS: {{ .Values.config.mcp.allowedOrigins | toJson | quote }}
MCP_MAX_REQUEST_BODY_BYTES: {{ printf "%.0f" .Values.config.mcp.maxRequestBodyBytes | quote }}

View File

@ -0,0 +1,42 @@
{{- if .Values.ingress.enabled }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "knowledge-platform.fullname" . }}
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
ingressClassName: {{ .Values.ingress.className }}
tls:
- hosts:
- {{ .Values.ingress.api.host | quote }}
secretName: {{ .Values.ingress.api.tlsSecretName }}
- hosts:
- {{ .Values.ingress.mcp.host | quote }}
secretName: {{ .Values.ingress.mcp.tlsSecretName }}
rules:
- host: {{ .Values.ingress.api.host | quote }}
http:
paths:
- path: {{ .Values.ingress.api.path }}
pathType: Prefix
backend:
service:
name: {{ include "knowledge-platform.fullname" . }}-api
port:
number: {{ .Values.api.service.port }}
- host: {{ .Values.ingress.mcp.host | quote }}
http:
paths:
- path: {{ .Values.ingress.mcp.path }}
pathType: Prefix
backend:
service:
name: {{ include "knowledge-platform.fullname" . }}-mcp
port:
number: {{ .Values.mcp.service.port }}
{{- end }}

View File

@ -0,0 +1,72 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-mcp
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: mcp
spec:
replicas: {{ .Values.mcp.replicaCount }}
minReadySeconds: 5
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: mcp
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: mcp
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: mcp
image: {{ include "knowledge-platform.image" .Values.images.mcp | quote }}
imagePullPolicy: {{ .Values.images.mcp.pullPolicy }}
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
ports:
- name: mcp
containerPort: 8766
protocol: TCP
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
tcpSocket:
port: mcp
{{- toYaml .Values.mcp.probes.startup | nindent 12 }}
readinessProbe:
tcpSocket:
port: mcp
{{- toYaml .Values.mcp.probes.readiness | nindent 12 }}
livenessProbe:
tcpSocket:
port: mcp
{{- toYaml .Values.mcp.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.mcp.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}

View File

@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "knowledge-platform.fullname" . }}-mcp
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: mcp
spec:
type: {{ .Values.mcp.service.type }}
ports:
- name: mcp
port: {{ .Values.mcp.service.port }}
targetPort: mcp
protocol: TCP
selector:
{{- include "knowledge-platform.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: mcp

View File

@ -0,0 +1,59 @@
{{- if .Values.migration.enabled }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "knowledge-platform.fullname" . }}-migration
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: migration
annotations:
# Helm waits for pre-install/pre-upgrade hooks. A failed migration blocks
# the application rollout, and the previous Job remains until the next run.
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-weight: "-10"
helm.sh/hook-delete-policy: before-hook-creation
spec:
backoffLimit: {{ .Values.migration.backoffLimit }}
activeDeadlineSeconds: {{ .Values.migration.activeDeadlineSeconds }}
ttlSecondsAfterFinished: {{ .Values.migration.ttlSecondsAfterFinished }}
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: migration
spec:
restartPolicy: Never
# Pre-install hooks run before the release ServiceAccount exists. The
# namespace default account is used with token mounting disabled; the
# migration needs no Kubernetes API permissions.
serviceAccountName: default
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: migration
image: {{ include "knowledge-platform.image" .Values.images.api | quote }}
imagePullPolicy: {{ .Values.images.api.pullPolicy }}
command: ["alembic"]
args: ["upgrade", "head"]
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
env:
- name: APP_ENV
value: production
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: {{ .Values.existingSecret }}
key: DATABASE_URL
resources:
{{- toYaml .Values.migration.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
{{- end }}

View File

@ -0,0 +1,71 @@
{{- if .Values.scheduler.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-scheduler
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: scheduler
spec:
# A single Celery Beat owns periodic durable-Job recovery. Recreate prevents
# two schedulers from overlapping during upgrades.
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: scheduler
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: scheduler
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: scheduler
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
workingDir: /tmp
command: ["celery"]
args:
- --app=ekp_api.worker:celery_app
- beat
- --loglevel=INFO
- --schedule=/tmp/celerybeat-schedule
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.scheduler.probes.startup | nindent 12 }}
livenessProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.scheduler.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.scheduler.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}
{{- end }}

View File

@ -0,0 +1,13 @@
{{- if .Values.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "knowledge-platform.serviceAccountName" . }}
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: false
{{- end }}

View File

@ -0,0 +1,17 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "knowledge-platform.fullname" . }}-preflight-references
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
annotations:
# The Make preflight renders this test hook to discover only resource
# names. It never renders or reads Secret data.
helm.sh/hook: test
data:
runtimeSecret: {{ .Values.existingSecret | quote }}
ingressEnabled: {{ .Values.ingress.enabled | quote }}
apiTlsSecret: {{ .Values.ingress.api.tlsSecretName | quote }}
mcpTlsSecret: {{ .Values.ingress.mcp.tlsSecretName | quote }}
rabbitmqEnabled: {{ .Values.rabbitmq.enabled | quote }}
rabbitmqSecret: {{ .Values.rabbitmq.existingSecret | quote }}

View File

@ -0,0 +1,71 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "knowledge-platform.fullname" . }}-worker
labels:
{{- include "knowledge-platform.labels" . | nindent 4 }}
app.kubernetes.io/component: worker
spec:
replicas: {{ .Values.worker.replicaCount }}
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 1
selector:
matchLabels:
{{- include "knowledge-platform.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: worker
template:
metadata:
labels:
{{- include "knowledge-platform.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: worker
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/runtime-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/runtime-secret-reference: {{ .Values.existingSecret | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "knowledge-platform.serviceAccountName" . }}
automountServiceAccountToken: false
terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
securityContext:
{{- include "knowledge-platform.podSecurityContext" . | nindent 8 }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: worker
image: {{ include "knowledge-platform.image" .Values.images.worker | quote }}
imagePullPolicy: {{ .Values.images.worker.pullPolicy }}
workingDir: /tmp
command: ["celery"]
args:
- --app=ekp_api.worker:celery_app
- worker
- --loglevel=INFO
- --queues=ingestion
securityContext:
{{- include "knowledge-platform.containerSecurityContext" . | nindent 12 }}
envFrom:
{{- include "knowledge-platform.envFrom" . | nindent 12 }}
startupProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.worker.probes.startup | nindent 12 }}
livenessProbe:
exec:
command: ["python", "-c", "import os; os.kill(1, 0)"]
{{- toYaml .Values.worker.probes.liveness | nindent 12 }}
resources:
{{- toYaml .Values.worker.resources | nindent 12 }}
volumeMounts:
{{- include "knowledge-platform.tmpMount" . | nindent 12 }}
volumes:
{{- include "knowledge-platform.tmpVolume" . | nindent 8 }}

View File

@ -0,0 +1,338 @@
{
"$schema": "https://json-schema.org/draft-07/schema#",
"title": "Enterprise Knowledge Platform Helm values",
"type": "object",
"additionalProperties": false,
"required": [
"existingSecret",
"images",
"serviceAccount",
"api",
"mcp",
"worker",
"scheduler",
"migration",
"ingress",
"config",
"rabbitmq"
],
"properties": {
"nameOverride": {"type": "string"},
"fullnameOverride": {"type": "string"},
"existingSecret": {"$ref": "#/definitions/kubernetesName"},
"imagePullSecrets": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["name"],
"properties": {"name": {"$ref": "#/definitions/kubernetesName"}}
}
},
"images": {
"type": "object",
"additionalProperties": false,
"required": ["api", "mcp", "worker"],
"properties": {
"api": {"$ref": "#/definitions/image"},
"mcp": {"$ref": "#/definitions/image"},
"worker": {"$ref": "#/definitions/image"}
}
},
"serviceAccount": {
"type": "object",
"additionalProperties": false,
"required": ["create", "name", "annotations"],
"properties": {
"create": {"type": "boolean"},
"name": {"type": "string"},
"annotations": {"$ref": "#/definitions/stringMap"}
}
},
"podAnnotations": {"$ref": "#/definitions/stringMap"},
"podLabels": {"$ref": "#/definitions/stringMap"},
"api": {"$ref": "#/definitions/httpWorkload"},
"mcp": {"$ref": "#/definitions/httpWorkload"},
"worker": {
"type": "object",
"additionalProperties": false,
"required": ["replicaCount", "terminationGracePeriodSeconds", "resources", "probes"],
"properties": {
"replicaCount": {"type": "integer", "minimum": 1, "maximum": 100},
"terminationGracePeriodSeconds": {"type": "integer", "minimum": 30, "maximum": 3600},
"resources": {"$ref": "#/definitions/resources"},
"probes": {"$ref": "#/definitions/processProbes"}
}
},
"scheduler": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "resources", "probes"],
"properties": {
"enabled": {"type": "boolean"},
"resources": {"$ref": "#/definitions/resources"},
"probes": {"$ref": "#/definitions/processProbes"}
}
},
"migration": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "activeDeadlineSeconds", "backoffLimit", "ttlSecondsAfterFinished", "resources"],
"properties": {
"enabled": {"type": "boolean"},
"activeDeadlineSeconds": {"type": "integer", "minimum": 60, "maximum": 3600},
"backoffLimit": {"type": "integer", "minimum": 0, "maximum": 3},
"ttlSecondsAfterFinished": {"type": "integer", "minimum": 0},
"resources": {"$ref": "#/definitions/resources"}
}
},
"ingress": {
"type": "object",
"additionalProperties": false,
"required": ["enabled", "className", "annotations", "api", "mcp"],
"properties": {
"enabled": {"type": "boolean"},
"className": {"type": "string", "minLength": 1},
"annotations": {"$ref": "#/definitions/stringMap"},
"api": {"$ref": "#/definitions/ingressRoute"},
"mcp": {"$ref": "#/definitions/ingressRoute"}
}
},
"config": {
"type": "object",
"additionalProperties": false,
"required": ["appName", "logLevel", "database", "ingestion", "objectStorage", "chunking", "embedding", "vectorStore", "retrieval", "reranker", "mcp"],
"properties": {
"appName": {"type": "string", "minLength": 1},
"logLevel": {"enum": ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]},
"database": {"$ref": "#/definitions/databaseConfig"},
"ingestion": {"$ref": "#/definitions/ingestionConfig"},
"objectStorage": {"$ref": "#/definitions/objectStorageConfig"},
"chunking": {"$ref": "#/definitions/chunkingConfig"},
"embedding": {"$ref": "#/definitions/embeddingConfig"},
"vectorStore": {"$ref": "#/definitions/vectorStoreConfig"},
"retrieval": {"$ref": "#/definitions/retrievalConfig"},
"reranker": {"$ref": "#/definitions/rerankerConfig"},
"mcp": {"$ref": "#/definitions/mcpConfig"}
}
},
"rabbitmq": {
"type": "object",
"required": ["enabled", "existingSecret"],
"properties": {
"enabled": {"type": "boolean"},
"existingSecret": {"$ref": "#/definitions/kubernetesName"}
}
}
},
"definitions": {
"kubernetesName": {
"type": "string",
"minLength": 1,
"maxLength": 253,
"pattern": "^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$"
},
"stringMap": {
"type": "object",
"additionalProperties": {"type": "string"}
},
"image": {
"type": "object",
"additionalProperties": false,
"required": ["repository", "tag", "digest", "pullPolicy"],
"properties": {
"repository": {"type": "string", "minLength": 1},
"tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}},
"digest": {"type": "string", "pattern": "^$|^sha256:[a-f0-9]{64}$"},
"pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]}
}
},
"probeTiming": {
"type": "object",
"additionalProperties": false,
"required": ["failureThreshold", "periodSeconds"],
"properties": {
"failureThreshold": {"type": "integer", "minimum": 1},
"periodSeconds": {"type": "integer", "minimum": 1},
"timeoutSeconds": {"type": "integer", "minimum": 1}
}
},
"httpProbes": {
"type": "object",
"additionalProperties": false,
"required": ["startup", "readiness", "liveness"],
"properties": {
"startup": {"$ref": "#/definitions/probeTiming"},
"readiness": {"$ref": "#/definitions/probeTiming"},
"liveness": {"$ref": "#/definitions/probeTiming"}
}
},
"processProbes": {
"type": "object",
"additionalProperties": false,
"required": ["startup", "liveness"],
"properties": {
"startup": {"$ref": "#/definitions/probeTiming"},
"liveness": {"$ref": "#/definitions/probeTiming"}
}
},
"resources": {
"type": "object",
"additionalProperties": false,
"required": ["requests", "limits"],
"properties": {
"requests": {"$ref": "#/definitions/resourcePair"},
"limits": {"$ref": "#/definitions/resourcePair"}
}
},
"resourcePair": {
"type": "object",
"additionalProperties": false,
"required": ["cpu", "memory"],
"properties": {
"cpu": {"type": ["string", "number"]},
"memory": {"type": "string", "minLength": 2}
}
},
"httpWorkload": {
"type": "object",
"additionalProperties": false,
"required": ["replicaCount", "service", "resources", "probes"],
"properties": {
"replicaCount": {"type": "integer", "minimum": 2, "maximum": 100},
"service": {
"type": "object",
"additionalProperties": false,
"required": ["type", "port"],
"properties": {
"type": {"enum": ["ClusterIP"]},
"port": {"type": "integer", "minimum": 1, "maximum": 65535}
}
},
"resources": {"$ref": "#/definitions/resources"},
"probes": {"$ref": "#/definitions/httpProbes"}
}
},
"ingressRoute": {
"type": "object",
"additionalProperties": false,
"required": ["host", "path", "tlsSecretName"],
"properties": {
"host": {"type": "string", "minLength": 1},
"path": {"type": "string", "pattern": "^/"},
"tlsSecretName": {"$ref": "#/definitions/kubernetesName"}
}
},
"databaseConfig": {
"type": "object",
"additionalProperties": false,
"required": ["poolSize", "maxOverflow", "poolRecycleSeconds", "poolTimeoutSeconds", "connectTimeoutSeconds"],
"properties": {
"poolSize": {"type": "integer", "minimum": 1, "maximum": 100},
"maxOverflow": {"type": "integer", "minimum": 0, "maximum": 100},
"poolRecycleSeconds": {"type": "integer", "minimum": 1},
"poolTimeoutSeconds": {"type": "number", "exclusiveMinimum": 0},
"connectTimeoutSeconds": {"type": "integer", "minimum": 1}
}
},
"ingestionConfig": {
"type": "object",
"additionalProperties": false,
"required": ["maxAttempts", "retryBackoffSeconds", "retryBackoffMaxSeconds", "workerLeaseSeconds", "recoveryIntervalSeconds", "batchMaxFiles", "batchMaxTotalBytes", "maxUploadBytes"],
"properties": {
"maxAttempts": {"type": "integer", "minimum": 1, "maximum": 10},
"retryBackoffSeconds": {"type": "integer", "minimum": 1},
"retryBackoffMaxSeconds": {"type": "integer", "minimum": 1},
"workerLeaseSeconds": {"type": "integer", "minimum": 30},
"recoveryIntervalSeconds": {"type": "integer", "minimum": 5},
"batchMaxFiles": {"type": "integer", "minimum": 1, "maximum": 100},
"batchMaxTotalBytes": {"type": "integer", "minimum": 1},
"maxUploadBytes": {"type": "integer", "minimum": 1}
}
},
"objectStorageConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "endpoint", "bucket", "keyPrefix", "timeoutSeconds"],
"properties": {
"provider": {"const": "bos"},
"endpoint": {"type": "string", "minLength": 1},
"bucket": {"type": "string", "minLength": 1},
"keyPrefix": {"type": "string", "minLength": 1},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"chunkingConfig": {
"type": "object",
"additionalProperties": false,
"required": ["maxTokens", "overlapTokens", "maxCharacters"],
"properties": {
"maxTokens": {"type": "integer", "minimum": 1},
"overlapTokens": {"type": "integer", "minimum": 0},
"maxCharacters": {"type": "integer", "minimum": 4}
}
},
"embeddingConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
"properties": {
"provider": {"const": "qianfan"},
"model": {"const": "bge-large-zh"},
"endpoint": {"type": "string", "minLength": 1},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"vectorStoreConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "uri", "user", "database", "table", "index", "batchSize", "timeoutSeconds"],
"properties": {
"provider": {"const": "oceanbase"},
"uri": {"type": "string", "minLength": 1},
"user": {"type": "string", "minLength": 1},
"database": {"type": "string", "minLength": 1},
"table": {"type": "string", "minLength": 1},
"index": {"type": "string", "minLength": 1},
"batchSize": {"type": "integer", "minimum": 1, "maximum": 1000},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"retrievalConfig": {
"type": "object",
"additionalProperties": false,
"required": ["indexProfileId", "denseTopN", "searchTopK"],
"properties": {
"indexProfileId": {"type": "string", "minLength": 1},
"denseTopN": {"type": "integer", "minimum": 1, "maximum": 64},
"searchTopK": {"type": "integer", "minimum": 1, "maximum": 64}
}
},
"rerankerConfig": {
"type": "object",
"additionalProperties": false,
"required": ["provider", "model", "endpoint", "timeoutSeconds"],
"properties": {
"provider": {"const": "qianfan"},
"model": {"const": "bce-reranker-base"},
"endpoint": {"type": "string", "minLength": 1},
"timeoutSeconds": {"type": "number", "exclusiveMinimum": 0}
}
},
"mcpConfig": {
"type": "object",
"additionalProperties": false,
"required": ["publicUrl", "issuerUrl", "clientId", "serviceIdentity", "requiredScope", "allowedHosts", "allowedOrigins", "maxRequestBodyBytes"],
"properties": {
"publicUrl": {"type": "string", "pattern": "^https://"},
"issuerUrl": {"type": "string", "pattern": "^https://"},
"clientId": {"type": "string", "minLength": 1},
"serviceIdentity": {"type": "string", "minLength": 1},
"requiredScope": {"const": "knowledge:search"},
"allowedHosts": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},
"allowedOrigins": {"type": "array", "minItems": 1, "items": {"type": "string", "pattern": "^https://"}},
"maxRequestBodyBytes": {"type": "integer", "minimum": 1024, "maximum": 4194304}
}
}
}
}

View File

@ -0,0 +1,218 @@
# DEPLOY-002 safe defaults. Real endpoints, immutable image coordinates and
# Secret names are supplied in an environment-specific values file.
nameOverride: ""
fullnameOverride: ""
existingSecret: knowledge-platform-runtime
imagePullSecrets: []
images:
api:
repository: knowledge-platform/enterprise-knowledge-api
tag: "0.1.0"
digest: ""
pullPolicy: IfNotPresent
mcp:
repository: knowledge-platform/enterprise-knowledge-mcp
tag: "0.1.0"
digest: ""
pullPolicy: IfNotPresent
worker:
repository: knowledge-platform/enterprise-knowledge-worker
tag: "0.1.0"
digest: ""
pullPolicy: IfNotPresent
serviceAccount:
create: true
name: ""
annotations: {}
podAnnotations: {}
podLabels: {}
api:
replicaCount: 2
service:
type: ClusterIP
port: 80
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
readiness:
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
mcp:
replicaCount: 2
service:
type: ClusterIP
port: 80
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
readiness:
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
worker:
replicaCount: 2
terminationGracePeriodSeconds: 60
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "2"
memory: 2Gi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
# Celery Beat is separated from scalable workers so only one recovery
# scheduler publishes due durable Jobs at a time.
scheduler:
enabled: true
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
probes:
startup:
failureThreshold: 30
periodSeconds: 2
liveness:
periodSeconds: 20
timeoutSeconds: 2
failureThreshold: 3
migration:
enabled: true
activeDeadlineSeconds: 600
backoffLimit: 0
ttlSecondsAfterFinished: 86400
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
ingress:
enabled: false
className: nginx
annotations: {}
api:
host: knowledge.example.invalid
path: /
tlsSecretName: knowledge-platform-api-tls
mcp:
host: mcp.example.invalid
path: /mcp
tlsSecretName: knowledge-platform-mcp-tls
# Only non-sensitive runtime settings belong here. The templates enumerate
# keys explicitly so credentials cannot be added to this ConfigMap by mistake.
config:
appName: enterprise-knowledge-platform
logLevel: INFO
database:
poolSize: 10
maxOverflow: 10
poolRecycleSeconds: 600
poolTimeoutSeconds: 30
connectTimeoutSeconds: 30
ingestion:
maxAttempts: 3
retryBackoffSeconds: 2
retryBackoffMaxSeconds: 60
workerLeaseSeconds: 300
recoveryIntervalSeconds: 30
batchMaxFiles: 20
batchMaxTotalBytes: 209715200
maxUploadBytes: 52428800
objectStorage:
provider: bos
endpoint: https://s3.example.invalid
bucket: knowledge-platform
keyPrefix: knowledge-platform
timeoutSeconds: 30
chunking:
maxTokens: 450
overlapTokens: 75
maxCharacters: 2000
embedding:
provider: qianfan
model: bge-large-zh
endpoint: https://qianfan.baidubce.com/v2/embeddings
timeoutSeconds: 30
vectorStore:
provider: oceanbase
uri: oceanbase.example.invalid:2881
user: knowledge-platform
database: knowledge_platform
table: knowledge_chunks_v1
index: vidx_knowledge_chunks_embedding
batchSize: 100
timeoutSeconds: 30
retrieval:
indexProfileId: demo-bge-large-zh-structure-v1-oceanbase-v1
denseTopN: 20
searchTopK: 5
reranker:
provider: qianfan
model: bce-reranker-base
endpoint: https://qianfan.baidubce.com/v2/rerank
timeoutSeconds: 30
mcp:
publicUrl: https://mcp.example.invalid/mcp
issuerUrl: https://mcp.example.invalid
clientId: workbuddy-enterprise
serviceIdentity: workbuddy-public-knowledge
requiredScope: knowledge:search
allowedHosts:
- mcp.example.invalid
allowedOrigins:
- https://mcp.example.invalid
maxRequestBodyBytes: 1048576
# Optional local dependency. Keep disabled for production and provide
# RABBITMQ_URL through existingSecret for an externally managed broker.
rabbitmq:
enabled: false
existingSecret: knowledge-platform-rabbitmq

View File

@ -0,0 +1,31 @@
-- ARCH-003 rebuildable OceanBase Vector index for bge-large-zh (1024 dimensions).
-- Canonical metadata is co-located in OceanBase relational tables; this vector
-- table remains a derived index and may be rebuilt from canonical chunks.
-- Requires OceanBase 4.3.5.4+; ARCH-006 verifies PRE-FILTER in the target test cluster.
CREATE TABLE IF NOT EXISTS knowledge_chunks_v1 (
index_record_id VARCHAR(36) NOT NULL,
chunk_id VARCHAR(36) NOT NULL,
document_id VARCHAR(36) NOT NULL,
filename VARCHAR(255) NOT NULL,
page INT NULL,
page_end INT NULL,
section VARCHAR(1024) NULL,
chunk_index INT NOT NULL,
content MEDIUMTEXT NOT NULL,
embedding VECTOR(1024) NOT NULL,
embedding_model VARCHAR(128) NOT NULL,
index_profile_id VARCHAR(128) NOT NULL,
PRIMARY KEY (index_record_id),
KEY idx_knowledge_chunks_document_profile (document_id, index_profile_id),
KEY idx_knowledge_chunks_profile (index_profile_id),
VECTOR KEY vidx_knowledge_chunks_embedding (embedding)
WITH (
DISTANCE = COSINE,
TYPE = HNSW,
LIB = VSAG,
M = 16,
EF_CONSTRUCTION = 256,
EF_SEARCH = 200
) BLOCK_SIZE 16384
) ORGANIZATION = HEAP;

View File

@ -0,0 +1,12 @@
{
"mcpServers": {
"enterprise-knowledge": {
"type": "http",
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
"headers": {
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
},
"description": "企业知识检索"
}
}
}

View File

@ -5,3 +5,139 @@ Version all public APIs under `/api/v1`.
Initial domains: auth, users/groups, knowledge-bases, documents, compiled-knowledge, search, evaluations, jobs, audit-logs, providers and MCP administration.
Use OpenAPI as executable source of truth once implementation starts. Provider-specific concepts must not leak into public contracts.
## Demo v0.1 P0
DEMO-009 公开以下最小接口。`docs/demo/DEMO_SCOPE.md` 中的 `/api/*` 逻辑路径按本契约统一
版本化为 `/api/v1/*`。
`DEMO-015` 已将单文件上传切换为持久化异步 Job;`DEMO-016` 已增加批量提交和聚合查询。
| Method | Path | 行为 |
|---|---|---|
| `POST` | `/api/v1/documents` | multipart 单文件接收;持久化 Source、Document、Job 后返回 `202` |
| `POST` | `/api/v1/ingestion-batches` | multipart `files` 多文件接收;返回批次与逐文件状态 |
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新 Job 状态 |
| `GET` | `/api/v1/documents` | 按创建时间倒序返回文档列表 |
| `GET` | `/api/v1/documents/{document_id}` | 返回单个文档状态与安全错误信息 |
| `DELETE` | `/api/v1/documents/{document_id}` | 直接删除向量、Metadata、解析产物和原始文件 |
| `GET` | `/api/v1/jobs/{job_id}` | 返回阶段、状态、attempt、最大尝试次数和安全错误信息 |
| `POST` | `/api/v1/search` | Dense Top20、Reranker、TopK、来源补全 |
上传字段名为 `file`,支持 PDF、DOCX、TXT 和 Markdown。成功返回 `202`、Document 字段及
`job_id`、`job_status=queued`、`max_attempts`;响应不等待 Parse、Clean、Chunk、Embedding 或
Index。Document 响应不暴露 `storage_key`、Provider 配置或索引实现。
Search 请求:
```json
{
"query": "产品A退款周期是多少?",
"top_k": 5
}
```
Search 响应:
```json
{
"results": [
{
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
"filename": "产品A说明书.pdf",
"page": 12,
"section": "退款政策",
"chunk_id": "76ba1c30-bff7-44e5-9989-fb8372f54f46",
"content": "……",
"score": 0.82,
"rerank_score": 0.94
}
]
}
```
公开错误语义:请求校验失败返回 `422`,不支持的上传类型返回 `415`,文件过大返回 `413`,
文档、Job 或批次不存在返回 `404`,活动 Job 的文档删除返回 `409`,运行时 Provider 未配置返回
`503`。上游 Provider 或持久化异常只返回经过收敛的 `5xx` 信息,不回显凭证、请求正文或
厂商响应。
## Demo Admin 客户端路由
DEMO-010 提供 `/admin/upload`、`/admin/documents` 和 `/admin/search` 三个客户端路由,
`/admin` 默认展示上传页。它们是同源静态页面,不是新增业务 API,也不改变上述公开 REST
契约;所有业务操作仍调用 `/api/v1`。
## 当前公共知识库部署边界(PLAN-002)
当前阶段所有 Document 属于同一个公共知识可见域,不新增用户、知识库或 ACL 字段,也不把
这些完整未实现能力伪装为默认放行。Admin 写接口只允许部署在企业受控网络或统一接入网关后;
Ingress 是否进一步认证属于部署配置,但前端隐藏按钮或隐藏路由不构成安全控制。
集群重启、扩缩容和滚动部署不得改变本契约或要求浏览器重新注入 Provider 配置。数据库、BOS、
模型与队列凭据由服务端 Secret 提供,任何 API 响应都不得返回这些值。
## 异步入库契约(DEMO-015~016 已实现)
| Method | Path | 行为 |
|---|---|---|
| `POST` | `/api/v1/documents` | 已实现:单文件接收并返回 Document/Job 标识 |
| `POST` | `/api/v1/ingestion-batches` | multipart 多文件接收,返回批次及逐文件 Document/Job 标识 |
| `GET` | `/api/v1/ingestion-batches/{batch_id}` | 返回批次汇总和逐文件最新状态 |
| `GET` | `/api/v1/jobs/{job_id}` | 已实现:返回持久化 Job 状态 |
单文件响应示意:
```json
{
"id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
"filename": "退款政策.md",
"media_type": "text/markdown",
"size_bytes": 1024,
"status": "uploaded",
"error_message": null,
"created_at": "2026-09-02T12:00:00Z",
"updated_at": "2026-09-02T12:00:00Z",
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
"job_status": "queued",
"max_attempts": 3
}
```
批量字段名为 `files`,成功返回 `202`。响应示意:
```json
{
"batch_id": "d21f8012-4c4e-4bb4-89af-e11c0f9f470e",
"status": "processing",
"total_files": 2,
"accepted_files": 1,
"rejected_files": 1,
"completed_files": 1,
"created_at": "2026-09-02T12:00:00Z",
"items": [
{
"item_index": 0,
"filename": "退款政策.md",
"acceptance_status": "accepted",
"document_id": "6e5ec370-5a45-4eb7-95d7-e84267ad8752",
"job_id": "7adc4278-1157-4c3d-95a4-6926ad77a04e",
"job_status": "queued",
"stage": null,
"attempt": 0,
"max_attempts": 3,
"error_message": null
}
]
}
```
Job 公开状态为 `queued | running | retrying | succeeded | failed | cancelled`,并通过 `stage`
表示 `parsing | cleaning | chunking | embedding | indexing`。错误字段只返回有界、安全摘要。
批次状态为 `processing | completed | partial_failure | failed`;逐文件接收状态为
`pending | accepted | rejected`。批次允许逐文件部分接收/失败,不以一个文件的处理结果回滚
其他文件。文件数量、单文件大小和请求总大小分别由 `BATCH_MAX_FILES`、`MAX_UPLOAD_BYTES`
和 `BATCH_MAX_TOTAL_BYTES` 限制。数量超限返回 `413`;类型、大小或内容不合规作为该文件的
`rejected` 安全结果返回,其他合规文件继续接收。
自动重试是服务端 Job 行为,不要求客户端再次上传,也不新增公开手动重试接口。每次重试前
清理派生数据但保留原始文件和稳定的 Document ID;达到次数上限或清理失败后进入 `failed`。

View File

@ -8,4 +8,44 @@ P0 business tools:
- `get_source`
- `list_knowledge_bases`
Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose Mochow, Qwen, BGE, MinerU or other infrastructure choices.
Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose OceanBase Vector, BOS, model names, parsers or other infrastructure choices.
## Demo priority slice(DEMO-012,已实现)
`DEMO-012` 已提升为当前最高实现优先级。首个可运行 Gateway 只注册
`search_knowledge`,其输入与 `/api/v1/search` 对齐:
```json
{
"query": "产品A退款周期是多少?",
"top_k": 5
}
```
输出保留业务来源字段:`document_id`、`filename`、`page`、`section`、`chunk_id`、`content`
和排序分数。Gateway 必须复用现有 Search Service 或正式 REST API,不得复制检索、来源补全
或排序逻辑。
Demo 验证也必须建立 user/service identity 映射边界,禁止使用共享超级管理员 Token。由于
Document ACL 尚未进入 Demo Scope,`DEMO-012` 只能证明身份传递、认证拒绝和业务服务复用,
不能声称完成 V1 ACL 授权闭环。错误响应和日志不得包含 Provider Endpoint、凭据、数据库表名
或模型名。
验收至少覆盖:MCP 工具发现、参数校验、成功检索、无结果、认证失败、安全错误收敛,以及
WorkBuddy Enterprise 的一次端到端调用。其余 P0 tools 继续留在完整 V1 `EKP-071`。
实现使用独立 Streamable HTTP Gateway,默认端点为 `http://127.0.0.1:8766/mcp`。请求必须
携带 `Authorization: Bearer <token>`;Token 只从运行环境读取,并映射为显式 service identity
和 `knowledge:search` scope。当前静态 service credential 是 Demo 兼容层,不替代 V1 OAuth、
ACL 或用户委托授权。
公开错误边界:缺失/无效 Token 为 HTTP `401`,scope 不足为 `403`;Tool 参数不合法返回 MCP
参数错误;检索运行失败只返回 `knowledge search failed`。Host/Origin allowlist 默认只允许
本机,远程部署必须显式注入公开域名。
## 当前公共知识库部署边界(PLAN-002)
集群阶段仍只暴露只读 `search_knowledge`,并为 WorkBuddy 配置稳定的 HTTPS MCP 地址。
Credential 由 Kubernetes Secret 在服务端运行时注入,客户端只保存管理员分发的最小
`knowledge:search` Token;Pod 重启或滚动升级不得要求重新生成配置。公共知识域不等于匿名
访问或超级管理员身份,未来引入非公共知识前必须升级为用户委托身份与 ACL-aware Search。

View File

@ -2,9 +2,40 @@
- ADR-001: API First + Admin Console — Accepted
- ADR-002: Modular monolith + workers + MCP gateway for V1 — Accepted
- ADR-003: PostgreSQL + object storage are canonical — Accepted
- ADR-004: Mochow is initial retrieval backend but replaceable — Accepted
- ADR-003: PostgreSQL + object storage are canonical — Superseded by ADR-012
- ADR-004: Mochow is initial retrieval backend but replaceable — Superseded by ADR-010
- ADR-005: Provider abstraction for embedding/reranker/parser/LLM — Accepted
- ADR-006: RAG + Knowledge Compilation — Accepted
- ADR-007: ACL before/during retrieval — Accepted
- ADR-008: MCP hides infrastructure details — Accepted
- ADR-009: Baidu BOS is the primary object-storage provider; local storage remains for tests and local development — Accepted
- ADR-010: OceanBase Vector is the primary rebuildable retrieval index behind `VectorStore`; PostgreSQL remains canonical — Superseded by ADR-012
- ADR-011: Qianfan `bge-large-zh` plus `bce-reranker-base` is the initial Demo model baseline — Accepted
- ADR-012: OceanBase MySQL relational tables + BOS are canonical; OceanBase vector tables remain rebuildable indexes — Accepted
## Architecture migration boundary
ARCH-006 的只读环境核验确认,目标平台没有独立 PostgreSQL,OceanBase MySQL 模式同时承载
关系数据和向量数据。因此 ADR-012 取代 ADR-003/010 中的 PostgreSQL 部署假设:OceanBase
关系表保存 canonical metadata,BOS 保存不可变源文件,OceanBase 专用向量表仍只保存可由
canonical Chunk 重建的检索行。共用数据库实例不改变两类数据的所有权边界,也不允许业务层
绕过 `VectorStore` 直接依赖 `pyobvector`。
目标 test 集群为 OceanBase 4.3.5.4 MySQL 模式。ARCH-006 已用只读 `EXPLAIN` 验证
`VECTOR_INDEX(... pre_filter)` 产生 `VECTOR INDEX ADAPTIVE SCAN (PRE-FILTER)`;版本下限与
运行检查据此调整为 4.3.5.4。生产升级和重新验收仍由运维流程控制。
## Public knowledge delivery boundary
PLAN-002 将当前交付阶段限定为企业受控网络内的单一公共知识可见域,并加入 Kubernetes/Helm
集群交付任务。这是部署范围和实施顺序调整,不是新的架构选型:继续采用 ADR-002 的模块化
单体 API、独立 Worker 和 MCP Gateway,以及架构既定的 Celery/RabbitMQ;OceanBase/BOS
边界继续遵循 ADR-012。
ADR-007 仍为 Accepted。当前阶段只是延后用户/组/角色和 Document ACL;开始接收任何非公共
知识前,必须先实现 Permission Resolver 和检索前/检索中 ACL 下推。公共知识库也不等于匿名
公网开放,Admin 写操作只允许经受控网络或统一接入网关访问,MCP 使用最小只读服务身份。
The migration retains the Demo's Dense Top20 → Reranker Top5 behavior and current lightweight
Parser scope. Top50 retrieval, distance thresholds, dual-query RRF, Marker/Pandoc and LLM-derived
index content require separate product and evaluation decisions.

View File

@ -63,10 +63,11 @@ Wiki / FAQ / SOP / Summary 等派生知识
┌────────────────────────┼────────────────────────┐
│ │ │
▼ ▼ ▼
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
│ PostgreSQL │ │ Object Store │ │ Mochow │
│ Canonical DB │ │ S3/MinIO │ │ Search Index │
└──────────────┘ └──────────────┘ └──────────────┘
┌────────────────────────┐ ┌──────────────┐
│ OceanBase MySQL │ │ Object Store │
│ Canonical + Vector │ │ BOS │
│ (logical separation) │ └──────────────┘
└────────────────────────┘
▲ ▲
│ │
┌───────┴────────────────────────┘
@ -88,7 +89,7 @@ Wiki / FAQ / SOP / Summary 等派生知识
1. **API First**:后台、MCP、其他业务系统均调用统一 API/Service。
2. **模块化单体优先**:V1 不拆大量微服务,API 服务 + 异步 Worker + MCP Gateway 足够。
3. **Canonical Data 独立**:PostgreSQL + 对象存储保存事实数据,Vector DB 仅保存可重建索引。
3. **Canonical Data 逻辑独立**:OceanBase 关系表 + BOS 保存事实数据,同库向量表仅保存可重建索引。
4. **Provider 可替换**:Embedding、Reranker、Vector DB、Parser、LLM 均通过 Adapter 抽象。
5. **MCP 不暴露底层实现**:客户端只看到 `search_knowledge` 等业务工具。
@ -99,14 +100,14 @@ Wiki / FAQ / SOP / Summary 等派生知识
| 模块 | V1 当前选型 | 说明 |
|---|---|---|
| Backend | Python + FastAPI | AI / 数据处理生态成熟,适合 MCP、异步任务和模型调用 |
| Metadata DB | PostgreSQL | 保存用户、权限、文档、版本、任务、Compiled Knowledge 等 Canonical Data |
| Object Storage | S3 Compatible / MinIO | 保存原始文件、解析产物、中间文件 |
| Metadata DB | OceanBase MySQL | 保存用户、权限、文档、版本、任务、Compiled Knowledge 等 Canonical Data |
| Object Storage | Baidu BOS | 保存不可变原始文件、解析产物和中间文件;Local Adapter 仅用于测试/本地开发 |
| Task Queue | Celery + RabbitMQ | 文件解析、Embedding、索引、Compilation 等异步任务 |
| PDF / OCR Parser | MinerU 优先 | 中文复杂 PDF、扫描件、表格、版面解析能力较强 |
| Office Parser | Docling 补充 | DOCX/PPTX/XLSX/HTML 等格式统一解析 |
| Vector/Search DB | Mochow | 延续公司现有经验,支持向量、BM25、Hybrid、Filter |
| Embedding | 多模型 Benchmark 决定 | 基线 bge-large-zh,对比 Qwen3 Embedding 等 |
| Reranker | Qwen3 Reranker 等候选 | V1 正式检索链必选能力 |
| Vector/Search DB | OceanBase Vector | 与关系表同库部署但逻辑隔离,必须通过 `VectorStore` 并可从 canonical Chunk 重建 |
| Embedding | Qianfan bge-large-zh | 当前 1024 维基线,后续模型变更使用版本化 Shadow Index |
| Reranker | Qianfan bce-reranker-base | 当前 Dense Top20 → Reranker Top5 基线 |
| LLM | Provider 抽象,云端优先 | 用于 Knowledge Compilation、分类、摘要等 |
| MCP | 独立 MCP Gateway | 面向 WorkBuddy 等客户端,隐藏内部技术栈 |
| Frontend | Admin Console | 知识库、文档、权限、检索调试、Benchmark、任务管理 |
@ -173,6 +174,46 @@ Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Fail
- 软删除;
- 来源页码和章节追踪。
### 4.2.1 批量异步提交与失败恢复
API 只负责校验并持久化不可变源文件、Document 和 Ingestion Job,然后返回 `202`。批次是
提交与聚合查询边界,不是跨文件数据库事务;每个文件独立排队、执行和失败。
DEMO-016 以 canonical `ingestion_batches` / `ingestion_batch_items` 保存输入顺序和接收引用;
处理阶段、Attempt 与最终结果继续以 `ingestion_jobs` / `ingestion_attempts` 为事实源,批次表
不复制 Worker 状态。
```text
Batch Request
├─ Source A + Document A + Job A ─→ Worker
├─ Source B + Document B + Job B ─→ Worker
└─ Source C + Document C + Job C ─→ Worker
```
Worker 对可重试错误执行有界退避。重新执行前必须通过独立的派生数据清理服务按以下顺序
恢复干净状态:
```text
Vector/Sparse rows
→ ChunkIndexState
→ Chunks
→ Parsed artifact + parsed_storage_key
→ reset processing state
→ next attempt from immutable source
```
清理服务与 Document 删除语义分离:重试清理保留原始 BOS 对象、Document 和 Job/Attempt
历史。清理失败时 Job 终止,不能在未知状态继续执行。Job 通过数据库状态、唯一有效 Attempt
和幂等键避免重复投递产生并发执行;具体队列实现保持可替换,不进入业务服务接口。
公共知识库集群交付阶段由 Celery/RabbitMQ 作为首个生产执行器。API 只调用队列执行器抽象,
Celery task 只负责恢复应用上下文并调用 Ingestion Application Service;重试分类、Attempt 状态、
清理顺序和业务幂等仍由应用层持有。Broker 消息不是 Job 的 source of truth,Worker 重启或消息
重复投递必须以 OceanBase 中的持久化 Job/Attempt 状态收敛。
DEPLOY-001 将 API/Admin、MCP Gateway 与 Async Worker 构建为三个共享锁定 Python 运行时的
非 root 镜像 target;进程边界与上述模块边界一致,不新增服务或把 Provider 配置烘焙进镜像。
---
### 4.3 文档结构与 Chunk
@ -326,21 +367,28 @@ profile_v2
避免直接原地覆盖现有生产索引。
ARCH-005/006 将 Profile 生命周期以 `shadow / active / retired` 记录在 OceanBase 关系表;任一时刻至多
一个 Active Profile。重建从 canonical Chunk 生成确定性 `chunk_id + profile_id` 索引行,
重复执行跳过已完整文档,激活前重新核对全部 ChunkIndexState,并在同一事务内锁定 Chunk
写入、退役旧 Profile、激活新 Profile。OceanBase 表本身仍不是 canonical source。
---
## 8. Vector Store 设计
### 当前默认:Mochow
### 当前默认:OceanBase Vector
选择理由:
- 公司已有使用经验;
- 已有百度云体系;
- 公司现有知识平台主链已有使用经验;
- 可复用现有 OceanBase 运维体系;
- 支持 Dense Vector;
- 支持 BM25 / Full Text;
- 支持 Hybrid Search;
- 支持 Metadata / Scalar Filter;
- 满足 V1 ACL 下推需求。
- 满足当前 Demo 的 Profile 过滤和后续 ACL 下推边界;
- `pyobvector` 仅允许出现在 Provider Adapter 内。
BM25/Hybrid 仍是 V1 产品能力,但不在本轮 Provider 迁移中提前扩展;相关实现必须经
独立 Task、契约测试和 RAG 回归后启用。
### 保留替换能力
@ -348,15 +396,18 @@ profile_v2
```text
VectorStoreProvider
├─ OceanBase Vector (Primary)
├─ Mochow
├─ Qdrant
├─ Milvus
└─ pgvector
```
业务 Domain Model 不依赖 Mochow SDK 数据结构。
业务 Domain Model 不依赖 OceanBase/`pyobvector` SDK 数据结构。
Mochow 中的数据全部视为 **可重建索引**,真实 Canonical Data 保存在 PostgreSQL / Object Storage。
OceanBase Vector 表中的数据全部视为 **可重建索引**。同一数据库的关系表保存文档、Chunk、
Profile 和索引状态等 canonical 数据,BOS 保存不可变原始文件和派生产物。物理共库不允许
向量表反向成为业务事实来源。
---
@ -441,7 +492,7 @@ Manage
User Identity
→ Permission Resolver
→ Allowed Scope / ACL Filter
→ Mochow Hybrid Retrieval
→ Provider-neutral Retrieval(OceanBase Vector 当前实现)
```
MCP / REST / Admin Console 必须复用同一权限服务。
@ -481,7 +532,8 @@ V1 建议工具:
MCP Response 不出现:
```text
Mochow
OceanBase Vector
BOS
Qdrant
bge-large-zh
Qwen3
@ -492,6 +544,17 @@ MinerU
首要验证客户端:**WorkBuddy Enterprise**。
### 11.1 DEMO-012 实现边界
当前已落地独立 Streamable HTTP Gateway,默认只绑定 `127.0.0.1:8766/mcp`,只注册
`search_knowledge`。Gateway 与 REST 共享 `SearchService`,同步检索通过工作线程执行,避免
阻塞 MCP ASGI 事件循环。
Demo 使用运行时专用 Bearer Token 映射到 client/service identity 和最小
`knowledge:search` scope,并启用 Host/Origin allowlist。该边界用于 WorkBuddy 接入验证,
后续 `EKP-070` 仍需用正式 OAuth/企业身份系统和 Permission Resolver 替换静态 Token;
`DEMO-012` 不改变 ADR-007 的 ACL 要求。
---
## 12. REST API
@ -592,6 +655,11 @@ ReindexJob
EvaluationJob
```
当前增量实现先提供单个 `IngestionJob` 编排既有 Parse/Clean/Chunk/Embedding/Indexing 阶段,
保留阶段级状态和 Attempt 历史;以后拆分为独立阶段 Job 时不得改变公开批次/Job 查询契约。
Demo 默认最多 3 次总尝试,只有瞬时 Provider、网络、存储或数据库错误进入自动重试;输入
校验和确定性内容错误直接终止。
GPU 节点优先承担:
- MinerU / OCR;
@ -604,7 +672,7 @@ LLM V1 可继续使用云端 Provider,不要求部署本地大模型。
## 15. 数据存储原则
### PostgreSQL
### OceanBase MySQL 关系表
保存:
@ -629,14 +697,14 @@ LLM V1 可继续使用云端 Provider,不要求部署本地大模型。
- Markdown / JSON 中间产物;
- 需要持久化的大对象。
### Mochow
### OceanBase Vector
仅保存:
- Dense Vector;
- Sparse/BM25 Index;
- Retrieval metadata;
- ACL 所需过滤字段。
- Profile/ACL 所需过滤字段;
- 经独立能力验证后启用的 Sparse/BM25 索引。
数据库损坏或更换时,可由 Canonical Data 全量重建。
@ -653,15 +721,13 @@ Reverse Proxy / Ingress
├─ Knowledge API
└─ MCP Gateway
PostgreSQL
OceanBase MySQL(Canonical + Vector,逻辑隔离)
RabbitMQ
Object Storage
Baidu BOS
CPU Worker
GPU Worker
Mochow
Observability
```
@ -678,6 +744,30 @@ Observability
未来根据负载再拆服务。
### 16.1 当前公共知识库集群交付基线
PLAN-002 将当前交付范围收敛为企业受控网络内的单一公共知识可见域。该范围不重开 ADR-007:
所有现有文档共享同一可见性,因而当前检索不执行事后 ACL 过滤;接收任何非公共知识前,必须
先实现 Permission Resolver 和数据库/向量检索前置过滤。Admin 写接口不得直接暴露到匿名公网,
MCP 使用运行时 Secret 注入的最小 `knowledge:search` 服务身份。
集群中 API/Admin、MCP Gateway 和 Async Worker 分别部署并独立扩缩容,业务核心继续保持模块化
单体。当前 Parser 及云端 Embedding/Reranker 不需要 GPU Worker,因此它不进入最小交付,但角色
边界保留。OceanBase、BOS 和 AI Provider 使用外部服务;RabbitMQ 默认使用外部地址,可为开发/
演示显式启用 Chart 依赖。
Kubernetes/Helm 必须提供:生产镜像、`existingSecret`、串行 Migration Job、Ingress/TLS、
readiness/liveness、资源限制、非 root 与只读根文件系统、多副本和滚动更新。仓库的一键入口只
编排已有集群和外部服务,不创建云账号或把凭据写入 Values、镜像、命令行及日志。现有
`docker-compose.dev.yml` 包含已被 ADR-012 取代的 PostgreSQL/MinIO 假设,只能视为待清理的
本地开发资产,不能作为集群模板。详细依赖和验收见
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
DEPLOY-002 的运行拓扑将 Celery Worker 与 Celery Beat 恢复调度器拆成独立 Pod:Worker 可横向
扩缩,Beat 固定单副本并使用 `Recreate`,避免每个 Worker 都发布周期恢复消息。二者仍调用同一
应用服务与持久化 Job/Attempt 状态,不形成新业务服务,也不改变 ADR-002。Migration 使用 Helm
`pre-install,pre-upgrade` Hook;失败会在业务 Deployment 更新前终止原子升级。
---
## 17. V1 验收重点
@ -732,13 +822,13 @@ WorkBuddy 可以通过 MCP 完成知识搜索和 Source 获取。
```text
Backend Python + FastAPI
Metadata PostgreSQL
Object Storage S3 Compatible / MinIO
Metadata OceanBase MySQL relational tables
Object Storage Baidu BOS(Local Adapter 仅测试/本地开发)
Queue RabbitMQ + Celery
Parser MinerU 主力 + Docling 补充
Vector Store Mochow Primary,Adapter 可替换
Embedding bge-large-zh 基线,多模型 Benchmark 后决定生产模型
Reranker Qwen3 Reranker 等候选,V1 必选
Vector Store OceanBase Vector Primary,Adapter 可替换,索引可重建
Embedding Qianfan bge-large-zh(1024 维)
Reranker Qianfan bce-reranker-base,Dense Top20 → Top5
LLM Provider 抽象,允许云端
Frontend Admin Console
Protocol REST API + MCP

View File

@ -6,7 +6,7 @@
该 Demo 的目标不是完成完整企业级知识平台,而是优先验证最核心的技术闭环:
> 文件上传 → 文档解析/清洗 → Chunk → Embedding → Mochow 入库 → 检索召回 → Reranker → 来源返回 → 简单管理页面
> 文件上传 → 文档解析/清洗 → Chunk → Embedding → OceanBase Vector 入库 → 检索召回 → Reranker → 来源返回 → 简单管理页面
当前 Demo Scope 的优先级高于完整 V1 PRD。
@ -20,7 +20,7 @@
1. 中文企业文档是否能够稳定上传和解析;
2. 清洗后的内容是否可以形成质量可接受的 Chunk;
3. 使用现有 Embedding + Mochow 能否完成稳定召回;
3. 使用现有 Embedding + OceanBase Vector 能否完成稳定召回;
4. Reranker 是否能提升最终 TopK 结果质量;
5. 检索结果是否能够准确返回文件名、页码、章节和原文;
6. 非研发人员是否能够通过简单页面完成上传和检索验证。
@ -49,13 +49,13 @@ FastAPI
│ ↓
│ Embedding
│ ↓
│ Mochow
│ OceanBase Vector
│
└── Search
↓
Query Embedding
↓
Mochow TopN
OceanBase Vector TopN
↓
Reranker
↓
@ -67,10 +67,10 @@ FastAPI
辅助存储:
```text
SQLite / PostgreSQL
└── document / chunk metadata
OceanBase MySQL
└── canonical document / chunk metadata
Local Storage / MinIO
Local Storage / BOS
└── original files
```
@ -83,16 +83,16 @@ Demo 阶段优先简单实现,不引入不必要的基础设施。
| 模块 | Demo 推荐选型 | 说明 |
|---|---|---|
| Backend | FastAPI | 简单、开发效率高 |
| Metadata DB | SQLite 或 PostgreSQL | 单机 Demo 可 SQLite;已有环境可直接 PostgreSQL |
| File Storage | 本地目录 | Demo 阶段优先简单;后续可替换 MinIO/S3 |
| Metadata DB | OceanBase MySQL | 目标环境使用 OceanBase;SQLite 仅用于离线测试 |
| File Storage | 百度 BOS | Local Adapter 仅用于测试和本地开发,目标环境使用 BOS |
| Parser | PyMuPDF + python-docx 或 MinerU | 优先快速跑通;复杂 PDF 后续增强 |
| Cleaner | 自定义基础清洗 | 处理空行、空白、异常换行等 |
| Chunker | 标题/段落优先 + Token 限制 | 暂不设计复杂 Chunk 策略 |
| Embedding | bge-large-zh | 延续公司现有百度千帆方案 |
| Vector DB | Mochow | 当前默认,不做多数据库实现 |
| Reranker | Qwen3 Reranker 或现有可调用 Reranker | Demo 中保留 |
| Vector DB | OceanBase Vector | 当前默认,通过 `VectorStore` 隔离 SDK,不做多数据库运行时切换 |
| Reranker | `bce-reranker-base` | 延续当前可调用的千帆 Reranker,Demo 中保留 |
| Frontend | 简单 React/Vue 或 FastAPI HTML | 以最小可用为目标 |
| Deployment | 本地 / 单机 Docker | 暂不考虑 K8s |
| Deployment | Kubernetes/Helm | PLAN-002 起面向企业受控网络交付;本地模式仅用于开发测试 |
---
@ -171,9 +171,9 @@ content
Demo 阶段不实现多 Embedding Benchmark 平台和动态模型切换,但代码层建议保留简单接口,避免业务逻辑直接写死调用细节。
### 5.6 Mochow 入库
### 5.6 OceanBase Vector 入库
将 Chunk 写入 Mochow,至少保存:
将 Chunk 写入 OceanBase Vector 专用索引表,至少保存:
```text
chunk_id
@ -187,7 +187,9 @@ embedding
支持:Upsert、按 document_id 删除相关 Chunk、Dense Search。
Demo 阶段不实现 Qdrant/Milvus/pgvector,也不实现完整 VectorStore Adapter 体系,但避免 Mochow SDK 调用散落在整个项目中。
Demo 阶段不实现 Qdrant/Milvus/pgvector,也不实现多 Vector Store 动态切换,但避免
OceanBase/pyobvector 调用散落在整个项目中。OceanBase Vector 行仍是可重建索引,不能替代
同库关系表中的 canonical Chunk。
### 5.7 Dense Retrieval
@ -196,7 +198,7 @@ Query
↓
bge-large-zh Embedding
↓
Mochow Dense Search
OceanBase Vector Dense Search
↓
Top 20 Candidates
```
@ -246,7 +248,8 @@ DELETE /api/documents/{id}
POST /api/search
```
Demo 阶段删除可以直接执行:删除原始文件 + 删除 Metadata + 删除 Mochow 对应 Chunk,暂不要求 Soft Delete。
Demo 阶段删除可以直接执行:删除 BOS/本地原始文件 + 删除 Metadata + 删除 OceanBase
Vector 对应 Chunk,暂不要求 Soft Delete。
Search Request 示例:
@ -302,7 +305,7 @@ Search Response 示例:
---
## 8. P1 可选增强
## 8. Demo 后续增强
### DEMO-011 Hybrid Search
@ -320,12 +323,26 @@ Reranker
### DEMO-012 MCP
只实现一个最小 MCP Tool:`search_knowledge`,内部调用 `/api/search`,用于验证 WorkBuddy 接入。
**已完成。** 只实现一个最小 MCP Tool:`search_knowledge`,复用现有
Search Service 或 `/api/v1/search`,用于验证标准 MCP、identity 边界和 WorkBuddy 接入。
不在本 Task 扩展其他 P0 tools 或声称完成尚未实施的 Document ACL。
### DEMO-013 OCR / MinerU
增加扫描 PDF、图片、OCR 和复杂中文 PDF 处理能力。
### DEMO-015 异步入库 Job 与自动重试清理
**已完成。** 将现有同步入库流水线放入持久化 Ingestion Job。可重试失败自动执行有界重试;每次重试前
删除旧解析产物、Chunk、索引状态和向量数据,但保留不可变原始文件、Document 和 Attempt
历史。清理失败时终止,不在未知脏状态继续执行。通过执行器抽象接入 Celery/RabbitMQ,使
API 与 Worker 可独立部署,并以持久化状态处理 Worker 重启和重复消息。
### DEMO-016 批量异步上传
**已完成。** 在 DEMO-015 基础上支持一次提交多个文件、逐文件 Document/Job 状态、批次聚合查询和
Admin Console 状态展示。批次内文件相互隔离,不包含目录上传、压缩包解包或断点续传。
---
## 9. 明确后置能力
@ -345,37 +362,67 @@ Reranker
- 多 Embedding Benchmark Platform;
- 多 Vector DB;
- 动态 Provider;
- RabbitMQ / Celery;
- Kubernetes;
- 高可用;
- 灾备;
- 完整 Observability。
用户/权限隔离属于完整 V1 后置能力,不因“公共知识库”而取消;本阶段所有已接收知识共享一个
可见域,且仅部署在企业受控网络或统一接入网关后。RabbitMQ/Celery 与最小 Kubernetes 可用性
已由 PLAN-002 纳入当前交付,跨区域容灾和完整平台级高可用仍不在范围内。
---
## 10. Demo Task 清单
```text
DEMO-001 项目骨架 + FastAPI + 配置
DEMO-002 Document 数据模型 + 文件存储
DEMO-002 Document 数据模型 + 本地文件存储
DEMO-003 PDF/DOCX/TXT/MD 上传
DEMO-004 Parser + Cleaner
DEMO-005 Chunk Pipeline
DEMO-006 bge-large-zh Embedding + Mochow 入库
DEMO-006 bge-large-zh Embedding + 初始 Mochow 入库(ARCH-003~005 已迁移为 OceanBase)
DEMO-007 Dense Retrieval
DEMO-008 Reranker + Source Result
DEMO-009 Document/Search REST API
DEMO-010 简单 Admin Console
DEMO-012 MCP search_knowledge(已提升优先级)
DEMO-014 地瓜机器人品牌化前端优化(用户新增)
DEMO-015 异步入库 Job + 自动重试前清理(用户新增)
DEMO-016 批量异步上传 + 状态展示(用户新增)
ARCH-001~006 Provider 架构迁移与真实验收
PLAN-002 公共知识库集群交付规划(已完成)
DEPLOY-001 API/Admin、MCP Gateway、Async Worker 生产镜像
DEPLOY-002 Kubernetes/Helm 一键部署
DEPLOY-003 可用性、滚动升级和远程 WorkBuddy 验收
```
实施状态(2026-09-02):`DEMO-001~DEMO-010`、用户新增的 `DEMO-014` 与
`ARCH-001~ARCH-006` 均已完成并通过当前 Scope 的闭环验收。`PLAN-001` 已完成现状审计和
任务拆分;`DEMO-012`、`DEMO-015~016` 已完成。`PLAN-002` 已将公共知识库集群交付纳入范围,
后续严格按 `DEPLOY-001~003` 顺序实施。`DEMO-011` 和 `DEMO-013` 仍为
可选项,不因本状态记录自动进入范围。
可选:
```text
DEMO-011 Hybrid Search
DEMO-012 MCP search_knowledge
DEMO-013 OCR / MinerU
```
已批准顺序:
```text
PLAN-002 Public Knowledge Cluster Plan
→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
→ DEMO-016 Batch Async Upload + UI
→ DEPLOY-001 Production Images
→ DEPLOY-002 Kubernetes/Helm One-click Deploy
→ DEPLOY-003 Availability + Remote Acceptance
```
详细 Task 映射、依赖与验收边界见
[`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`](../plans/ASYNC_INGESTION_MCP_PLAN.md) 和
[`docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
---
## 11. Demo 测试数据
@ -422,7 +469,7 @@ Upload
→ Clean
→ Chunk
→ Embedding
→ Mochow
→ OceanBase Vector
```
完整跑通。
@ -432,7 +479,7 @@ Upload
```text
Query
→ Embedding
→ Mochow Dense Top20
→ OceanBase Vector Dense Top20
→ Reranker
→ Top5
```
@ -464,9 +511,13 @@ Demo 阶段不设严格生产指标。
## 14. Codex 开发约束
Codex 当前只能实现 `DEMO-001 ~ DEMO-010`。
Codex 当前只能实现 `DEMO-001 ~ DEMO-010`、`DEMO-012`、`DEMO-014 ~ DEMO-016`、
`ARCH-001 ~ ARCH-006`、`PLAN-001 ~ PLAN-002` 和 `DEPLOY-001 ~ DEPLOY-003`。
PLAN-002 只授权实现 Celery/RabbitMQ 异步执行器与最小 Kubernetes/Helm 集群交付,不授权
拆分额外微服务或扩大业务能力。
除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、微服务、Kubernetes 和企业级 Job Queue。
除非明确要求,否则禁止提前实现权限、审计、Knowledge Compilation、多数据库、多模型平台、
额外微服务、跨区域容灾和完整 Observability 平台。
当本文件与完整 PRD 的实现优先级冲突时:
@ -495,20 +546,29 @@ docs/demo/DEMO_SCOPE.md
>
完整 V1 PRD
只允许实现 DEMO-001 ~ DEMO-010。
只允许实现本文件明确列出的 Demo、Architecture、Plan 与 Deploy Task,并且一次只执行一个
Task。当前严格顺序为 DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003。
当前 Demo 的目标只有:
当前公共知识库阶段的业务目标保持为:
Upload
→ Parse/Clean
→ Chunk
→ bge-large-zh
→ Mochow
→ OceanBase Vector
→ Dense Retrieval
→ Reranker
→ Source Result
→ Search UI
并补齐以下交付能力:
Async Job / Retry Cleanup
→ Batch Async Upload
→ Production Images
→ Kubernetes/Helm One-click Deploy
→ Availability + Remote WorkBuddy Acceptance
不要提前实现:
- ACL
@ -518,9 +578,9 @@ Upload
- Knowledge Compilation
- Multi Vector Store
- Embedding Benchmark Platform
- Celery/RabbitMQ
- Kubernetes
- 企业级高可用
- 额外微服务
- 跨区域容灾
- 完整 Observability 平台
先不要修改代码。
@ -529,7 +589,7 @@ Upload
1. 当前仓库状态;
2. 为完成 Demo 需要保留哪些现有结构;
3. 哪些完整 V1 设计可以暂时不实现;
4. DEMO-001 ~ DEMO-010 的建议执行顺序;
4. 当前已批准 Task 的执行顺序;
5. 下一步应该实施的第一个 Task。
```
@ -540,10 +600,20 @@ Upload
```text
Demo
↓
Hybrid Search
↓
MCP
↓
Async Job + Retry Cleanup
↓
Batch Async Upload
↓
Production Images
↓
Kubernetes/Helm One-click Deploy
↓
Availability + Remote WorkBuddy Acceptance
↓
Hybrid Search
↓
OCR / Complex Parsing
↓
Document Version

View File

@ -0,0 +1,59 @@
# Demo Admin Console 使用说明
DEMO-010 在 FastAPI 服务中提供一个轻量、同源的管理客户端。页面只负责展示和交互,上传、
文档管理与检索仍全部通过 `/api/v1` 公开业务接口完成,不复制服务端业务逻辑。
## 启动与访问
完成运行时配置后启动 Worker 与 API:
```bash
make run-worker
make run
```
页面地址:
| 页面 | 地址 | 对应业务接口 |
|---|---|---|
| 文档上传 | `/admin/upload` | `POST/GET /api/v1/ingestion-batches*` |
| 文档列表与详情 | `/admin/documents` | `GET/DELETE /api/v1/documents*` |
| Search Lab | `/admin/search` | `POST /api/v1/search` |
`/admin` 默认展示上传页。OpenAPI 文档仍位于 `/api/v1/docs`。
## 品牌视觉基线
DEMO-014 参考地瓜机器人公开官网的设计语言,使用品牌橙 `#FF3C00`、黑白高对比、中文
无衬线标题、紧凑数据字体、低圆角容器和宽松留白。贯穿页面标题的“计算信号轨”是唯一的
强调性图形,序号对应上传、文档、检索三步真实操作顺序。
品牌图形使用仓库内的代码原生 SVG/CSS 绘制,不复制或热链官网图片、字体和脚本;页面
运行时不依赖品牌官网。前端设计方法参考 GitHub `anthropics/skills` 的 Apache-2.0
`frontend-design` Skill,并以现有 Demo 信息架构、可访问性和业务正确性为优先约束。
## Demo 操作闭环
1. 在上传页选择一个或多个 PDF、DOCX、TXT 或 Markdown 文件并提交;
2. 接口返回 `202` 后,页面自动轮询批次,逐文件显示接收状态、处理阶段、尝试次数和安全错误;
3. 批次进入 `Completed`、`Partial failure` 或 `Failed` 后停止轮询,再在文档列表查看元数据;
4. 在 Search Lab 输入问题并查看 Top-K 的文件名、页码、章节、Chunk 内容、Dense Score 和
Rerank Score;
5. 如需清理 Demo 数据,在文档列表确认后直接删除。
## 范围与安全边界
- 页面静态文件位于 `apps/admin-web`,由同一个 FastAPI 进程提供,避免额外前端运行时;
- API 返回内容仅通过 DOM `textContent` 渲染,不作为 HTML 注入;
- 页面不保存或展示 Provider 凭据、内部 Endpoint、storage key 或 Index Profile;
- Console 只轮询当前批次的聚合接口,不读取 Provider/队列细节,也不提供人工重试按钮;
- 批量上传不包含目录上传、压缩包解包、断点续传或跨批次去重;
- 本任务不包含 ACL/RBAC、审计、Document Version、Compiled Knowledge、MCP、Hybrid/BM25
或完整 V1 Admin Console 导航;
- 删除是 Demo 的直接删除语义,确认后会清理 canonical Metadata、派生产物、源文件与向量行。
## 验证
静态路由和范围契约由 `tests/integration/test_admin_console.py` 覆盖;现有 REST 全链路测试覆盖
上传、列表、详情、搜索与删除。浏览器验收使用确定性本地 Provider,不读取真实企业文档,
并覆盖桌面与移动布局。

View File

@ -0,0 +1,97 @@
# 异步入库与失败恢复说明
## 范围
DEMO-015 将单文件上传从 API 请求线程迁移到独立 Celery Worker,DEMO-016 在其上增加逐文件
隔离的批量接收和状态聚合。OceanBase 中的
`ingestion_jobs` 和 `ingestion_attempts` 是执行状态事实源;RabbitMQ 消息只携带 Job ID,
不携带文件内容、Provider 凭据或业务状态。
批次只聚合接收与 Job 状态,不改变每个文件的独立执行和重试边界。
## 运行时配置
以下值必须通过环境变量或部署 Secret 注入,不得提交真实值:
```text
RABBITMQ_URL=amqp://<user>:<password>@<host>:5672/<vhost>
INGESTION_MAX_ATTEMPTS=3
INGESTION_RETRY_BACKOFF_SECONDS=2
INGESTION_RETRY_BACKOFF_MAX_SECONDS=60
INGESTION_WORKER_LEASE_SECONDS=300
INGESTION_RECOVERY_INTERVAL_SECONDS=30
BATCH_MAX_FILES=20
BATCH_MAX_TOTAL_BYTES=209715200
```
Worker 与 API 还需要相同的 OceanBase、BOS、Embedding、VectorStore 和 Index Profile 配置。
RabbitMQ 用户应只具备指定 vhost/queue 的最小权限。
## 启动顺序
```bash
uv run alembic upgrade head
make run-worker
make run
```
`make run-worker` 在当前 Demo 中同时启动 Worker 和恢复调度器。恢复调度器周期扫描:
- 尚未成功投递的 `queued` Job;
- 已到重试时间的 `retrying` Job;
- Worker 丢失后租约已过期的 `running` Job。
任务投递使用 late acknowledgement、worker-lost reject 和单任务 prefetch。重复消息会先锁定
Job 行;有效租约内不会创建第二个 Attempt。租约过期后,旧 Attempt 以安全错误关闭,再按重试
规则恢复。
## 重试与清理
确定性的内容错误(无法解析、清洗后为空、无 Chunk、派生数据身份不一致)直接进入 `failed`。
其余运行时/Provider 错误按指数退避重试,默认最多三次总 Attempt。
第二次及后续 Attempt 创建前严格执行:
```text
OceanBase Vector rows
→ ChunkIndexState
→ Chunk
→ parsed JSON object
→ reset Document processing state
```
不可变源文件、Document、Job 和已结束 Attempt 始终保留。任一清理步骤失败时不创建新 Attempt,
Job 直接以 `retry cleanup failed` 终止,禁止在未知脏状态继续执行。
## API 状态
`POST /api/v1/documents` 成功返回 `202` 和 `job_id`。查询:
```text
GET /api/v1/jobs/{job_id}
```
Job 状态为 `queued | running | retrying | succeeded | failed | cancelled`;阶段为
`parsing | cleaning | chunking | embedding | indexing`。公开错误只返回固定安全摘要,不保存或
返回 Broker/Provider 异常正文。活动 Job 的 Document 暂不允许删除,API 返回 `409`。
批量提交与查询:
```text
POST /api/v1/ingestion-batches # multipart 字段 files,可重复
GET /api/v1/ingestion-batches/{batch_id}
```
批次创建后逐文件返回 `accepted` 或 `rejected`;已接收文件继续通过上述 Job 状态异步推进。
批次聚合状态为 `processing | completed | partial_failure | failed`。请求文件数超过
`BATCH_MAX_FILES` 时整体返回 `413`;单文件或共享总字节预算超限只拒绝当前文件,不删除此前
已接收文件。默认总预算 200 MiB,且配置不得小于单文件 `MAX_UPLOAD_BYTES`。
## 故障检查
1. Job 长期为 `queued`:检查 RabbitMQ 连通性、恢复调度器以及 `ingestion` queue;
2. Job 长期为 `running`:检查 Worker,租约到期后恢复调度器会重新投递;
3. Job 为 `retrying`:等待 `next_run_at`,客户端不应重新上传同一文件;
4. Job 为 `failed` 且错误为 `retry cleanup failed`:停止人工重试,先核对向量、关系数据与 BOS
派生对象的一致性;
5. 不得通过日志打印 `RABBITMQ_URL`、数据库 URL、BOS/模型凭据或原始 Provider 响应。

View File

@ -0,0 +1,63 @@
# ARCH-002 BOS 配置与对象契约
ARCH-002 在既有 `ObjectStorage` 契约后新增百度 BOS Adapter。业务服务仍只使用逻辑 key,
不知道 Bucket、Prefix、SDK 类型或凭据。ARCH-005 已将 BOS 设为正式运行默认值;Local
Adapter 继续用于单元测试和显式配置的本地开发。
## 运行配置
配置只能由部署环境或 Secret Manager 注入,禁止把真实值写入 `.env`、镜像、日志、测试
快照或提交记录。
```text
OBJECT_STORAGE_PROVIDER=bos
BOS_ENDPOINT=<已授权的 BOS Host 或 HTTP(S) Origin>
BOS_ACCESS_KEY_ID=<运行时注入>
BOS_SECRET_ACCESS_KEY=<运行时注入>
BOS_BUCKET=<预创建的 Bucket>
BOS_KEY_PREFIX=knowledge-platform
BOS_TIMEOUT_SECONDS=30
```
- Endpoint 必须是无用户名、密码、Path、Query 和 Fragment 的 Host 或 HTTP(S) Origin;旧配置
的纯 Host 会在 Provider 边界强制规范化为 HTTPS,禁止沿用 SDK 的默认 HTTP;
- Bucket 名必须符合 BOS 的小写字母、数字和连字符规则;
- Prefix 可为空;非空时使用安全的 POSIX 相对 key,不允许空段、`.`、`..` 或反斜杠;
- 应用不会创建、删除或修改 Bucket 配置。
## 对象与不可变约束
canonical `Document.storage_key` 保存与 Provider 无关的逻辑 key,例如:
```text
documents/<document_id>/<filename>
```
BOS Adapter 发起请求时才映射为:
```text
<BOS_KEY_PREFIX>/documents/<document_id>/<filename>
```
上传先写入可回卷的临时流,以满足官方 SDK 对长度和 CRC 计算的要求;随后使用
`If-None-Match: *` 执行单对象 Put,服务端返回 412 时统一映射为
`ObjectAlreadyExistsError`。这样不会用“先 HEAD、再 PUT”的竞态检查破坏不可变来源约束。
下载返回 SDK 的流式 Response Body,由调用方负责关闭;存在性检查只发 HEAD,不下载正文。
404、412 和其他 SDK 异常映射为稳定的 Provider-neutral 错误,外层错误不包含 Endpoint、
SDK Response Body 或凭据。
## 权限与验收边界
运行身份只授予目标 Bucket/Prefix 所需的 Put、Get/Head 和 Delete 权限,不授予 Bucket 管理或
全局 List 权限。若生产策略要求来源不可物理删除,应在后续 Document Version/Retention
设计中单独处理;当前 Demo 的直接删除行为保持不变。
默认测试使用 Fake Client,不连接 BOS。真实连接、权限和临时对象清理已在 ARCH-006 的
受控 test 资源中验证;凭据只允许运行时注入,不得写入仓库、日志或测试快照。
官方参考:
- [BOS Python SDK 上传文件](https://cloud.baidu.com/doc/BOS/s/yl5jfscvi)
- [BOS 获取 Object](https://cloud.baidu.com/doc/BOS/s/Mk4xtzv8v)
- [BOS 访问控制](https://cloud.baidu.com/doc/BOS/s/Tjwvysda9)

View File

@ -0,0 +1,91 @@
# 生产容器镜像说明
## 范围
DEPLOY-001 使用同一个锁定运行时构建三个独立镜像目标:
| Target | 默认镜像名 | 入口 | 端口 | 镜像健康检查 |
|---|---|---|---|---|
| `api` | `enterprise-knowledge-api` | Uvicorn + API/Admin | `8000` | `GET /healthz` |
| `mcp` | `enterprise-knowledge-mcp` | `ekp-mcp` | `8766` | PID 1 存活 |
| `worker` | `enterprise-knowledge-worker` | `ekp-worker` | 无 | PID 1 存活 |
镜像没有内置数据库迁移、Helm、Ingress 或 Secret 对象;DEPLOY-002 已在独立 Chart 中提供这些
编排能力。API 镜像包含 Alembic 和 migrations,由一次性 Migration Job 覆盖命令后复用。详见
[`KUBERNETES_DEPLOYMENT.md`](KUBERNETES_DEPLOYMENT.md)。
## 可重复构建基线
- Python 基础镜像固定为 `3.12.14-slim-bookworm`,uv 固定为 `0.12.5`,两者同时锁定 manifest
digest;
- Python 依赖只使用提交的 `uv.lock`,以 `uv sync --frozen --no-dev --no-editable` 安装;
- 三个目标共享相同依赖层,不复制不同版本的业务运行时;
- 进程均以固定 UID/GID `10001:10001` 运行,不具备 root 权限;
- `.dockerignore` 排除 `.env`、Git、虚拟环境、缓存、本地存储、测试和开发产物;
- Bake 计划同时声明 `linux/amd64` 与 `linux/arm64`,不下载或打包平台专属模型。
基础镜像 tag/digest 与应用依赖均已固定;镜像发布系统仍应保存本项目镜像的构建 digest,并让
Helm 按 digest 部署。本项目镜像 digest 由实际 Registry 发布产生,不把某个环境的 Registry
地址写入仓库。
## 构建命令
先确认 Docker Buildx builder 支持两个目标平台:
```bash
make images-print
```
构建全部目标:
```bash
IMAGE_REGISTRY=registry.example.internal/knowledge \
IMAGE_TAG=<immutable-release-tag> \
make images
```
只有发布流水线才执行推送:
```bash
IMAGE_REGISTRY=registry.example.internal/knowledge \
IMAGE_TAG=<immutable-release-tag> \
make images-push
```
不要把 Registry 密码放入变量或命令行;构建节点应提前通过其凭据助手登录。禁止使用
`latest` 作为集群部署版本。
## 运行时配置
镜像只内置以下非敏感进程默认值:`APP_ENV=production`、容器监听地址、静态页面绝对路径和
临时缓存目录。OceanBase、BOS、RabbitMQ、Embedding、Reranker 与 MCP Token 必须在容器启动
时由 Secret 注入。镜像构建参数、Layer、标签、健康检查和启动命令都不得包含这些值。
API 的 `/healthz` 只证明进程存活,不读取云端凭据或探测依赖。Kubernetes readiness、资源限制、
只读根文件系统所需临时卷及依赖就绪策略由 DEPLOY-002 Chart 配置。MCP endpoint 需要认证,Worker
没有 HTTP server,因此这两个镜像的 Docker 健康检查只验证主进程;业务可恢复性仍以持久化
Job/Attempt 和 DEPLOY-003 故障验收为准。
## 本地单目标检查
需要加载到本地 Docker 时,可直接构建当前平台的一个 target:
```bash
docker build --target api -f deploy/docker/Dockerfile -t ekp-api:test .
docker build --target mcp -f deploy/docker/Dockerfile -t ekp-mcp:test .
docker build --target worker -f deploy/docker/Dockerfile -t ekp-worker:test .
```
启动时使用 `--env-file` 或编排系统的 Secret 引用,不在命令中展开敏感值。API/Admin smoke test
至少检查 `/healthz`、`/admin/upload` 和 `/api/v1/openapi.json`;MCP 与 Worker 的真实依赖启动
验收在具备 RabbitMQ 和目标 Provider 的受控环境执行。
## DEPLOY-001 构建验收
2026-09-02 已使用 Buildx 分别构建并加载三个 `linux/arm64` target 和三个 `linux/amd64`
target。API/Admin 以非 root 用户完成健康、静态页面和 OpenAPI smoke test;MCP/Worker 完成非
root 安装入口检查。构建历史与容器文件系统检查未发现 `.env`、Git、测试文件或敏感配置。
这次验收只生成本地 `deploy001-*` 测试 tag,没有登录或推送 Registry,因此不产生可供集群
引用的项目镜像 digest。DEPLOY-002 必须使用操作者指定的不可变 Registry tag/digest,不得把
本地测试 tag 当作发布版本。

View File

@ -1,3 +1,19 @@
# Definition of Done
A task is done only when scope is implemented, architecture invariants hold, format/lint/typecheck pass, relevant tests pass, migrations/contracts/docs are updated as needed, no secrets are committed, retrieval changes include RAG regression results, and known limitations are documented.
For asynchronous ingestion or retry tasks, relevant tests must also prove durable Job/Attempt state,
bounded retry classification, idempotent duplicate delivery, cleanup-before-retry, preservation of the
immutable source, and terminal behavior when cleanup fails. Batch ingestion must cover per-file partial
failure and configured limits.
For MCP tasks, contract tests must cover tool discovery, identity propagation, authorization/error
boundaries, REST/application-service parity, infrastructure redaction, and the explicitly targeted MCP
client compatibility check.
For cluster delivery tasks, completion also requires reproducible production images, Secret-only
sensitive configuration, serialized migration, readiness/liveness probes, resource and pod-security
defaults, idempotent Helm install/upgrade, rolling-restart verification, durable worker recovery, stable
HTTPS Admin/MCP endpoints, and remote WorkBuddy acceptance. Manifests, rendered output, logs and images
must be checked for credential leakage. Public-knowledge delivery does not complete or waive V1
user/ACL tasks.

View File

@ -0,0 +1,53 @@
# ARCH-005 Shadow Index 重建与切换
ARCH-005 将正式运行默认值切换为 BOS、OceanBase Vector 和千帆 BCE Reranker;ARCH-006
进一步确认 canonical 关系数据与向量索引同处 OceanBase MySQL 数据库。关系表保存 Document、
Chunk、ChunkIndexState 和 Index Profile;向量表只保存可以重建的检索行。旧 Mochow 运行配置、依赖和代码已移除,但本流程不会自动删除
旧外部索引,物理退役必须由受控运维流程另行确认。
## Profile 与迁移前提
默认新 Profile 为:
```text
demo-bge-large-zh-structure-v1-oceanbase-v1
```
它与旧 Demo Profile 使用不同 ID,避免已有 Mochow `ChunkIndexState` 被误认为 OceanBase
写入证据。自定义 ID 仍由 `INDEX_PROFILE_ID` 注入,并且必须是全新的、稳定的版本标识。
执行前必须:
1. 运行 `uv run alembic upgrade head`,创建 canonical `index_profiles` 表;
2. 使用 `deploy/oceanbase/001_knowledge_chunks_v1.sql` 预创建目标表和 HNSW 索引;
3. 仅在运行时注入已授权的 OceanBase 与千帆凭据,不写入仓库或日志;
4. 确认目标 OceanBase 为 4.3.5.4+,且应用账号只有目标数据库/表所需权限;
5. 在重建期间避免修改 Chunk 策略;激活事务会阻止 canonical Chunk 并发写入穿过最终核对。
## 顺序执行
每一步只输出 Profile ID 和计数,不输出 Endpoint、SQL、Chunk 内容或凭据。
```bash
uv run ekp-index-rebuild build
uv run ekp-index-rebuild verify
uv run ekp-index-rebuild activate
```
- `build` 注册 Shadow Profile,从 OceanBase canonical Chunk 分文档生成 Embedding 并写入
OceanBase。命令中断后可以原样重试;已完整文档会跳过,部分目标 Profile 数据会先定向
清理再重建,不触碰 Active Profile;
- `verify` 要求目标策略下至少存在一个 canonical Chunk,并核对全部 Chunk 与匹配模型、
维度的 ChunkIndexState。输出中的 `complete` 必须为 `true`;
- `activate` 在 OceanBase 事务内再次验证,锁定 Chunk 写入,将原 Active Profile 标为
`retired`,再将目标 Shadow Profile 标为 `active`。重复执行已成功的激活是幂等的。
激活后 API 从 OceanBase canonical 关系表解析 Active Profile,不再仅依赖进程环境中的 ID。新上传与查询
统一使用 OceanBase Active Profile。部署仍应保留前一版本制品和旧索引回滚窗口;本 Task
不自动销毁任何旧外部数据。
## 验证与已知边界
离线测试覆盖多文档构建、重试跳过、完整性失败关闭、单 Active 约束、原子激活、迁移和
CLI 输出脱敏。真实 BOS/OceanBase/Qianfan 连通性、DDL 权限、30~50 条脱敏 Golden Query、
召回质量和延迟属于 ARCH-006,且只能使用运行时授权凭据与受控测试资源。

View File

@ -0,0 +1,143 @@
# Kubernetes / Helm 部署说明
## 1. 交付范围
DEPLOY-002 提供单一 Helm Chart,将以下运行角色部署到已有 Kubernetes 集群:
- API/Admin:默认两个副本,提供 `/healthz`、`/admin/*` 和 `/api/v1/*`;
- MCP Gateway:默认两个副本,提供经 Bearer Token 保护的 `/mcp`;
- Async Worker:默认两个副本,只消费 `ingestion` 队列;
- Recovery Scheduler:固定单副本,以 `Recreate` 策略运行 Celery Beat;
- Migration Job:每次安装/升级前串行执行 `alembic upgrade head`;
- 可选 RabbitMQ:只供开发/演示集群显式启用,生产默认使用外部 Broker。
OceanBase MySQL、BOS、Embedding/Reranker 服务继续使用集群外已有资源。Chart 不创建数据库、
BOS Bucket、域名、证书或云账号,也没有扩大公共知识库阶段的业务范围。
## 2. 前置条件
操作者需要提前准备:
1. 可用的 Kubernetes Namespace 和对应 `kubectl` 上下文;
2. Helm 3/4 与 Kubernetes 客户端;
3. 已推送到目标 Registry 的 API、MCP、Worker 不可变 tag 或 digest;
4. Namespace 内的运行时 Secret、TLS Secret,以及私有 Registry 所需的 ImagePullSecret;
5. 集群到 OceanBase、BOS、RabbitMQ 和模型服务的网络连通性。
Chart 默认引用 `knowledge-platform-runtime`,但不会创建它。该 Secret 至少包含以下键:
```text
DATABASE_URL
RABBITMQ_URL
BOS_ACCESS_KEY_ID
BOS_SECRET_ACCESS_KEY
BAIDU_QIANFAN_API_KEY
OCEANBASE_VECTOR_PASSWORD
MCP_AUTH_TOKEN
```
`RERANKER_API_KEY` 可单独提供;未提供时应用按既有契约复用
`BAIDU_QIANFAN_API_KEY`。`MCP_AUTH_TOKEN` 至少 32 个字符,只具备
`knowledge:search` 服务范围。不要使用 `kubectl --from-literal`、Shell 参数或 Helm
`--set` 传递以上值;应由受控 Secret 管理系统或运维流程预先写入 Namespace。
## 3. 环境 Values
复制 [`values.example.yaml`](../../deploy/helm/environments/values.example.yaml) 到仓库外的受控
路径,填写以下非敏感信息:
- 三个镜像的 Registry 路径与不可变 tag/digest;
- `existingSecret` 和 `imagePullSecrets` 的资源名称;
- OceanBase/BOS 的非敏感 Endpoint、数据库标识和 Bucket;
- Admin/API 与 MCP 的 Ingress Host、TLS Secret 名称;
- 与外部地址一致的 MCP Public URL、Issuer、Allowed Host/Origin;
- 副本、资源、超时和批量上传限制。
`values.schema.json` 拒绝未知顶层字段、`latest` tag、无效 digest、单副本 API/MCP、错误类型和
空的关键引用;模板还会校验 Chunk、检索和入库限制之间的大小关系。Values 只允许非敏感配置,
模板逐项生成 ConfigMap,不提供任意 `extraEnv` 入口,避免把凭据误写进渲染产物。
## 4. 安装与幂等升级
先做离线校验和清单检查:
```bash
make deploy-lint VALUES=/secure/path/environment-values.yaml
make deploy-render VALUES=/secure/path/environment-values.yaml
```
安装或升级:
```bash
NAMESPACE=knowledge-platform \
RELEASE=knowledge-platform \
make deploy VALUES=/secure/path/environment-values.yaml
```
`make deploy` 只接收 Values 文件路径。脚本会先验证 Chart、Namespace、运行时 Secret 及必需
键名、可选 RabbitMQ Secret 键名,以及启用 Ingress 时的两个 TLS Secret 引用,然后执行带失败
回滚和等待语义的 `helm upgrade --install`;脚本对 Helm 3 使用 `--atomic`,对 Helm 4 使用等价
的 `--rollback-on-failure`。它只验证 Secret 资源和键是否存在,不读取或输出 Secret 值。
Migration 是 `pre-install,pre-upgrade` Hook,固定名称配合
`before-hook-creation` 保证同一 Release 内不会并发执行。Helm 等待 Migration 成功后才更新业务
工作负载;失败时 Helm 保留上一版本。完成的 Job 最长保留一天供日志追查,并在下次升级前删除。
安装命令可以重复执行;数据库迁移必须保持 forward-only 与幂等。
## 5. Ingress、探针与 Pod 安全
生产环境启用 Ingress 后,Admin/API 与 MCP 使用独立稳定 HTTPS Host,也可由同一 Ingress
Controller 托管。证书只通过预建 TLS Secret 引用,不进入 Values。
- API 的 startup/readiness/liveness 调用无凭据 `/healthz`;
- MCP 的三类探针使用 TCP 端口,不发送或记录 Token;
- Worker/Scheduler 使用 PID 1 存活探针,业务恢复正确性由持久化 Job/Attempt 保证;
- 全部应用容器固定 UID/GID `10001`、禁止提权、丢弃 Linux capabilities、使用
`RuntimeDefault` seccomp 和只读根文件系统;
- `/tmp` 使用有大小上限的 `emptyDir`,ServiceAccount 不挂载 API Token;
- API/MCP 滚动更新使用 `maxUnavailable: 0`,未 Ready Pod 不进入 Service Endpoint。
`/healthz` 当前只表示 API 进程存活,不探测所有外部 Provider;真实依赖故障、滚动重启和
Worker 恢复行为由 DEPLOY-003 在目标集群验收。
## 6. 可选 Demo RabbitMQ
生产默认 `rabbitmq.enabled=false`,`RABBITMQ_URL` 指向外部受管 Broker。仅开发/演示时可开启
本地子 Chart;它使用单节点 StatefulSet 和 PVC,并要求预建 Secret 包含:
```text
username
password
erlang-cookie
```
同时必须让运行时 Secret 中的 `RABBITMQ_URL` 指向该 Service。子 Chart 使用固定
`rabbitmq:4.1.4-alpine` tag 与 multi-architecture manifest digest;它不具备生产 RabbitMQ 的
高可用、备份或升级保证,禁止作为生产默认值。卸载 Release 时 StatefulSet PVC 默认保留,
删除前必须由数据责任人确认。
## 7. 回滚、卸载与 Secret 轮换
查看历史并回滚时不要输出 Pod 环境变量:
```bash
helm -n knowledge-platform history knowledge-platform
helm -n knowledge-platform rollback knowledge-platform <revision> --wait
```
更新外部 Secret 后,Secret 名称不变;用户和 WorkBuddy 配置无需重新注入。当前应通过受控
滚动重启让 Pod 读取新值,目标集群的无中断验证属于 DEPLOY-003。
卸载只删除本 Release 管理的 Kubernetes 对象:
```bash
helm -n knowledge-platform uninstall knowledge-platform
```
外部 OceanBase、BOS、RabbitMQ、TLS/运行时 Secret 和可选 RabbitMQ PVC 不由卸载命令删除。
## 8. DEPLOY-002 验收边界
本任务完成 Chart lint、双路径渲染、严格 Kubernetes Schema、Values 失败关闭、Secret 泄漏扫描
以及本地集群两次安装/升级验证。目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实
Provider 和远程 WorkBuddy 验收统一属于后续 `DEPLOY-003`,不得提前宣称完成。

View File

@ -0,0 +1,67 @@
# ARCH-006 受控真实链路验收
本验收覆盖 OceanBase canonical relational metadata → BOS immutable source → 千帆
`bge-large-zh` → OceanBase Vector → 千帆 `bce-reranker-base` → canonical Source Result。数据集包含
32 条完全合成、脱敏 Query,不含真实客户、产品、政策或内部知识。
## 强制安全门禁
只有同时满足以下条件才允许执行:
1. 数据库、BOS 和千帆凭据已经所有者明确授权;配置清单中的现有值只允许本次 test
验收临时注入,禁止保存为 `.env`、写入日志或用于 fat/prod,完成后仍必须轮换;
2. OceanBase Database/Table 和 BOS Bucket/Prefix 已明确授权用于测试;
3. `APP_ENV=test`;
4. `BOS_KEY_PREFIX` 以 `live-acceptance/` 开头,避免写入共享业务前缀;
5. 显式设置 `EKP_RUN_LIVE_TESTS=1`、`EKP_CREDENTIALS_AUTHORIZED=1` 和
`EKP_LIVE_RESOURCE_ACK=1`。
任一条件不满足时,命令在创建 Provider 或外部连接前失败关闭,并且不输出 Endpoint、用户
名、SQL、响应正文、Chunk 内容或 Secret。仓库 `.env.example` 中三个开关固定为 `0`。
## 执行顺序
先完成 ARCH-005 切换:
```bash
uv run alembic upgrade head
uv run ekp-index-rebuild build
uv run ekp-index-rebuild verify
uv run ekp-index-rebuild activate
```
确认 `verify` 和 `activate` 输出的 `complete` 均为 `true` 后,再运行:
```bash
make live-acceptance
```
命令会使用随机运行前缀上传两份合成 Markdown,执行全部 32 条 Query,并在 `finally` 清理
本次运行的 OceanBase Vector 行、canonical Document/Chunk/State 以及 BOS 对象。清理失败会使验收
失败,且不会尝试删除不带本次随机文件名前缀的数据。
## 验收指标
默认门槛:
```text
Recall@1 >= 0.90
Recall@5 = 1.00
MRR >= 0.90
NDCG@5 >= 0.90
Search P95 <= 10000 ms
```
输出只包含 Query 数、Recall@1/3/5、MRR、NDCG@5、P50/P95 和 `passed`。如受控环境需要调整
门槛,只能通过 `EKP_LIVE_MIN_*` / `EKP_LIVE_MAX_P95_MS` 显式注入,并把实际门槛与结果记录
到受保护的验收日志;不要把 Endpoint、账号、凭据或原始响应提交到仓库。
## 验收结果(2026-09-02)
目标 test OceanBase 已完成 migration、专用向量表/HNSW、PRE-FILTER、BOS、Embedding、
Reranker 和 canonical Source Result 验收。32 条查询结果:Recall@1/3/5、MRR、NDCG@5 均为
`1.0`,P50 `3787.61 ms`,P95 `4759.21 ms`,`passed=true`。正式命令退出码为 `0`,数据库
复核 `live-` Document/Chunk/Vector 残留为 `0`。
首次正式运行出现过一次无细节瞬时失败;分阶段 Provider/单文档/32 查询诊断以及随后正式
运行均通过。后续稳定性任务应补充有界重试、限流指标与告警;这不阻塞本次受控验收。

View File

@ -0,0 +1,102 @@
# DEMO-012 MCP Gateway 使用说明
## 范围
DEMO-012 提供独立的 Enterprise Knowledge MCP Gateway,通过标准 Streamable HTTP 暴露一个
只读工具:`search_knowledge`。Gateway 直接复用 API 进程使用的 `SearchService` 组合,不复制
Dense Retrieval、Reranker 或 Source Enrichment 逻辑。
当前 Demo 只建立认证和身份传递边界:一个运行时注入的专用 Bearer Token 映射到一个
user/service identity,并且只有 `knowledge:search` scope。它不是共享超级管理员 Token,也不
代表 V1 OAuth、Document ACL 或完整 MCP P0 工具已经完成。
## 配置
必须通过运行环境提供至少 32 字符的随机 Token:
```bash
export MCP_AUTH_TOKEN="<runtime-secret-with-at-least-32-characters>"
make run-mcp
```
默认地址为 `http://127.0.0.1:8766/mcp`。除现有数据库、BOS、Embedding、Vector 和 Reranker
配置外,Gateway 支持:
| 变量 | 默认值 | 说明 |
|---|---|---|
| `MCP_HOST` | `127.0.0.1` | 监听地址;本地默认不暴露到其他网卡 |
| `MCP_PORT` | `8766` | Gateway 端口 |
| `MCP_PATH` | `/mcp` | Streamable HTTP 单一端点 |
| `MCP_PUBLIC_URL` | `http://127.0.0.1:8766/mcp` | MCP Resource Server URL |
| `MCP_ISSUER_URL` | `http://127.0.0.1:8766` | Demo Token 的身份颁发方标识 |
| `MCP_CLIENT_ID` | `workbuddy-enterprise` | MCP Client 类型标识 |
| `MCP_SERVICE_IDENTITY` | `workbuddy-demo` | 传入业务边界的服务身份 |
| `MCP_REQUIRED_SCOPE` | `knowledge:search` | Tool 所需最小 scope |
| `MCP_ALLOWED_HOSTS` | 仅本机/test | DNS rebinding Host allowlist,JSON 数组 |
| `MCP_ALLOWED_ORIGINS` | 仅本机 | Origin allowlist,JSON 数组 |
远程部署必须使用 TLS 反向代理,并把公开域名加入 `MCP_ALLOWED_HOSTS`;有 `Origin` 的客户端还
必须显式配置 `MCP_ALLOWED_ORIGINS`。Token、真实域名和内部 Endpoint 不得提交 Git。
## WorkBuddy / CodeBuddy 配置
WorkBuddy Enterprise 官方客户端支持 `type: "http"`、远程 URL、自定义 Header 和环境变量
展开。可复制 `deploy/workbuddy-mcp.example.json` 作为项目配置,内容如下:
```jsonc
{
"mcpServers": {
"enterprise-knowledge": {
"type": "http",
"url": "${EKP_MCP_URL:-http://127.0.0.1:8766/mcp}",
"headers": {
"Authorization": "Bearer ${EKP_MCP_TOKEN}"
},
"description": "企业知识检索"
}
}
}
```
只提交包含环境变量占位符的配置;`EKP_MCP_TOKEN` 必须由每个运行环境安全注入。连接后应只
发现 `mcp__enterprise_knowledge__search_knowledge` 一个工具。
## 安全和错误语义
- 缺失或错误 Token 返回 `401`,缺少 scope 返回 `403`;
- 未批准 Host 返回 `421`,未批准 Origin 返回 `403`;
- Query 长度和 `top_k` 在 MCP schema 层校验;
- 预期业务错误返回安全 Tool Error;未知 Provider/数据库错误收敛为
`knowledge search failed`;
- MCP 输出和 Gateway 日志不得包含 Token、Provider Endpoint、数据库表名、模型名或
Index Profile。
## 验证
```bash
uv run pytest tests/unit/test_mcp_auth.py tests/unit/test_mcp_search.py
uv run pytest tests/integration/test_mcp_server.py
```
集成测试使用官方 MCP Python Client 通过 Streamable HTTP 完成 initialize、tools/list 和
tools/call,并使用与 WorkBuddy 相同的 Bearer Header 配置验证身份传递、无结果、参数错误、
认证失败、Origin 防护和错误脱敏。
### WorkBuddy 现场验收记录
2026-09-02 使用 WorkBuddy Desktop `5.3.14` 自带的 CodeBuddy CLI `2.115.0` 完成兼容性
验收。客户端直接加载 `deploy/workbuddy-mcp.example.json`,健康检查为 `Connected`;工具
发现结果只有 `mcp__enterprise-knowledge__search_knowledge`,随后以字符串 `query` 和
`top_k=1` 完成 Tool 调用并读取结构化 `filename`、`section`、`content` 等来源字段。
现场验收使用 DEMO-012 的真实 Gateway、Bearer 验证、service identity 传递、输入 Schema 和
Tool 注册代码。为避免外部数据库与模型波动掩盖客户端兼容性,检索应用使用确定性验收返回;
真实 Provider 检索链路由 `ARCH-006` 的受控验收负责。非交互 CLI 需要批准 MCP Tool 调用,
本次只在单次会话中允许 `DeferExecuteTool`,没有永久启用跳过权限。测试结束后已删除临时 MCP
配置并停止本地 Gateway,未写入真实凭据。
参考:
- [MCP Python SDK](https://github.com/modelcontextprotocol/python-sdk)
- [MCP Streamable HTTP 规范](https://modelcontextprotocol.io/specification/2025-11-25/basic/transports)
- [WorkBuddy Enterprise MCP 使用文档](https://cloud.tencent.cn/document/product/1831/137039)

View File

@ -0,0 +1,72 @@
# ARCH-003 OceanBase Vector 配置与索引契约
ARCH-003 在既有 `VectorStore` 契约后新增 OceanBase Vector Adapter。ARCH-006 确认 OceanBase
MySQL 关系表保存 canonical Document、Chunk、Index Profile 和 Chunk Index State;同库向量表
只保存可由这些数据重建的检索行。ARCH-005 已完成运行时默认切换,并移除 Mochow 运行配置与依赖。
## 数据库前提
- OceanBase MySQL 租户版本至少为 4.3.5.4:ARCH-006 已在目标 test 集群用只读
`EXPLAIN` 验证 `VECTOR_INDEX(... pre_filter)` 产生 PRE-FILTER 执行计划;
- 目标数据库和最小权限账号由受控部署流程预创建;
- Vector 内存、资源规格和 HNSW 参数由 DBA 在受控环境确认;
- 应用启动和请求路径不会创建、修改或删除表结构。
使用 [001_knowledge_chunks_v1.sql](../../deploy/oceanbase/001_knowledge_chunks_v1.sql)
在专用数据库中创建索引表。脚本固定:
```text
VECTOR(1024)
DISTANCE = COSINE
TYPE = HNSW
EF_CONSTRUCTION = 256
EF_SEARCH = 200
```
表的 `index_record_id` 是确定性的 Chunk/Profile 主键;`document_id + index_profile_id` 和
`index_profile_id` 是清理与 Profile 过滤的标量索引。表名或向量维度变化必须建立新表或
Shadow Profile,验证后再激活,禁止在生产索引上原地修改。
## 运行配置
配置只能由部署环境或 Secret Manager 注入,禁止把真实地址、用户名和密码写入 `.env`、
镜像、日志、测试快照或提交记录。
```text
VECTOR_STORE_PROVIDER=oceanbase
OCEANBASE_VECTOR_URI=<host:port>
OCEANBASE_VECTOR_USER=<运行时注入>
OCEANBASE_VECTOR_PASSWORD=<运行时注入>
OCEANBASE_VECTOR_DATABASE=<预创建数据库>
OCEANBASE_VECTOR_TABLE=knowledge_chunks_v1
OCEANBASE_VECTOR_INDEX=vidx_knowledge_chunks_embedding
OCEANBASE_VECTOR_BATCH_SIZE=100
OCEANBASE_VECTOR_TIMEOUT_SECONDS=30
```
- URI 只接受 `host:port`,不得包含 Scheme、用户信息、Path、Query 或 Fragment;
- Database/Table/Index 只接受安全标识符;
- Password 使用 `SecretStr`,Provider 异常只返回收敛后的错误,不回显 SQL、连接信息、
Chunk 内容或 SDK Response;
- `pyobvector` 只存在于 Adapter/Factory 内,业务服务不依赖其 Row、Engine 或距离函数类型。
## 写入、删除和检索语义
- Upsert 使用 `index_record_id` 主键,按可配置批次写入;
- 删除使用参数化的 `document_id + index_profile_id` 条件,不拼接用户输入;
- Dense Search 在同一 ANN 查询中使用 `VECTOR_INDEX(... pre_filter)`,保证
`index_profile_id` 在候选近邻搜索前过滤,并返回 Dense Top20;禁止依赖普通 `WHERE`
的自适应过滤语义,也禁止全局 TopK 后再过滤;
- OceanBase `cosine_distance` 是低值优先距离,Adapter 统一转换为
`score = 1 - cosine_distance`,保持 `VectorStore` 的高分优先契约;
- 返回 UUID、Profile、字段数、距离范围和排序不满足契约时失败关闭,不生成 Source Result。
默认测试使用 Fake Client,不连接 OceanBase。真实版本、DDL、权限、Cosine HNSW 召回、过滤
行为与 30~50 条脱敏 Golden Query 统一在 ARCH-006 的受控资源中验证。凭据只能在运行时
临时注入,禁止进入仓库、日志或测试快照。
官方参考:
- [pyobvector 官方仓库](https://github.com/oceanbase/pyobvector)
- [OceanBase 创建向量索引](https://en.oceanbase.com/docs/common-oceanbase-database-10000000003075357)
- [OceanBase 向量索引查询 Hint](https://en.oceanbase.com/docs/common-oceanbase-database-10000000004311163)

View File

@ -0,0 +1,76 @@
# ARCH-004 千帆 BCE Reranker 与 Source Result 配置
本文记录 Knowledge Platform Demo 的重排序连接方式、请求边界和 canonical Source Result
契约。真实 API Key 只允许由运行环境注入,不得写入代码、示例配置、日志或提交记录。
## 运行配置
```dotenv
RERANKER_PROVIDER=qianfan
RERANKER_MODEL=bce-reranker-base
RERANKER_ENDPOINT=https://qianfan.baidubce.com/v2/rerank
RERANKER_API_KEY=<运行环境注入,可空>
RERANKER_TIMEOUT_SECONDS=30
DENSE_TOP_N=20
SEARCH_TOP_K=5
```
`RERANKER_API_KEY` 未单独配置时,Demo 会复用 `BAIDU_QIANFAN_API_KEY`。两者都不存在时
Provider 创建失败。凭证使用 `SecretStr`,HTTP/JSON 异常不会包含请求头、请求正文或
响应正文。
## Reranker 请求契约
ARCH-004 固定使用千帆 `POST /v2/rerank`,模型基线为
`bce-reranker-base`:
- Query 去除首尾空白后不能为空,最多 1600 字符;
- documents 不能为空,最多 64 条,每条最多 4096 字符;
- 当前标准链路发送 Dense Top20,并请求 Reranker Top5;
- 响应必须恰好返回请求的 Top-N,`index` 唯一且指向原输入位置;
- `relevance_score` 必须是有限数值,并保持降序;
- 响应若回显 document,必须与对应输入文本完全一致。
上述边界与千帆重排序接口当前约束一致。BCE 和原 Demo Qwen3 模型共享请求/响应结构,
因此本次变更不修改 `RerankerProvider`、Search REST API 或最终 Top5 行为。Qwen3-Reranker
保留为 PRD 中的后续 Benchmark 候选,不作为当前运行默认值。
Adapter 只向业务层返回原候选索引和 Reranker 分数,不暴露千帆响应结构。默认测试使用
`FakeRerankerProvider` 的确定性字符覆盖率排序,不发起网络请求。
## Source Result 契约
Reranker 对 provider-neutral Dense Candidate 的 `content` 评分;选出 Top5 后,
`SearchService` 使用
`chunk_id` 一次性回查 OceanBase canonical `chunks` 关系表,并由 canonical row 生成:
```text
document_id
filename
page
section
chunk_id
content
score
rerank_score
```
其中 `score` 保留 Dense 分数,`rerank_score` 保存精排分数。若 Chunk 缺失、document_id
不一致或向量索引 content 已与 canonical content 不一致,搜索会失败关闭,避免返回来源
与评分文本不一致的结果。
DEMO-009 已通过 `POST /api/v1/search` 提供序列化和错误映射,参见
`docs/development/REST_API_SETUP.md`。Demo 仍不实现 RBAC/Document ACL,正式 V1 权限
范围必须在 Dense 召回阶段下推,不能在 Reranker 后过滤。
## 验证边界
默认测试离线覆盖 Dense Top20 → Reranker → Top5 → canonical Source Result,并使用合成
Golden Dataset 验证预期文档和章节升至首位。真实千帆联调前还需在受控运行环境确认:
- API Key 已开通目标 Reranker 模型权限;
- Endpoint 网络可达;
- 模型 API 名称与控制台服务一致;
- 真实中文 Query 的延迟、配额和 Top5 质量满足 Demo 演示需求。
官方接口参考:[千帆重排序 API](https://cloud.baidu.com/doc/qianfan-api/s/2m7u4zt74)。

View File

@ -0,0 +1,76 @@
# Demo REST API 使用说明
## 范围
DEMO-009 将入库和检索服务公开为版本化 REST API,DEMO-015 将单文件入库切换为持久化异步
Job,DEMO-016 增加批量接收与聚合查询。API 层只负责输入校验、
错误映射和响应序列化,不重复实现 Parser、Embedding、VectorStore 或 Reranker 业务逻辑。
当前不包含 ACL/RBAC、软删除、目录/压缩包上传、断点续传或 Hybrid Search。
## 启动前提
1. 按 `docs/development/BOS_SETUP.md` 配置 BOS;
2. 按 `docs/development/OCEANBASE_VECTOR_SETUP.md` 配置 OceanBase Vector 和千帆 Embedding;
3. 按 `docs/development/RERANKER_SETUP.md` 配置 Reranker;
4. 按 `docs/development/ASYNC_INGESTION_SETUP.md` 配置 RabbitMQ 并启动 Worker;
5. 执行 `uv run alembic upgrade head`;
6. 新旧索引迁移按 `docs/development/INDEX_REBUILD.md` 构建、验证并激活 Profile;
7. 执行 `make run`。
`/healthz` 和 OpenAPI 可以在没有云端凭证时启动和访问。业务依赖在首次访问 P0 业务接口时
延迟创建;配置不完整时返回 `503`,不会将凭证名以外的配置值暴露给客户端。
OpenAPI 地址:`/api/v1/openapi.json`
Swagger UI:`/api/v1/docs`
## 示例
上传并异步排队:
```bash
curl -F 'file=@./policy.md;type=text/markdown' \
http://127.0.0.1:8000/api/v1/documents
```
使用响应中的 `job_id` 查询进度:
```bash
curl http://127.0.0.1:8000/api/v1/jobs/<job-id>
```
批量上传并查询聚合进度:
```bash
curl -F 'files=@./policy.md;type=text/markdown' \
-F 'files=@./manual.pdf;type=application/pdf' \
http://127.0.0.1:8000/api/v1/ingestion-batches
curl http://127.0.0.1:8000/api/v1/ingestion-batches/<batch-id>
```
检索:
```bash
curl -H 'Content-Type: application/json' \
-d '{"query":"产品A退款周期是多少?","top_k":5}' \
http://127.0.0.1:8000/api/v1/search
```
上传只等待不可变源文件、Document 与 Job 持久化,成功返回 `202`。Worker 后台执行 Parse、
Clean、Chunk、Embedding 和 OceanBase Upsert;客户端通过 Job 状态判断何时可检索。
## 删除一致性
删除按以下顺序执行:
1. 删除所有已记录 Index Profile 以及当前 Profile 的 OceanBase 行;
2. 删除 OceanBase canonical 关系表中的 ChunkIndexState、Chunk 和 Document;
3. 删除解析 JSON 与不可变原始文件。
`queued`、`running` 或 `retrying` Job 对应的 Document 返回 `409`,避免删除与 Worker 并发。
向量清理失败时保留 canonical Metadata 和原始文件。Metadata 提交后若文件清理失败,接口
返回明确的 `500`,可能留下不可访问的孤立文件;这是 Demo 直接删除的已知边界,正式 V1
应通过可重试清理任务和审计记录强化。

View File

@ -1,6 +1,6 @@
# Test Strategy
Unit tests must run without Mochow, Qianfan, internet, GPU or production credentials.
Unit tests must run without OceanBase, BOS, Qianfan, internet, GPU or production credentials.
Provide deterministic fakes:
- FakeEmbeddingProvider
@ -11,6 +11,11 @@ Provide deterministic fakes:
External integrations use separate integration suites/markers.
Async ingestion tests run with an in-process recording dispatcher and deterministic providers. They
must cover persisted Job/Attempt history, safe error summaries, bounded retry, cleanup-before-retry,
active-lease duplicate delivery, expired-lease recovery, immutable-source preservation and terminal
cleanup failure without requiring a live RabbitMQ broker.
E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval.
Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation.

View File

@ -6,4 +6,20 @@ Benchmark the full pipeline: Embedding × Retrieval Strategy × Fusion × Rerank
Initial embedding candidates: bge-large-zh, Qwen3-Embedding-0.6B, Qwen3-Embedding-4B, optional 8B.
Use synthetic/public/anonymized fixtures only.
Use synthetic/public/anonymized fixtures only. ARCH-006 expands the checked-in dataset to 32
fully synthetic queries across refund, technical support, lodging, and reimbursement-material
sources. Offline Fake Provider regression must remain deterministic; target-provider acceptance
uses the same records through `make live-acceptance` and reports aggregate metrics only.
Current live gates:
```text
Recall@1 >= 0.90
Recall@5 = 1.00
MRR >= 0.90
NDCG@5 >= 0.90
Latency P95 <= 10000 ms
```
Threshold overrides are allowed only as explicit test-run environment inputs and must be recorded
with the run result. See `docs/development/LIVE_ACCEPTANCE.md` for safety and cleanup requirements.

View File

@ -0,0 +1,85 @@
# Provider 架构迁移计划
## 目标基线
本计划依据 2026-09-02 对最新内部配置与 test 环境的只读核验调整 EKP Provider 和数据库
选型,同时保留 canonical-data 和 API-first 原则。
```text
OceanBase MySQL canonical Metadata / Chunk / Index State
Baidu BOS immutable source and derived artifacts
OceanBase Vector co-located rebuildable Dense retrieval index
Qianfan bge-large-zh 1024-dimensional Embedding
Qianfan BCE Reranker bce-reranker-base
```
Provider、内部地址、数据库账户和凭据不得出现在 API/MCP Contract、日志、镜像或提交中。
本地测试继续使用 LocalObjectStorage、FakeVectorStore、FakeEmbeddingProvider 和
FakeRerankerProvider。
## 明确保留的行为
- OceanBase 关系表保持 canonical database,专用向量表只保存可重建索引;
- 原始文件不可变;
- Dense Top20 → Reranker Top5;
- 当前 PDF/DOCX/TXT/Markdown Parser 和 `structure_v1` Chunk;
- `/api/v1` 和 MCP 继续只暴露业务语义;
- `index_profile_id` 过滤在召回阶段下推;
- 新索引通过 shadow 表/Profile 构建和验证,不原地覆盖旧索引。
## 明确不随迁移引入
- 将 OceanBase 向量表当作 canonical source;
- Top50、距离阈值、标准答案捷径、双查询和 RRF;
- Marker/Pandoc、LLM 摘要/问题/关键词索引;
- RBAC/ACL、异步队列、MCP、Hybrid/BM25 或 Admin Console 扩展;
- 旧系统 `.env`/镜像配置加载方式或任何现有敏感凭据。
## Task 与验收
### ARCH-001 架构和执行基线
更新 ADR、PRD、Architecture、Demo Scope、Tasks 和 Progress,消除目标 Provider 歧义。
### ARCH-002 BOS ObjectStorage
新增 provider-neutral BOS Adapter、配置、脱敏错误和离线单元测试。Local Adapter 保留为
测试/开发实现。
依赖:ARCH-001。
### ARCH-003 OceanBase VectorStore
新增 OceanBase Vector Adapter 和专用索引 DDL,覆盖 Upsert、文档/Profile 删除、Dense
TopN、Cosine Distance 到统一高分优先 `score` 的转换,以及 profile 过滤。
依赖:ARCH-002。
### ARCH-004 BCE Reranker
将运行基线改为 `bce-reranker-base`,按真实接口约束更新配置、Adapter、单元测试和 RAG
Evaluation;保持 Top5 API 行为。
依赖:ARCH-003。
### ARCH-005 Runtime Cutover
将 API 运行时切换到 BOS、OceanBase Vector 和 BCE Reranker;提供从 canonical
Chunk 构建新 shadow index 的可重试命令,验证后激活新 Profile,再清理 Mochow 运行配置。
依赖:ARCH-004。
### ARCH-006 真实验收
修正先前错误的 PostgreSQL 前提:将 canonical SQLAlchemy/Alembic 迁移到 OceanBase MySQL,
同库保留逻辑隔离的可重建向量表;在受控 test 资源中执行 OceanBase canonical → BOS →
Qianfan Embedding → OceanBase Vector → BCE Reranker → canonical Source Result 全链路,并完成
30~50 条脱敏 Golden Query 验收。
依赖:ARCH-005、已授权的 test 运行凭据、数据库/表和 BOS 专用前缀。
## 安全边界
内部配置清单中的现有凭据按已暴露处理。经所有者明确要求,本次只允许把 test 凭据临时
注入 ARCH-006 进程;不得生成 `.env`、测试快照或调试日志,不得连接 fat/prod,验收后仍应
轮换。真实集成测试必须使用专用表/BOS 前缀并清理本次临时对象。

View File

@ -0,0 +1,123 @@
# MCP 与异步批量入库优先级方案
**Task:** PLAN-001
**Status:** Accepted planning baseline
**Date:** 2026-09-02
## 1. 现状判断
| 能力 | 当前实现 | 结论 |
|---|---|---|
| 单文件上传 | `POST /api/v1/documents` 持久化 Source/Document/Job 后返回 `202`,Worker 后台处理 | DEMO-015 已实现 |
| 批量上传 | 批次 API、逐文件 Job 状态和 Admin Console 自动轮询 | DEMO-016 已实现 |
| 失败补偿 | 阶段内补偿 + 重试前清理全部派生数据,保留不可变源和执行历史 | DEMO-015 已实现 |
| 自动重试 | OceanBase Job/Attempt + Celery/RabbitMQ + 有界退避 + Worker 租约恢复 | DEMO-015 已实现 |
| 重试前清理 | 重试前删除向量、索引状态、Chunk、解析产物,保留源文件与历史 | DEMO-015 已实现 |
| MCP | 独立 Gateway、认证边界、`search_knowledge` 与 WorkBuddy 协议级验收 | DEMO-012 已实现 |
PLAN-001 的异步执行缺口已由 DEMO-015 补齐,批量提交与状态 UI 已由 DEMO-016 补齐。
## 2. Task 归并结论
### DEMO-012:MCP `search_knowledge`(提升为最高优先级)
保留既有 Task ID,不新建重复任务。范围限定为:
- 提供可运行的标准 MCP Gateway;
- 只暴露 `search_knowledge`;
- 复用现有 Search Service 或正式 `/api/v1/search`,不复制检索逻辑;
- 建立 WorkBuddy 所需的 user/service identity 边界,不使用共享超级管理员凭据;
- 完成协议契约、错误收敛、凭据脱敏和 WorkBuddy 兼容性验证。
该 Task 是 `EKP-070`、`EKP-071`、`EKP-072` 的最小垂直切片,不能据此把三个完整 V1 Task 标记为完成。
### DEMO-015:异步入库 Job 与自动重试清理(新增)
从 `EKP-017` 和 `EKP-023` 抽取 Demo 所需的最小基础能力,并复用 DEMO-003~006 的处理服务:
- 新增持久化 Ingestion Job/Attempt,记录阶段、状态、尝试次数、安全错误和时间戳;
- 源文件成功写入 BOS 且 Document 已持久化后立即排队,API 不等待解析和索引;
- 同一 Document 同时最多一个有效执行,任务投递和消费具备幂等保护;
- 对可重试的瞬时错误自动执行有界重试和退避;Demo 默认最多 3 次总尝试,允许通过非敏感配置调整;
- 格式不支持、文件为空、无可提取内容等确定性错误直接失败,不进行无意义重试;
- 每次重新执行前,按顺序清理该 Document 的向量行、`ChunkIndexState`、Chunk 和解析产物,再从不可变原始文件重新处理;
- 原始文件、Document 标识和审计所需的 Job/Attempt 历史不得被重试清理删除;
- 任一清理步骤失败时禁止启动下一次尝试,并记录为终止失败,避免在未知脏状态上重复执行。
- 通过执行器抽象接入 Celery/RabbitMQ,使 API 和 Worker 可独立部署;业务应用服务不得直接
依赖 Celery API,确保执行器仍可替换;
- Worker 重启、消息重复投递和确认超时必须保持持久化 Job 状态一致,不能产生双重有效
Attempt 或重复派生数据。
该 Task 只完成公共知识库入库所需的 Job/Retry 与队列基线,不扩展到完整审计系统、
Compilation、OCR、跨区域高可用或通用企业级 Job 平台。
### DEMO-016:批量异步上传与状态展示(已完成)
建立在 DEMO-015 之上,不与 Job 基础合并:
- 新增批量上传 API,一次接收多个 Demo 支持类型文件;
- 每个文件独立生成 Document 和 Ingestion Job,单个文件失败不回滚已成功接收的其他文件;
- 返回批次 ID、逐文件 Document ID、Job ID 和接收状态,不等待后台处理完成;
- 提供批次/Job 查询能力,并在 Admin Console 展示逐文件阶段、重试次数和最终结果;
- 文件数量、单文件大小和请求总大小均受可配置限制;
- 不在本 Task 引入目录上传、压缩包解包、断点续传或跨批次去重。
该 Task 对应 `EKP-021` 的批量异步扩展及 `EKP-080`/`EKP-081` 的最小状态 UI 切片,不能据此把完整 V1 Task 标记为完成。
## 3. 实施顺序与依赖
```text
DEMO-009 + ARCH-006 → DEMO-012(已完成)
PLAN-002(已完成)
└──→ DEMO-003 → DEMO-004 → DEMO-005 → DEMO-006
└──→ DEMO-015 Async Job + Celery/RabbitMQ + Retry Cleanup
└──→ DEMO-016 Batch Async Upload + UI
└──→ DEPLOY-001 → DEPLOY-002 → DEPLOY-003
```
推荐严格按以下顺序执行并逐项闭环:
1. `PLAN-002`(已完成)
2. `DEMO-015`(已完成)
3. `DEMO-016`(已完成)
4. `DEPLOY-001`
5. `DEPLOY-002`
6. `DEPLOY-003`
MCP 与异步入库没有代码级前置依赖,因此提高 MCP 优先级不会阻塞后续批量上传。`DEMO-016` 必须等待 `DEMO-015`,否则会把同步流水线简单并发放大,缺少可靠的状态、隔离和恢复边界。
## 4. 验收边界
### DEMO-012
- MCP Inspector/标准客户端能发现并调用 `search_knowledge`;
- WorkBuddy 使用已映射的 user/service identity 完成一次端到端检索;
- REST 与 MCP 对同一查询复用同一业务逻辑并返回一致的来源字段;
- 响应与日志不泄露 OceanBase、BOS、模型名、Endpoint 或凭据。
### DEMO-015
- 上传接口在源文件和 Job 持久化后返回 `202`,后台完成现有全流水线;
- 注入一次可重试故障后可观察到自动重试,并最终成功;
- 第二次尝试开始前,旧解析产物、Chunk、索引状态和向量均已清理,原始源文件仍存在;
- 永久错误不重试,达到次数上限后稳定进入 `failed`;
- 并发或重复投递不会产生双重有效执行和重复派生数据。
- RabbitMQ 消息确认超时或 Worker 重启后,Job 可恢复到终态且不会重复生成派生数据。
### DEMO-016
- 一次提交多个文件后立即得到逐文件 Document/Job 标识;
- 批次内成功、处理中、重试中和失败状态互不影响且可查询;
- Admin Console 无需刷新即可轮询展示逐文件进度和最终状态;
- 批次边界、大小限制、部分失败和重试清理均有集成测试。
## 5. 非目标与 ADR 影响
- PLAN-002 已把 Celery/RabbitMQ 纳入 `DEMO-015`,并以独立 `DEPLOY-*` Task 纳入最小
Kubernetes、多副本和滚动升级能力;跨区域容灾、自动云资源编排和完整 Observability
仍为非目标。
- 不改变 OceanBase/BOS canonical、向量索引可重建、原始文档不可变、Provider 抽象和 MCP 隐藏基础设施等 Accepted ADR。
- 公共知识库只是阶段性单一可见域;ADR-007 保持 Accepted,用户/ACL 实现后置且不得用全局
Top-K 后过滤替代。
- 本次任务拆分和交付边界不需要新增或重开 Accepted ADR。

View File

@ -1,15 +1,15 @@
# 项目进度
当前阶段:Knowledge Retrieval Demo v0.1
当前阶段:生产容器镜像闭环完成,准备实施 Kubernetes/Helm 一键部署
已完成:DEMO-001、DEMO-002
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001
进行中:无
阻塞:无
下一任务:DEMO-003
下一任务:`DEPLOY-002`,完成并合并后执行 `DEPLOY-003`。
完整 V1 待办将在 Demo Scope 完成后继续执行。当前任务顺序以
`docs/demo/DEMO_SCOPE.md` 为准。
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
## 任务记录
@ -29,3 +29,398 @@
不安全路径并清理写入中断产生的半文件。
- 新增模型、数据库、存储、迁移及 Schema Drift 测试;无需连接生产数据库即可
验证 PostgreSQL 方言的迁移 SQL。
### DEMO-003
- 新增 PDF、DOCX、TXT、MD 单文件上传应用服务,完成文件名、扩展名、媒体类型、
空文件和流式大小限制校验。
- 上传使用 UUID 隔离的不可覆盖 storage key,并将原始文件与 `uploaded` 状态的
Document 元数据关联保存。
- 数据库写入失败时自动回滚并删除已保存文件;新增四类文件、拒绝规则、大小限制
和失败补偿集成测试。
- Parser 触发与公开上传 REST API 分别留在 DEMO-004 和 DEMO-009,不在本任务
提前实现。
### DEMO-004
- 新增统一的 `ParsedDocument` / `ParsedBlock` 领域模型和 Parser Provider 接口,
使用 PyMuPDF、python-docx 及轻量文本解析器支持 PDF、DOCX、UTF-8 TXT、Markdown。
- PDF 保留一基页码,DOCX/Markdown 保留基础标题层级;基础 Cleaner 统一处理多余
空白、空行、异常换行和空块。
- 新增上传后同步触发解析的应用层编排,将版本化解析 JSON 作为可重建派生文件保存,
并通过数据库迁移增加 `parsed_storage_key` 追踪结果;原始文件保持不可变。
- 解析完成后文档保持 `processing`,等待 DEMO-005/006 完成后续流水线;解析失败记录
`failed` 和有界错误信息,派生结果写库失败时执行文件补偿。
- 新增四类格式、页码/章节、清洗、解析持久化、禁止覆盖和失败状态测试。公开 REST API
仍按计划留在 DEMO-009。
### DEMO-005
- 新增可注入 `TokenCounter` 的 `structure_v1` Chunk 策略,默认使用 450 tokens 上限和
75 tokens overlap;标题或章节变化形成硬边界,同章节段落优先组合,超长内容按 token
窗口切分。
- 默认轻量计数器按 CJK 字符、标点和保守 ASCII 子词估算 token,避免引入模型运行时;
精确模型 tokenizer 可通过既有接口注入,留待 DEMO-006 与 bge-large-zh 联调。
- 新增 canonical Chunk 元数据与只向前迁移,原子保存确定性 `chunk_id`、文档/文件、
一基页码范围、章节路径、顺序、token 数和策略版本;禁止静默覆盖同策略结果。
- 新增上传 → 解析/清洗 → Chunk 的同步流水线编排。Chunk 完成后仍保持 `processing`,
等待 DEMO-006 Embedding/Mochow;失败时不产生半套 Chunk,并保留原始文件。
- 新增完全合成的 Markdown 测试文档和 Golden Dataset Chunk 回归,覆盖章节追踪、
token 上限、overlap、页码范围、迁移、持久化、重复执行和失败状态。
### DEMO-006
- 新增与厂商无关的 `EmbeddingProvider` / `VectorStore` 契约及确定性 Fake,业务编排
不依赖 Mochow SDK 数据结构。
- 新增千帆 v2 `bge-large-zh` Adapter,使用 Bearer API Key,支持最多 16 条批量请求,
Chunk 同时限制 450 tokens/2000 字符,并严格校验响应顺序、1024 维度和有限数值;
凭证使用 `SecretStr`,
不进入错误消息或提交记录。
- 新增官方 `pymochow` Adapter,支持分批 Upsert 和按 `document_id + index_profile_id`
删除;Mochow 只保存向量与检索元数据,PostgreSQL 继续保存 canonical Chunk 和索引状态。
- 新增版本化 `IndexProfile` 与 `chunk_index_states` 向前迁移,防止模型/Chunk 策略原地
覆盖;所有 Chunk 成功入库后文档才进入 `ready`。
- 新增上传 → 解析/清洗 → Chunk → Embedding → Mochow Upsert 同步编排。Mochow 批次
或 canonical 状态写入失败时按文档/profile 清理可能的半批索引并记录 `failed`。
- 新增 Qianfan、Mochow 映射、配置脱敏、索引状态、失败补偿和 Golden Dataset 全链路
回归测试。真实凭证/网络集成未在默认测试中执行;专用 Mochow 表需按
`docs/development/MOCHOW_SETUP.md` 预创建。
### DEMO-007
- 新增 `DenseSearchRequest` / `DenseSearchCandidate` 与 `DenseRetrievalService`,完成
Query 清洗、同模型 Embedding、版本化 `index_profile_id` 过滤和默认 Top20 候选召回;
此阶段不加入 Reranker、最终 Source Result 或 REST API。
- `FakeVectorStore` 新增离线精确余弦检索,先按 profile 过滤再排序,并使用稳定的 UUID
tie-break,支持无网络的单元/集成/Evaluation 回归。
- `MochowVectorStore` 新增官方 `VectorTopkSearchRequest` 调用,固定检索 `embedding` 字段,
下推 profile 标量过滤,支持可配置 HNSW `ef`,严格校验响应字段、UUID、有限分数和
Top-N 数量,异常不泄露 SDK 请求或凭证。
- 新增 Golden Dataset Dense Retrieval 回归,完整覆盖上传 → 解析/清洗 → Chunk →
Embedding/入库 → Query Embedding → Dense Top20,确认两个合成 Query 均召回预期文件和
章节。
- 默认自动化测试不连接真实千帆/Mochow;真实环境仍需按
`docs/development/MOCHOW_SETUP.md` 注入凭证并验证网络、表 Schema 与索引状态。
### DEMO-008
- 新增与厂商无关的 `RerankerProvider` / `RerankResult` 契约及确定性 Fake,使用原始
candidate index 映射精排结果,业务服务不依赖千帆响应结构。
- 新增千帆 v2 Reranker Adapter(初始为 `qwen3-reranker-0.6b`,ARCH-004 已迁移至
`bce-reranker-base`),调用 `/v2/rerank`,严格限制 Query、
documents 和 Top-N,校验响应数量、唯一索引、回显文本、有限分数与降序关系;错误不会
回显请求内容或凭证。
- 新增 `SearchService`,完成 Dense Top20 → Reranker → Top5。Reranker 选中候选后按
`chunk_id` 回查 PostgreSQL canonical Chunk,再生成包含 document、filename、page、
section、chunk、content、Dense score 和 rerank score 的最终 Source Result。
- Source Enrichment 对缺失 Chunk、document_id 不一致和 stale vector content 采用失败
关闭,防止 Mochow 副本与 canonical 来源不一致时返回错误证据。
- 新增 Provider、配置脱敏、排序映射、canonical 回查、一致性失败和 Golden Dataset
全链路回归。默认测试不调用真实千帆;真实联调边界记录在
`docs/development/RERANKER_SETUP.md`。公开 Search REST API 留在 DEMO-009。
### DEMO-009
- 新增 `/api/v1/documents` 上传/列表、`/api/v1/documents/{document_id}` 详情/直接删除和
`/api/v1/search`,通过 OpenAPI 固化 multipart 上传、Document 状态和最小 Source Result
契约;API 响应不暴露 storage key、Index Profile 或厂商实现。
- 上传接口同步复用 DEMO-003~006 的完整入库流水线,并在完成后回读 PostgreSQL canonical
Document,确保返回最终 `ready` 状态和数据库生成时间戳。
- 新增文档查询/删除应用服务。删除覆盖当前及历史已记录 Index Profile 的向量行、
ChunkIndexState、Chunk、Document、解析 JSON 和原始文件;向量清理失败时保留 canonical
数据,来源清理失败返回明确错误并记录 Demo 一致性边界。
- 业务依赖延迟创建,因此健康检查和 OpenAPI 启动不要求云端凭证;Provider 配置不完整时
业务接口返回 `503`,上游异常使用收敛后的错误响应,避免泄露凭证或厂商正文。
- 新增完全离线的 REST 全链路测试,覆盖上传、列表、详情、Dense/Reranker 检索、来源字段、
直接删除、OpenAPI 路径、输入校验、404、Provider 延迟加载及删除失败保护。
### ARCH-001
- 接受 BOS、OceanBase Vector、千帆 `bge-large-zh` 和 `bce-reranker-base` 作为新的 V1
Provider 基线;PostgreSQL 继续保存 canonical 数据,OceanBase 只保存可重建索引。
- 新增 ADR-009~ADR-011 和独立迁移计划,将 ARCH-002~ARCH-006 的范围、依赖、验收顺序
与安全边界写入任务清单。
- 保留 Demo 的 Dense Top20 → Reranker Top5、现有 Parser/Chunk、API/MCP 业务契约;不随迁移
引入 Top50、RRF、Marker/Pandoc、LLM 派生索引、ACL 或 Hybrid/BM25 扩展。
- 本任务只调整架构和产品文档,不修改运行时代码;旧系统地址、账号和凭据均未写入仓库,
真实验收只允许使用轮换后的 Secret 与受控测试资源。
### ARCH-002
- 新增官方 `bce-python-sdk` 驱动的 `BosObjectStorage`,保持逻辑 key 与 Bucket/Prefix 分离,
覆盖 Put、流式 Open、HEAD Exists 和 Delete。
- 上传先使用可回卷临时流确定长度,再通过 `If-None-Match: *` 原子拒绝覆盖;404/412 和
其他 SDK 异常映射为不泄露 Endpoint、响应正文或凭据的稳定存储错误。
- 新增 `local|bos` 类型化配置和运行时工厂;BOS AK/SK 使用 `SecretStr`,Bucket、Endpoint、
Prefix 和超时均由环境注入。Local 仍是测试/开发及 ARCH-005 前的默认实现。
- 新增完全离线的 BOS Adapter、配置、Provider wiring 测试和中文配置说明;未使用内部配置
清单中的真实地址或凭据,真实 BOS 联调留在 ARCH-006。
### ARCH-003
- 新增官方 `pyobvector` 驱动的 `OceanBaseVectorStore`,覆盖批量 Upsert、按 Document/Profile
参数化删除和 Profile 条件下推的 Cosine ANN TopN;SDK 类型不进入业务服务。检索固定使用
OceanBase 4.6 的 `VECTOR_INDEX(... pre_filter)`,不依赖普通 `WHERE` 的自适应过滤,也不做
全局 TopK 后过滤。
- 将低值优先的 `cosine_distance` 转为 `score = 1 - distance`,并对向量维度、有限值、
非零范数、返回字段、UUID、Profile、距离范围和排序执行失败关闭校验。
- 新增 `VECTOR(1024)`、Cosine HNSW、Profile/Document 标量索引的预置 DDL;应用不自动执行
外部索引 DDL,PostgreSQL 继续保存 canonical 数据,OceanBase 表可重建。
- 新增 `mochow|oceanbase` 类型化配置、SecretStr 密码和运行时工厂;Mochow 保持 ARCH-005
前的默认实现。全部 Adapter、配置、Schema 和 wiring 测试均离线运行,真实数据库验收留在
ARCH-006。
### ARCH-004
- 将千帆 Reranker 运行基线从 `qwen3-reranker-0.6b` 切换为已接受的
`bce-reranker-base`;配置使用受限 Literal,业务 API、Provider 抽象和 Dense Top20 →
Reranker Top5 行为保持不变。
- 按千帆 `POST /v2/rerank` 官方约束复核并保留 Query 1600 字符、64 个候选、单文档
4096 字符、Top-N 及响应排序/索引/分数失败关闭校验;真实 API Key 不进入代码和测试。
- 更新 Adapter、环境示例、Provider wiring、配置与请求契约测试;Golden Dataset 继续使用
确定性 Fake 离线评估,但显式绑定 `bce-reranker-base` 当前模型标识。真实 BCE 质量与延迟
统一留在 ARCH-006 使用轮换凭据验收。
### ARCH-005
- 将正式运行默认值切换为 BOS、OceanBase Vector 和 `bce-reranker-base`,新 Profile ID 为
`demo-bge-large-zh-structure-v1-oceanbase-v1`;Local/Fake 仍用于显式本地和离线测试。
- 新增 PostgreSQL canonical `index_profiles` 迁移与 `shadow / active / retired` 生命周期,
通过局部唯一索引保证至多一个 Active Profile;API 启动时优先解析数据库 Active Profile,
无记录的新部署才使用配置基线。
- 新增 `ekp-index-rebuild build|verify|activate`:从 canonical Chunk 分文档重建,确定性 Upsert,
重试时跳过完整文档并清理目标 Profile 的部分写入;激活前核对全部 ChunkIndexState,并在
PostgreSQL 事务内锁定 Chunk 写入、退役旧 Profile、激活新 Profile。
- 移除 Mochow 运行配置、公开 Adapter、测试、配置文档及 `pymochow` 依赖;不自动删除旧
外部索引,保留受控回滚和运维退役边界。
- 新增重建/重试/完整性失败/单 Active/激活/迁移/CLI 脱敏测试及中文操作文档。真实资源
构建、BOS/OceanBase/Qianfan 连通性和 30~50 条脱敏查询验收仍属于 ARCH-006。
### ARCH-006(已完成)
- Golden Dataset 已扩展为 32 条完全合成 Query,覆盖退款、技术支持、住宿标准和报销材料;
Dense、Reranker、canonical Source Result 的离线回归全部通过。
- 新增 `ekp-live-acceptance` / `make live-acceptance`,实际覆盖上传、解析、Chunk、Embedding、
OceanBase Dense Top20、BCE Reranker Top5、canonical 来源核对及本次运行数据清理。
- live 命令强制要求 `APP_ENV=test`、专用 `live-acceptance/` BOS Prefix、凭据授权确认、资源
授权确认和显式运行开关;当前环境下已验证会在创建 Provider 前失败关闭且不泄露细节。
- 根据最新配置清单和 test 环境只读探测,确认没有独立 PostgreSQL;目标为 OceanBase
4.3.5.4 MySQL 模式,同一数据库承载 canonical 关系数据和逻辑隔离的向量表。ADR-012 已
取代先前 PostgreSQL 部署假设,业务事实来源与可重建索引的逻辑边界保持不变。
- 已在目标 test 集群只读验证数据库连通、版本与兼容模式,并通过 `EXPLAIN` 确认
`VECTOR_INDEX(... pre_filter)` 生成 `VECTOR INDEX ADAPTIVE SCAN (PRE-FILTER)`;未读取
业务正文或输出任何连接信息/凭据。
- canonical SQLAlchemy 默认已切换为 `mysql+pymysql`,数据库 URL 使用 `SecretStr`,新增
有界连接池/超时配置;新增 forward-only migration,以生成列唯一索引保证 OceanBase 与
SQLite 下至多一个 Active Profile;PostgreSQL 驱动已移除。
- 配置清单中的 test 凭据仅按用户明确授权临时注入本次验收,不保存为 `.env`、不提交、
不连接 fat/prod;完成后仍记录为待轮换风险。
- 已在 test OceanBase 执行全部 forward-only migration,并对本项目表完成列、类型、默认值、
主键的 scoped schema drift 核对;OceanBase 扩展语法导致标准 MySQL 反射器不能可靠比较的
二级索引/唯一约束/外键,由 SQLite 全量 drift、真实唯一约束事务和 live 级联删除补充验证。创建
`knowledge_chunks_v1` 12 列专用向量表和 Cosine HNSW,确认 PRE-FILTER 执行计划。迁移前
已确认所有目标表均不存在,没有覆盖原有 31 张业务表;事务内双 Active 写入实测被唯一
索引拒绝并完整回滚。
- 已完成 OceanBase canonical → BOS → `bge-large-zh` → OceanBase Vector →
`bce-reranker-base` → canonical Source Result 正式验收:32 条查询的 Recall@1/3/5、MRR、
NDCG@5 均为 `1.0`,P50 `3787.61 ms`、P95 `4759.21 ms`,通过 P95 `10000 ms` 门槛。
- 正式命令退出码为 `0`;两份合成文档、canonical Chunk/State、向量行与 BOS 对象均走随机
前缀清理,数据库复核 `live-` Document/Chunk/Vector 残留均为 `0`。首次 live 运行曾出现
一次瞬时失败,分阶段诊断及随后完整 32 查询均成功,仍需在后续稳定性任务关注 Provider
重试、限流和告警。
### DEMO-010(已完成)
- 新增同源轻量 Admin Console,提供 `/admin/upload`、`/admin/documents` 和
`/admin/search` 三个直接可访问页面;FastAPI 只负责静态文件路由,页面业务操作全部复用
`/api/v1`,没有复制上传、文档管理或检索逻辑。
- 上传页支持 PDF、DOCX、TXT、Markdown 文件选择、处理状态和结果摘要;文档页展示文件名、
类型、上传时间、状态,支持详情与确认删除;Search Lab 显式展示 Rank、Filename、Page、
Section、Chunk Content、Dense Score 和 Rerank Score。
- 前端使用安全 DOM 文本节点渲染 API 数据,不注入 HTML,不展示 storage key、Index Profile、
Provider Endpoint 或凭据;只实现 Demo 规定的三个页面,没有引入 ACL/RBAC、任务、审计、
Compiled Knowledge、MCP、Hybrid/BM25 或额外前端构建工具链。
- 新增静态路由、资产、范围和 API First 契约测试;浏览器使用本地确定性 Provider 完成
`README.md` 上传 → Ready → 列表/详情 → Top5 检索,并验证桌面和 390px 移动布局无横向
溢出、无前端警告或错误。删除的 UI 确认和服务端清理语义由现有 REST 全链路测试覆盖。
- 完整验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、Git diff、依赖锁、Alembic Head/
离线升级均通过;Unit + Integration `150 passed`,RAG Evaluation `5 passed`。本任务没有
修改检索算法或 Provider,不重复调用真实资源验收;ARCH-006 的 32 条真实查询结果继续
作为底层链路基线。
- 已知非阻塞风险保持不变:配置清单中的历史 test 凭据仍应轮换;真实 Provider 瞬时失败的
重试、限流与告警留待后续明确 Task。Demo Console 本身没有新增凭据或生产 Endpoint。
### DEMO-014(已完成)
- 将 DEMO-010 通用绿色界面升级为地瓜机器人品牌化 Admin Console,使用唯一品牌强调色
`#FF3C00`、黑白高对比、无衬线中文排版、低圆角工业组件和与三步操作对应的计算信号轨;
没有增加页面、接口或业务能力。
- 品牌标记由内联 SVG 与 CSS 原生绘制,不下载、复制或热链官网资产。页面没有新增外部
Runtime、字体请求或第三方脚本,继续保持原生 HTML/CSS/JS 和同源 `/api/v1` 调用。
- GitHub 调研确认官方 `openai/skills` 当前没有通用前端设计 Skill;选择并安装了 Apache-2.0
的 `anthropics/skills/skills/frontend-design`。该 Skill 只影响视觉策划和自检方法,不进入
项目运行依赖或仓库产物。
- 浏览器使用本地确定性 Provider 再次完成 README 上传 → Ready → 文档列表 → Top5 检索;
1440px 桌面端和移动端均无根页面横向溢出、无前端警告或错误。复验中发现隐藏表头将移动
页面撑宽至 503px,已改为 `aria-label`,最终根宽度与 463px 视口一致,表格只在容器内滚动。
- 完整验证结果:Ruff Format/Lint、Mypy、JavaScript 语法与 Git diff 通过;Unit、Integration
和 RAG Evaluation 合计 `155 passed`;Alembic Head 与离线升级通过。本任务不修改 API、
检索算法、Provider 或数据库,不重复调用真实资源验收。
### PLAN-001(已完成)
- 审计确认当前 `/api/v1/documents` 仅接受一个文件,并在请求线程同步执行解析、Chunk、
Embedding 和索引;API 与 Admin Console 均没有批量提交、Job 查询或后台状态轮询。
- 当前上传、解析和向量索引具备阶段内失败补偿,影子索引重建具备目标 Profile 的重试安全
清理,但正常上传流水线没有持久化 Job/Attempt、自动调度退避或统一的重试前派生数据清理,
因而不能视为已实现自动重试。
- 保留既有 `DEMO-012` 并提升为最高优先级;新增 `DEMO-015` 承担 Async Job、自动重试和
重试前清理,新增 `DEMO-016` 承担批量异步上传和逐文件状态 UI。严格执行顺序为
`DEMO-012 → DEMO-015 → DEMO-016`。
- 已更新 PRD、Architecture、API/MCP Contract、Execution Plan、Task List、Demo Scope 与 DoD;
本任务只形成规划和待实现契约,没有修改业务代码、数据库 Migration 或 Accepted ADR。
- 验证结果:Ruff Format/Lint、Mypy、Git diff 与 Alembic Head/离线升级通过;Unit + Integration
`150 passed`,RAG Evaluation `5 passed`。首次受限网络运行因无法解析 PyPI 失败,获准联网
获取已锁定的构建依赖后全套复验通过。
- 下一任务:`DEMO-012`。详细依赖、范围和验收标准见
`docs/plans/ASYNC_INGESTION_MCP_PLAN.md`。
### DEMO-012(已完成)
- 新增独立 `ekp-mcp` Streamable HTTP Gateway 和 `make run-mcp` 入口,使用官方 MCP Python
SDK v2;默认只监听 `127.0.0.1:8766/mcp`,且只注册 `search_knowledge` 一个业务 Tool。
- Gateway 直接复用现有 `SearchService`,没有复制 Dense Retrieval、Reranker 或 Source
Enrichment;结构化输出与 REST 保持相同来源字段,不返回 Index Profile 或 Provider 信息。
- 新增运行时专用 Bearer Token 验证,将凭据映射为显式 client/service identity 和最小
`knowledge:search` scope;Token 至少 32 字符并通过 `SecretStr` 管理,不允许写入仓库。
当前身份层是 Demo 边界,不宣称完成 V1 OAuth、Document ACL 或用户委托授权。
- 启用 Host/Origin allowlist 和请求体限制;缺失/错误认证、scope、参数和检索异常均按安全
边界收敛,响应和日志不回显 Token、内部 Endpoint、数据库、模型或索引实现。
- 新增官方 MCP Client 的协议级测试,覆盖 initialize、tools/list、tools/call、唯一 Tool、
identity 传递、REST 字段一致性、无结果、参数拒绝、认证失败、Origin 防护和错误脱敏;
WorkBuddy 的 HTTP + Bearer + 环境变量配置记录于 `docs/development/MCP_SETUP.md`。
- 验证结果:Ruff Format/Lint、Mypy、Git diff、Alembic Head/离线升级全部通过;Unit +
Integration `165 passed`,RAG Evaluation `5 passed`。本任务未改变检索算法或数据库结构,
不重复消耗真实 Embedding/OceanBase/Reranker 资源。
- 2026-09-02 补充完成 WorkBuddy 现场验收:本机 WorkBuddy Desktop `5.3.14` 自带的
CodeBuddy CLI `2.115.0` 通过仓库交付的 `deploy/workbuddy-mcp.example.json` 建立
Streamable HTTP + Bearer 连接,健康检查为 `Connected`,仅发现
`mcp__enterprise-knowledge__search_knowledge`,并以 `query` + `top_k=1` 完成真实 Tool
调用、读取结构化来源字段。验收使用真实 Gateway、认证、身份传递、Schema 和 Tool
注册代码,检索应用返回确定性数据,以隔离 WorkBuddy 兼容性与外部 Provider 波动。
- 非交互客户端调用使用仅对当前会话放行 `DeferExecuteTool` 的权限设置,没有持久化放宽权限;
临时客户端配置已删除、Gateway 已停止,仓库和用户配置中均未留下测试 Token。至此
`DEMO-012` 约定的 WorkBuddy 客户端验收项闭环;公网 HTTPS 部署验证属于部署环境验收,
不阻塞本地 Demo Task 完成。
- 下一任务:`DEMO-015`。
### PLAN-002(已完成)
- 将当前交付目标明确为企业受控网络内的单一公共知识库:本阶段所有知识共享一个可见域,
暂缓用户、角色、知识库空间和 Document ACL,但不废弃 ADR-007,也不把未实现项标记完成。
- 保持 Accepted ADR 的模块化单体边界,确定 API/Admin、MCP Gateway 和 Async Worker 独立
部署;OceanBase/BOS/AI Provider 使用外部服务,当前云端模型链路不启用 GPU Worker。
- 将 Celery/RabbitMQ 正式并入 `DEMO-015`,并新增 `DEPLOY-001~003`,严格顺序为
`DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003`。
- 定义 `make deploy VALUES=...` 的一键部署边界:操作者预先提供集群访问与 Secret,仓库负责
生产镜像、串行 Migration、Helm 安装/升级、Ingress/TLS、探针、资源与 Pod 安全配置;不得
将凭据写入 Git、镜像、命令行或日志。
- 明确现有 `docker-compose.dev.yml` 的 PostgreSQL/MinIO 假设已被 ADR-012 取代,不作为集群
交付模板;新增 `docs/plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md` 作为任务范围和验收基线。
- 本任务仅修改规划、产品、架构、API/MCP 契约、Demo Scope 和 DoD 文档,没有修改业务代码、
数据库 Migration 或 Accepted ADR 状态。
- 验证结果:Markdown 相对链接、敏感值模式和 Git diff 检查通过;Ruff Format/Lint、Mypy、
Alembic Head/离线升级通过;Unit + Integration `165 passed`,RAG Evaluation `5 passed`。
- 下一任务:`DEMO-015`。
### DEMO-015(已完成)
- 新增 OceanBase canonical `ingestion_jobs` / `ingestion_attempts` 与 forward-only migration;
Document 唯一 Job、Attempt 编号上限和单一运行中 Attempt 均由数据库约束保护。
- `POST /api/v1/documents` 改为持久化不可变 Source、Document、Job 后返回 `202`,新增
`GET /api/v1/jobs/{job_id}`;活动 Job 的 Document 删除返回 `409`,避免与 Worker 竞态。
- 新增队列无关的 Dispatcher 与 Job Runner,首个适配器使用 Celery/RabbitMQ;消息只包含 Job
ID,启用 late ack、worker-lost reject、单任务 prefetch 与周期性持久化 Job 恢复扫描。
- 瞬时错误按配置执行指数退避和最多三次总 Attempt;Worker 租约有效时重复投递不创建新
Attempt,租约过期后关闭旧 Attempt 并恢复。错误记录为固定安全摘要,不保存 Provider 正文。
- 第二次及后续 Attempt 前依次清理向量、ChunkIndexState、Chunk 与 parsed JSON,再从不可变
源文件重建;清理失败直接终止且不创建新 Attempt。Job 创建失败会补偿 Document 与源对象。
- Admin Console 仅将上传提示调整为“已接收并进入队列”;批量选择、Job 轮询和逐文件状态 UI
没有提前实现,继续属于 `DEMO-016`。
- 验证结果:Ruff Format/Lint、Mypy、JavaScript 语法、依赖锁、Alembic Head/离线 MySQL SQL
与 SQLite schema drift 均通过;Unit + Integration `179 passed`,RAG Evaluation `5 passed`;
Worker 入口确认注册 `ekp.ingestion.run` 与 `ekp.ingestion.recover`。
- 环境限制:当前工作站没有 Docker/RabbitMQ,也没有持久化注入目标 OceanBase URL,因此未做
真实 Broker 断连/确认超时和目标库 migration;这些真实环境项必须在 DEPLOY-003 集群验收
重跑。本任务以 Celery 配置测试、SQLite 持久化/唯一约束和故障注入集成测试闭环代码语义。
- 下一任务:`DEMO-016`。
### DEMO-016(已完成)
- 新增 canonical `ingestion_batches` / `ingestion_batch_items` 及 forward-only migration,保存
输入顺序、逐文件接收结果与 Document/Job 引用;异步阶段和最终结果仍以 Job 为事实源,
批次不是跨文件事务。
- 新增 `POST /api/v1/ingestion-batches` 与 `GET /api/v1/ingestion-batches/{batch_id}`;批量
请求字段为 `files`,成功返回 `202`、聚合状态和逐文件接收/Job 状态。类型、大小或内容错误
只拒绝当前文件,错误映射为固定安全摘要,其他文件继续接收。
- 新增 `BATCH_MAX_FILES` 与 `BATCH_MAX_TOTAL_BYTES`,复用 `MAX_UPLOAD_BYTES` 约束单文件;
上传流共享总字节预算,不把整批文件读入内存。数量超限在创建批次前返回 `413`。
- Admin Console 支持多选提交,显示当前批次的接收数量、完成数量、逐文件阶段、Attempt 与安全
错误,并每 1.5 秒自动轮询;进入 `completed`、`partial_failure` 或 `failed` 后停止轮询。
DOM 数据继续只通过文本节点渲染。
- 集成测试覆盖顺序稳定、部分拒绝不回滚、数量/单文件/批次总大小、全成功/部分失败聚合、
未知批次、OpenAPI 和前端契约;迁移 drift 增加两张新表。Ruff Format/Lint、Mypy、
JavaScript 语法、依赖锁、Alembic Head/离线升级均通过;Unit + Integration `183 passed`,
RAG Evaluation `5 passed`。
- 浏览器验收确认桌面布局和 390px 窄屏布局正常、无根页面横向溢出、无控制台警告或错误;
本地页面不注入云端凭据,批次结果行为由确定性 API 集成测试验证。
- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续
复用 DEMO-015,Accepted ADR 未变更。
- 下一任务:`DEPLOY-001`。
### DEPLOY-001(已完成)
- 新增共享生产 Dockerfile 的 `api`、`mcp`、`worker` 三个 target,使用精确 Python/uv 版本、
frozen production dependencies、固定非 root UID/GID、exec-form 启动命令和逐 target 健康检查。
- 新增 amd64/arm64 Buildx Bake 计划、镜像构建/渲染/推送 Make 入口和 `.dockerignore`;构建上下文
排除凭据、本地状态、缓存、测试与开发产物,镜像不包含环境 Endpoint 或 Secret。
- Admin 静态目录新增类型化运行时覆盖,使非 editable 安装后的 API 镜像仍能提供同源页面;
源码运行继续使用既有默认目录。
- 新增不依赖 Docker daemon 的镜像契约测试与中文运行文档;基础 Python/uv 镜像同时固定精确
tag 与 manifest digest,Registry 地址和本项目发布 digest 继续由部署环境提供。
- 使用 Docker Buildx `0.36.1` + Colima 分别实际构建并加载全部三个 `linux/arm64` 和三个
`linux/amd64` target;最终 digest 固定版本再次构建命中相同业务 layer。没有执行 Registry
push,也没有在构建参数、history 或镜像文件系统中发现凭据。
- API 镜像以 `10001:10001` 启动并进入 healthy,`/healthz`、`/admin/upload`、OpenAPI 及
DEMO-016 批量路径 smoke test 通过;MCP/Worker 镜像以同一非 root 用户完成安装入口导入,
默认 CMD 与 HEALTHCHECK 元数据核对通过。真实 Provider/Broker 启动仍属于 DEPLOY-003。
- Ruff Format/Lint、Mypy、Docker Bake 渲染、JavaScript/依赖锁/Alembic 与完整测试门禁通过;
Unit + Integration `187 passed`,RAG Evaluation `5 passed`。下一任务:`DEPLOY-002`。
### DEPLOY-002(已完成)
- 新增 `deploy/helm/knowledge-platform` Helm Chart,部署 API/Admin、MCP Gateway、Async Worker、
单实例 Recovery Scheduler、串行 Migration Job、Service、Ingress/TLS 和可选 Demo RabbitMQ;
API/MCP 默认各两个副本,Worker 可独立扩缩,Beat 使用 `Recreate` 避免重复调度。
- 新增严格 `values.schema.json` 与仓库外环境 Values 示例,拒绝未知顶层字段、`latest` tag、
非法 digest、API/MCP 单副本和错误类型;跨字段模板校验继续约束批量大小、重试、Chunk 和
Top-K 关系。应用和可选 RabbitMQ 镜像均支持 digest,Demo RabbitMQ 已固定 manifest digest。
- 敏感配置只通过预建 `existingSecret` 注入;ConfigMap 仅渲染显式白名单内的非敏感字段,
没有任意 `extraEnv` 或 Secret 对象。`scripts/deploy.sh` 在部署前校验 Values、Namespace、运行时
Secret/必需键、可选 RabbitMQ Secret/键和启用 Ingress 时的 TLS Secret 引用,不读取 Secret
值;Helm 3/4 分别使用等价的原子失败回滚参数。
- Migration 使用 `pre-install,pre-upgrade` Hook、固定名称和 `before-hook-creation`,安装/升级在
Migration 成功前不更新业务 Pod。应用容器固定非 root UID/GID、只读根文件系统、丢弃全部
capabilities、禁用 ServiceAccount Token,并为临时目录、RabbitMQ 日志和 Beat 状态提供
`emptyDir`;API/MCP/Worker/Scheduler 均配置启动和存活/就绪探针及资源 requests/limits。
- 本机使用 K3s `v1.35.0+k3s1`、Helm `4.2.4` 和 DEPLOY-001 ARM64 镜像完成真实安装;Migration
成功,API/MCP 为 `2/2`,Worker、Scheduler 与可选 RabbitMQ 全部 Ready。首次安装暴露的
RabbitMQ 只读日志目录和 Beat 调度文件权限问题已修复,随后相同 Values 连续升级至 Revision
3~5 均成功;API `/healthz` 和 Admin 上传页 smoke test 通过。
- 默认、环境示例和 RabbitMQ 三种 Helm lint 均通过;含 Ingress/TLS/RabbitMQ 的 14 个最终
资源通过 kubeconform strict 校验。仓库、Helm manifest、Pod 日志三路 Secret 值扫描均为 `0`;
Ruff Format/Lint、Mypy、Alembic、Bash 语法与完整门禁通过,Unit + Integration `195 passed`,
RAG Evaluation `5 passed`。
- 目标企业集群的稳定 HTTPS、滚动重启、Worker 故障恢复、真实外部依赖监测与远程 WorkBuddy
仍属于 `DEPLOY-003`;本任务没有把这些项提前标记完成,也没有改变 Accepted ADR 或公共知识库
范围。下一任务:`DEPLOY-003`。

View File

@ -0,0 +1,123 @@
# 公共知识库集群交付计划
**Task:** PLAN-002
**Status:** Accepted planning baseline
**Date:** 2026-09-02
## 1. 阶段目标与边界
当前阶段只交付一个面向企业内部受控网络的公共知识库,先保证服务持续可用、可重复部署,
并让使用者打开 Admin Console 或连接 WorkBuddy 后即可使用,无需感知后端进程启停。
“公共”表示本阶段所有已接收知识处于同一个可见域,不代表面向公网匿名开放,也不代表
ADR-007 被废弃。用户、用户组、角色、知识库多空间、Document ACL、SSO、OAuth 用户委托等
完整 V1 能力继续后置;一旦需要接收非公共知识,必须先实现 Permission Resolver 和检索前/
检索中 ACL 下推,禁止先做全局 Top-K 再过滤。
Admin 写操作只部署在企业受控网络或统一接入网关后。MCP 继续使用运行时注入、只具备
`knowledge:search` scope 的服务凭据;静态 Demo credential 不得演变为共享超级管理员 Token。
## 2. 目标部署拓扑
```text
Ingress / TLS
│
┌──────────────┴──────────────┐
│ │
Admin Console + Knowledge API MCP Gateway
replicas replicas
│ │
└──────── Application Services ────────┐
│
┌─────────────────────────────┼─────────────┐
│ │ │
OceanBase MySQL BOS RabbitMQ
canonical + vector │
▼
Async Worker(s)
```
沿用 ADR-002:业务核心仍为模块化单体,不拆分额外微服务;API、MCP Gateway 和异步 Worker
分别作为可独立扩缩容的进程/容器交付。当前 Parser 与云端 Embedding/Reranker 不需要本地 GPU,
因此 GPU Worker 不进入本轮最小部署,未来引入本地 OCR/模型时再按既有架构角色启用。
OceanBase、BOS 和模型服务使用集群外已有服务。RabbitMQ 默认使用外部地址;Helm Chart 可以
提供显式开关安装开发/演示依赖,但生产值不得把队列数据生命周期绑定到应用 Pod。
## 3. “一键部署”的定义
在操作者已具备 Kubernetes 访问权限、目标 Namespace 和 Secret 的前提下,仓库提供单一入口:
```text
make deploy VALUES=<environment-values.yaml>
```
该入口负责:
- 校验非敏感配置和 `existingSecret` 引用,不把凭据写入 Git、镜像、命令行或日志;
- 运行一次性、可追踪且串行的数据库 Migration Job;
- 安装或升级 API/Admin、MCP Gateway、Async Worker、Service 和可选 RabbitMQ 依赖;
- 配置 Ingress/TLS、资源 requests/limits、非 root、只读根文件系统和最小 ServiceAccount;
- 配置 readiness/liveness probe、副本数和滚动升级策略;
- 支持重复执行和幂等升级,并提供对应卸载/回滚说明。
本定义不承诺自动创建 OceanBase、BOS、域名证书或云厂商账号,也不允许在仓库内保存这些
资源的敏感凭据。
## 4. Task 顺序与依赖
```text
PLAN-002
└──→ DEMO-015 Celery/RabbitMQ Async Job + Retry Cleanup
└──→ DEMO-016 Batch Async Upload + Status UI
└──→ DEPLOY-001 Production Images
└──→ DEPLOY-002 Kubernetes/Helm One-click Deploy
└──→ DEPLOY-003 Availability + Remote Acceptance
```
### DEMO-015
实现持久化 Job/Attempt、重试前清理、幂等消费和有界自动重试,并把执行器正式接入
Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务直接依赖 Celery API。
### DEMO-016
**已完成。** 基于 DEMO-015 提供逐文件隔离的批量异步提交、批次/Job 查询和 Admin Console
状态展示;下一交付门槛为 DEPLOY-001 的生产镜像。
### DEPLOY-001
**已完成。** 交付 API/Admin、MCP Gateway、Async Worker 的生产容器镜像与启动约定;镜像使用固定依赖、
非 root 用户、健康检查和多架构可构建基线,不内置凭据。
### DEPLOY-002
**已完成。** 交付 Kubernetes/Helm Chart、严格环境 Values Schema、`existingSecret`、串行
Migration Hook、Ingress/TLS、Pod 安全、资源限制、探针和 `make deploy` 入口;Worker 与单实例
恢复调度器已分离。现有 `docker-compose.dev.yml` 仍仅作为旧的本地开发文件,因其
PostgreSQL/MinIO 假设与 ADR-012 不一致,不得作为集群交付模板。下一交付门槛为 DEPLOY-003
的目标集群可用性与远程验收。
### DEPLOY-003
在目标集群完成安装、幂等升级、滚动重启、Worker 故障恢复和远程 WorkBuddy 验收;形成运行
手册、备份/恢复责任边界和最小日志/指标检查。灾备演练与完整 Observability 平台仍属后置项。
## 5. 最小验收标准
- 新 Namespace 可由单一命令完成安装,重复执行不会破坏已有数据;
- Migration 在应用升级前串行成功,失败时新版本不继续接管流量;
- Admin/API 与 MCP 使用稳定 HTTPS 地址,重启或滚动升级不要求最终用户重新注入配置;
- API/MCP 支持至少两个副本,readiness 未通过的 Pod 不接收流量;
- Worker 重启后持久化 Job 可继续恢复,重复消息不产生双重有效执行或重复派生数据;
- 远程 WorkBuddy 仅发现批准的 `search_knowledge` Tool,并完成一次真实检索;
- 清单、镜像、日志和验收产物不包含凭据,敏感值均来自 Secret;
- 关键请求、Job 和外部依赖失败具备可定位的结构化日志和最小指标。
## 6. 明确后置能力
- 用户、用户组、角色、多租户、知识库空间隔离和 Document ACL;
- LDAP/SSO、OAuth 用户委托、公共互联网匿名访问;
- GPU Worker、本地模型、OCR/MinerU;
- 跨区域容灾、自动云资源编排和完整 Observability 平台;
- 未被既有 Demo/V1 明确批准的业务能力。

View File

@ -1,5 +1,52 @@
# V1 Tasks
## Architecture provider migration
- [x] ARCH-001 Architecture decision, PRD, Scope and execution baseline
- [x] ARCH-002 BOS ObjectStorage adapter and runtime configuration
- [x] ARCH-003 OceanBase VectorStore adapter and index schema
- [x] ARCH-004 Qianfan `bce-reranker-base` alignment
- [x] ARCH-005 Runtime cutover, shadow rebuild workflow and legacy configuration cleanup
- [x] ARCH-006 OceanBase canonical cutover, controlled live integration and RAG acceptance
Dependencies:
```text
ARCH-001
→ ARCH-002
→ ARCH-003
→ ARCH-004
→ ARCH-005
→ ARCH-006
```
Each architecture task uses an independent branch and is merged into `main` only after its own
validation. Provider credentials and internal endpoints are runtime inputs and never task artifacts.
## Post-Demo priority plan
- [x] PLAN-001 MCP 与异步批量入库现状审计、任务拆分和契约基线
- [x] DEMO-012 MCP `search_knowledge` Gateway + identity + WorkBuddy verification
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
- [x] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
- [x] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance
Dependencies and acceptance boundaries are defined in
[`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) and
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md). These Demo/Deploy tasks are
vertical slices of the V1 backlog below; completing a slice does not automatically complete its
corresponding `EKP-*` task. The current public-knowledge phase defers, but does not complete or
supersede, the unchecked user/ACL tasks.
```text
DEMO-009 + ARCH-006 → DEMO-012
PLAN-002
└→ DEMO-003~006 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
```
## Phase 0
- [ ] EKP-001 Repository bootstrap
- [ ] EKP-002 Backend Python toolchain
@ -11,14 +58,14 @@
## Canonical domain / ingestion
- [ ] EKP-010 Core domain package
- [ ] EKP-011 PostgreSQL schema and migrations
- [ ] EKP-011 OceanBase MySQL schema and migrations
- [ ] EKP-012 User/group/role model
- [ ] EKP-013 Knowledge Base model
- [ ] EKP-014 Folder model
- [ ] EKP-015 Document + DocumentVersion model
- [ ] EKP-016 ACL model
- [ ] EKP-017 Job + audit models
- [ ] EKP-020 ObjectStorage abstraction + S3 adapter
- [ ] EKP-020 ObjectStorage abstraction + BOS adapter
- [ ] EKP-021 Document upload API
- [ ] EKP-022 Source hash/dedup/version activation
- [ ] EKP-023 Async job framework
@ -35,13 +82,13 @@
- [ ] EKP-042 Qwen embedding adapter
- [ ] EKP-043 IndexProfile model
- [ ] EKP-044 VectorStore + fake
- [ ] EKP-045 Mochow adapter
- [ ] EKP-045 OceanBase Vector adapter
- [ ] EKP-046 Index build + shadow index lifecycle
- [ ] EKP-050 Permission resolver + ACL retrieval filters
- [ ] EKP-051 Dense retrieval
- [ ] EKP-052 BM25/sparse retrieval
- [ ] EKP-053 Hybrid fusion
- [ ] EKP-054 RerankerProvider + fake + Qwen adapter
- [ ] EKP-054 RerankerProvider + fake + Qianfan BCE adapter
- [ ] EKP-055 Source enrichment + Search service/API
- [ ] EKP-056 Retrieval diagnostics / Search Lab backend

View File

@ -3,9 +3,32 @@
1. Repository / developer harness
2. Canonical domain and ACL
3. Document ingestion and parsing
4. Embedding / Index Profile / Mochow adapter
4. Embedding / Index Profile / OceanBase Vector adapter
5. ACL-aware hybrid retrieval + reranker
6. MCP gateway and WorkBuddy verification
7. Admin Console
8. Knowledge Compilation
9. Evaluation, security and release hardening
## 当前增量实施顺序(2026-09-02)
`DEMO-012` 已完成 MCP `search_knowledge` 与本地 WorkBuddy 验收。根据 PLAN-002,当前阶段按
“单一公共知识可见域、企业受控网络、持续可用和一键集群部署”推进,不提前实现用户/权限隔离:
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
4. `DEPLOY-001`(已完成):API/Admin、MCP Gateway、Async Worker 生产镜像;
5. `DEPLOY-002`(已完成):Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。
严格依赖为:
```text
PLAN-002 → DEMO-015 → DEMO-016 → DEPLOY-001 → DEPLOY-002 → DEPLOY-003
```
`DEMO-016` 不得在缺少持久化状态、幂等消费和恢复边界时直接并发调用同步流水线;集群交付也
不得继续使用 `docker-compose.dev.yml` 中已被 ADR-012 取代的 PostgreSQL/MinIO 假设。详细
范围和验收标准见 [`ASYNC_INGESTION_MCP_PLAN.md`](ASYNC_INGESTION_MCP_PLAN.md) 与
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。

View File

@ -10,6 +10,21 @@
---
## 当前阶段交付覆盖(PLAN-002)
为优先达成服务可用和一键集群部署,当前阶段只交付企业受控网络内的单一公共知识库:所有
已接收知识处于同一个可见域,用户打开 Admin Console 或连接 WorkBuddy 即可使用,无需感知
后端进程启停。用户、用户组、角色、知识库多空间、Document ACL、LDAP/SSO 和 OAuth 用户
委托仍是本 PRD 的后续 V1 要求,本阶段不实现,也不得标记为完成。
该阶段覆盖不会取代安全原则:公共知识库不等于匿名公网开放;Admin 写操作必须位于受控网络
或统一接入网关后,MCP 使用运行时注入的最小 `knowledge:search` 服务凭据。一旦平台需要接收
非公共知识,必须先完成 Permission Resolver 与检索前/检索中 ACL 下推,禁止全局 Top-K 后置
过滤。实施顺序和集群验收边界见
[`PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md`](../plans/PUBLIC_KNOWLEDGE_CLUSTER_PLAN.md)。
---
# 1. 项目背景
企业内部存在大量分散于 PDF、Word、PPT、Excel、Markdown、图片、扫描件及其他文件中的知识资产。
@ -112,7 +127,8 @@ Admin Console 是官方管理客户端,主要解决:
WorkBuddy、Cursor、Claude Code 或其他 Agent 不应该知道:
- Mochow;
- OceanBase Vector;
- BOS;
- Qdrant;
- Milvus;
- bge-large-zh;
@ -446,6 +462,19 @@ V1 必须支持:
Document processing 必须异步执行。
源文件与 Document/Job 成功持久化后返回 `202 Accepted`。批量上传中的每个文件拥有独立的
Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批次内其他已接收文件。
当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、
断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。
公共知识库集群交付使用分别可扩缩的 API/Admin、MCP Gateway 和 Async Worker 生产镜像;
镜像只包含锁定代码与非敏感默认值,环境凭据必须在部署时注入。
集群编排由 Helm Chart 统一管理:API/MCP 默认多副本,Worker 与单实例恢复调度器分离;敏感值
只引用预建 Kubernetes Secret,数据库 Migration 在安装/升级前串行完成。Ingress/TLS、探针、
资源限制和 Pod 安全默认值属于部署契约,不改变 REST/MCP 业务接口。
**REQ-INGEST-004**
展示处理阶段:
@ -467,10 +496,18 @@ Failed
失败必须展示:
- Failed Stage;
- Attempt Count;
- 是否仍会自动重试;
- Error Code;
- Error Message;
- Retry。
**REQ-INGEST-006**
可重试的瞬时执行失败必须自动进行有界重试。每次重新执行前必须清理该 Document 的旧解析
产物、Chunk、Chunk Index State 和向量/稀疏索引数据,再从不可变原始文件开始处理。清理不得
删除原始文件、Document 标识或 Job/Attempt 历史;清理失败时不得继续重试。
---
# 11. 文件版本和去重
@ -800,21 +837,24 @@ VectorStoreProvider
抽象。
默认首选:
当前默认:
```text
Mochow
OceanBase Vector
```
原因:
公司已有使用经验,同时 Mochow 当前官方能力已经支持向量检索、BM25、Hybrid Search 和标量过滤。
公司现有知识平台主检索链已经使用 OceanBase Vector,并可与既有 OceanBase 运维体系复用。
V1 通过标量条件下推满足 Profile 与 ACL 过滤;BM25/Hybrid 能力必须继续通过统一检索契约
演进,不把特定数据库查询结构写入业务层。
但系统不得使用 Mochow-specific data model 污染业务 Domain Model。
系统不得使用 OceanBase/`pyobvector`-specific data model 污染业务 Domain Model。
未来可以新增:
```text
Mochow
Qdrant
Milvus
pgvector
@ -913,15 +953,13 @@ Provider:
RerankerProvider
```
初始候选:
当前基线与后续候选:
- Qwen3-Reranker-0.6B;
- Qwen3-Reranker-4B;
- Qwen3-Reranker-8B;
- bce-reranker-base;
- bce-reranker-base(当前基线);
- Qwen3-Reranker 系列(后续 Benchmark 候选);
- Local Reranker。
千帆当前已经提供 Qwen3 Reranker 系列,最大输入支持 32K。
模型与参数由版本化 Index/Profile 配置记录,业务 API 不暴露模型名。
---
@ -1812,6 +1850,9 @@ Cancelled
Retry
```
Retry 必须记录 Attempt 历史并区分可重试的瞬时错误与确定性永久错误;重试执行前遵循
`REQ-INGEST-006` 的派生数据清理规则。
---
# 57. 删除语义
@ -1871,15 +1912,15 @@ Embedding × Retrieval × Reranker
```text
bge-large-zh
+ Mochow Dense
+ OceanBase Vector Dense
+ No Reranker
bge-large-zh
+ Mochow Hybrid
+ Qwen3 Reranker
+ OceanBase Vector Dense
+ bce-reranker-base
Qwen3 Embedding
+ Mochow Hybrid
+ OceanBase Vector Hybrid(能力验证后)
+ Qwen3 Reranker
```
@ -2168,7 +2209,7 @@ MCP Error Rate
10. Document Version;
11. Hash 去重;
12. Embedding Provider;
13. Mochow Provider;
13. OceanBase Vector Provider;
14. Dense Search;
15. BM25;
16. Hybrid Retrieval;
@ -2291,9 +2332,9 @@ Embedding 和 Reranker 可以通过 Provider 配置替换。
## Vector DB
Mochow 可以作为默认生产 Vector Store。
OceanBase Vector 作为当前默认生产 Vector Store。
业务代码不存在必须依赖 Mochow SDK 数据结构的 API Contract。
业务代码不存在必须依赖 OceanBase/`pyobvector` SDK 数据结构的 API Contract。
---
@ -2338,11 +2379,14 @@ get_source
# 73. 当前技术基线
PRD 阶段暂定但不最终锁定:
当前 V1 架构基线:
```text
Primary Vector Store:
Mochow
OceanBase Vector
Primary Object Storage:
Baidu BOS
Embedding Baseline:
bge-large-zh
@ -2351,7 +2395,7 @@ Embedding Benchmark:
Qwen3 Embedding
Reranker:
Qwen3 Reranker / equivalent
bce-reranker-base
Document Parsing:
Benchmark MinerU / Docling
@ -2369,6 +2413,10 @@ Backend:
API First
```
运行时切换约束:索引 Provider 或 Profile 变更必须以新 Profile 构建 Shadow Index;Profile
生命周期和激活状态保存在 OceanBase canonical 关系表。只有 canonical Chunk 与 ChunkIndexState 完整核对
通过后才能原子激活,禁止复用旧 Profile ID 或原地覆盖当前 Active Index。
---
# 74. 已确认产品决策
@ -2387,7 +2435,8 @@ API First + Admin Console
**DEC-002**
Mochow 继续作为生产首选候选,同时必须保留 Vector Store 替换能力。
OceanBase Vector 作为当前生产主检索索引,同时必须保留 Vector Store 替换能力。
OceanBase 关系表保存 canonical 数据,同库 Vector Store 表只保存可重建索引。
---
@ -2425,7 +2474,7 @@ RAG + Knowledge Compilation
**DEC-007**
MCP 只能暴露业务能力,禁止泄露 Mochow、Embedding Model 等底层实现。
MCP 只能暴露业务能力,禁止泄露 OceanBase Vector、BOS、Embedding Model 等底层实现。
---
@ -2443,13 +2492,13 @@ PRD 评审通过后进入《总体技术架构设计》。
1. Backend 技术栈;
2. Admin Console 技术栈;
3. PostgreSQL 数据模型;
3. OceanBase MySQL canonical 数据模型;
4. Object Storage;
5. Task Queue;
6. MinerU vs Docling;
7. OCR Pipeline;
8. Chunk Strategy;
9. Mochow Schema;
9. OceanBase Vector Index Schema;
10. VectorStore Adapter;
11. Embedding Provider;
12. Reranker Serving;
@ -2473,7 +2522,9 @@ PRD 评审通过后进入《总体技术架构设计》。
```text
Qianfan
+
Mochow
BOS
+
OceanBase Vector
+
Local Parsing
```
@ -2511,7 +2562,9 @@ Replaceable Embedding
+
Replaceable Reranker
+
Mochow Primary
BOS Object Storage
+
OceanBase Vector Primary
+
Cloud LLM
```
@ -2524,9 +2577,12 @@ Cloud LLM
# 77. 调研依据备注
当前百度千帆模型列表表明 bge-large-zh 与 Qwen3 Embedding、Qwen3 Reranker 可以作为同平台 Benchmark 候选,因此早期无需为模型实验同时引入多个云厂商。
内部最新配置基线确认现有知识平台主链使用千帆 `bge-large-zh`、`bce-reranker-base`、
BOS 与 OceanBase MySQL 模式;OceanBase 同时承载 canonical 关系表和逻辑隔离的可重建
向量表。连接地址、账号和密钥仍只能作为运行时 Secret,不进入产品文档或提交记录。
Mochow 当前官方 API 已支持向量检索与 BM25 全文检索组合的 Hybrid Search,同时可施加标量 Filter,与企业知识库的 Dense + Sparse + ACL Retrieval 需求相匹配。
OceanBase Vector 的生产接入必须验证向量检索、标量 Filter、索引参数和版本兼容性;
BM25/Hybrid 在相应任务验收前仍按产品能力要求保留,不在本轮 Provider 迁移中提前扩展。
WorkBuddy Enterprise 当前公开 MCP 文档支持自定义 MCP Server,并提供传输、权限、安全、远程 API Server 以及 iWiki 配置示例,因此“企业知识服务 → MCP → WorkBuddy”的产品路线具备正式产品支持基础。
@ -2534,4 +2590,4 @@ Karpathy 的 LLM Wiki 原始思路强调 immutable raw sources 与由 LLM 持续
---
**PRD v0.1 END**
**PRD v0.1 END**

View File

@ -1,3 +1,6 @@
# Test Documents
Only commit synthetic, public, or properly anonymized fixtures. Never commit real internal enterprise documents, HR data, customer data, contracts, credentials, or confidential material.
The `sample-*.md` files are entirely synthetic and provide stable parser/chunker
inputs for the Golden Dataset regression checks.

View File

@ -0,0 +1,11 @@
# 示例产品 A 服务政策
> 本文件完全由测试内容构造,不描述任何真实产品或客户承诺。
## 退款
示例产品 A 的测试退款周期为提交申请后的七个自然日。本规则只用于检索效果验证。
## 技术支持
测试用户可以通过虚构的支持入口提交问题,本文不包含真实联系方式。

View File

@ -0,0 +1,12 @@
# 示例出差管理制度
> 本文件完全由测试内容构造,不代表任何真实企业政策。
## 住宿标准
员工出差时应在批准的预算范围内选择安全、交通便利的住宿。示例标准仅用于验证知识检索,
不作为实际报销依据。
## 报销材料
提交报销时需附上示例行程单和合规票据。本段内容同样只用于自动化测试。

View File

@ -1,2 +1,32 @@
{"id":"sample-001","query":"员工出差住宿标准是多少?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","自然语言"]}
{"id":"sample-002","query":"产品A退款周期","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","短查询"]}
{"id":"refund-001","query":"产品A退款周期","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","短查询"]}
{"id":"refund-002","query":"示例产品 A 的退款需要几个自然日?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","自然语言"]}
{"id":"refund-003","query":"申请产品A退款后多久完成?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","同义表达"]}
{"id":"refund-004","query":"退款规则里写的处理周期是什么?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","规则"]}
{"id":"refund-005","query":"产品 A 提交退款申请后的等待时间","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","长查询"]}
{"id":"refund-006","query":"退款是否按自然日计算?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","精确术语"]}
{"id":"refund-007","query":"请查找产品A有关退款时限的说明","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","指令式"]}
{"id":"refund-008","query":"示例服务政策中的退款周期是多少天?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","文档定位"]}
{"id":"support-001","query":"产品A遇到问题如何联系技术支持?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","自然语言"]}
{"id":"support-002","query":"测试用户在哪里提交支持问题?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","入口"]}
{"id":"support-003","query":"技术支持入口是否提供真实联系方式?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","边界"]}
{"id":"support-004","query":"我想提交产品问题,支持说明是什么?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","同义表达"]}
{"id":"support-005","query":"测试用户的虚构支持入口","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","短查询"]}
{"id":"support-006","query":"支持入口能否用于测试用户反馈问题?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","多条件"]}
{"id":"support-007","query":"请定位服务政策里关于提交问题的段落","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","文档定位"]}
{"id":"support-008","query":"产品技术支持部分有没有联系电话?","expected_documents":["sample-product-a-policy.md"],"expected_sections":["技术支持"],"tags":["支持","否定事实"]}
{"id":"lodging-001","query":"员工出差住宿标准是多少?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","自然语言"]}
{"id":"lodging-002","query":"出差时应如何选择住宿?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","行为"]}
{"id":"lodging-003","query":"住宿地点需要满足哪些安全和交通条件?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","多条件"]}
{"id":"lodging-004","query":"出差住宿是否必须在批准预算内?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","预算"]}
{"id":"lodging-005","query":"批准预算范围内的安全住宿","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","短查询"]}
{"id":"lodging-006","query":"员工出差选酒店时要考虑什么?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","同义表达"]}
{"id":"lodging-007","query":"请找出关于安全且交通便利住宿的说明","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","文档定位"]}
{"id":"lodging-008","query":"出差住宿示例标准能否作为实际报销依据?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","边界"]}
{"id":"expense-001","query":"出差报销需要提交哪些材料?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","自然语言"]}
{"id":"expense-002","query":"报销时是否需要行程单?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","材料"]}
{"id":"expense-003","query":"合规票据是不是报销必需材料?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","票据"]}
{"id":"expense-004","query":"提交报销要附上什么凭证?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","同义表达"]}
{"id":"expense-005","query":"提交报销的示例行程单和合规票据","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","短查询"]}
{"id":"expense-006","query":"行程单和票据应在什么时候提交?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","多条件"]}
{"id":"expense-007","query":"请定位管理制度中有关合规票据的段落","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","文档定位"]}
{"id":"expense-008","query":"报销材料段落是否只用于自动化测试?","expected_documents":["sample-travel-policy.md"],"expected_sections":["报销材料"],"tags":["报销","边界"]}

View File

@ -1,6 +1,27 @@
# Database migrations
Migrations are forward-only and operate on PostgreSQL canonical metadata. Run
Migrations are forward-only and operate on OceanBase MySQL canonical metadata. Run
`make migration-check` to render the full upgrade path without changing a
database. Integration tests also apply the migration to a temporary database
and compare it with SQLAlchemy metadata.
The target OceanBase database is shared. Alembic autogenerate and drift checks
are intentionally scoped to tables present in EKP `Base.metadata`; unrelated
application tables and the separately provisioned vector-index table are never
reflected or treated as migration targets.
OceanBase adds `BLOCK_SIZE ... LOCAL` syntax that SQLAlchemy's stock MySQL
reflector does not recognize for secondary indexes, unique constraints, and
foreign keys. Real OceanBase drift checks therefore compare columns, types,
defaults, and primary keys while excluding those three falsely reported object
types. Full constraint comparison still runs against SQLite in integration
tests; target-cluster checks separately verify HNSW/PRE-FILTER and the
single-active generated unique index.
DEMO-015 adds `ingestion_jobs` and `ingestion_attempts`. The former is the canonical execution state;
the latter preserves numbered history and uses a generated nullable active marker plus a unique index
to reject two running Attempts for the same Job on both OceanBase MySQL and SQLite.
DEMO-016 adds `ingestion_batches` and `ingestion_batch_items`. They retain input order and per-file
acceptance references while Job rows remain the processing state source of truth; a batch is an aggregate,
not a cross-file transaction.

View File

@ -1,6 +1,7 @@
"""Alembic environment for canonical document metadata migrations."""
from logging.config import fileConfig
from typing import Any
from alembic import context
from sqlalchemy import engine_from_config, pool
@ -13,12 +14,47 @@ if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
TARGET_TABLE_NAMES = frozenset(target_metadata.tables)
def _include_name(
name: str | None,
type_: str,
parent_names: dict[str, str | None],
) -> bool:
"""Keep Alembic reflection inside EKP-owned canonical tables.
The target OceanBase database is shared with another application and
contains provider-specific VECTOR columns. Excluding unrelated tables
before reflection prevents schema checks from inspecting or proposing
changes to objects outside this repository's ownership boundary.
"""
del parent_names
return type_ != "table" or name in TARGET_TABLE_NAMES
def _include_oceanbase_object(
object_: Any,
name: str | None,
type_: str,
reflected: bool,
compare_to: Any,
) -> bool:
"""Skip constraints the stock MySQL reflector cannot parse from OceanBase.
OceanBase appends `BLOCK_SIZE ... LOCAL` to secondary constraints. SQLAlchemy
consequently reports model constraints as missing even when they exist.
Columns, types, defaults and primary keys remain in the real drift check;
constraints are covered by migration tests and targeted live invariants.
"""
del object_, name, reflected, compare_to
return type_ not in {"index", "unique_constraint", "foreign_key_constraint"}
def _database_url() -> str:
"""Allow tests to override the URL without changing runtime settings."""
override = config.attributes.get("database_url")
return override if isinstance(override, str) else get_settings().database_url
return override if isinstance(override, str) else get_settings().database_url.get_secret_value()
def run_migrations_offline() -> None:
@ -29,6 +65,7 @@ def run_migrations_offline() -> None:
literal_binds=True,
dialect_opts={"paramstyle": "named"},
compare_type=True,
include_name=_include_name,
)
with context.begin_transaction():
@ -48,6 +85,10 @@ def run_migrations_online() -> None:
connection=connection,
target_metadata=target_metadata,
compare_type=True,
include_name=_include_name,
include_object=(
_include_oceanbase_object if connection.dialect.name == "mysql" else None
),
)
with context.begin_transaction():

View File

@ -0,0 +1,29 @@
"""Add parsed-result object reference to documents.
Revision ID: 20260901_0002
Revises: 20260901_0001
Create Date: 2026-09-01
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260901_0002"
down_revision: str | None = "20260901_0001"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Reference the rebuildable parsed JSON object after parsing succeeds."""
op.add_column(
"documents",
sa.Column("parsed_storage_key", sa.String(length=512), nullable=True),
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -0,0 +1,73 @@
"""Create canonical chunk metadata table.
Revision ID: 20260901_0003
Revises: 20260901_0002
Create Date: 2026-09-01
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260901_0003"
down_revision: str | None = "20260901_0002"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Persist traceable chunks without introducing embedding/index fields."""
op.create_table(
"chunks",
sa.Column("chunk_id", sa.Uuid(), nullable=False),
sa.Column("document_id", sa.Uuid(), nullable=False),
sa.Column("filename", sa.String(length=255), nullable=False),
sa.Column("page", sa.Integer(), nullable=True),
sa.Column("page_end", sa.Integer(), nullable=True),
sa.Column("section", sa.Text(), nullable=True),
sa.Column("section_path", sa.JSON(), nullable=False),
sa.Column("chunk_index", sa.Integer(), nullable=False),
sa.Column("content", sa.Text(), nullable=False),
sa.Column("token_count", sa.Integer(), nullable=False),
sa.Column("strategy_version", sa.String(length=64), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
sa.CheckConstraint("length(content) > 0", name="content_not_empty"),
sa.CheckConstraint(
"length(strategy_version) > 0",
name="strategy_version_not_empty",
),
sa.CheckConstraint("chunk_index >= 0", name="chunk_index_non_negative"),
sa.CheckConstraint("token_count > 0", name="token_count_positive"),
sa.CheckConstraint("page IS NULL OR page > 0", name="page_positive"),
sa.CheckConstraint("page_end IS NULL OR page_end > 0", name="page_end_positive"),
sa.CheckConstraint(
"page IS NULL OR page_end IS NULL OR page_end >= page",
name="page_range_valid",
),
sa.ForeignKeyConstraint(
["document_id"],
["documents.id"],
name="fk_chunks_document_id_documents",
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("chunk_id", name="pk_chunks"),
sa.UniqueConstraint(
"document_id",
"strategy_version",
"chunk_index",
name="uq_chunks_document_strategy_index",
),
)
op.create_index("ix_chunks_document_id", "chunks", ["document_id"], unique=False)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -0,0 +1,58 @@
"""Create versioned canonical chunk-index state.
Revision ID: 20260901_0004
Revises: 20260901_0003
Create Date: 2026-09-01
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260901_0004"
down_revision: str | None = "20260901_0003"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Track successful index writes without storing canonical vectors in Mochow."""
op.create_table(
"chunk_index_states",
sa.Column("chunk_id", sa.Uuid(), nullable=False),
sa.Column("index_profile_id", sa.String(length=128), nullable=False),
sa.Column("embedding_model", sa.String(length=128), nullable=False),
sa.Column("vector_dimension", sa.Integer(), nullable=False),
sa.Column(
"indexed_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.CheckConstraint(
"length(index_profile_id) > 0",
name="profile_not_empty",
),
sa.CheckConstraint(
"length(embedding_model) > 0",
name="model_not_empty",
),
sa.CheckConstraint("vector_dimension > 0", name="dimension_positive"),
sa.ForeignKeyConstraint(
["chunk_id"],
["chunks.chunk_id"],
name="fk_chunk_index_states_chunk_id_chunks",
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint(
"chunk_id",
"index_profile_id",
name="pk_chunk_index_states",
),
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -0,0 +1,77 @@
"""Create canonical index-profile lifecycle records.
Revision ID: 20260902_0005
Revises: 20260901_0004
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0005"
down_revision: str | None = "20260901_0004"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Persist shadow/active/retired profiles without storing vector data."""
op.create_table(
"index_profiles",
sa.Column("profile_id", sa.String(length=128), nullable=False),
sa.Column("chunk_strategy_version", sa.String(length=64), nullable=False),
sa.Column("embedding_model", sa.String(length=128), nullable=False),
sa.Column("vector_dimension", sa.Integer(), nullable=False),
sa.Column(
"status",
sa.Enum(
"shadow",
"active",
"retired",
name="index_profile_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column("activated_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("length(profile_id) > 0", name="profile_id_not_empty"),
sa.CheckConstraint(
"length(chunk_strategy_version) > 0",
name="profile_chunk_strategy_not_empty",
),
sa.CheckConstraint(
"length(embedding_model) > 0",
name="profile_embedding_model_not_empty",
),
sa.CheckConstraint(
"vector_dimension > 0",
name="profile_dimension_positive",
),
sa.CheckConstraint(
"status IN ('shadow', 'active', 'retired')",
name="profile_status_valid",
),
sa.PrimaryKeyConstraint("profile_id", name="pk_index_profiles"),
)
op.create_index(
"uq_index_profiles_single_active",
"index_profiles",
["status"],
unique=True,
postgresql_where=sa.text("status = 'active'"),
sqlite_where=sa.text("status = 'active'"),
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -0,0 +1,41 @@
"""Make single-active profile enforcement portable to OceanBase MySQL.
Revision ID: 20260902_0006
Revises: 20260902_0005
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0006"
down_revision: str | None = "20260902_0005"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Replace dialect-specific partial uniqueness with a generated marker."""
op.drop_index("uq_index_profiles_single_active", table_name="index_profiles")
op.add_column(
"index_profiles",
sa.Column(
"active_marker",
sa.String(length=6),
sa.Computed("CASE WHEN status = 'active' THEN 'active' ELSE NULL END"),
nullable=True,
),
)
op.create_index(
"uq_index_profiles_single_active",
"index_profiles",
["active_marker"],
unique=True,
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -0,0 +1,174 @@
"""Create durable ingestion jobs and attempt history.
Revision ID: 20260902_0007
Revises: 20260902_0006
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0007"
down_revision: str | None = "20260902_0006"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Persist queue-independent Job state and bounded execution attempts."""
op.create_table(
"ingestion_jobs",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("document_id", sa.Uuid(), nullable=False),
sa.Column(
"status",
sa.Enum(
"queued",
"running",
"retrying",
"succeeded",
"failed",
"cancelled",
name="ingestion_job_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column(
"stage",
sa.Enum(
"parsing",
"cleaning",
"chunking",
"embedding",
"indexing",
name="ingestion_stage",
native_enum=False,
create_constraint=False,
),
nullable=True,
),
sa.Column("attempt_count", sa.Integer(), nullable=False),
sa.Column("max_attempts", sa.Integer(), nullable=False),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("attempt_count >= 0", name="attempt_count_non_negative"),
sa.CheckConstraint("max_attempts > 0", name="max_attempts_positive"),
sa.CheckConstraint("attempt_count <= max_attempts", name="attempt_count_bounded"),
sa.CheckConstraint(
"status IN ('queued', 'running', 'retrying', 'succeeded', 'failed', 'cancelled')",
name="status_valid",
),
sa.CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
sa.ForeignKeyConstraint(
["document_id"],
["documents.id"],
name="fk_ingestion_jobs_document_id_documents",
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_jobs"),
sa.UniqueConstraint("document_id", name="uq_ingestion_jobs_document_id"),
)
op.create_index(
"ix_ingestion_jobs_status_next_run",
"ingestion_jobs",
["status", "next_run_at"],
unique=False,
)
op.create_table(
"ingestion_attempts",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("job_id", sa.Uuid(), nullable=False),
sa.Column("attempt_number", sa.Integer(), nullable=False),
sa.Column(
"status",
sa.Enum(
"running",
"succeeded",
"failed",
name="ingestion_attempt_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column(
"stage",
sa.Enum(
"parsing",
"cleaning",
"chunking",
"embedding",
"indexing",
name="ingestion_attempt_stage",
native_enum=False,
create_constraint=False,
),
nullable=True,
),
sa.Column("retryable", sa.Boolean(), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column(
"active_marker",
sa.Integer(),
sa.Computed("CASE WHEN status = 'running' THEN 1 ELSE NULL END"),
nullable=True,
),
sa.Column(
"started_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("attempt_number > 0", name="attempt_number_positive"),
sa.CheckConstraint(
"status IN ('running', 'succeeded', 'failed')",
name="status_valid",
),
sa.CheckConstraint(
"stage IS NULL OR stage IN "
"('parsing', 'cleaning', 'chunking', 'embedding', 'indexing')",
name="stage_valid",
),
sa.ForeignKeyConstraint(
["job_id"],
["ingestion_jobs.id"],
name="fk_ingestion_attempts_job_id_ingestion_jobs",
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_attempts"),
sa.UniqueConstraint("job_id", "attempt_number", name="uq_ingestion_attempts_job_number"),
)
op.create_index(
"uq_ingestion_attempts_active",
"ingestion_attempts",
["job_id", "active_marker"],
unique=True,
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -0,0 +1,109 @@
"""Create ingestion batch aggregation metadata.
Revision ID: 20260902_0008
Revises: 20260902_0007
Create Date: 2026-09-02
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260902_0008"
down_revision: str | None = "20260902_0007"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Persist ordered per-file acceptance without making a batch transactional."""
op.create_table(
"ingestion_batches",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("total_files", sa.Integer(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.CheckConstraint("total_files > 0", name="total_files_positive"),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batches"),
)
op.create_table(
"ingestion_batch_items",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("batch_id", sa.Uuid(), nullable=False),
sa.Column("item_index", sa.Integer(), nullable=False),
sa.Column("filename", sa.String(length=255), nullable=False),
sa.Column(
"acceptance_status",
sa.Enum(
"pending",
"accepted",
"rejected",
name="batch_item_acceptance_status",
native_enum=False,
create_constraint=False,
),
nullable=False,
),
sa.Column("document_id", sa.Uuid(), nullable=True),
sa.Column("job_id", sa.Uuid(), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
nullable=False,
),
sa.CheckConstraint("item_index >= 0", name="item_index_non_negative"),
sa.CheckConstraint("length(filename) > 0", name="filename_not_empty"),
sa.CheckConstraint(
"acceptance_status IN ('pending', 'accepted', 'rejected')",
name="acceptance_status_valid",
),
sa.ForeignKeyConstraint(
["batch_id"],
["ingestion_batches.id"],
name="fk_ingestion_batch_items_batch_id_ingestion_batches",
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["document_id"],
["documents.id"],
name="fk_ingestion_batch_items_document_id_documents",
ondelete="SET NULL",
),
sa.ForeignKeyConstraint(
["job_id"],
["ingestion_jobs.id"],
name="fk_ingestion_batch_items_job_id_ingestion_jobs",
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name="pk_ingestion_batch_items"),
)
op.create_index(
"uq_ingestion_batch_items_order",
"ingestion_batch_items",
["batch_id", "item_index"],
unique=True,
)
op.create_index(
"uq_ingestion_batch_items_job_id",
"ingestion_batch_items",
["job_id"],
unique=True,
)
def downgrade() -> None:
"""Reject downgrade because project migrations are forward-only."""
raise NotImplementedError("Database migrations are forward-only")

View File

@ -1 +1,26 @@
"""Shared core utilities for the Enterprise Knowledge Platform."""
from ekp_core.dense_retrieval import DenseRetrievalResult, DenseRetrievalService
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
from ekp_core.document_cleaner import clean_parsed_document
from ekp_core.document_indexing import (
DocumentIndexingPipelineService,
DocumentIndexingService,
)
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
from ekp_core.search import SearchResult, SearchService, SourceResult
__all__ = [
"DocumentChunkingService",
"DenseRetrievalResult",
"DenseRetrievalService",
"DocumentIngestionService",
"DocumentIndexingPipelineService",
"DocumentIndexingService",
"DocumentPipelineService",
"DocumentParsingService",
"SearchResult",
"SearchService",
"SourceResult",
"clean_parsed_document",
]

View File

@ -0,0 +1,223 @@
"""Structure-aware, token-window chunking for the retrieval Demo."""
import re
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Protocol
from ekp_domain import ParsedBlock, ParsedBlockType, ParsedDocument
DEFAULT_CHUNK_STRATEGY_VERSION = "structure_v1"
TOKEN_PATTERN = re.compile(
r"[\u3400-\u4dbf\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]"
r"|[A-Za-z0-9_]+"
r"|[^\s]"
)
ASCII_SUBTOKEN_CHARACTERS = 4
@dataclass(frozen=True, slots=True)
class TokenSpan:
"""Character offsets for one conservatively estimated model token."""
start: int
end: int
class TokenCounter(Protocol):
"""Injectable token boundary contract for future model-specific counters."""
def spans(self, text: str) -> tuple[TokenSpan, ...]:
"""Return ordered, non-overlapping token character spans."""
def count(self, text: str) -> int:
"""Return the token estimate for text."""
class ApproximateTokenCounter:
"""Conservatively estimate Chinese/BGE-sized token windows without a model SDK.
CJK characters and punctuation count individually. ASCII words are divided
into four-character pieces so unusually long terms cannot bypass the limit.
DEMO-006 may inject a model-specific counter without changing chunk policy.
"""
def spans(self, text: str) -> tuple[TokenSpan, ...]:
token_spans: list[TokenSpan] = []
for match in TOKEN_PATTERN.finditer(text):
value = match.group()
if value.isascii() and (value.isalnum() or "_" in value):
token_spans.extend(
TokenSpan(start, min(start + ASCII_SUBTOKEN_CHARACTERS, match.end()))
for start in range(match.start(), match.end(), ASCII_SUBTOKEN_CHARACTERS)
)
else:
token_spans.append(TokenSpan(match.start(), match.end()))
return tuple(token_spans)
def count(self, text: str) -> int:
"""Count the same boundaries used to create chunk windows."""
return len(self.spans(text))
@dataclass(frozen=True, slots=True)
class ChunkDraft:
"""Persistence-neutral chunk content produced by one strategy version."""
chunk_index: int
content: str
token_count: int
page: int | None
page_end: int | None
section_path: tuple[str, ...]
@property
def section(self) -> str | None:
"""Return a compact section label for Demo retrieval results."""
return " / ".join(self.section_path) or None
class ChunkingStrategy(Protocol):
"""Provider-neutral strategy contract consumed by the pipeline service."""
@property
def version(self) -> str:
"""Return a stable strategy identifier for rebuilds and shadow indexes."""
def chunk(self, document: ParsedDocument) -> tuple[ChunkDraft, ...]:
"""Build ordered chunks from a cleaned parsed document."""
@dataclass(frozen=True, slots=True)
class _BlockRange:
start: int
end: int
page_number: int | None
@dataclass(frozen=True, slots=True)
class _StructuralGroup:
content: str
section_path: tuple[str, ...]
block_ranges: tuple[_BlockRange, ...]
class StructureAwareChunker:
"""Combine paragraphs inside structural boundaries, then apply token overlap."""
def __init__(
self,
max_tokens: int = 450,
overlap_tokens: int = 75,
max_characters: int = 2000,
token_counter: TokenCounter | None = None,
strategy_version: str = DEFAULT_CHUNK_STRATEGY_VERSION,
) -> None:
if max_tokens <= 0:
raise ValueError("max_tokens must be positive")
if overlap_tokens < 0 or overlap_tokens >= max_tokens:
raise ValueError("overlap_tokens must be non-negative and less than max_tokens")
if max_characters < ASCII_SUBTOKEN_CHARACTERS:
raise ValueError("max_characters must be at least four")
if not strategy_version:
raise ValueError("strategy_version must not be empty")
self._max_tokens = max_tokens
self._overlap_tokens = overlap_tokens
self._max_characters = max_characters
self._token_counter = token_counter or ApproximateTokenCounter()
self._version = strategy_version
@property
def version(self) -> str:
"""Identify the chunk policy used by persisted rows."""
return self._version
def chunk(self, document: ParsedDocument) -> tuple[ChunkDraft, ...]:
"""Create deterministic chunks without crossing heading/section boundaries."""
drafts: list[ChunkDraft] = []
for group in _structural_groups(document.blocks):
for content, token_count, page, page_end in self._windows(group):
drafts.append(
ChunkDraft(
chunk_index=len(drafts),
content=content,
token_count=token_count,
page=page,
page_end=page_end,
section_path=group.section_path,
)
)
return tuple(drafts)
def _windows(
self,
group: _StructuralGroup,
) -> Iterable[tuple[str, int, int | None, int | None]]:
spans = self._token_counter.spans(group.content)
start_token = 0
while start_token < len(spans):
end_token = min(start_token + self._max_tokens, len(spans))
char_start = spans[start_token].start
while (
end_token > start_token + 1
and spans[end_token - 1].end - char_start > self._max_characters
):
end_token -= 1
char_end = spans[end_token - 1].end
content = group.content[char_start:char_end].strip()
pages = [
block_range.page_number
for block_range in group.block_ranges
if block_range.page_number is not None
and block_range.end > char_start
and block_range.start < char_end
]
yield (
content,
end_token - start_token,
min(pages, default=None),
max(pages, default=None),
)
if end_token == len(spans):
break
start_token = end_token - self._overlap_tokens
def _structural_groups(blocks: tuple[ParsedBlock, ...]) -> Iterable[_StructuralGroup]:
"""Group content without crossing explicit headings or section changes."""
current: list[ParsedBlock] = []
current_section: tuple[str, ...] | None = None
for block in blocks:
is_new_boundary = bool(current) and (
block.block_type is ParsedBlockType.HEADING or block.section_path != current_section
)
if is_new_boundary:
yield _build_group(current)
current = []
if not current:
current_section = block.section_path
current.append(block)
if current:
yield _build_group(current)
def _build_group(blocks: list[ParsedBlock]) -> _StructuralGroup:
"""Join blocks with readable separators and retain their character ranges."""
parts: list[str] = []
block_ranges: list[_BlockRange] = []
offset = 0
for block in blocks:
if parts:
parts.append("\n\n")
offset += 2
start = offset
parts.append(block.text)
offset += len(block.text)
block_ranges.append(_BlockRange(start=start, end=offset, page_number=block.page_number))
return _StructuralGroup(
content="".join(parts),
section_path=blocks[0].section_path,
block_ranges=tuple(block_ranges),
)

View File

@ -4,8 +4,9 @@ from functools import lru_cache
from pathlib import Path
from typing import Literal
from pydantic import Field
from pydantic import AnyHttpUrl, Field, SecretStr, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
from sqlalchemy.engine import make_url
class Settings(BaseSettings):
@ -24,9 +25,105 @@ class Settings(BaseSettings):
api_host: str = "0.0.0.0"
api_port: int = Field(default=8000, ge=1, le=65535)
api_prefix: str = "/api/v1"
database_url: str = "postgresql+psycopg://localhost:5432/ekp"
object_storage_provider: Literal["local"] = "local"
admin_web_root: Path | None = None
# SQLAlchemy URLs commonly embed a password, so configuration rendering
# must mask this value and unwrap it only at the database driver boundary.
database_url: SecretStr = SecretStr("mysql+pymysql://localhost:2881/ekp")
database_pool_size: int = Field(default=10, ge=1, le=100)
database_max_overflow: int = Field(default=10, ge=0, le=100)
database_pool_recycle_seconds: int = Field(default=600, ge=1)
database_pool_timeout_seconds: float = Field(default=30.0, gt=0)
database_connect_timeout_seconds: int = Field(default=30, ge=1)
rabbitmq_url: SecretStr | None = None
ingestion_max_attempts: int = Field(default=3, ge=1, le=10)
ingestion_retry_backoff_seconds: int = Field(default=2, ge=1, le=3600)
ingestion_retry_backoff_max_seconds: int = Field(default=60, ge=1, le=86400)
ingestion_worker_lease_seconds: int = Field(default=300, ge=30, le=86400)
ingestion_recovery_interval_seconds: int = Field(default=30, ge=5, le=3600)
batch_max_files: int = Field(default=20, ge=1, le=100)
batch_max_total_bytes: int = Field(default=200 * 1024 * 1024, gt=0)
object_storage_provider: Literal["local", "bos"] = "bos"
object_storage_path: Path = Path("storage")
bos_endpoint: str | None = None
bos_access_key_id: SecretStr | None = None
bos_secret_access_key: SecretStr | None = None
bos_bucket: str | None = None
bos_key_prefix: str = "knowledge-platform"
bos_timeout_seconds: float = Field(default=30.0, gt=0)
max_upload_bytes: int = Field(default=50 * 1024 * 1024, gt=0)
chunk_max_tokens: int = Field(default=450, gt=0)
chunk_overlap_tokens: int = Field(default=75, ge=0)
chunk_max_characters: int = Field(default=2000, ge=4)
embedding_provider: Literal["qianfan"] = "qianfan"
embedding_model: Literal["bge-large-zh"] = "bge-large-zh"
embedding_endpoint: str = "https://qianfan.baidubce.com/v2/embeddings"
baidu_qianfan_api_key: SecretStr | None = None
embedding_timeout_seconds: float = Field(default=30.0, gt=0)
vector_store_provider: Literal["oceanbase"] = "oceanbase"
oceanbase_vector_uri: str | None = None
oceanbase_vector_user: str | None = None
oceanbase_vector_password: SecretStr | None = None
oceanbase_vector_database: str | None = None
oceanbase_vector_table: str = "knowledge_chunks_v1"
oceanbase_vector_index: str = "vidx_knowledge_chunks_embedding"
oceanbase_vector_batch_size: int = Field(default=100, ge=1, le=1000)
oceanbase_vector_timeout_seconds: float = Field(default=30.0, gt=0)
index_profile_id: str = "demo-bge-large-zh-structure-v1-oceanbase-v1"
dense_top_n: int = Field(default=20, ge=1, le=64)
reranker_provider: Literal["qianfan"] = "qianfan"
reranker_model: Literal["bce-reranker-base"] = "bce-reranker-base"
reranker_endpoint: str = "https://qianfan.baidubce.com/v2/rerank"
reranker_api_key: SecretStr | None = None
reranker_timeout_seconds: float = Field(default=30.0, gt=0)
search_top_k: int = Field(default=5, ge=1, le=64)
mcp_host: str = "127.0.0.1"
mcp_port: int = Field(default=8766, ge=1, le=65535)
mcp_path: str = "/mcp"
mcp_public_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766/mcp")
mcp_issuer_url: AnyHttpUrl = AnyHttpUrl("http://127.0.0.1:8766")
mcp_auth_token: SecretStr | None = None
mcp_client_id: str = Field(default="workbuddy-enterprise", min_length=1, max_length=200)
mcp_service_identity: str = Field(default="workbuddy-demo", min_length=1, max_length=200)
mcp_required_scope: str = Field(default="knowledge:search", min_length=1, max_length=200)
mcp_allowed_hosts: list[str] = ["127.0.0.1:*", "localhost:*", "[::1]:*", "testserver"]
mcp_allowed_origins: list[str] = [
"http://127.0.0.1:*",
"http://localhost:*",
"http://[::1]:*",
]
mcp_max_request_body_bytes: int = Field(default=1024 * 1024, ge=1024, le=4 * 1024 * 1024)
@model_validator(mode="after")
def validate_chunk_window(self) -> "Settings":
"""Require forward progress for every overlapping chunk window."""
if self.chunk_overlap_tokens >= self.chunk_max_tokens:
raise ValueError("chunk_overlap_tokens must be less than chunk_max_tokens")
if self.search_top_k > self.dense_top_n:
raise ValueError("search_top_k must not exceed dense_top_n")
if self.ingestion_retry_backoff_max_seconds < self.ingestion_retry_backoff_seconds:
raise ValueError(
"ingestion_retry_backoff_max_seconds must be at least the initial backoff"
)
longest_provider_timeout = max(
self.bos_timeout_seconds,
self.embedding_timeout_seconds,
self.oceanbase_vector_timeout_seconds,
)
if self.ingestion_worker_lease_seconds <= longest_provider_timeout:
raise ValueError("ingestion_worker_lease_seconds must exceed provider timeouts")
if self.batch_max_total_bytes < self.max_upload_bytes:
raise ValueError("batch_max_total_bytes must be at least max_upload_bytes")
if not self.mcp_path.startswith("/") or self.mcp_path.endswith("/"):
raise ValueError("mcp_path must start with '/' and must not end with '/'")
if not self.mcp_allowed_hosts:
raise ValueError("mcp_allowed_hosts must not be empty")
database_url = make_url(self.database_url.get_secret_value())
if database_url.get_backend_name() == "sqlite":
if self.app_env != "test":
raise ValueError("SQLite database URLs are restricted to APP_ENV=test")
elif database_url.drivername != "mysql+pymysql":
raise ValueError("canonical database must use OceanBase through mysql+pymysql")
return self
@lru_cache

View File

@ -1,14 +1,35 @@
"""SQLAlchemy engine and session construction for canonical metadata."""
from sqlalchemy import Engine, create_engine
from sqlalchemy.engine import make_url
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.config import Settings
def create_database_engine(settings: Settings) -> Engine:
"""Create the metadata engine without opening a connection eagerly."""
return create_engine(settings.database_url, pool_pre_ping=True)
"""Create the canonical engine without opening a connection eagerly.
OceanBase uses the synchronous PyMySQL driver in this Demo. Pool and
socket timeouts are explicit so an outage cannot hold API workers
indefinitely. SQLite remains available only for isolated tests.
"""
database_url = settings.database_url.get_secret_value()
url = make_url(database_url)
options: dict[str, object] = {"pool_pre_ping": True}
if url.get_backend_name() == "mysql":
options.update(
pool_size=settings.database_pool_size,
max_overflow=settings.database_max_overflow,
pool_recycle=settings.database_pool_recycle_seconds,
pool_timeout=settings.database_pool_timeout_seconds,
connect_args={
"connect_timeout": settings.database_connect_timeout_seconds,
"read_timeout": settings.database_connect_timeout_seconds,
"write_timeout": settings.database_connect_timeout_seconds,
},
)
return create_engine(database_url, **options)
def create_session_factory(engine: Engine) -> sessionmaker[Session]:

View File

@ -0,0 +1,90 @@
"""Query embedding and profile-scoped dense candidate retrieval."""
from dataclasses import dataclass
from math import isfinite
from ekp_domain import IndexProfile
from ekp_embedding import EmbeddingProvider
from ekp_vector_store import DenseSearchCandidate, DenseSearchRequest, VectorStore
DEFAULT_DENSE_TOP_N = 20
class InvalidRetrievalQueryError(ValueError):
"""Raised when a query cannot be embedded safely."""
class DenseRetrievalContractError(RuntimeError):
"""Raised when an embedding or vector provider violates its contract."""
@dataclass(frozen=True, slots=True)
class DenseRetrievalResult:
"""Dense candidates before DEMO-008 reranking and source-result assembly."""
query: str
index_profile_id: str
candidates: tuple[DenseSearchCandidate, ...]
class DenseRetrievalService:
"""Embed one query and request Top-N from the active versioned index profile."""
def __init__(
self,
embedding_provider: EmbeddingProvider,
vector_store: VectorStore,
index_profile: IndexProfile,
default_top_n: int = DEFAULT_DENSE_TOP_N,
) -> None:
if embedding_provider.model_name != index_profile.embedding_model:
raise ValueError("embedding provider model does not match index profile")
if embedding_provider.dimensions != index_profile.vector_dimension:
raise ValueError("embedding dimensions do not match index profile")
if not 1 <= default_top_n <= 1000:
raise ValueError("default_top_n must be between 1 and 1000")
self._embedding_provider = embedding_provider
self._vector_store = vector_store
self._index_profile = index_profile
self._default_top_n = default_top_n
def search(self, query: str, top_n: int | None = None) -> DenseRetrievalResult:
"""Return raw dense candidates without reranking or final API shaping."""
normalized_query = query.strip()
if not normalized_query:
raise InvalidRetrievalQueryError("retrieval query must not be empty")
requested_top_n = self._default_top_n if top_n is None else top_n
if not 1 <= requested_top_n <= 1000:
raise InvalidRetrievalQueryError("top_n must be between 1 and 1000")
embedded = self._embedding_provider.embed((normalized_query,))
if len(embedded) != 1:
raise DenseRetrievalContractError("embedding provider returned the wrong vector count")
query_vector = embedded[0]
if len(query_vector) != self._index_profile.vector_dimension:
raise DenseRetrievalContractError("embedding provider returned the wrong dimensions")
if any(not isfinite(value) for value in query_vector):
raise DenseRetrievalContractError("embedding provider returned non-finite values")
if not any(value != 0 for value in query_vector):
raise DenseRetrievalContractError("embedding provider returned a zero vector")
candidates = self._vector_store.dense_search(
DenseSearchRequest(
query_vector=query_vector,
index_profile_id=self._index_profile.profile_id,
top_n=requested_top_n,
)
)
if len(candidates) > requested_top_n:
raise DenseRetrievalContractError("vector store returned too many candidates")
if any(
candidate.index_profile_id != self._index_profile.profile_id for candidate in candidates
):
raise DenseRetrievalContractError("vector store crossed the active index profile")
if any(not isfinite(candidate.score) for candidate in candidates):
raise DenseRetrievalContractError("vector store returned a non-finite score")
return DenseRetrievalResult(
query=normalized_query,
index_profile_id=self._index_profile.profile_id,
candidates=candidates,
)

View File

@ -0,0 +1,194 @@
"""Application services for building and persisting document chunks."""
from collections.abc import Callable
from dataclasses import dataclass
from hashlib import sha256
from uuid import UUID, uuid5
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.chunking import ChunkDraft, ChunkingStrategy
from ekp_core.document_parsing import DocumentIngestionService, DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import deserialize_parsed_document
from ekp_domain import Chunk, Document, DocumentStatus, IngestionStage, ParsedDocument
from ekp_object_storage import BinarySource, ObjectStorage
MAX_CHUNKING_ERROR_LENGTH = 2000
class DocumentNotFoundError(LookupError):
"""Raised when chunking is requested for an unknown document id."""
class DocumentNotParsedError(RuntimeError):
"""Raised when chunking is requested before a parsed artifact exists."""
class DocumentAlreadyChunkedError(RuntimeError):
"""Raised instead of silently replacing an existing strategy result."""
class ParsedDocumentMismatchError(ValueError):
"""Raised when a parsed artifact does not belong to its Document metadata."""
class EmptyChunkSetError(ValueError):
"""Raised when a parsed document produces no usable chunks."""
class ChunkingStatePersistenceError(RuntimeError):
"""Raised when a chunking failure cannot be recorded in metadata."""
class DocumentChunkingService:
"""Load a parsed artifact, build chunks, and atomically persist their metadata."""
def __init__(
self,
session_factory: sessionmaker[Session],
storage: ObjectStorage,
strategy: ChunkingStrategy,
) -> None:
self._session_factory = session_factory
self._storage = storage
self._strategy = strategy
def process(
self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> tuple[Chunk, ...]:
"""Build chunks while optionally reporting the durable Job stage."""
filename, parsed_key = self._start_chunking(document_id)
try:
if stage_callback is not None:
stage_callback(IngestionStage.CHUNKING)
with self._storage.open(parsed_key) as source:
parsed = deserialize_parsed_document(source.read())
if parsed.document_id != document_id or parsed.filename != filename:
raise ParsedDocumentMismatchError(
f"parsed artifact identity does not match document {document_id}"
)
drafts = self._strategy.chunk(parsed)
if not drafts:
raise EmptyChunkSetError(f"{filename!r} produced no chunks")
return self._persist_chunks(document_id, filename, drafts)
except Exception as error:
self._record_failure(document_id, error)
raise
def _start_chunking(self, document_id: UUID) -> tuple[str, str]:
"""Validate prerequisites and guard against in-place strategy replacement."""
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise DocumentNotFoundError(str(document_id))
if document.parsed_storage_key is None:
raise DocumentNotParsedError(str(document_id))
existing_chunk_id = session.scalar(
select(Chunk.chunk_id)
.where(
Chunk.document_id == document_id,
Chunk.strategy_version == self._strategy.version,
)
.limit(1)
)
if existing_chunk_id is not None:
raise DocumentAlreadyChunkedError(str(document_id))
document.status = DocumentStatus.PROCESSING
document.error_message = None
session.commit()
return document.filename, document.parsed_storage_key
def _persist_chunks(
self,
document_id: UUID,
filename: str,
drafts: tuple[ChunkDraft, ...],
) -> tuple[Chunk, ...]:
"""Persist all chunks in one transaction so partial sets are never visible."""
chunks: list[Chunk] = []
for draft in drafts:
content_digest = sha256(draft.content.encode("utf-8")).hexdigest()
chunks.append(
Chunk(
chunk_id=uuid5(
document_id,
f"{self._strategy.version}:{draft.chunk_index}:{content_digest}",
),
document_id=document_id,
filename=filename,
page=draft.page,
page_end=draft.page_end,
section=draft.section,
section_path=list(draft.section_path),
chunk_index=draft.chunk_index,
content=draft.content,
token_count=draft.token_count,
strategy_version=self._strategy.version,
)
)
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise DocumentNotFoundError(str(document_id))
session.add_all(chunks)
document.status = DocumentStatus.PROCESSING
document.error_message = None
session.commit()
return tuple(chunks)
def _record_failure(self, document_id: UUID, error: Exception) -> None:
"""Persist a bounded chunking diagnostic after transaction rollback."""
try:
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
return
document.status = DocumentStatus.FAILED
message = str(error).strip() or type(error).__name__
document.error_message = message[:MAX_CHUNKING_ERROR_LENGTH]
session.commit()
except Exception as state_error:
raise ChunkingStatePersistenceError(
f"could not record chunking failure for {document_id}"
) from state_error
@dataclass(frozen=True, slots=True)
class DocumentPipelineResult:
"""Current synchronous upload-to-chunk result before embedding is added."""
document: Document
parsed_document: ParsedDocument
chunks: tuple[Chunk, ...]
class DocumentPipelineService:
"""Compose upload, parse/clean, and chunk stages for the current Demo scope."""
def __init__(
self,
upload_service: DocumentUploadService,
parsing_service: DocumentParsingService,
chunking_service: DocumentChunkingService,
) -> None:
self._ingestion_service = DocumentIngestionService(upload_service, parsing_service)
self._chunking_service = chunking_service
def ingest(
self,
filename: str,
media_type: str | None,
source: BinarySource,
) -> DocumentPipelineResult:
"""Run every completed Demo ingestion stage in dependency order."""
document, parsed = self._ingestion_service.ingest(filename, media_type, source)
chunks = self._chunking_service.process(document.id)
document.status = DocumentStatus.PROCESSING
return DocumentPipelineResult(document=document, parsed_document=parsed, chunks=chunks)

View File

@ -0,0 +1,34 @@
"""Basic deterministic cleanup for provider-neutral parsed documents."""
from ekp_domain import ParsedBlock, ParsedDocument
def clean_parsed_document(document: ParsedDocument) -> ParsedDocument:
"""Normalize whitespace, remove empty blocks, and clean section labels."""
cleaned_blocks: list[ParsedBlock] = []
for block in document.blocks:
text = _normalize_whitespace(block.text)
if not text:
continue
section_path = tuple(
cleaned for section in block.section_path if (cleaned := _normalize_whitespace(section))
)
cleaned_blocks.append(
ParsedBlock(
text=text,
block_type=block.block_type,
page_number=block.page_number,
section_path=section_path,
)
)
return ParsedDocument(
document_id=document.document_id,
filename=document.filename,
blocks=tuple(cleaned_blocks),
)
def _normalize_whitespace(value: str) -> str:
"""Collapse Unicode whitespace, including abnormal intra-block line breaks."""
return " ".join(value.split())

View File

@ -0,0 +1,258 @@
"""Embedding and vector-index orchestration for canonical document chunks."""
from collections.abc import Callable
from dataclasses import dataclass
from uuid import UUID, uuid5
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_chunking import DocumentPipelineResult, DocumentPipelineService
from ekp_domain import (
Chunk,
ChunkIndexState,
Document,
DocumentStatus,
IndexProfile,
IngestionStage,
)
from ekp_embedding import EmbeddingProvider
from ekp_object_storage import BinarySource
from ekp_vector_store import VectorRecord, VectorStore
MAX_INDEXING_ERROR_LENGTH = 2000
class DocumentNotFoundError(LookupError):
"""Raised when indexing is requested for an unknown document id."""
class DocumentHasNoChunksError(RuntimeError):
"""Raised when the selected chunk strategy has no canonical rows."""
class DocumentAlreadyIndexedError(RuntimeError):
"""Raised instead of overwriting an existing versioned index profile."""
class EmbeddingResultError(RuntimeError):
"""Raised when provider output violates count or dimension contracts."""
class VectorIndexRollbackError(RuntimeError):
"""Raised when failed indexing also cannot delete partial vector rows."""
class IndexingStatePersistenceError(RuntimeError):
"""Raised when an indexing failure cannot be stored in canonical metadata."""
@dataclass(frozen=True, slots=True)
class DocumentIndexingResult:
"""Summary of one completed versioned index build."""
document_id: UUID
index_profile_id: str
indexed_chunks: int
class DocumentIndexingService:
"""Embed canonical chunks and upsert a rebuildable VectorStore index."""
def __init__(
self,
session_factory: sessionmaker[Session],
embedding_provider: EmbeddingProvider,
vector_store: VectorStore,
index_profile: IndexProfile,
) -> None:
if embedding_provider.model_name != index_profile.embedding_model:
raise ValueError("embedding provider model does not match index profile")
if embedding_provider.dimensions != index_profile.vector_dimension:
raise ValueError("embedding dimensions do not match index profile")
self._session_factory = session_factory
self._embedding_provider = embedding_provider
self._vector_store = vector_store
self._index_profile = index_profile
def process(
self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> DocumentIndexingResult:
"""Embed/index chunks while optionally reporting durable Job stages."""
chunks = self._start_indexing(document_id)
upsert_attempted = False
try:
vectors = self._embed_chunks(chunks, stage_callback)
records = tuple(
VectorRecord(
index_record_id=uuid5(
chunk.chunk_id,
self._index_profile.profile_id,
),
chunk_id=chunk.chunk_id,
document_id=chunk.document_id,
filename=chunk.filename,
page=chunk.page,
page_end=chunk.page_end,
section=chunk.section,
chunk_index=chunk.chunk_index,
content=chunk.content,
embedding=vector,
embedding_model=self._index_profile.embedding_model,
index_profile_id=self._index_profile.profile_id,
)
for chunk, vector in zip(chunks, vectors, strict=True)
)
if stage_callback is not None:
stage_callback(IngestionStage.INDEXING)
upsert_attempted = True
self._vector_store.upsert(records)
self._record_success(document_id, chunks)
return DocumentIndexingResult(
document_id=document_id,
index_profile_id=self._index_profile.profile_id,
indexed_chunks=len(chunks),
)
except Exception as error:
cleanup_error: Exception | None = None
if upsert_attempted:
try:
self._vector_store.delete_document(
document_id,
self._index_profile.profile_id,
)
except Exception as caught_cleanup_error:
cleanup_error = caught_cleanup_error
self._record_failure(document_id, error)
if cleanup_error is not None:
raise VectorIndexRollbackError(
f"indexing failed and vector cleanup failed for {document_id}"
) from cleanup_error
raise
def _start_indexing(self, document_id: UUID) -> tuple[Chunk, ...]:
"""Load the selected chunk version and reject in-place profile replacement."""
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise DocumentNotFoundError(str(document_id))
chunks = tuple(
session.scalars(
select(Chunk)
.where(
Chunk.document_id == document_id,
Chunk.strategy_version == self._index_profile.chunk_strategy_version,
)
.order_by(Chunk.chunk_index)
).all()
)
if not chunks:
raise DocumentHasNoChunksError(str(document_id))
chunk_ids = [chunk.chunk_id for chunk in chunks]
existing_state = session.scalar(
select(ChunkIndexState.chunk_id)
.where(
ChunkIndexState.index_profile_id == self._index_profile.profile_id,
ChunkIndexState.chunk_id.in_(chunk_ids),
)
.limit(1)
)
if existing_state is not None:
raise DocumentAlreadyIndexedError(str(document_id))
document.status = DocumentStatus.PROCESSING
document.error_message = None
session.commit()
return chunks
def _embed_chunks(
self,
chunks: tuple[Chunk, ...],
stage_callback: Callable[[IngestionStage], None] | None,
) -> tuple[tuple[float, ...], ...]:
"""Embed bounded batches and renew an optional Job lease before each call."""
vectors: list[tuple[float, ...]] = []
batch_size = self._embedding_provider.max_batch_size
if batch_size <= 0:
raise EmbeddingResultError("embedding provider has an invalid batch size")
for start in range(0, len(chunks), batch_size):
if stage_callback is not None:
stage_callback(IngestionStage.EMBEDDING)
batch = chunks[start : start + batch_size]
embedded = self._embedding_provider.embed([chunk.content for chunk in batch])
if len(embedded) != len(batch):
raise EmbeddingResultError("embedding provider returned the wrong vector count")
if any(len(vector) != self._index_profile.vector_dimension for vector in embedded):
raise EmbeddingResultError("embedding provider returned the wrong dimensions")
vectors.extend(embedded)
return tuple(vectors)
def _record_success(self, document_id: UUID, chunks: tuple[Chunk, ...]) -> None:
"""Atomically record every indexed chunk before marking the document ready."""
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise DocumentNotFoundError(str(document_id))
session.add_all(
[
ChunkIndexState(
chunk_id=chunk.chunk_id,
index_profile_id=self._index_profile.profile_id,
embedding_model=self._index_profile.embedding_model,
vector_dimension=self._index_profile.vector_dimension,
)
for chunk in chunks
]
)
document.status = DocumentStatus.READY
document.error_message = None
session.commit()
def _record_failure(self, document_id: UUID, error: Exception) -> None:
"""Store a bounded error without exposing credentials or request bodies."""
try:
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
return
document.status = DocumentStatus.FAILED
message = str(error).strip() or type(error).__name__
document.error_message = message[:MAX_INDEXING_ERROR_LENGTH]
session.commit()
except Exception as state_error:
raise IndexingStatePersistenceError(
f"could not record indexing failure for {document_id}"
) from state_error
@dataclass(frozen=True, slots=True)
class IndexedDocumentPipelineResult:
"""Upload-to-index result for all Demo stages completed through DEMO-006."""
ingestion: DocumentPipelineResult
indexing: DocumentIndexingResult
class DocumentIndexingPipelineService:
"""Compose upload, parsing, chunking, embedding, and vector upsert."""
def __init__(
self,
document_pipeline: DocumentPipelineService,
indexing_service: DocumentIndexingService,
) -> None:
self._document_pipeline = document_pipeline
self._indexing_service = indexing_service
def ingest(
self,
filename: str,
media_type: str | None,
source: BinarySource,
) -> IndexedDocumentPipelineResult:
"""Run the complete current ingestion pipeline and return trace metadata."""
ingestion = self._document_pipeline.ingest(filename, media_type, source)
indexing = self._indexing_service.process(ingestion.document.id)
ingestion.document.status = DocumentStatus.READY
return IndexedDocumentPipelineResult(ingestion=ingestion, indexing=indexing)

View File

@ -0,0 +1,119 @@
"""Document query and direct-delete services for the Demo REST API."""
from uuid import UUID
from sqlalchemy import delete, select
from sqlalchemy.orm import Session, sessionmaker
from ekp_domain import Chunk, ChunkIndexState, Document, IngestionJob, IngestionJobStatus
from ekp_object_storage import ObjectStorage
from ekp_vector_store import VectorStore
class DocumentNotFoundError(LookupError):
"""Raised when a requested document does not exist."""
class DocumentArtifactCleanupError(RuntimeError):
"""Raised when deleted metadata leaves an external storage artifact behind."""
class DocumentIngestionActiveError(RuntimeError):
"""Raised when direct deletion would race an active asynchronous Job."""
class DocumentManagementService:
"""Read canonical documents and coordinate the Demo's direct deletion.
Vector rows are removed before canonical metadata. Object bytes are removed
after the metadata commit, so a storage failure can only leave an unreachable
orphan rather than expose a searchable row without canonical provenance.
"""
def __init__(
self,
session_factory: sessionmaker[Session],
storage: ObjectStorage,
vector_store: VectorStore,
active_index_profile_id: str,
) -> None:
if not active_index_profile_id:
raise ValueError("active_index_profile_id must not be empty")
self._session_factory = session_factory
self._storage = storage
self._vector_store = vector_store
self._active_index_profile_id = active_index_profile_id
def list_documents(self) -> tuple[Document, ...]:
"""Return documents newest first with a deterministic UUID tie-break."""
with self._session_factory() as session:
return tuple(
session.scalars(
select(Document).order_by(Document.created_at.desc(), Document.id.desc())
).all()
)
def get_document(self, document_id: UUID) -> Document:
"""Return one canonical document or raise a stable not-found error."""
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise DocumentNotFoundError(str(document_id))
return document
def delete_document(self, document_id: UUID) -> None:
"""Delete vector rows, canonical metadata, and immutable source artifacts."""
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise DocumentNotFoundError(str(document_id))
active_job_id = session.scalar(
select(IngestionJob.id)
.where(
IngestionJob.document_id == document_id,
IngestionJob.status.in_(
(
IngestionJobStatus.QUEUED,
IngestionJobStatus.RUNNING,
IngestionJobStatus.RETRYING,
)
),
)
.limit(1)
)
if active_job_id is not None:
raise DocumentIngestionActiveError(str(document_id))
source_keys = tuple(
key
for key in (document.parsed_storage_key, document.storage_key)
if key is not None
)
profile_ids = set(
session.scalars(
select(ChunkIndexState.index_profile_id)
.join(Chunk, Chunk.chunk_id == ChunkIndexState.chunk_id)
.where(Chunk.document_id == document_id)
.distinct()
).all()
)
# Also clean the active profile when an interrupted index write did
# not reach canonical ChunkIndexState persistence.
profile_ids.add(self._active_index_profile_id)
for profile_id in sorted(profile_ids):
self._vector_store.delete_document(document_id, profile_id)
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
session.execute(delete(ChunkIndexState).where(ChunkIndexState.chunk_id.in_(chunk_ids)))
session.execute(delete(Chunk).where(Chunk.document_id == document_id))
session.execute(delete(Document).where(Document.id == document_id))
session.commit()
try:
for source_key in source_keys:
if self._storage.exists(source_key):
self._storage.delete(source_key)
except Exception as error:
raise DocumentArtifactCleanupError(
f"external artifact cleanup failed for document {document_id}"
) from error

View File

@ -0,0 +1,159 @@
"""Application services for parsing and persisting derived document content."""
from collections.abc import Callable
from io import BytesIO
from uuid import UUID
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_cleaner import clean_parsed_document
from ekp_core.document_upload import DocumentUploadService
from ekp_core.parsed_document_artifact import serialize_parsed_document
from ekp_domain import Document, DocumentStatus, IngestionStage, ParsedDocument
from ekp_object_storage import BinarySource, ObjectStorage
from ekp_parser import DocumentParseError, Parser
MAX_ERROR_MESSAGE_LENGTH = 2000
class DocumentNotFoundError(LookupError):
"""Raised when parsing is requested for an unknown document id."""
class DocumentAlreadyParsedError(RuntimeError):
"""Raised when immutable parsed output already exists for a document."""
class EmptyParsedDocumentError(DocumentParseError):
"""Raised when a source contains no extractable text after cleanup."""
class ParsedResultRollbackError(RuntimeError):
"""Raised when failed metadata persistence also leaves an undeletable object."""
class ParsingStatePersistenceError(RuntimeError):
"""Raised when a parsing failure cannot be reflected in document metadata."""
class DocumentParsingService:
"""Parse one uploaded source and persist a traceable, rebuildable JSON result."""
def __init__(
self,
session_factory: sessionmaker[Session],
storage: ObjectStorage,
parser: Parser,
) -> None:
self._session_factory = session_factory
self._storage = storage
self._parser = parser
def process(
self,
document_id: UUID,
stage_callback: Callable[[IngestionStage], None] | None = None,
) -> ParsedDocument:
"""Parse and clean while optionally reporting durable Job stage boundaries."""
filename, source_key = self._start_processing(document_id)
parsed_key = f"parsed/{document_id}/document.json"
parsed_object_stored = False
try:
if stage_callback is not None:
stage_callback(IngestionStage.PARSING)
with self._storage.open(source_key) as source:
parsed = self._parser.parse(document_id, filename, source)
if stage_callback is not None:
stage_callback(IngestionStage.CLEANING)
cleaned = clean_parsed_document(parsed)
if not cleaned.blocks:
raise EmptyParsedDocumentError(
f"{filename!r} contains no extractable text after cleanup"
)
payload = serialize_parsed_document(cleaned)
self._storage.put(parsed_key, BytesIO(payload))
parsed_object_stored = True
self._record_success(document_id, parsed_key)
return cleaned
except Exception as error:
cleanup_error: Exception | None = None
if parsed_object_stored:
try:
self._storage.delete(parsed_key)
except Exception as caught_cleanup_error:
cleanup_error = caught_cleanup_error
self._record_failure(document_id, error)
if cleanup_error is not None:
raise ParsedResultRollbackError(
f"parsing failed and parsed-result cleanup failed for {parsed_key}"
) from cleanup_error
raise
def _start_processing(self, document_id: UUID) -> tuple[str, str]:
"""Validate state and durably mark a document as being processed."""
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise DocumentNotFoundError(str(document_id))
if document.parsed_storage_key is not None:
raise DocumentAlreadyParsedError(str(document_id))
document.status = DocumentStatus.PROCESSING
document.error_message = None
session.commit()
return document.filename, document.storage_key
def _record_success(self, document_id: UUID, parsed_key: str) -> None:
"""Attach parsed output while later pipeline stages remain processing."""
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
raise DocumentNotFoundError(str(document_id))
document.parsed_storage_key = parsed_key
# READY is reserved for completion of chunking and indexing.
document.status = DocumentStatus.PROCESSING
document.error_message = None
session.commit()
def _record_failure(self, document_id: UUID, error: Exception) -> None:
"""Persist a bounded diagnostic while preserving the original exception."""
try:
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
return
document.status = DocumentStatus.FAILED
message = str(error).strip() or type(error).__name__
document.error_message = message[:MAX_ERROR_MESSAGE_LENGTH]
session.commit()
except Exception as state_error:
raise ParsingStatePersistenceError(
f"could not record parsing failure for {document_id}"
) from state_error
class DocumentIngestionService:
"""Compose upload and synchronous parsing for the Demo ingestion trigger."""
def __init__(
self,
upload_service: DocumentUploadService,
parsing_service: DocumentParsingService,
) -> None:
self._upload_service = upload_service
self._parsing_service = parsing_service
def ingest(
self,
filename: str,
media_type: str | None,
source: BinarySource,
) -> tuple[Document, ParsedDocument]:
"""Persist an upload, then immediately trigger the parsing stage."""
document = self._upload_service.upload(filename, media_type, source)
parsed = self._parsing_service.process(document.id)
document.status = DocumentStatus.PROCESSING
document.parsed_storage_key = f"parsed/{document.id}/document.json"
return document, parsed

View File

@ -0,0 +1,184 @@
"""Validated, transactional source-document upload application service."""
from pathlib import Path
from uuid import UUID, uuid4
from sqlalchemy import delete
from sqlalchemy.orm import Session, sessionmaker
from ekp_domain import Document
from ekp_object_storage import BinarySource, ObjectStorage
GENERIC_MEDIA_TYPE = "application/octet-stream"
SUPPORTED_MEDIA_TYPES = {
".pdf": ("application/pdf", frozenset({"application/pdf", GENERIC_MEDIA_TYPE})),
".docx": (
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
frozenset(
{
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
GENERIC_MEDIA_TYPE,
}
),
),
".txt": ("text/plain", frozenset({"text/plain", GENERIC_MEDIA_TYPE})),
".md": (
"text/markdown",
frozenset({"text/markdown", "text/plain", GENERIC_MEDIA_TYPE}),
),
}
class DocumentUploadError(ValueError):
"""Base error for rejected source-document uploads."""
class InvalidDocumentFilenameError(DocumentUploadError):
"""Raised when a filename is blank, unsafe, or too long."""
class UnsupportedDocumentTypeError(DocumentUploadError):
"""Raised when a filename extension is outside the Demo scope."""
class MediaTypeMismatchError(DocumentUploadError):
"""Raised when the declared media type conflicts with the extension."""
class EmptyDocumentError(DocumentUploadError):
"""Raised when an uploaded source contains no bytes."""
class DocumentTooLargeError(DocumentUploadError):
"""Raised when a source exceeds the configured streaming limit."""
class UploadRollbackError(RuntimeError):
"""Raised when database failure compensation cannot remove source bytes."""
class _SizeLimitedSource:
"""Count streamed bytes and stop before an oversized object is committed."""
def __init__(self, source: BinarySource, limit_bytes: int) -> None:
self._source = source
self._limit_bytes = limit_bytes
self._consumed_bytes = 0
def read(self, size: int = -1, /) -> bytes:
remaining_with_probe = self._limit_bytes - self._consumed_bytes + 1
requested = remaining_with_probe if size < 0 else min(size, remaining_with_probe)
chunk = self._source.read(requested)
self._consumed_bytes += len(chunk)
if self._consumed_bytes > self._limit_bytes:
raise DocumentTooLargeError(
f"document exceeds the {self._limit_bytes}-byte upload limit"
)
return chunk
class DocumentUploadService:
"""Persist immutable source bytes and their canonical Document metadata."""
def __init__(
self,
session_factory: sessionmaker[Session],
storage: ObjectStorage,
max_upload_bytes: int,
) -> None:
if max_upload_bytes <= 0:
raise ValueError("max_upload_bytes must be positive")
self._session_factory = session_factory
self._storage = storage
self._max_upload_bytes = max_upload_bytes
def upload(
self,
filename: str,
media_type: str | None,
source: BinarySource,
) -> Document:
"""Validate and persist one source, compensating if metadata commit fails."""
safe_filename = _validate_filename(filename)
canonical_media_type = _resolve_media_type(safe_filename, media_type)
document_id = uuid4()
storage_key = f"documents/{document_id}/{safe_filename}"
limited_source = _SizeLimitedSource(source, self._max_upload_bytes)
with self._session_factory() as session:
stored = self._storage.put(storage_key, limited_source)
if stored.size_bytes == 0:
self._storage.delete(stored.key)
raise EmptyDocumentError("document must contain at least one byte")
try:
document = Document(
id=document_id,
filename=safe_filename,
media_type=canonical_media_type,
size_bytes=stored.size_bytes,
storage_key=stored.key,
)
session.add(document)
session.commit()
except Exception:
session.rollback()
try:
self._storage.delete(stored.key)
except Exception as cleanup_error:
raise UploadRollbackError(
f"metadata failed and source cleanup failed for {stored.key}"
) from cleanup_error
raise
return document
def rollback_acceptance(self, document_id: UUID) -> None:
"""Compensate a just-uploaded source when durable Job creation fails.
This narrow method is only valid before parsing starts; normal document
deletion remains coordinated by ``DocumentManagementService``.
"""
with self._session_factory() as session:
document = session.get(Document, document_id)
if document is None:
return
storage_key = document.storage_key
session.execute(delete(Document).where(Document.id == document.id))
session.commit()
try:
self._storage.delete(storage_key)
except Exception as cleanup_error:
raise UploadRollbackError(
f"job creation failed and source cleanup failed for {storage_key}"
) from cleanup_error
def _validate_filename(filename: str) -> str:
"""Accept a plain basename that can be traced without path ambiguity."""
if (
not filename.strip()
or len(filename) > 255
or "\x00" in filename
or "/" in filename
or "\\" in filename
):
raise InvalidDocumentFilenameError(filename)
return filename
def _resolve_media_type(filename: str, media_type: str | None) -> str:
"""Return the canonical media type after extension and declaration checks."""
# File signatures and actual parseability are validated by the DEMO-004 parser.
extension = Path(filename).suffix.lower()
type_config = SUPPORTED_MEDIA_TYPES.get(extension)
if type_config is None:
raise UnsupportedDocumentTypeError(extension or "<missing>")
canonical_media_type, accepted_media_types = type_config
declared_media_type = (media_type or GENERIC_MEDIA_TYPE).partition(";")[0].strip().lower()
if declared_media_type not in accepted_media_types:
raise MediaTypeMismatchError(
f"media type {declared_media_type!r} does not match extension {extension!r}"
)
return canonical_media_type

View File

@ -0,0 +1,279 @@
"""Retry-safe shadow index rebuild, verification, and activation services."""
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import UUID
from sqlalchemy import delete, select
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.document_indexing import DocumentIndexingService
from ekp_domain import (
Chunk,
ChunkIndexState,
IndexProfile,
IndexProfileStatus,
StoredIndexProfile,
)
from ekp_embedding import EmbeddingProvider
from ekp_vector_store import VectorStore
class IndexProfileLifecycleError(RuntimeError):
"""Raised when a shadow profile cannot safely move through its lifecycle."""
class IndexProfileMetadataConflictError(IndexProfileLifecycleError):
"""Raised when a profile ID is reused with different immutable metadata."""
class IndexProfileNotFoundError(IndexProfileLifecycleError):
"""Raised when verification or activation targets an unknown profile."""
class IndexProfileIncompleteError(IndexProfileLifecycleError):
"""Raised when activation is attempted before every canonical chunk is indexed."""
@dataclass(frozen=True, slots=True)
class IndexBuildSummary:
"""Safe progress counters for rebuild commands and deployment checks."""
profile_id: str
expected_documents: int
built_documents: int
skipped_documents: int
expected_chunks: int
indexed_chunks: int
complete: bool
class IndexRebuildService:
"""Build a new profile from canonical chunks without mutating an active index."""
def __init__(
self,
session_factory: sessionmaker[Session],
embedding_provider: EmbeddingProvider,
vector_store: VectorStore,
target_profile: IndexProfile,
) -> None:
self._session_factory = session_factory
self._vector_store = vector_store
self._target_profile = target_profile
self._indexing = DocumentIndexingService(
session_factory,
embedding_provider,
vector_store,
target_profile,
)
def rebuild(self) -> IndexBuildSummary:
"""Register and populate a shadow profile, skipping verified documents on retry."""
self._register_shadow()
document_ids = self._target_document_ids()
built_documents = 0
skipped_documents = 0
for document_id in document_ids:
if self._document_is_complete(document_id):
skipped_documents += 1
continue
self._reset_partial_document(document_id)
self._indexing.process(document_id)
built_documents += 1
verified = self.verify()
return IndexBuildSummary(
profile_id=verified.profile_id,
expected_documents=verified.expected_documents,
built_documents=built_documents,
skipped_documents=skipped_documents,
expected_chunks=verified.expected_chunks,
indexed_chunks=verified.indexed_chunks,
complete=verified.complete,
)
def verify(self) -> IndexBuildSummary:
"""Compare canonical chunks and successful index states for the target profile."""
with self._session_factory() as session:
self._require_registered_profile(session)
return self._verification_summary(session)
def activate(self) -> IndexBuildSummary:
"""Atomically retire the previous profile after target completeness is proven."""
with self._session_factory() as session:
target = self._require_registered_profile(session, for_update=True)
self._lock_canonical_chunks(session)
summary = self._verification_summary(session)
if not summary.complete:
raise IndexProfileIncompleteError(
"shadow index is incomplete and cannot be activated"
)
if target.status is IndexProfileStatus.ACTIVE:
return summary
if target.status is not IndexProfileStatus.SHADOW:
raise IndexProfileLifecycleError("only a shadow profile can be activated")
active_profiles = session.scalars(
select(StoredIndexProfile)
.where(StoredIndexProfile.status == IndexProfileStatus.ACTIVE)
.with_for_update()
).all()
for active in active_profiles:
active.status = IndexProfileStatus.RETIRED
target.status = IndexProfileStatus.ACTIVE
target.activated_at = datetime.now(UTC)
session.commit()
return summary
def _register_shadow(self) -> None:
"""Create an idempotent shadow registration without reusing active IDs."""
with self._session_factory() as session:
stored = session.get(StoredIndexProfile, self._target_profile.profile_id)
if stored is None:
session.add(
StoredIndexProfile(
profile_id=self._target_profile.profile_id,
chunk_strategy_version=self._target_profile.chunk_strategy_version,
embedding_model=self._target_profile.embedding_model,
vector_dimension=self._target_profile.vector_dimension,
status=IndexProfileStatus.SHADOW,
)
)
session.commit()
return
self._validate_stored_metadata(stored)
if stored.status is not IndexProfileStatus.SHADOW:
raise IndexProfileLifecycleError(
"rebuild target must be a shadow profile with a new profile ID"
)
def _target_document_ids(self) -> tuple[UUID, ...]:
"""List documents that have canonical chunks for the target strategy."""
with self._session_factory() as session:
return tuple(
session.scalars(
select(Chunk.document_id)
.where(Chunk.strategy_version == self._target_profile.chunk_strategy_version)
.distinct()
.order_by(Chunk.document_id)
).all()
)
def _document_is_complete(self, document_id: UUID) -> bool:
"""Return true only when every selected chunk has matching profile evidence."""
with self._session_factory() as session:
chunks = session.scalars(
select(Chunk.chunk_id).where(
Chunk.document_id == document_id,
Chunk.strategy_version == self._target_profile.chunk_strategy_version,
)
).all()
states = session.scalars(
select(ChunkIndexState).where(
ChunkIndexState.chunk_id.in_(chunks),
ChunkIndexState.index_profile_id == self._target_profile.profile_id,
)
).all()
return len(states) == len(chunks) and all(
self._state_matches_profile(state) for state in states
)
def _reset_partial_document(self, document_id: UUID) -> None:
"""Remove only target-profile remnants before retrying one document."""
self._vector_store.delete_document(document_id, self._target_profile.profile_id)
with self._session_factory() as session:
chunk_ids = select(Chunk.chunk_id).where(Chunk.document_id == document_id)
session.execute(
delete(ChunkIndexState).where(
ChunkIndexState.chunk_id.in_(chunk_ids),
ChunkIndexState.index_profile_id == self._target_profile.profile_id,
)
)
session.commit()
def _require_registered_profile(
self,
session: Session,
*,
for_update: bool = False,
) -> StoredIndexProfile:
statement = select(StoredIndexProfile).where(
StoredIndexProfile.profile_id == self._target_profile.profile_id
)
if for_update:
statement = statement.with_for_update()
stored = session.scalar(statement)
if stored is None:
raise IndexProfileNotFoundError(self._target_profile.profile_id)
self._validate_stored_metadata(stored)
return stored
def _validate_stored_metadata(self, stored: StoredIndexProfile) -> None:
expected = self._target_profile
if (
stored.chunk_strategy_version != expected.chunk_strategy_version
or stored.embedding_model != expected.embedding_model
or stored.vector_dimension != expected.vector_dimension
):
raise IndexProfileMetadataConflictError(
"index profile ID is already registered with different metadata"
)
def _verification_summary(self, session: Session) -> IndexBuildSummary:
expected_rows = session.execute(
select(Chunk.chunk_id, Chunk.document_id).where(
Chunk.strategy_version == self._target_profile.chunk_strategy_version
)
).all()
expected_chunk_ids = {row.chunk_id for row in expected_rows}
expected_document_ids = {row.document_id for row in expected_rows}
states = session.scalars(
select(ChunkIndexState).where(
ChunkIndexState.index_profile_id == self._target_profile.profile_id
)
).all()
indexed_chunk_ids = {
state.chunk_id
for state in states
if state.chunk_id in expected_chunk_ids and self._state_matches_profile(state)
}
complete = (
bool(expected_chunk_ids)
and len(states) == len(expected_chunk_ids)
and indexed_chunk_ids == expected_chunk_ids
)
return IndexBuildSummary(
profile_id=self._target_profile.profile_id,
expected_documents=len(expected_document_ids),
built_documents=0,
skipped_documents=0,
expected_chunks=len(expected_chunk_ids),
indexed_chunks=len(indexed_chunk_ids),
complete=complete,
)
@staticmethod
def _lock_canonical_chunks(session: Session) -> None:
"""Prevent canonical chunk writes between verification and activation."""
bind = session.get_bind()
if bind.dialect.name == "mysql":
session.execute(select(Chunk.chunk_id).with_for_update()).all()
def _state_matches_profile(self, state: ChunkIndexState) -> bool:
return (
state.embedding_model == self._target_profile.embedding_model
and state.vector_dimension == self._target_profile.vector_dimension
)
def resolve_active_index_profile(
session_factory: sessionmaker[Session],
fallback: IndexProfile,
) -> IndexProfile:
"""Resolve the canonical active profile, with a fresh-deployment fallback."""
with session_factory() as session:
stored = session.scalar(
select(StoredIndexProfile).where(StoredIndexProfile.status == IndexProfileStatus.ACTIVE)
)
return fallback if stored is None else stored.to_profile()

Some files were not shown because too many files have changed in this diff Show More